diff --git a/.mailmap b/.mailmap index db58eedb44f1..a35ae244dfda 100644 --- a/.mailmap +++ b/.mailmap @@ -102,6 +102,7 @@ Felipe W Damasio Felix Kuhling Felix Moeller Filipe Lautert +Finn Thain Franck Bui-Huu Frank Rowand Frank Rowand diff --git a/Documentation/ABI/obsolete/sysfs-cpuidle b/Documentation/ABI/obsolete/sysfs-cpuidle index e398fb5e542f..972cc11d3434 100644 --- a/Documentation/ABI/obsolete/sysfs-cpuidle +++ b/Documentation/ABI/obsolete/sysfs-cpuidle @@ -6,4 +6,4 @@ Description: with the update that cpuidle governor can be changed at runtime in default, both current_governor and current_governor_ro co-exist under /sys/devices/system/cpu/cpuidle/ file, it's duplicate so make - current_governor_ro obselete. + current_governor_ro obsolete. diff --git a/Documentation/ABI/removed/sysfs-kernel-uids b/Documentation/ABI/removed/sysfs-kernel-uids index dc4463f190a7..85a90b86ce1e 100644 --- a/Documentation/ABI/removed/sysfs-kernel-uids +++ b/Documentation/ABI/removed/sysfs-kernel-uids @@ -5,7 +5,7 @@ Contact: Dhaval Giani Description: The /sys/kernel/uids//cpu_shares tunable is used to set the cpu bandwidth a user is allowed. This is a - propotional value. What that means is that if there + proportional value. What that means is that if there are two users logged in, each with an equal number of shares, then they will get equal CPU bandwidth. Another example would be, if User A has shares = 1024 and user diff --git a/Documentation/ABI/stable/sysfs-bus-vmbus b/Documentation/ABI/stable/sysfs-bus-vmbus index 42599d9fa161..3066feae1d8d 100644 --- a/Documentation/ABI/stable/sysfs-bus-vmbus +++ b/Documentation/ABI/stable/sysfs-bus-vmbus @@ -61,7 +61,7 @@ Date: September. 2017 KernelVersion: 4.14 Contact: Stephen Hemminger Description: Directory for per-channel information - NN is the VMBUS relid associtated with the channel. + NN is the VMBUS relid associated with the channel. What: /sys/bus/vmbus/devices//channels//cpu Date: September. 2017 diff --git a/Documentation/ABI/stable/sysfs-bus-xen-backend b/Documentation/ABI/stable/sysfs-bus-xen-backend index e8b60bd766f7..480a89edfa05 100644 --- a/Documentation/ABI/stable/sysfs-bus-xen-backend +++ b/Documentation/ABI/stable/sysfs-bus-xen-backend @@ -19,7 +19,7 @@ Date: April 2011 KernelVersion: 3.0 Contact: Konrad Rzeszutek Wilk Description: - The major:minor number (in hexidecimal) of the + The major:minor number (in hexadecimal) of the physical device providing the storage for this backend block device. diff --git a/Documentation/ABI/stable/sysfs-devices-system-cpu b/Documentation/ABI/stable/sysfs-devices-system-cpu index 33c133e2a631..516dafea03eb 100644 --- a/Documentation/ABI/stable/sysfs-devices-system-cpu +++ b/Documentation/ABI/stable/sysfs-devices-system-cpu @@ -23,3 +23,86 @@ Description: Default value for the Data Stream Control Register (DSCR) on here). If set by a process it will be inherited by child processes. Values: 64 bit unsigned integer (bit field) + +What: /sys/devices/system/cpu/cpuX/topology/physical_package_id +Description: physical package id of cpuX. Typically corresponds to a physical + socket number, but the actual value is architecture and platform + dependent. +Values: integer + +What: /sys/devices/system/cpu/cpuX/topology/die_id +Description: the CPU die ID of cpuX. Typically it is the hardware platform's + identifier (rather than the kernel's). The actual value is + architecture and platform dependent. +Values: integer + +What: /sys/devices/system/cpu/cpuX/topology/core_id +Description: the CPU core ID of cpuX. Typically it is the hardware platform's + identifier (rather than the kernel's). The actual value is + architecture and platform dependent. +Values: integer + +What: /sys/devices/system/cpu/cpuX/topology/book_id +Description: the book ID of cpuX. Typically it is the hardware platform's + identifier (rather than the kernel's). The actual value is + architecture and platform dependent. it's only used on s390. +Values: integer + +What: /sys/devices/system/cpu/cpuX/topology/drawer_id +Description: the drawer ID of cpuX. Typically it is the hardware platform's + identifier (rather than the kernel's). The actual value is + architecture and platform dependent. it's only used on s390. +Values: integer + +What: /sys/devices/system/cpu/cpuX/topology/core_cpus +Description: internal kernel map of CPUs within the same core. + (deprecated name: "thread_siblings") +Values: hexadecimal bitmask. + +What: /sys/devices/system/cpu/cpuX/topology/core_cpus_list +Description: human-readable list of CPUs within the same core. + The format is like 0-3, 8-11, 14,17. + (deprecated name: "thread_siblings_list"). +Values: decimal list. + +What: /sys/devices/system/cpu/cpuX/topology/package_cpus +Description: internal kernel map of the CPUs sharing the same physical_package_id. + (deprecated name: "core_siblings"). +Values: hexadecimal bitmask. + +What: /sys/devices/system/cpu/cpuX/topology/package_cpus_list +Description: human-readable list of CPUs sharing the same physical_package_id. + The format is like 0-3, 8-11, 14,17. + (deprecated name: "core_siblings_list") +Values: decimal list. + +What: /sys/devices/system/cpu/cpuX/topology/die_cpus +Description: internal kernel map of CPUs within the same die. +Values: hexadecimal bitmask. + +What: /sys/devices/system/cpu/cpuX/topology/die_cpus_list +Description: human-readable list of CPUs within the same die. + The format is like 0-3, 8-11, 14,17. +Values: decimal list. + +What: /sys/devices/system/cpu/cpuX/topology/book_siblings +Description: internal kernel map of cpuX's hardware threads within the same + book_id. it's only used on s390. +Values: hexadecimal bitmask. + +What: /sys/devices/system/cpu/cpuX/topology/book_siblings_list +Description: human-readable list of cpuX's hardware threads within the same + book_id. + The format is like 0-3, 8-11, 14,17. it's only used on s390. +Values: decimal list. + +What: /sys/devices/system/cpu/cpuX/topology/drawer_siblings +Description: internal kernel map of cpuX's hardware threads within the same + drawer_id. it's only used on s390. +Values: hexadecimal bitmask. + +What: /sys/devices/system/cpu/cpuX/topology/drawer_siblings_list +Description: human-readable list of cpuX's hardware threads within the same + drawer_id. + The format is like 0-3, 8-11, 14,17. it's only used on s390. +Values: decimal list. diff --git a/Documentation/ABI/stable/sysfs-driver-dma-idxd b/Documentation/ABI/stable/sysfs-driver-dma-idxd index 55285c136cf0..d431e2d00472 100644 --- a/Documentation/ABI/stable/sysfs-driver-dma-idxd +++ b/Documentation/ABI/stable/sysfs-driver-dma-idxd @@ -173,7 +173,7 @@ What: /sys/bus/dsa/devices/wq./priority Date: Oct 25, 2019 KernelVersion: 5.6.0 Contact: dmaengine@vger.kernel.org -Description: The priority value of this work queue, it is a vlue relative to +Description: The priority value of this work queue, it is a value relative to other work queue in the same group to control quality of service for dispatching work from multiple workqueues in the same group. diff --git a/Documentation/ABI/stable/sysfs-driver-mlxreg-io b/Documentation/ABI/stable/sysfs-driver-mlxreg-io index fd9a8045bb0c..b2553df2e786 100644 --- a/Documentation/ABI/stable/sysfs-driver-mlxreg-io +++ b/Documentation/ABI/stable/sysfs-driver-mlxreg-io @@ -137,7 +137,7 @@ Contact: Vadim Pasternak Description: These files show the system reset cause, as following: COMEX thermal shutdown; wathchdog power off or reset was derived by one of the next components: COMEX, switch board or by Small Form - Factor mezzanine, reset requested from ASIC, reset cuased by BIOS + Factor mezzanine, reset requested from ASIC, reset caused by BIOS reload. Value 1 in file means this is reset cause, 0 - otherwise. Only one of the above causes could be 1 at the same time, representing only last reset cause. @@ -183,7 +183,7 @@ What: /sys/devices/platform/mlxplat/mlxreg-io/hwmon/hwmon*/vpd_wp Date: January 2020 KernelVersion: 5.6 Contact: Vadim Pasternak -Description: This file allows to overwrite system VPD hardware wrtie +Description: This file allows to overwrite system VPD hardware write protection when attribute is set 1. The file is read/write. diff --git a/Documentation/ABI/testing/configfs-iio b/Documentation/ABI/testing/configfs-iio index aebda53ec0f7..1637fcb50f56 100644 --- a/Documentation/ABI/testing/configfs-iio +++ b/Documentation/ABI/testing/configfs-iio @@ -31,4 +31,4 @@ Date: April 2016 KernelVersion: 4.7 Description: Dummy IIO devices directory. Creating a directory here will result - in creating a dummy IIO device in the IIO subystem. + in creating a dummy IIO device in the IIO subsystem. diff --git a/Documentation/ABI/testing/configfs-most b/Documentation/ABI/testing/configfs-most index bc6b8bd18da4..0a4b8649aa5a 100644 --- a/Documentation/ABI/testing/configfs-most +++ b/Documentation/ABI/testing/configfs-most @@ -20,7 +20,7 @@ Description: subbuffer_size configure the sub-buffer size for this channel - (needed for synchronous and isochrnous data) + (needed for synchronous and isochronous data) num_buffers @@ -75,7 +75,7 @@ Description: subbuffer_size configure the sub-buffer size for this channel - (needed for synchronous and isochrnous data) + (needed for synchronous and isochronous data) num_buffers @@ -130,7 +130,7 @@ Description: subbuffer_size configure the sub-buffer size for this channel - (needed for synchronous and isochrnous data) + (needed for synchronous and isochronous data) num_buffers @@ -196,7 +196,7 @@ Description: subbuffer_size configure the sub-buffer size for this channel - (needed for synchronous and isochrnous data) + (needed for synchronous and isochronous data) num_buffers diff --git a/Documentation/ABI/testing/configfs-usb-gadget b/Documentation/ABI/testing/configfs-usb-gadget index dc351e9af80a..b7943aa7e997 100644 --- a/Documentation/ABI/testing/configfs-usb-gadget +++ b/Documentation/ABI/testing/configfs-usb-gadget @@ -137,7 +137,7 @@ Description: This group contains "OS String" extension handling attributes. ============= =============================================== - use flag turning "OS Desctiptors" support on/off + use flag turning "OS Descriptors" support on/off b_vendor_code one-byte value used for custom per-device and per-interface requests qw_sign an identifier to be reported as "OS String" diff --git a/Documentation/ABI/testing/configfs-usb-gadget-uvc b/Documentation/ABI/testing/configfs-usb-gadget-uvc index ac5e11af79a8..889ed45be4ca 100644 --- a/Documentation/ABI/testing/configfs-usb-gadget-uvc +++ b/Documentation/ABI/testing/configfs-usb-gadget-uvc @@ -170,7 +170,7 @@ Description: Default color matching descriptors bMatrixCoefficients matrix used to compute luma and chroma values from the color primaries bTransferCharacteristics optoelectronic transfer - characteristic of the source picutre, + characteristic of the source picture, also called the gamma function bColorPrimaries color primaries and the reference white @@ -311,7 +311,7 @@ Description: Specific streaming header descriptors a hardware trigger interrupt event bTriggerSupport flag specifying if hardware triggering is supported - bStillCaptureMethod method of still image caputre + bStillCaptureMethod method of still image capture supported bTerminalLink id of the output terminal to which the video endpoint of this interface diff --git a/Documentation/ABI/testing/debugfs-driver-genwqe b/Documentation/ABI/testing/debugfs-driver-genwqe index 1c2f25674e8c..b45b016545d8 100644 --- a/Documentation/ABI/testing/debugfs-driver-genwqe +++ b/Documentation/ABI/testing/debugfs-driver-genwqe @@ -31,7 +31,7 @@ What: /sys/kernel/debug/genwqe/genwqe_card/prev_regs Date: Oct 2013 Contact: haver@linux.vnet.ibm.com Description: Dump of the error registers before the last reset of - the card occured. + the card occurred. Only available for PF. What: /sys/kernel/debug/genwqe/genwqe_card/prev_dbg_uid0 diff --git a/Documentation/ABI/testing/debugfs-driver-habanalabs b/Documentation/ABI/testing/debugfs-driver-habanalabs index c78fc9282876..e89c6351503c 100644 --- a/Documentation/ABI/testing/debugfs-driver-habanalabs +++ b/Documentation/ABI/testing/debugfs-driver-habanalabs @@ -153,7 +153,7 @@ KernelVersion: 5.1 Contact: ogabbay@kernel.org Description: Triggers an I2C transaction that is generated by the device's CPU. Writing to this file generates a write transaction while - reading from the file generates a read transcation + reading from the file generates a read transaction What: /sys/kernel/debug/habanalabs/hl/i2c_reg Date: Jan 2019 diff --git a/Documentation/ABI/testing/evm b/Documentation/ABI/testing/evm index 3c477ba48a31..553fd8a33e56 100644 --- a/Documentation/ABI/testing/evm +++ b/Documentation/ABI/testing/evm @@ -24,7 +24,7 @@ Description: 1 Enable digital signature validation 2 Permit modification of EVM-protected metadata at runtime. Not supported if HMAC validation and - creation is enabled. + creation is enabled (deprecated). 31 Disable further runtime modification of EVM policy === ================================================== @@ -47,10 +47,38 @@ Description: will enable digital signature validation, permit modification of EVM-protected metadata and - disable all further modification of policy + disable all further modification of policy. This option is now + deprecated in favor of:: - Note that once a key has been loaded, it will no longer be - possible to enable metadata modification. + echo 0x80000002 >/evm + + as the outstanding issues that prevent the usage of EVM portable + signatures have been solved. + + Echoing a value is additive, the new value is added to the + existing initialization flags. + + For example, after:: + + echo 2 >/evm + + another echo can be performed:: + + echo 1 >/evm + + and the resulting value will be 3. + + Note that once an HMAC key has been loaded, it will no longer + be possible to enable metadata modification. Signaling that an + HMAC key has been loaded will clear the corresponding flag. + For example, if the current value is 6 (2 and 4 set):: + + echo 1 >/evm + + will set the new value to 3 (4 cleared). + + Loading an HMAC key is the only way to disable metadata + modification. Until key loading has been signaled EVM can not create or validate the 'security.evm' xattr, but returns diff --git a/Documentation/ABI/testing/sysfs-bus-fsi b/Documentation/ABI/testing/sysfs-bus-fsi index d148214181a1..76e0caa0c2b3 100644 --- a/Documentation/ABI/testing/sysfs-bus-fsi +++ b/Documentation/ABI/testing/sysfs-bus-fsi @@ -12,7 +12,7 @@ KernelVersion: 4.12 Contact: linux-fsi@lists.ozlabs.org Description: Sends an FSI BREAK command on a master's communication - link to any connnected slaves. A BREAK resets connected + link to any connected slaves. A BREAK resets connected device's logic and preps it to receive further commands from the master. diff --git a/Documentation/ABI/testing/sysfs-bus-iio b/Documentation/ABI/testing/sysfs-bus-iio index 267973541e72..6f98b6a9b785 100644 --- a/Documentation/ABI/testing/sysfs-bus-iio +++ b/Documentation/ABI/testing/sysfs-bus-iio @@ -786,7 +786,7 @@ What: /sys/.../events/in_capacitanceY_adaptive_thresh_rising_en What: /sys/.../events/in_capacitanceY_adaptive_thresh_falling_en KernelVersion: 5.13 Contact: linux-iio@vger.kernel.org -Descrption: +Description: Adaptive thresholds are similar to normal fixed thresholds but the value is expressed as an offset from a value which provides a low frequency approximation of the channel itself. @@ -798,10 +798,10 @@ What: /sys/.../in_capacitanceY_adaptive_thresh_rising_timeout What: /sys/.../in_capacitanceY_adaptive_thresh_falling_timeout KernelVersion: 5.11 Contact: linux-iio@vger.kernel.org -Descrption: +Description: When adaptive thresholds are used, the tracking signal may adjust too slowly to step changes in the raw signal. - *_timeout (in seconds) specifies a time for which the + Thus these specify the time in seconds for which the difference between the slow tracking signal and the raw signal is allowed to remain out-of-range before a reset event occurs in which the tracking signal is made equal diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci index ef00fada2efb..793cbb76cd25 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci +++ b/Documentation/ABI/testing/sysfs-bus-pci @@ -139,8 +139,8 @@ Description: binary file containing the Vital Product Data for the device. It should follow the VPD format defined in PCI Specification 2.1 or 2.2, but users should consider - that some devices may have malformatted data. If the - underlying VPD has a writable section then the + that some devices may have incorrectly formatted data. + If the underlying VPD has a writable section then the corresponding section of this file will be writable. What: /sys/bus/pci/devices/.../virtfnN diff --git a/Documentation/ABI/testing/sysfs-class-backlight b/Documentation/ABI/testing/sysfs-class-backlight index 1fc86401bf95..c453646b06e2 100644 --- a/Documentation/ABI/testing/sysfs-class-backlight +++ b/Documentation/ABI/testing/sysfs-class-backlight @@ -84,3 +84,103 @@ Description: It can be enabled by writing the value stored in /sys/class/backlight//max_brightness to /sys/class/backlight//brightness. + +What: /sys/class/backlight//_max +Date: Sep, 2009 +KernelVersion: v2.6.32 +Contact: device-drivers-devel@blackfin.uclinux.org +Description: + Control the maximum brightness for + on this . Values are between 0 and 127. This file + will also show the brightness level stored for this + . + + The is device-driver specific: + + For ADP5520 and ADP5501, can be: + + =========== ================================================ + Ambient sysfs entry + light zone + =========== ================================================ + daylight /sys/class/backlight//daylight_max + office /sys/class/backlight//office_max + dark /sys/class/backlight//dark_max + =========== ================================================ + + For ADP8860, can be: + + =========== ================================================ + Ambient sysfs entry + light zone + =========== ================================================ + l1_daylight /sys/class/backlight//l1_daylight_max + l2_office /sys/class/backlight//l2_office_max + l3_dark /sys/class/backlight//l3_dark_max + =========== ================================================ + + For ADP8870, can be: + + =========== ================================================ + Ambient sysfs entry + light zone + =========== ================================================ + l1_daylight /sys/class/backlight//l1_daylight_max + l2_bright /sys/class/backlight//l2_bright_max + l3_office /sys/class/backlight//l3_office_max + l4_indoor /sys/class/backlight//l4_indoor_max + l5_dark /sys/class/backlight//l5_dark_max + =========== ================================================ + + See also: /sys/class/backlight//ambient_light_zone. + +What: /sys/class/backlight//_dim +Date: Sep, 2009 +KernelVersion: v2.6.32 +Contact: device-drivers-devel@blackfin.uclinux.org +Description: + Control the dim brightness for + on this . Values are between 0 and 127, typically + set to 0. Full off when the backlight is disabled. + This file will also show the dim brightness level stored for + this . + + The is device-driver specific: + + For ADP5520 and ADP5501, can be: + + =========== ================================================ + Ambient sysfs entry + light zone + =========== ================================================ + daylight /sys/class/backlight//daylight_dim + office /sys/class/backlight//office_dim + dark /sys/class/backlight//dark_dim + =========== ================================================ + + For ADP8860, can be: + + =========== ================================================ + Ambient sysfs entry + light zone + =========== ================================================ + l1_daylight /sys/class/backlight//l1_daylight_dim + l2_office /sys/class/backlight//l2_office_dim + l3_dark /sys/class/backlight//l3_dark_dim + =========== ================================================ + + For ADP8870, can be: + + =========== ================================================ + Ambient sysfs entry + light zone + =========== ================================================ + l1_daylight /sys/class/backlight//l1_daylight_dim + l2_bright /sys/class/backlight//l2_bright_dim + l3_office /sys/class/backlight//l3_office_dim + l4_indoor /sys/class/backlight//l4_indoor_dim + l5_dark /sys/class/backlight//l5_dark_dim + =========== ================================================ + + See also: /sys/class/backlight//ambient_light_zone. + diff --git a/Documentation/ABI/testing/sysfs-class-backlight-adp5520 b/Documentation/ABI/testing/sysfs-class-backlight-adp5520 deleted file mode 100644 index 34b6ebafa210..000000000000 --- a/Documentation/ABI/testing/sysfs-class-backlight-adp5520 +++ /dev/null @@ -1,31 +0,0 @@ -sysfs interface for analog devices adp5520(01) backlight driver ---------------------------------------------------------------- - -The backlight brightness control operates at three different levels for the -adp5520 and adp5501 devices: daylight (level 1), office (level 2) and dark -(level 3). By default the brightness operates at the daylight brightness level. - -What: /sys/class/backlight//daylight_max -What: /sys/class/backlight//office_max -What: /sys/class/backlight//dark_max -Date: Sep, 2009 -KernelVersion: v2.6.32 -Contact: Michael Hennerich -Description: - (RW) Maximum current setting for the backlight when brightness - is at one of the three levels (daylight, office or dark). This - is an input code between 0 and 127, which is transformed to a - value between 0 mA and 30 mA using linear or non-linear - algorithms. - -What: /sys/class/backlight//daylight_dim -What: /sys/class/backlight//office_dim -What: /sys/class/backlight//dark_dim -Date: Sep, 2009 -KernelVersion: v2.6.32 -Contact: Michael Hennerich -Description: - (RW) Dim current setting for the backlight when brightness is at - one of the three levels (daylight, office or dark). This is an - input code between 0 and 127, which is transformed to a value - between 0 mA and 30 mA using linear or non-linear algorithms. diff --git a/Documentation/ABI/testing/sysfs-class-backlight-adp8860 b/Documentation/ABI/testing/sysfs-class-backlight-adp8860 deleted file mode 100644 index 6610ac73f9ba..000000000000 --- a/Documentation/ABI/testing/sysfs-class-backlight-adp8860 +++ /dev/null @@ -1,37 +0,0 @@ -sysfs interface for analog devices adp8860 backlight driver ------------------------------------------------------------ - -The backlight brightness control operates at three different levels for the -adp8860, adp8861 and adp8863 devices: daylight (level 1), office (level 2) and -dark (level 3). By default the brightness operates at the daylight brightness -level. - -See also /sys/class/backlight//ambient_light_level and -/sys/class/backlight//ambient_light_zone. - - -What: /sys/class/backlight//l1_daylight_max -What: /sys/class/backlight//l2_office_max -What: /sys/class/backlight//l3_dark_max -Date: Apr, 2010 -KernelVersion: v2.6.35 -Contact: Michael Hennerich -Description: - (RW) Maximum current setting for the backlight when brightness - is at one of the three levels (daylight, office or dark). This - is an input code between 0 and 127, which is transformed to a - value between 0 mA and 30 mA using linear or non-linear - algorithms. - - -What: /sys/class/backlight//l1_daylight_dim -What: /sys/class/backlight//l2_office_dim -What: /sys/class/backlight//l3_dark_dim -Date: Apr, 2010 -KernelVersion: v2.6.35 -Contact: Michael Hennerich -Description: - (RW) Dim current setting for the backlight when brightness is at - one of the three levels (daylight, office or dark). This is an - input code between 0 and 127, which is transformed to a value - between 0 mA and 30 mA using linear or non-linear algorithms. diff --git a/Documentation/ABI/testing/sysfs-class-backlight-driver-adp8870 b/Documentation/ABI/testing/sysfs-class-backlight-driver-adp8870 deleted file mode 100644 index b08ca912cad4..000000000000 --- a/Documentation/ABI/testing/sysfs-class-backlight-driver-adp8870 +++ /dev/null @@ -1,32 +0,0 @@ -See also /sys/class/backlight//ambient_light_level and -/sys/class/backlight//ambient_light_zone. - -What: /sys/class/backlight//_max -What: /sys/class/backlight//l1_daylight_max -What: /sys/class/backlight//l2_bright_max -What: /sys/class/backlight//l3_office_max -What: /sys/class/backlight//l4_indoor_max -What: /sys/class/backlight//l5_dark_max -Date: May 2011 -KernelVersion: 3.0 -Contact: device-drivers-devel@blackfin.uclinux.org -Description: - Control the maximum brightness for - on this . Values are between 0 and 127. This file - will also show the brightness level stored for this - . - -What: /sys/class/backlight//_dim -What: /sys/class/backlight//l2_bright_dim -What: /sys/class/backlight//l3_office_dim -What: /sys/class/backlight//l4_indoor_dim -What: /sys/class/backlight//l5_dark_dim -Date: May 2011 -KernelVersion: 3.0 -Contact: device-drivers-devel@blackfin.uclinux.org -Description: - Control the dim brightness for - on this . Values are between 0 and 127, typically - set to 0. Full off when the backlight is disabled. - This file will also show the dim brightness level stored for - this . diff --git a/Documentation/ABI/testing/sysfs-class-led-driver-el15203000 b/Documentation/ABI/testing/sysfs-class-led-driver-el15203000 deleted file mode 100644 index 04f3ffdc5936..000000000000 --- a/Documentation/ABI/testing/sysfs-class-led-driver-el15203000 +++ /dev/null @@ -1,9 +0,0 @@ -What: /sys/class/leds//repeat -Date: September 2019 -KernelVersion: 5.5 -Description: - EL15203000 supports only indefinitely patterns, - so this file should always store -1. - - For more info, please see: - Documentation/ABI/testing/sysfs-class-led-trigger-pattern diff --git a/Documentation/ABI/testing/sysfs-class-led-trigger-pattern b/Documentation/ABI/testing/sysfs-class-led-trigger-pattern index d91a07767adf..8c57d2780554 100644 --- a/Documentation/ABI/testing/sysfs-class-led-trigger-pattern +++ b/Documentation/ABI/testing/sysfs-class-led-trigger-pattern @@ -35,3 +35,6 @@ Description: This file will always return the originally written repeat number. + + It should be noticed that some leds, like EL15203000 may + only support indefinitely patterns, so they always store -1. diff --git a/Documentation/ABI/testing/sysfs-devices-system-cpu b/Documentation/ABI/testing/sysfs-devices-system-cpu index fe13baa53c59..160b10c029c0 100644 --- a/Documentation/ABI/testing/sysfs-devices-system-cpu +++ b/Documentation/ABI/testing/sysfs-devices-system-cpu @@ -50,7 +50,7 @@ Description: Dynamic addition and removal of CPU's. This is not hotplug architecture specific. release: writes to this file dynamically remove a CPU from - the system. Information writtento the file to remove CPU's + the system. Information written to the file to remove CPU's is architecture specific. What: /sys/devices/system/cpu/cpu#/node @@ -97,7 +97,7 @@ Description: CPU topology files that describe a logical CPU's relationship corresponds to a physical socket number, but the actual value is architecture and platform dependent. - thread_siblings: internel kernel map of cpu#'s hardware + thread_siblings: internal kernel map of cpu#'s hardware threads within the same core as cpu# thread_siblings_list: human-readable list of cpu#'s hardware @@ -280,7 +280,7 @@ Description: Disable L3 cache indices on a processor with this functionality will return the currently disabled index for that node. There is one L3 structure per node, or per internal node on MCM machines. Writing a valid - index to one of these files will cause the specificed cache + index to one of these files will cause the specified cache index to be disabled. All AMD processors with L3 caches provide this functionality. @@ -295,7 +295,7 @@ Description: Processor frequency boosting control This switch controls the boost setting for the whole system. Boosting allows the CPU and the firmware to run at a frequency - beyound it's nominal limit. + beyond it's nominal limit. More details can be found in Documentation/admin-guide/pm/cpufreq.rst @@ -532,7 +532,7 @@ What: /sys/devices/system/cpu/smt /sys/devices/system/cpu/smt/control Date: June 2018 Contact: Linux kernel mailing list -Description: Control Symetric Multi Threading (SMT) +Description: Control Symmetric Multi Threading (SMT) active: Tells whether SMT is active (enabled and siblings online) diff --git a/Documentation/ABI/testing/sysfs-driver-ufs b/Documentation/ABI/testing/sysfs-driver-ufs index d1bc23cb6a9d..eaac6898f0c0 100644 --- a/Documentation/ABI/testing/sysfs-driver-ufs +++ b/Documentation/ABI/testing/sysfs-driver-ufs @@ -168,7 +168,7 @@ Description: This file shows the manufacturing date in BCD format. What: /sys/bus/platform/drivers/ufshcd/*/device_descriptor/manufacturer_id Date: February 2018 Contact: Stanislav Nijnikov -Description: This file shows the manufacturee ID. This is one of the +Description: This file shows the manufacturer ID. This is one of the UFS device descriptor parameters. The full information about the descriptor could be found at UFS specifications 2.1. @@ -521,7 +521,7 @@ Description: This file shows maximum VCC, VCCQ and VCCQ2 value for What: /sys/bus/platform/drivers/ufshcd/*/string_descriptors/manufacturer_name Date: February 2018 Contact: Stanislav Nijnikov -Description: This file contains a device manufactureer name string. +Description: This file contains a device manufacturer name string. The full information about the descriptor could be found at UFS specifications 2.1. diff --git a/Documentation/ABI/testing/sysfs-fs-f2fs b/Documentation/ABI/testing/sysfs-fs-f2fs index 4849b8e84e42..5d9ae27bd462 100644 --- a/Documentation/ABI/testing/sysfs-fs-f2fs +++ b/Documentation/ABI/testing/sysfs-fs-f2fs @@ -238,7 +238,7 @@ Description: Shows current reserved blocks in system, it may be temporarily What: /sys/fs/f2fs//gc_urgent Date: August 2017 Contact: "Jaegeuk Kim" -Description: Do background GC agressively when set. When gc_urgent = 1, +Description: Do background GC aggressively when set. When gc_urgent = 1, background thread starts to do GC by given gc_urgent_sleep_time interval. When gc_urgent = 2, F2FS will lower the bar of checking idle in order to process outstanding discard commands diff --git a/Documentation/ABI/testing/sysfs-kernel-iommu_groups b/Documentation/ABI/testing/sysfs-kernel-iommu_groups index 0fedbb0f94e4..eae2f1c1e11e 100644 --- a/Documentation/ABI/testing/sysfs-kernel-iommu_groups +++ b/Documentation/ABI/testing/sysfs-kernel-iommu_groups @@ -25,14 +25,10 @@ Description: /sys/kernel/iommu_groups/reserved_regions list IOVA the base IOVA, the second is the end IOVA and the third field describes the type of the region. -What: /sys/kernel/iommu_groups/reserved_regions -Date: June 2019 -KernelVersion: v5.3 -Contact: Eric Auger -Description: In case an RMRR is used only by graphics or USB devices - it is now exposed as "direct-relaxable" instead of "direct". - In device assignment use case, for instance, those RMRR - are considered to be relaxable and safe. + Since kernel 5.3, in case an RMRR is used only by graphics or + USB devices it is now exposed as "direct-relaxable" instead + of "direct". In device assignment use case, for instance, + those RMRR are considered to be relaxable and safe. What: /sys/kernel/iommu_groups//type Date: November 2020 diff --git a/Documentation/Makefile b/Documentation/Makefile index 9c42dde97671..c3feb657b654 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -76,7 +76,7 @@ quiet_cmd_sphinx = SPHINX $@ --> file://$(abspath $(BUILDDIR)/$3/$4) PYTHONDONTWRITEBYTECODE=1 \ BUILDDIR=$(abspath $(BUILDDIR)) SPHINX_CONF=$(abspath $(srctree)/$(src)/$5/$(SPHINX_CONF)) \ $(PYTHON3) $(srctree)/scripts/jobserver-exec \ - $(SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \ + $(CONFIG_SHELL) $(srctree)/Documentation/sphinx/parallel-wrapper.sh \ $(SPHINXBUILD) \ -b $2 \ -c $(abspath $(srctree)/$(src)) \ diff --git a/Documentation/PCI/acpi-info.rst b/Documentation/PCI/acpi-info.rst index 060217081c79..34c64a5a66ec 100644 --- a/Documentation/PCI/acpi-info.rst +++ b/Documentation/PCI/acpi-info.rst @@ -22,9 +22,9 @@ or if the device has INTx interrupts connected by platform interrupt controllers and a _PRT is needed to describe those connections. ACPI resource description is done via _CRS objects of devices in the ACPI -namespace [2].   The _CRS is like a generalized PCI BAR: the OS can read +namespace [2]. The _CRS is like a generalized PCI BAR: the OS can read _CRS and figure out what resource is being consumed even if it doesn't have -a driver for the device [3].  That's important because it means an old OS +a driver for the device [3]. That's important because it means an old OS can work correctly even on a system with new devices unknown to the OS. The new devices might not do anything, but the OS can at least make sure no resources conflict with them. @@ -41,15 +41,15 @@ ACPI, that device will have a specific _HID/_CID that tells the OS what driver to bind to it, and the _CRS tells the OS and the driver where the device's registers are. -PCI host bridges are PNP0A03 or PNP0A08 devices.  Their _CRS should -describe all the address space they consume.  This includes all the windows +PCI host bridges are PNP0A03 or PNP0A08 devices. Their _CRS should +describe all the address space they consume. This includes all the windows they forward down to the PCI bus, as well as registers of the host bridge -itself that are not forwarded to PCI.  The host bridge registers include +itself that are not forwarded to PCI. The host bridge registers include things like secondary/subordinate bus registers that determine the bus range below the bridge, window registers that describe the apertures, etc. These are all device-specific, non-architected things, so the only way a PNP0A03/PNP0A08 driver can manage them is via _PRS/_CRS/_SRS, which contain -the device-specific details.  The host bridge registers also include ECAM +the device-specific details. The host bridge registers also include ECAM space, since it is consumed by the host bridge. ACPI defines a Consumer/Producer bit to distinguish the bridge registers @@ -66,7 +66,7 @@ the PNP0A03/PNP0A08 device itself. The workaround was to describe the bridge registers (including ECAM space) in PNP0C02 catch-all devices [6]. With the exception of ECAM, the bridge register space is device-specific anyway, so the generic PNP0A03/PNP0A08 driver (pci_root.c) has no need to -know about it.   +know about it. New architectures should be able to use "Consumer" Extended Address Space descriptors in the PNP0A03 device for bridge registers, including ECAM, @@ -75,9 +75,9 @@ ia64 kernels assume all address space descriptors, including "Consumer" Extended Address Space ones, are windows, so it would not be safe to describe bridge registers this way on those architectures. -PNP0C02 "motherboard" devices are basically a catch-all.  There's no +PNP0C02 "motherboard" devices are basically a catch-all. There's no programming model for them other than "don't use these resources for -anything else."  So a PNP0C02 _CRS should claim any address space that is +anything else." So a PNP0C02 _CRS should claim any address space that is (1) not claimed by _CRS under any other device object in the ACPI namespace and (2) should not be assigned by the OS to something else. diff --git a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst index 696f8eeb4738..db609b97ad58 100644 --- a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst +++ b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst @@ -125,4 +125,4 @@ all the EPF devices are created and linked with the EPC device. | interrupt_pin | function -[1] :doc:`pci-endpoint` +[1] Documentation/PCI/endpoint/pci-endpoint.rst diff --git a/Documentation/PCI/pci.rst b/Documentation/PCI/pci.rst index 814b40f8360b..fa651e25d98c 100644 --- a/Documentation/PCI/pci.rst +++ b/Documentation/PCI/pci.rst @@ -265,7 +265,7 @@ Set the DMA mask size --------------------- .. note:: If anything below doesn't make sense, please refer to - :doc:`/core-api/dma-api`. This section is just a reminder that + Documentation/core-api/dma-api.rst. This section is just a reminder that drivers need to indicate DMA capabilities of the device and is not an authoritative source for DMA interfaces. @@ -291,7 +291,7 @@ Many 64-bit "PCI" devices (before PCI-X) and some PCI-X devices are Setup shared control data ------------------------- Once the DMA masks are set, the driver can allocate "consistent" (a.k.a. shared) -memory. See :doc:`/core-api/dma-api` for a full description of +memory. See Documentation/core-api/dma-api.rst for a full description of the DMA APIs. This section is just a reminder that it needs to be done before enabling DMA on the device. @@ -421,7 +421,7 @@ owners if there is one. Then clean up "consistent" buffers which contain the control data. -See :doc:`/core-api/dma-api` for details on unmapping interfaces. +See Documentation/core-api/dma-api.rst for details on unmapping interfaces. Unregister from other subsystems diff --git a/Documentation/admin-guide/cputopology.rst b/Documentation/admin-guide/cputopology.rst index b90dafcc8237..8632a1db36e4 100644 --- a/Documentation/admin-guide/cputopology.rst +++ b/Documentation/admin-guide/cputopology.rst @@ -2,87 +2,10 @@ How CPU topology info is exported via sysfs =========================================== -Export CPU topology info via sysfs. Items (attributes) are similar -to /proc/cpuinfo output of some architectures. They reside in -/sys/devices/system/cpu/cpuX/topology/: - -physical_package_id: - - physical package id of cpuX. Typically corresponds to a physical - socket number, but the actual value is architecture and platform - dependent. - -die_id: - - the CPU die ID of cpuX. Typically it is the hardware platform's - identifier (rather than the kernel's). The actual value is - architecture and platform dependent. - -core_id: - - the CPU core ID of cpuX. Typically it is the hardware platform's - identifier (rather than the kernel's). The actual value is - architecture and platform dependent. - -book_id: - - the book ID of cpuX. Typically it is the hardware platform's - identifier (rather than the kernel's). The actual value is - architecture and platform dependent. - -drawer_id: - - the drawer ID of cpuX. Typically it is the hardware platform's - identifier (rather than the kernel's). The actual value is - architecture and platform dependent. - -core_cpus: - - internal kernel map of CPUs within the same core. - (deprecated name: "thread_siblings") - -core_cpus_list: - - human-readable list of CPUs within the same core. - (deprecated name: "thread_siblings_list"); - -package_cpus: - - internal kernel map of the CPUs sharing the same physical_package_id. - (deprecated name: "core_siblings") - -package_cpus_list: - - human-readable list of CPUs sharing the same physical_package_id. - (deprecated name: "core_siblings_list") - -die_cpus: - - internal kernel map of CPUs within the same die. - -die_cpus_list: - - human-readable list of CPUs within the same die. - -book_siblings: - - internal kernel map of cpuX's hardware threads within the same - book_id. - -book_siblings_list: - - human-readable list of cpuX's hardware threads within the same - book_id. - -drawer_siblings: - - internal kernel map of cpuX's hardware threads within the same - drawer_id. - -drawer_siblings_list: - - human-readable list of cpuX's hardware threads within the same - drawer_id. +CPU topology info is exported via sysfs. Items (attributes) are similar +to /proc/cpuinfo output of some architectures. They reside in +/sys/devices/system/cpu/cpuX/topology/. Please refer to the ABI file: +Documentation/ABI/stable/sysfs-devices-system-cpu. Architecture-neutral, drivers/base/topology.c, exports these attributes. However, the book and drawer related sysfs files will only be created if diff --git a/Documentation/admin-guide/ext4.rst b/Documentation/admin-guide/ext4.rst index d2795ca6821e..4c559e08d11e 100644 --- a/Documentation/admin-guide/ext4.rst +++ b/Documentation/admin-guide/ext4.rst @@ -392,7 +392,7 @@ When mounting an ext4 filesystem, the following option are accepted: dax Use direct access (no page cache). See - Documentation/filesystems/dax.txt. Note that this option is + Documentation/filesystems/dax.rst. Note that this option is incompatible with data=journal. inlinecrypt diff --git a/Documentation/admin-guide/hw-vuln/special-register-buffer-data-sampling.rst b/Documentation/admin-guide/hw-vuln/special-register-buffer-data-sampling.rst index 3b1ce68d2456..966c9b3296ea 100644 --- a/Documentation/admin-guide/hw-vuln/special-register-buffer-data-sampling.rst +++ b/Documentation/admin-guide/hw-vuln/special-register-buffer-data-sampling.rst @@ -3,7 +3,8 @@ SRBDS - Special Register Buffer Data Sampling ============================================= -SRBDS is a hardware vulnerability that allows MDS :doc:`mds` techniques to +SRBDS is a hardware vulnerability that allows MDS +Documentation/admin-guide/hw-vuln/mds.rst techniques to infer values returned from special register accesses. Special register accesses are accesses to off core registers. According to Intel's evaluation, the special register reads that have a security expectation of privacy are diff --git a/Documentation/admin-guide/kdump/kdump.rst b/Documentation/admin-guide/kdump/kdump.rst index 75a9dd98e76e..cb30ca3df27c 100644 --- a/Documentation/admin-guide/kdump/kdump.rst +++ b/Documentation/admin-guide/kdump/kdump.rst @@ -2,7 +2,7 @@ Documentation for Kdump - The kexec-based Crash Dumping Solution ================================================================ -This document includes overview, setup and installation, and analysis +This document includes overview, setup, installation, and analysis information. Overview @@ -13,9 +13,9 @@ dump of the system kernel's memory needs to be taken (for example, when the system panics). The system kernel's memory image is preserved across the reboot and is accessible to the dump-capture kernel. -You can use common commands, such as cp and scp, to copy the -memory image to a dump file on the local disk, or across the network to -a remote system. +You can use common commands, such as cp, scp or makedumpfile to copy +the memory image to a dump file on the local disk, or across the network +to a remote system. Kdump and kexec are currently supported on the x86, x86_64, ppc64, ia64, s390x, arm and arm64 architectures. @@ -26,13 +26,15 @@ the dump-capture kernel. This ensures that ongoing Direct Memory Access The kexec -p command loads the dump-capture kernel into this reserved memory. -On x86 machines, the first 640 KB of physical memory is needed to boot, -regardless of where the kernel loads. Therefore, kexec backs up this -region just before rebooting into the dump-capture kernel. +On x86 machines, the first 640 KB of physical memory is needed for boot, +regardless of where the kernel loads. For simpler handling, the whole +low 1M is reserved to avoid any later kernel or device driver writing +data into this area. Like this, the low 1M can be reused as system RAM +by kdump kernel without extra handling. -Similarly on PPC64 machines first 32KB of physical memory is needed for -booting regardless of where the kernel is loaded and to support 64K page -size kexec backs up the first 64KB memory. +On PPC64 machines first 32KB of physical memory is needed for booting +regardless of where the kernel is loaded and to support 64K page size +kexec backs up the first 64KB memory. For s390x, when kdump is triggered, the crashkernel region is exchanged with the region [0, crashkernel region size] and then the kdump kernel @@ -46,14 +48,14 @@ passed to the dump-capture kernel through the elfcorehdr= boot parameter. Optionally the size of the ELF header can also be passed when using the elfcorehdr=[size[KMG]@]offset[KMG] syntax. - With the dump-capture kernel, you can access the memory image through /proc/vmcore. This exports the dump as an ELF-format file that you can -write out using file copy commands such as cp or scp. Further, you can -use analysis tools such as the GNU Debugger (GDB) and the Crash tool to -debug the dump file. This method ensures that the dump pages are correctly -ordered. - +write out using file copy commands such as cp or scp. You can also use +makedumpfile utility to analyze and write out filtered contents with +options, e.g with '-d 31' it will only write out kernel data. Further, +you can use analysis tools such as the GNU Debugger (GDB) and the Crash +tool to debug the dump file. This method ensures that the dump pages are +correctly ordered. Setup and Installation ====================== @@ -125,9 +127,18 @@ dump-capture kernels for enabling kdump support. System kernel config options ---------------------------- -1) Enable "kexec system call" in "Processor type and features.":: +1) Enable "kexec system call" or "kexec file based system call" in + "Processor type and features.":: - CONFIG_KEXEC=y + CONFIG_KEXEC=y or CONFIG_KEXEC_FILE=y + + And both of them will select KEXEC_CORE:: + + CONFIG_KEXEC_CORE=y + + Subsequently, CRASH_CORE is selected by KEXEC_CORE:: + + CONFIG_CRASH_CORE=y 2) Enable "sysfs file system support" in "Filesystem" -> "Pseudo filesystems." This is usually enabled by default:: @@ -175,17 +186,19 @@ Dump-capture kernel config options (Arch Dependent, i386 and x86_64) CONFIG_HIGHMEM4G -2) On i386 and x86_64, disable symmetric multi-processing support - under "Processor type and features":: +2) With CONFIG_SMP=y, usually nr_cpus=1 need specified on the kernel + command line when loading the dump-capture kernel because one + CPU is enough for kdump kernel to dump vmcore on most of systems. - CONFIG_SMP=n + However, you can also specify nr_cpus=X to enable multiple processors + in kdump kernel. In this case, "disable_cpu_apicid=" is needed to + tell kdump kernel which cpu is 1st kernel's BSP. Please refer to + admin-guide/kernel-parameters.txt for more details. - (If CONFIG_SMP=y, then specify maxcpus=1 on the kernel command line - when loading the dump-capture kernel, see section "Load the Dump-capture - Kernel".) + With CONFIG_SMP=n, the above things are not related. -3) If one wants to build and use a relocatable kernel, - Enable "Build a relocatable kernel" support under "Processor type and +3) A relocatable kernel is suggested to be built by default. If not yet, + enable "Build a relocatable kernel" support under "Processor type and features":: CONFIG_RELOCATABLE=y @@ -232,7 +245,7 @@ Dump-capture kernel config options (Arch Dependent, ia64) as a dump-capture kernel if desired. The crashkernel region can be automatically placed by the system - kernel at run time. This is done by specifying the base address as 0, + kernel at runtime. This is done by specifying the base address as 0, or omitting it all together:: crashkernel=256M@0 @@ -241,10 +254,6 @@ Dump-capture kernel config options (Arch Dependent, ia64) crashkernel=256M - If the start address is specified, note that the start address of the - kernel will be aligned to 64Mb, so if the start address is not then - any space below the alignment point will be wasted. - Dump-capture kernel config options (Arch Dependent, arm) ---------------------------------------------------------- @@ -260,46 +269,82 @@ Dump-capture kernel config options (Arch Dependent, arm64) on non-VHE systems even if it is configured. This is because the CPU will not be reset to EL2 on panic. -Extended crashkernel syntax +crashkernel syntax =========================== +1) crashkernel=size@offset -While the "crashkernel=size[@offset]" syntax is sufficient for most -configurations, sometimes it's handy to have the reserved memory dependent -on the value of System RAM -- that's mostly for distributors that pre-setup -the kernel command line to avoid a unbootable system after some memory has -been removed from the machine. - -The syntax is:: - - crashkernel=:[,:,...][@offset] - range=start-[end] - -For example:: - - crashkernel=512M-2G:64M,2G-:128M - -This would mean: - - 1) if the RAM is smaller than 512M, then don't reserve anything - (this is the "rescue" case) - 2) if the RAM size is between 512M and 2G (exclusive), then reserve 64M - 3) if the RAM size is larger than 2G, then reserve 128M - - - -Boot into System Kernel -======================= - -1) Update the boot loader (such as grub, yaboot, or lilo) configuration - files as necessary. - -2) Boot the system kernel with the boot parameter "crashkernel=Y@X", - where Y specifies how much memory to reserve for the dump-capture kernel - and X specifies the beginning of this reserved memory. For example, + Here 'size' specifies how much memory to reserve for the dump-capture kernel + and 'offset' specifies the beginning of this reserved memory. For example, "crashkernel=64M@16M" tells the system kernel to reserve 64 MB of memory starting at physical address 0x01000000 (16MB) for the dump-capture kernel. - On x86 and x86_64, use "crashkernel=64M@16M". + The crashkernel region can be automatically placed by the system + kernel at run time. This is done by specifying the base address as 0, + or omitting it all together:: + + crashkernel=256M@0 + + or:: + + crashkernel=256M + + If the start address is specified, note that the start address of the + kernel will be aligned to a value (which is Arch dependent), so if the + start address is not then any space below the alignment point will be + wasted. + +2) range1:size1[,range2:size2,...][@offset] + + While the "crashkernel=size[@offset]" syntax is sufficient for most + configurations, sometimes it's handy to have the reserved memory dependent + on the value of System RAM -- that's mostly for distributors that pre-setup + the kernel command line to avoid a unbootable system after some memory has + been removed from the machine. + + The syntax is:: + + crashkernel=:[,:,...][@offset] + range=start-[end] + + For example:: + + crashkernel=512M-2G:64M,2G-:128M + + This would mean: + + 1) if the RAM is smaller than 512M, then don't reserve anything + (this is the "rescue" case) + 2) if the RAM size is between 512M and 2G (exclusive), then reserve 64M + 3) if the RAM size is larger than 2G, then reserve 128M + +3) crashkernel=size,high and crashkernel=size,low + + If memory above 4G is preferred, crashkernel=size,high can be used to + fulfill that. With it, physical memory is allowed to be allocated from top, + so could be above 4G if system has more than 4G RAM installed. Otherwise, + memory region will be allocated below 4G if available. + + When crashkernel=X,high is passed, kernel could allocate physical memory + region above 4G, low memory under 4G is needed in this case. There are + three ways to get low memory: + + 1) Kernel will allocate at least 256M memory below 4G automatically + if crashkernel=Y,low is not specified. + 2) Let user specify low memory size instead. + 3) Specified value 0 will disable low memory allocation:: + + crashkernel=0,low + +Boot into System Kernel +----------------------- +1) Update the boot loader (such as grub, yaboot, or lilo) configuration + files as necessary. + +2) Boot the system kernel with the boot parameter "crashkernel=Y@X". + + On x86 and x86_64, use "crashkernel=Y[@X]". Most of the time, the + start address 'X' is not necessary, kernel will search a suitable + area. Unless an explicit start address is expected. On ppc64, use "crashkernel=128M@32M". @@ -331,8 +376,8 @@ of dump-capture kernel. Following is the summary. For i386 and x86_64: - - Use vmlinux if kernel is not relocatable. - Use bzImage/vmlinuz if kernel is relocatable. + - Use vmlinux if kernel is not relocatable. For ppc64: @@ -392,7 +437,7 @@ loading dump-capture kernel. For i386, x86_64 and ia64: - "1 irqpoll maxcpus=1 reset_devices" + "1 irqpoll nr_cpus=1 reset_devices" For ppc64: @@ -400,7 +445,7 @@ For ppc64: For s390x: - "1 maxcpus=1 cgroup_disable=memory" + "1 nr_cpus=1 cgroup_disable=memory" For arm: @@ -408,7 +453,7 @@ For arm: For arm64: - "1 maxcpus=1 reset_devices" + "1 nr_cpus=1 reset_devices" Notes on loading the dump-capture kernel: @@ -488,6 +533,10 @@ the following command:: cp /proc/vmcore +You can also use makedumpfile utility to write out the dump file +with specified options to filter out unwanted contents, e.g:: + + makedumpfile -l --message-level 1 -d 31 /proc/vmcore Analysis ======== @@ -535,8 +584,7 @@ This will cause a kdump to occur at the add_taint()->panic() call. Contact ======= -- Vivek Goyal (vgoyal@redhat.com) -- Maneesh Soni (maneesh@in.ibm.com) +- kexec@lists.infradead.org GDB macros ========== diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index eb85ec270023..9cd0352c2feb 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -3522,6 +3522,9 @@ nr_uarts= [SERIAL] maximum number of UARTs to be registered. + numa=off [KNL, ARM64, PPC, RISCV, SPARC, X86] Disable NUMA, Only + set up a single NUMA node spanning all memory. + numa_balancing= [KNL,ARM64,PPC,RISCV,S390,X86] Enable or disable automatic NUMA balancing. Allowed values are enable and disable @@ -4784,11 +4787,6 @@ Reserves a hole at the top of the kernel virtual address space. - reservelow= [X86] - Format: nn[K] - Set the amount of memory to reserve for BIOS at - the bottom of the address space. - reset_devices [KNL] Force drivers to reset the underlying device during initialization. @@ -5292,6 +5290,14 @@ exception. Default behavior is by #AC if both features are enabled in hardware. + ratelimit:N - + Set system wide rate limit to N bus locks + per second for bus lock detection. + 0 < N <= 1000. + + N/A for split lock detection. + + If an #AC exception is hit in the kernel or in firmware (i.e. not while executing in user mode) the kernel will oops in either "warn" or "fatal" diff --git a/Documentation/admin-guide/media/bt8xx.rst b/Documentation/admin-guide/media/bt8xx.rst index 1382ada1e38e..3589f6ab7e46 100644 --- a/Documentation/admin-guide/media/bt8xx.rst +++ b/Documentation/admin-guide/media/bt8xx.rst @@ -15,11 +15,12 @@ Authors: General information ------------------- -This class of cards has a bt878a as the PCI interface, and require the bttv driver -for accessing the i2c bus and the gpio pins of the bt8xx chipset. +This class of cards has a bt878a as the PCI interface, and require the bttv +driver for accessing the i2c bus and the gpio pins of the bt8xx chipset. -Please see :doc:`bttv-cardlist` for a complete list of Cards based on the -Conexant Bt8xx PCI bridge supported by the Linux Kernel. +Please see Documentation/admin-guide/media/bttv-cardlist.rst for a complete +list of Cards based on the Conexant Bt8xx PCI bridge supported by the +Linux Kernel. In order to be able to compile the kernel, some config options should be enabled:: @@ -80,7 +81,7 @@ for dvb-bt8xx drivers by passing modprobe parameters may be necessary. Running TwinHan and Clones ~~~~~~~~~~~~~~~~~~~~~~~~~~ -As shown at :doc:`bttv-cardlist`, TwinHan and +As shown at Documentation/admin-guide/media/bttv-cardlist.rst, TwinHan and clones use ``card=113`` modprobe parameter. So, in order to properly detect it for devices without EEPROM, you should use:: @@ -105,12 +106,12 @@ The autodetected values are determined by the cards' "response string". In your logs see f. ex.: dst_get_device_id: Recognize [DSTMCI]. For bug reports please send in a complete log with verbose=4 activated. -Please also see :doc:`ci`. +Please also see Documentation/admin-guide/media/ci.rst. Running multiple cards ~~~~~~~~~~~~~~~~~~~~~~ -See :doc:`bttv-cardlist` for a complete list of +See Documentation/admin-guide/media/bttv-cardlist.rst for a complete list of Card ID. Some examples: =========================== === diff --git a/Documentation/admin-guide/media/bttv.rst b/Documentation/admin-guide/media/bttv.rst index 0ef1f203104d..125f6f47123d 100644 --- a/Documentation/admin-guide/media/bttv.rst +++ b/Documentation/admin-guide/media/bttv.rst @@ -24,7 +24,8 @@ If your board has digital TV, you'll also need:: ./scripts/config -m DVB_BT8XX -In this case, please see :doc:`bt8xx` for additional notes. +In this case, please see Documentation/admin-guide/media/bt8xx.rst +for additional notes. Make bttv work with your card ----------------------------- @@ -39,7 +40,7 @@ If it doesn't bttv likely could not autodetect your card and needs some insmod options. The most important insmod option for bttv is "card=n" to select the correct card type. If you get video but no sound you've very likely specified the wrong (or no) card type. A list of supported -cards is in :doc:`bttv-cardlist`. +cards is in Documentation/admin-guide/media/bttv-cardlist.rst. If bttv takes very long to load (happens sometimes with the cheap cards which have no tuner), try adding this to your modules configuration @@ -57,8 +58,8 @@ directory should be enough for it to be autoload during the driver's probing mode (e. g. when the Kernel boots or when the driver is manually loaded via ``modprobe`` command). -If your card isn't listed in :doc:`bttv-cardlist` or if you have -trouble making audio work, please read :ref:`still_doesnt_work`. +If your card isn't listed in Documentation/admin-guide/media/bttv-cardlist.rst +or if you have trouble making audio work, please read :ref:`still_doesnt_work`. Autodetecting cards @@ -77,8 +78,8 @@ the Subsystem ID in the second line, looks like this: only bt878-based cards can have a subsystem ID (which does not mean that every card really has one). bt848 cards can't have a Subsystem ID and therefore can't be autodetected. There is a list with the ID's -at :doc:`bttv-cardlist` (in case you are interested or want to mail -patches with updates). +at Documentation/admin-guide/media/bttv-cardlist.rst +(in case you are interested or want to mail patches with updates). .. _still_doesnt_work: @@ -259,15 +260,15 @@ bug. It is very helpful if you can tell where exactly it broke With a hard freeze you probably doesn't find anything in the logfiles. The only way to capture any kernel messages is to hook up a serial console and let some terminal application log the messages. /me uses -screen. See :doc:`/admin-guide/serial-console` for details on setting -up a serial console. +screen. See Documentation/admin-guide/serial-console.rst for details on +setting up a serial console. -Read :doc:`/admin-guide/bug-hunting` to learn how to get any useful +Read Documentation/admin-guide/bug-hunting.rst to learn how to get any useful information out of a register+stack dump printed by the kernel on protection faults (so-called "kernel oops"). If you run into some kind of deadlock, you can try to dump a call trace -for each process using sysrq-t (see :doc:`/admin-guide/sysrq`). +for each process using sysrq-t (see Documentation/admin-guide/sysrq.rst). This way it is possible to figure where *exactly* some process in "D" state is stuck. diff --git a/Documentation/admin-guide/media/index.rst b/Documentation/admin-guide/media/index.rst index 6e0d2bae7154..c676af665111 100644 --- a/Documentation/admin-guide/media/index.rst +++ b/Documentation/admin-guide/media/index.rst @@ -11,12 +11,14 @@ its supported drivers. Please see: -- :doc:`/userspace-api/media/index` - for the userspace APIs used on media devices. +Documentation/userspace-api/media/index.rst -- :doc:`/driver-api/media/index` - for driver development information and Kernel APIs used by - media devices; + - for the userspace APIs used on media devices. + +Documentation/driver-api/media/index.rst + + - for driver development information and Kernel APIs used by + media devices; The media subsystem =================== diff --git a/Documentation/admin-guide/media/ipu3.rst b/Documentation/admin-guide/media/ipu3.rst index f59697c7b374..52c1c04173da 100644 --- a/Documentation/admin-guide/media/ipu3.rst +++ b/Documentation/admin-guide/media/ipu3.rst @@ -234,22 +234,23 @@ The IPU3 ImgU pipelines can be configured using the Media Controller, defined at Running mode and firmware binary selection ------------------------------------------ -ImgU works based on firmware, currently the ImgU firmware support run 2 pipes in -time-sharing with single input frame data. Each pipe can run at certain mode - -"VIDEO" or "STILL", "VIDEO" mode is commonly used for video frames capture, and -"STILL" is used for still frame capture. However, you can also select "VIDEO" to -capture still frames if you want to capture images with less system load and -power. For "STILL" mode, ImgU will try to use smaller BDS factor and output -larger bayer frame for further YUV processing than "VIDEO" mode to get high -quality images. Besides, "STILL" mode need XNR3 to do noise reduction, hence -"STILL" mode will need more power and memory bandwidth than "VIDEO" mode. TNR -will be enabled in "VIDEO" mode and bypassed by "STILL" mode. ImgU is running at -“VIDEO” mode by default, the user can use v4l2 control V4L2_CID_INTEL_IPU3_MODE -(currently defined in drivers/staging/media/ipu3/include/intel-ipu3.h) to query -and set the running mode. For user, there is no difference for buffer queueing -between the "VIDEO" and "STILL" mode, mandatory input and main output node -should be enabled and buffers need be queued, the statistics and the view-finder -queues are optional. +ImgU works based on firmware, currently the ImgU firmware support run 2 pipes +in time-sharing with single input frame data. Each pipe can run at certain mode +- "VIDEO" or "STILL", "VIDEO" mode is commonly used for video frames capture, +and "STILL" is used for still frame capture. However, you can also select +"VIDEO" to capture still frames if you want to capture images with less system +load and power. For "STILL" mode, ImgU will try to use smaller BDS factor and +output larger bayer frame for further YUV processing than "VIDEO" mode to get +high quality images. Besides, "STILL" mode need XNR3 to do noise reduction, +hence "STILL" mode will need more power and memory bandwidth than "VIDEO" mode. +TNR will be enabled in "VIDEO" mode and bypassed by "STILL" mode. ImgU is +running at "VIDEO" mode by default, the user can use v4l2 control +V4L2_CID_INTEL_IPU3_MODE (currently defined in +drivers/staging/media/ipu3/include/uapi/intel-ipu3.h) to query and set the +running mode. For user, there is no difference for buffer queueing between the +"VIDEO" and "STILL" mode, mandatory input and main output node should be +enabled and buffers need be queued, the statistics and the view-finder queues +are optional. The firmware binary will be selected according to current running mode, such log "using binary if_to_osys_striped " or "using binary if_to_osys_primary_striped" @@ -586,7 +587,7 @@ preserved. References ========== -.. [#f5] drivers/staging/media/ipu3/include/intel-ipu3.h +.. [#f5] drivers/staging/media/ipu3/include/uapi/intel-ipu3.h .. [#f1] https://github.com/intel/nvt diff --git a/Documentation/admin-guide/media/saa7134.rst b/Documentation/admin-guide/media/saa7134.rst index 7ab9c70b9abe..51eae7eb5ab7 100644 --- a/Documentation/admin-guide/media/saa7134.rst +++ b/Documentation/admin-guide/media/saa7134.rst @@ -50,7 +50,8 @@ To build and install, you should run:: Once the new Kernel is booted, saa7134 driver should be loaded automatically. Depending on the card you might have to pass ``card=`` as insmod option. -If so, please check :doc:`saa7134-cardlist` for valid choices. +If so, please check Documentation/admin-guide/media/saa7134-cardlist.rst +for valid choices. Once you have your card type number, you can pass a modules configuration via a file (usually, it is either ``/etc/modules.conf`` or some file at diff --git a/Documentation/admin-guide/pm/intel_idle.rst b/Documentation/admin-guide/pm/intel_idle.rst index 89309e1b0e48..b799a43da62e 100644 --- a/Documentation/admin-guide/pm/intel_idle.rst +++ b/Documentation/admin-guide/pm/intel_idle.rst @@ -20,8 +20,8 @@ Nehalem and later generations of Intel processors, but the level of support for a particular processor model in it depends on whether or not it recognizes that processor model and may also depend on information coming from the platform firmware. [To understand ``intel_idle`` it is necessary to know how ``CPUIdle`` -works in general, so this is the time to get familiar with :doc:`cpuidle` if you -have not done that yet.] +works in general, so this is the time to get familiar with +Documentation/admin-guide/pm/cpuidle.rst if you have not done that yet.] ``intel_idle`` uses the ``MWAIT`` instruction to inform the processor that the logical CPU executing it is idle and so it may be possible to put some of the @@ -53,7 +53,8 @@ processor) corresponding to them depends on the processor model and it may also depend on the configuration of the platform. In order to create a list of available idle states required by the ``CPUIdle`` -subsystem (see :ref:`idle-states-representation` in :doc:`cpuidle`), +subsystem (see :ref:`idle-states-representation` in +Documentation/admin-guide/pm/cpuidle.rst), ``intel_idle`` can use two sources of information: static tables of idle states for different processor models included in the driver itself and the ACPI tables of the system. The former are always used if the processor model at hand is @@ -98,7 +99,8 @@ states may not be enabled by default if there are no matching entries in the preliminary list of idle states coming from the ACPI tables. In that case user space still can enable them later (on a per-CPU basis) with the help of the ``disable`` idle state attribute in ``sysfs`` (see -:ref:`idle-states-representation` in :doc:`cpuidle`). This basically means that +:ref:`idle-states-representation` in +Documentation/admin-guide/pm/cpuidle.rst). This basically means that the idle states "known" to the driver may not be enabled by default if they have not been exposed by the platform firmware (through the ACPI tables). @@ -186,7 +188,8 @@ be desirable. In practice, it is only really necessary to do that if the idle states in question cannot be enabled during system startup, because in the working state of the system the CPU power management quality of service (PM QoS) feature can be used to prevent ``CPUIdle`` from touching those idle states -even if they have been enumerated (see :ref:`cpu-pm-qos` in :doc:`cpuidle`). +even if they have been enumerated (see :ref:`cpu-pm-qos` in +Documentation/admin-guide/pm/cpuidle.rst). Setting ``max_cstate`` to 0 causes the ``intel_idle`` initialization to fail. The ``no_acpi`` and ``use_acpi`` module parameters (recognized by ``intel_idle`` @@ -202,7 +205,8 @@ Namely, the positions of the bits that are set in the ``states_off`` value are the indices of idle states to be disabled by default (as reflected by the names of the corresponding idle state directories in ``sysfs``, :file:`state0`, :file:`state1` ... :file:`state` ..., where ```` is the index of the given -idle state; see :ref:`idle-states-representation` in :doc:`cpuidle`). +idle state; see :ref:`idle-states-representation` in +Documentation/admin-guide/pm/cpuidle.rst). For example, if ``states_off`` is equal to 3, the driver will disable idle states 0 and 1 by default, and if it is equal to 8, idle state 3 will be diff --git a/Documentation/admin-guide/pm/intel_pstate.rst b/Documentation/admin-guide/pm/intel_pstate.rst index df29b4f1f219..7a7d4b041eac 100644 --- a/Documentation/admin-guide/pm/intel_pstate.rst +++ b/Documentation/admin-guide/pm/intel_pstate.rst @@ -18,8 +18,8 @@ General Information (``CPUFreq``). It is a scaling driver for the Sandy Bridge and later generations of Intel processors. Note, however, that some of those processors may not be supported. [To understand ``intel_pstate`` it is necessary to know -how ``CPUFreq`` works in general, so this is the time to read :doc:`cpufreq` if -you have not done that yet.] +how ``CPUFreq`` works in general, so this is the time to read +Documentation/admin-guide/pm/cpufreq.rst if you have not done that yet.] For the processors supported by ``intel_pstate``, the P-state concept is broader than just an operating frequency or an operating performance point (see the @@ -445,8 +445,9 @@ Interpretation of Policy Attributes ----------------------------------- The interpretation of some ``CPUFreq`` policy attributes described in -:doc:`cpufreq` is special with ``intel_pstate`` as the current scaling driver -and it generally depends on the driver's `operation mode `_. +Documentation/admin-guide/pm/cpufreq.rst is special with ``intel_pstate`` +as the current scaling driver and it generally depends on the driver's +`operation mode `_. First of all, the values of the ``cpuinfo_max_freq``, ``cpuinfo_min_freq`` and ``scaling_cur_freq`` attributes are produced by applying a processor-specific diff --git a/Documentation/admin-guide/pstore-blk.rst b/Documentation/admin-guide/pstore-blk.rst index 49d8149f8d32..2d22ead9520e 100644 --- a/Documentation/admin-guide/pstore-blk.rst +++ b/Documentation/admin-guide/pstore-blk.rst @@ -45,15 +45,18 @@ blkdev The block device to use. Most of the time, it is a partition of block device. It's required for pstore/blk. It is also used for MTD device. -It accepts the following variants for block device: +When pstore/blk is built as a module, "blkdev" accepts the following variants: -1. device number in hexadecimal represents itself; no - leading 0x, for example b302. -#. /dev/ represents the device number of disk +1. /dev/ represents the device number of disk #. /dev/ represents the device number of partition - device number of disk plus the partition number #. /dev/p - same as the above; this form is used when disk name of partitioned disk ends with a digit. + +When pstore/blk is built into the kernel, "blkdev" accepts the following variants: + +#. device number in hexadecimal representation, + with no leading 0x, for example b302. #. PARTUUID=00112233-4455-6677-8899-AABBCCDDEEFF represents the unique id of a partition if the partition table provides it. The UUID may be either an EFI/GPT UUID, or refer to an MSDOS partition using the format SSSSSSSS-PP, @@ -227,8 +230,5 @@ For developer reference, here are all the important structures and APIs: .. kernel-doc:: include/linux/pstore_zone.h :internal: -.. kernel-doc:: fs/pstore/blk.c - :internal: - .. kernel-doc:: include/linux/pstore_blk.h :internal: diff --git a/Documentation/admin-guide/reporting-issues.rst b/Documentation/admin-guide/reporting-issues.rst index 18d8e25ba9df..d7ac13f789cc 100644 --- a/Documentation/admin-guide/reporting-issues.rst +++ b/Documentation/admin-guide/reporting-issues.rst @@ -1248,7 +1248,7 @@ paragraph makes the severeness obvious. In case you performed a successful bisection, use the title of the change that introduced the regression as the second part of your subject. Make the report -also mention the commit id of the culprit. In case of an unsuccessful bisection, +also mention the commit id of the culprit. In case of an unsuccessful bisection, make your report mention the latest tested version that's working fine (say 5.7) and the oldest where the issue occurs (say 5.8-rc1). diff --git a/Documentation/admin-guide/sysctl/abi.rst b/Documentation/admin-guide/sysctl/abi.rst index 77b1d1b2ad42..4e6db0a2a4c0 100644 --- a/Documentation/admin-guide/sysctl/abi.rst +++ b/Documentation/admin-guide/sysctl/abi.rst @@ -11,7 +11,7 @@ Documentation for /proc/sys/abi/ Copyright (c) 2020, Stephen Kitt -For general info, see :doc:`index`. +For general info, see Documentation/admin-guide/sysctl/index.rst. ------------------------------------------------------------------------------ diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index 0ef05750dadc..10dd4b111e5c 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -9,7 +9,8 @@ Copyright (c) 1998, 1999, Rik van Riel Copyright (c) 2009, Shen Feng -For general info and legal blurb, please look in :doc:`index`. +For general info and legal blurb, please look in +Documentation/admin-guide/sysctl/index.rst. ------------------------------------------------------------------------------ @@ -54,7 +55,7 @@ free space valid for 30 seconds. acpi_video_flags ================ -See :doc:`/power/video`. This allows the video resume mode to be set, +See Documentation/power/video.rst. This allows the video resume mode to be set, in a similar fashion to the ``acpi_sleep`` kernel parameter, by combining the following values: @@ -89,7 +90,7 @@ is 0x15 and the full version number is 0x234, this file will contain the value 340 = 0x154. See the ``type_of_loader`` and ``ext_loader_type`` fields in -:doc:`/x86/boot` for additional information. +Documentation/x86/boot.rst for additional information. bootloader_version (x86 only) @@ -99,7 +100,7 @@ The complete bootloader version number. In the example above, this file will contain the value 564 = 0x234. See the ``type_of_loader`` and ``ext_loader_ver`` fields in -:doc:`/x86/boot` for additional information. +Documentation/x86/boot.rst for additional information. bpf_stats_enabled @@ -269,7 +270,7 @@ see the ``hostname(1)`` man page. firmware_config =============== -See :doc:`/driver-api/firmware/fallback-mechanisms`. +See Documentation/driver-api/firmware/fallback-mechanisms.rst. The entries in this directory allow the firmware loader helper fallback to be controlled: @@ -297,7 +298,7 @@ crashes and outputting them to a serial console. ftrace_enabled, stack_tracer_enabled ==================================== -See :doc:`/trace/ftrace`. +See Documentation/trace/ftrace.rst. hardlockup_all_cpu_backtrace @@ -325,7 +326,7 @@ when a hard lockup is detected. 1 Panic on hard lockup. = =========================== -See :doc:`/admin-guide/lockup-watchdogs` for more information. +See Documentation/admin-guide/lockup-watchdogs.rst for more information. This can also be set using the nmi_watchdog kernel parameter. @@ -333,7 +334,12 @@ hotplug ======= Path for the hotplug policy agent. -Default value is "``/sbin/hotplug``". +Default value is ``CONFIG_UEVENT_HELPER_PATH``, which in turn defaults +to the empty string. + +This file only exists when ``CONFIG_UEVENT_HELPER`` is enabled. Most +modern systems rely exclusively on the netlink-based uevent source and +don't need this. hung_task_all_cpu_backtrace @@ -582,7 +588,8 @@ in a KVM virtual machine. This default can be overridden by adding:: nmi_watchdog=1 -to the guest kernel command line (see :doc:`/admin-guide/kernel-parameters`). +to the guest kernel command line (see +Documentation/admin-guide/kernel-parameters.rst). numa_balancing @@ -1067,7 +1074,7 @@ that support this feature. real-root-dev ============= -See :doc:`/admin-guide/initrd`. +See Documentation/admin-guide/initrd.rst. reboot-cmd (SPARC only) @@ -1161,7 +1168,7 @@ will take effect. seccomp ======= -See :doc:`/userspace-api/seccomp_filter`. +See Documentation/userspace-api/seccomp_filter.rst. sg-big-buff @@ -1332,7 +1339,7 @@ the boot PROM. sysrq ===== -See :doc:`/admin-guide/sysrq`. +See Documentation/admin-guide/sysrq.rst. tainted @@ -1362,15 +1369,16 @@ ORed together. The letters are seen in "Tainted" line of Oops reports. 131072 `(T)` The kernel was built with the struct randomization plugin ====== ===== ============================================================== -See :doc:`/admin-guide/tainted-kernels` for more information. +See Documentation/admin-guide/tainted-kernels.rst for more information. Note: writes to this sysctl interface will fail with ``EINVAL`` if the kernel is booted with the command line option ``panic_on_taint=,nousertaint`` and any of the ORed together values being written to ``tainted`` match with the bitmask declared on panic_on_taint. - See :doc:`/admin-guide/kernel-parameters` for more details on that particular - kernel command line option and its optional ``nousertaint`` switch. + See Documentation/admin-guide/kernel-parameters.rst for more details on + that particular kernel command line option and its optional + ``nousertaint`` switch. threads-max =========== @@ -1394,7 +1402,7 @@ If a value outside of this range is written to ``threads-max`` an traceoff_on_warning =================== -When set, disables tracing (see :doc:`/trace/ftrace`) when a +When set, disables tracing (see Documentation/trace/ftrace.rst) when a ``WARN()`` is hit. @@ -1414,8 +1422,8 @@ will send them to printk() again. This only works if the kernel was booted with ``tp_printk`` enabled. -See :doc:`/admin-guide/kernel-parameters` and -:doc:`/trace/boottime-trace`. +See Documentation/admin-guide/kernel-parameters.rst and +Documentation/trace/boottime-trace.rst. .. _unaligned-dump-stack: diff --git a/Documentation/arm/marvell.rst b/Documentation/arm/marvell.rst index c50be711ec72..db2246493d18 100644 --- a/Documentation/arm/marvell.rst +++ b/Documentation/arm/marvell.rst @@ -259,7 +259,7 @@ Storage family https://web.archive.org/web/20191129073953/http://www.marvell.com/storage/armada-sp/ Core: - Sheeva ARMv7 comatible Quad-core PJ4C + Sheeva ARMv7 compatible Quad-core PJ4C (not supported in upstream Linux kernel) diff --git a/Documentation/arm64/booting.rst b/Documentation/arm64/booting.rst index 18b8cc1bf32c..a9192e7a231b 100644 --- a/Documentation/arm64/booting.rst +++ b/Documentation/arm64/booting.rst @@ -277,6 +277,12 @@ Before jumping into the kernel, the following conditions must be met: - SCR_EL3.FGTEn (bit 27) must be initialised to 0b1. + For CPUs with support for HCRX_EL2 (FEAT_HCX) present: + + - If EL3 is present and the kernel is entered at EL2: + + - SCR_EL3.HXEn (bit 38) must be initialised to 0b1. + For CPUs with Advanced SIMD and floating point support: - If EL3 is present: diff --git a/Documentation/block/biodoc.rst b/Documentation/block/biodoc.rst index 1d4d71e391af..2098477851a4 100644 --- a/Documentation/block/biodoc.rst +++ b/Documentation/block/biodoc.rst @@ -196,7 +196,7 @@ a virtual address mapping (unlike the earlier scheme of virtual address do not have a corresponding kernel virtual address space mapping) and low-memory pages. -Note: Please refer to :doc:`/core-api/dma-api-howto` for a discussion +Note: Please refer to Documentation/core-api/dma-api-howto.rst for a discussion on PCI high mem DMA aspects and mapping of scatter gather lists, and support for 64 bit PCI. diff --git a/Documentation/block/blk-mq.rst b/Documentation/block/blk-mq.rst index a980d23af48c..d96118c73954 100644 --- a/Documentation/block/blk-mq.rst +++ b/Documentation/block/blk-mq.rst @@ -62,7 +62,7 @@ queue, to be sent in the future, when the hardware is able. Software staging queues ~~~~~~~~~~~~~~~~~~~~~~~ -The block IO subsystem adds requests in the software staging queues +The block IO subsystem adds requests in the software staging queues (represented by struct blk_mq_ctx) in case that they weren't sent directly to the driver. A request is one or more BIOs. They arrived at the block layer through the data structure struct bio. The block layer @@ -132,7 +132,7 @@ In order to indicate which request has been completed, every request is identified by an integer, ranging from 0 to the dispatch queue size. This tag is generated by the block layer and later reused by the device driver, removing the need to create a redundant identifier. When a request is completed in the -drive, the tag is sent back to the block layer to notify it of the finalization. +driver, the tag is sent back to the block layer to notify it of the finalization. This removes the need to do a linear search to find out which IO has been completed. diff --git a/Documentation/block/stat.rst b/Documentation/block/stat.rst index 77311335c08b..a1cd9db2058f 100644 --- a/Documentation/block/stat.rst +++ b/Documentation/block/stat.rst @@ -18,7 +18,7 @@ A. each, it would be impossible to guarantee that a set of readings represent a single point in time. -The stat file consists of a single line of text containing 11 decimal +The stat file consists of a single line of text containing 17 decimal values separated by whitespace. The fields are summarized in the following table, and described in more detail below. diff --git a/Documentation/bpf/bpf_lsm.rst b/Documentation/bpf/bpf_lsm.rst index 1c0a75a51d79..0dc3fb0d9544 100644 --- a/Documentation/bpf/bpf_lsm.rst +++ b/Documentation/bpf/bpf_lsm.rst @@ -20,10 +20,10 @@ LSM hook: Other LSM hooks which can be instrumented can be found in ``include/linux/lsm_hooks.h``. -eBPF programs that use :doc:`/bpf/btf` do not need to include kernel headers -for accessing information from the attached eBPF program's context. They can -simply declare the structures in the eBPF program and only specify the fields -that need to be accessed. +eBPF programs that use Documentation/bpf/btf.rst do not need to include kernel +headers for accessing information from the attached eBPF program's context. +They can simply declare the structures in the eBPF program and only specify +the fields that need to be accessed. .. code-block:: c @@ -88,8 +88,9 @@ example: The ``__attribute__((preserve_access_index))`` is a clang feature that allows the BPF verifier to update the offsets for the access at runtime using the -:doc:`/bpf/btf` information. Since the BPF verifier is aware of the types, it -also validates all the accesses made to the various types in the eBPF program. +Documentation/bpf/btf.rst information. Since the BPF verifier is aware of the +types, it also validates all the accesses made to the various types in the +eBPF program. Loading ------- diff --git a/Documentation/conf.py b/Documentation/conf.py index 879e86dbea66..7d92ec3e5b6e 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -41,15 +41,7 @@ extensions = ['kerneldoc', 'rstFlatTable', 'kernel_include', 'maintainers_include', 'sphinx.ext.autosectionlabel', 'kernel_abi', 'kernel_feat'] -# -# cdomain is badly broken in Sphinx 3+. Leaving it out generates *most* -# of the docs correctly, but not all. Scream bloody murder but allow -# the process to proceed; hopefully somebody will fix this properly soon. -# if major >= 3: - sys.stderr.write('''WARNING: The kernel documentation build process - support for Sphinx v3.0 and above is brand new. Be prepared for - possible issues in the generated output.\n''') if (major > 3) or (minor > 0 or patch >= 2): # Sphinx c function parser is more pedantic with regards to type # checking. Due to that, having macros at c:function cause problems. @@ -353,6 +345,8 @@ latex_elements = { # Additional stuff for the LaTeX preamble. 'preamble': ''' + % Prevent column squeezing of tabulary. + \\setlength{\\tymin}{20em} % Use some font with UTF-8 support with XeLaTeX \\usepackage{fontspec} \\setsansfont{DejaVu Sans} @@ -366,11 +360,23 @@ latex_elements = { cjk_cmd = check_output(['fc-list', '--format="%{family[0]}\n"']).decode('utf-8', 'ignore') if cjk_cmd.find("Noto Sans CJK SC") >= 0: - print ("enabling CJK for LaTeX builder") latex_elements['preamble'] += ''' % This is needed for translations \\usepackage{xeCJK} \\setCJKmainfont{Noto Sans CJK SC} + % Define custom macros to on/off CJK + \\newcommand{\\kerneldocCJKon}{\\makexeCJKactive} + \\newcommand{\\kerneldocCJKoff}{\\makexeCJKinactive} + % To customize \sphinxtableofcontents + \\usepackage{etoolbox} + % Inactivate CJK after tableofcontents + \\apptocmd{\\sphinxtableofcontents}{\\kerneldocCJKoff}{}{} + ''' +else: + latex_elements['preamble'] += ''' + % Custom macros to on/off CJK (Dummy) + \\newcommand{\\kerneldocCJKon}{} + \\newcommand{\\kerneldocCJKoff}{} ''' # Fix reference escape troubles with Sphinx 1.4.x diff --git a/Documentation/core-api/bus-virt-phys-mapping.rst b/Documentation/core-api/bus-virt-phys-mapping.rst index c7bc99cd2e21..c72b24a7d52c 100644 --- a/Documentation/core-api/bus-virt-phys-mapping.rst +++ b/Documentation/core-api/bus-virt-phys-mapping.rst @@ -8,7 +8,7 @@ How to access I/O mapped memory from within device drivers The virt_to_bus() and bus_to_virt() functions have been superseded by the functionality provided by the PCI DMA interface - (see :doc:`/core-api/dma-api-howto`). They continue + (see Documentation/core-api/dma-api-howto.rst). They continue to be documented below for historical purposes, but new code must not use them. --davidm 00/12/12 diff --git a/Documentation/core-api/dma-api.rst b/Documentation/core-api/dma-api.rst index 00a1d4fa3f9e..6d6d0edd2d27 100644 --- a/Documentation/core-api/dma-api.rst +++ b/Documentation/core-api/dma-api.rst @@ -5,7 +5,7 @@ Dynamic DMA mapping using the generic device :Author: James E.J. Bottomley This document describes the DMA API. For a more gentle introduction -of the API (and actual examples), see :doc:`/core-api/dma-api-howto`. +of the API (and actual examples), see Documentation/core-api/dma-api-howto.rst. This API is split into two pieces. Part I describes the basic API. Part II describes extensions for supporting non-consistent memory @@ -479,7 +479,8 @@ without the _attrs suffixes, except that they pass an optional dma_attrs. The interpretation of DMA attributes is architecture-specific, and -each attribute should be documented in :doc:`/core-api/dma-attributes`. +each attribute should be documented in +Documentation/core-api/dma-attributes.rst. If dma_attrs are 0, the semantics of each of these functions is identical to those of the corresponding function diff --git a/Documentation/core-api/dma-isa-lpc.rst b/Documentation/core-api/dma-isa-lpc.rst index e59a3d35a93d..17b193603f0a 100644 --- a/Documentation/core-api/dma-isa-lpc.rst +++ b/Documentation/core-api/dma-isa-lpc.rst @@ -17,7 +17,7 @@ To do ISA style DMA you need to include two headers:: #include The first is the generic DMA API used to convert virtual addresses to -bus addresses (see :doc:`/core-api/dma-api` for details). +bus addresses (see Documentation/core-api/dma-api.rst for details). The second contains the routines specific to ISA DMA transfers. Since this is not present on all platforms make sure you construct your diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index f1c9d20bd42d..5de2c7a4b1b3 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -48,7 +48,7 @@ Concurrency primitives ====================== How Linux keeps everything from happening at the same time. See -:doc:`/locking/index` for more related documentation. +Documentation/locking/index.rst for more related documentation. .. toctree:: :maxdepth: 1 @@ -77,7 +77,7 @@ Memory management ================= How to allocate and use memory in the kernel. Note that there is a lot -more memory-management documentation in :doc:`/vm/index`. +more memory-management documentation in Documentation/vm/index.rst. .. toctree:: :maxdepth: 1 diff --git a/Documentation/core-api/printk-formats.rst b/Documentation/core-api/printk-formats.rst index f063a384c7c8..385c0cc52f1f 100644 --- a/Documentation/core-api/printk-formats.rst +++ b/Documentation/core-api/printk-formats.rst @@ -37,14 +37,13 @@ Integer types u64 %llu or %llx -If is dependent on a config option for its size (e.g., sector_t, -blkcnt_t) or is architecture-dependent for its size (e.g., tcflag_t), use a -format specifier of its largest possible type and explicitly cast to it. +If is architecture-dependent for its size (e.g., cycles_t, tcflag_t) or +is dependent on a config option for its size (e.g., blk_status_t), use a format +specifier of its largest possible type and explicitly cast to it. Example:: - printk("test: sector number/total blocks: %llu/%llu\n", - (unsigned long long)sector, (unsigned long long)blockcount); + printk("test: latency: %llu cycles\n", (unsigned long long)time); Reminder: sizeof() returns type size_t. diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst index 51fed1bd72ec..f0956e9ea2d8 100644 --- a/Documentation/dev-tools/checkpatch.rst +++ b/Documentation/dev-tools/checkpatch.rst @@ -246,6 +246,7 @@ Allocation style The first argument for kcalloc or kmalloc_array should be the number of elements. sizeof() as the first argument is generally wrong. + See: https://www.kernel.org/doc/html/latest/core-api/memory-allocation.html **ALLOC_SIZEOF_STRUCT** @@ -264,6 +265,7 @@ Allocation style **ALLOC_WITH_MULTIPLY** Prefer kmalloc_array/kcalloc over kmalloc/kzalloc with a sizeof multiply. + See: https://www.kernel.org/doc/html/latest/core-api/memory-allocation.html @@ -284,6 +286,7 @@ API usage BUG() or BUG_ON() should be avoided totally. Use WARN() and WARN_ON() instead, and handle the "impossible" error condition as gracefully as possible. + See: https://www.kernel.org/doc/html/latest/process/deprecated.html#bug-and-bug-on **CONSIDER_KSTRTO** @@ -292,12 +295,161 @@ API usage may lead to unexpected results in callers. The respective kstrtol(), kstrtoll(), kstrtoul(), and kstrtoull() functions tend to be the correct replacements. + See: https://www.kernel.org/doc/html/latest/process/deprecated.html#simple-strtol-simple-strtoll-simple-strtoul-simple-strtoull + **CONSTANT_CONVERSION** + Use of __constant_ form is discouraged for the following functions:: + + __constant_cpu_to_be[x] + __constant_cpu_to_le[x] + __constant_be[x]_to_cpu + __constant_le[x]_to_cpu + __constant_htons + __constant_ntohs + + Using any of these outside of include/uapi/ is not preferred as using the + function without __constant_ is identical when the argument is a + constant. + + In big endian systems, the macros like __constant_cpu_to_be32(x) and + cpu_to_be32(x) expand to the same expression:: + + #define __constant_cpu_to_be32(x) ((__force __be32)(__u32)(x)) + #define __cpu_to_be32(x) ((__force __be32)(__u32)(x)) + + In little endian systems, the macros __constant_cpu_to_be32(x) and + cpu_to_be32(x) expand to __constant_swab32 and __swab32. __swab32 + has a __builtin_constant_p check:: + + #define __swab32(x) \ + (__builtin_constant_p((__u32)(x)) ? \ + ___constant_swab32(x) : \ + __fswab32(x)) + + So ultimately they have a special case for constants. + Similar is the case with all of the macros in the list. Thus + using the __constant_... forms are unnecessarily verbose and + not preferred outside of include/uapi. + + See: https://lore.kernel.org/lkml/1400106425.12666.6.camel@joe-AO725/ + + **DEPRECATED_API** + Usage of a deprecated RCU API is detected. It is recommended to replace + old flavourful RCU APIs by their new vanilla-RCU counterparts. + + The full list of available RCU APIs can be viewed from the kernel docs. + + See: https://www.kernel.org/doc/html/latest/RCU/whatisRCU.html#full-list-of-rcu-apis + + **DEPRECATED_VARIABLE** + EXTRA_{A,C,CPP,LD}FLAGS are deprecated and should be replaced by the new + flags added via commit f77bf01425b1 ("kbuild: introduce ccflags-y, + asflags-y and ldflags-y"). + + The following conversion scheme maybe used:: + + EXTRA_AFLAGS -> asflags-y + EXTRA_CFLAGS -> ccflags-y + EXTRA_CPPFLAGS -> cppflags-y + EXTRA_LDFLAGS -> ldflags-y + + See: + + 1. https://lore.kernel.org/lkml/20070930191054.GA15876@uranus.ravnborg.org/ + 2. https://lore.kernel.org/lkml/1313384834-24433-12-git-send-email-lacombar@gmail.com/ + 3. https://www.kernel.org/doc/html/latest/kbuild/makefiles.html#compilation-flags + + **DEVICE_ATTR_FUNCTIONS** + The function names used in DEVICE_ATTR is unusual. + Typically, the store and show functions are used with _store and + _show, where is a named attribute variable of the device. + + Consider the following examples:: + + static DEVICE_ATTR(type, 0444, type_show, NULL); + static DEVICE_ATTR(power, 0644, power_show, power_store); + + The function names should preferably follow the above pattern. + + See: https://www.kernel.org/doc/html/latest/driver-api/driver-model/device.html#attributes + + **DEVICE_ATTR_RO** + The DEVICE_ATTR_RO(name) helper macro can be used instead of + DEVICE_ATTR(name, 0444, name_show, NULL); + + Note that the macro automatically appends _show to the named + attribute variable of the device for the show method. + + See: https://www.kernel.org/doc/html/latest/driver-api/driver-model/device.html#attributes + + **DEVICE_ATTR_RW** + The DEVICE_ATTR_RW(name) helper macro can be used instead of + DEVICE_ATTR(name, 0644, name_show, name_store); + + Note that the macro automatically appends _show and _store to the + named attribute variable of the device for the show and store methods. + + See: https://www.kernel.org/doc/html/latest/driver-api/driver-model/device.html#attributes + + **DEVICE_ATTR_WO** + The DEVICE_AATR_WO(name) helper macro can be used instead of + DEVICE_ATTR(name, 0200, NULL, name_store); + + Note that the macro automatically appends _store to the + named attribute variable of the device for the store method. + + See: https://www.kernel.org/doc/html/latest/driver-api/driver-model/device.html#attributes + + **DUPLICATED_SYSCTL_CONST** + Commit d91bff3011cf ("proc/sysctl: add shared variables for range + check") added some shared const variables to be used instead of a local + copy in each source file. + + Consider replacing the sysctl range checking value with the shared + one in include/linux/sysctl.h. The following conversion scheme may + be used:: + + &zero -> SYSCTL_ZERO + &one -> SYSCTL_ONE + &int_max -> SYSCTL_INT_MAX + + See: + + 1. https://lore.kernel.org/lkml/20190430180111.10688-1-mcroce@redhat.com/ + 2. https://lore.kernel.org/lkml/20190531131422.14970-1-mcroce@redhat.com/ + + **ENOSYS** + ENOSYS means that a nonexistent system call was called. + Earlier, it was wrongly used for things like invalid operations on + otherwise valid syscalls. This should be avoided in new code. + + See: https://lore.kernel.org/lkml/5eb299021dec23c1a48fa7d9f2c8b794e967766d.1408730669.git.luto@amacapital.net/ + + **ENOTSUPP** + ENOTSUPP is not a standard error code and should be avoided in new patches. + EOPNOTSUPP should be used instead. + + See: https://lore.kernel.org/netdev/20200510182252.GA411829@lunn.ch/ + + **EXPORT_SYMBOL** + EXPORT_SYMBOL should immediately follow the symbol to be exported. + + **IN_ATOMIC** + in_atomic() is not for driver use so any such use is reported as an ERROR. + Also in_atomic() is often used to determine if sleeping is permitted, + but it is not reliable in this use model. Therefore its use is + strongly discouraged. + + However, in_atomic() is ok for core kernel use. + + See: https://lore.kernel.org/lkml/20080320201723.b87b3732.akpm@linux-foundation.org/ + **LOCKDEP** The lockdep_no_validate class was added as a temporary measure to prevent warnings on conversion of device->sem to device->mutex. It should not be used for any other purpose. + See: https://lore.kernel.org/lkml/1268959062.9440.467.camel@laptop/ **MALFORMED_INCLUDE** @@ -308,14 +460,21 @@ API usage **USE_LOCKDEP** lockdep_assert_held() annotations should be preferred over assertions based on spin_is_locked() + See: https://www.kernel.org/doc/html/latest/locking/lockdep-design.html#annotations **UAPI_INCLUDE** No #include statements in include/uapi should use a uapi/ path. + **USLEEP_RANGE** + usleep_range() should be preferred over udelay(). The proper way of + using usleep_range() is mentioned in the kernel docs. -Comment style -------------- + See: https://www.kernel.org/doc/html/latest/timers/timers-howto.html#delays-information-on-the-various-kernel-delay-sleep-mechanisms + + +Comments +-------- **BLOCK_COMMENT_STYLE** The comment style is incorrect. The preferred style for multi- @@ -338,8 +497,24 @@ Comment style **C99_COMMENTS** C99 style single line comments (//) should not be used. Prefer the block comment style instead. + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#commenting + **DATA_RACE** + Applications of data_race() should have a comment so as to document the + reasoning behind why it was deemed safe. + + See: https://lore.kernel.org/lkml/20200401101714.44781-1-elver@google.com/ + + **FSF_MAILING_ADDRESS** + Kernel maintainers reject new instances of the GPL boilerplate paragraph + directing people to write to the FSF for a copy of the GPL, since the + FSF has moved in the past and may do so again. + So do not write paragraphs about writing to the Free Software Foundation's + mailing address. + + See: https://lore.kernel.org/lkml/20131006222342.GT19510@leaf/ + Commit message -------------- @@ -347,6 +522,7 @@ Commit message **BAD_SIGN_OFF** The signed-off-by line does not fall in line with the standards specified by the community. + See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#developer-s-certificate-of-origin-1-1 **BAD_STABLE_ADDRESS_STYLE** @@ -368,12 +544,33 @@ Commit message **COMMIT_MESSAGE** The patch is missing a commit description. A brief description of the changes made by the patch should be added. + See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes + **EMAIL_SUBJECT** + Naming the tool that found the issue is not very useful in the + subject line. A good subject line summarizes the change that + the patch brings. + + See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#describe-your-changes + + **FROM_SIGN_OFF_MISMATCH** + The author's email does not match with that in the Signed-off-by: + line(s). This can be sometimes caused due to an improperly configured + email client. + + This message is emitted due to any of the following reasons:: + + - The email names do not match. + - The email addresses do not match. + - The email subaddresses do not match. + - The email comments do not match. + **MISSING_SIGN_OFF** The patch is missing a Signed-off-by line. A signed-off-by line should be added according to Developer's certificate of Origin. + See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin **NO_AUTHOR_SIGN_OFF** @@ -382,6 +579,7 @@ Commit message end of explanation of the patch to denote that the author has written it or otherwise has the rights to pass it on as an open source patch. + See: https://www.kernel.org/doc/html/latest/process/submitting-patches.html#sign-your-work-the-developer-s-certificate-of-origin **DIFF_IN_COMMIT_MSG** @@ -389,6 +587,7 @@ Commit message This causes problems when one tries to apply a file containing both the changelog and the diff because patch(1) tries to apply the diff which it found in the changelog. + See: https://lore.kernel.org/lkml/20150611134006.9df79a893e3636019ad2759e@linux-foundation.org/ **GERRIT_CHANGE_ID** @@ -431,6 +630,7 @@ Comparison style **BOOL_COMPARISON** Comparisons of A to true and false are better written as A and !A. + See: https://lore.kernel.org/lkml/1365563834.27174.12.camel@joe-AO722/ **COMPARISON_TO_NULL** @@ -442,6 +642,87 @@ Comparison style side of the test should be avoided. +Indentation and Line Breaks +--------------------------- + + **CODE_INDENT** + Code indent should use tabs instead of spaces. + Outside of comments, documentation and Kconfig, + spaces are never used for indentation. + + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#indentation + + **DEEP_INDENTATION** + Indentation with 6 or more tabs usually indicate overly indented + code. + + It is suggested to refactor excessive indentation of + if/else/for/do/while/switch statements. + + See: https://lore.kernel.org/lkml/1328311239.21255.24.camel@joe2Laptop/ + + **SWITCH_CASE_INDENT_LEVEL** + switch should be at the same indent as case. + Example:: + + switch (suffix) { + case 'G': + case 'g': + mem <<= 30; + break; + case 'M': + case 'm': + mem <<= 20; + break; + case 'K': + case 'k': + mem <<= 10; + fallthrough; + default: + break; + } + + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#indentation + + **LONG_LINE** + The line has exceeded the specified maximum length. + To use a different maximum line length, the --max-line-length=n option + may be added while invoking checkpatch. + + Earlier, the default line length was 80 columns. Commit bdc48fa11e46 + ("checkpatch/coding-style: deprecate 80-column warning") increased the + limit to 100 columns. This is not a hard limit either and it's + preferable to stay within 80 columns whenever possible. + + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#breaking-long-lines-and-strings + + **LONG_LINE_STRING** + A string starts before but extends beyond the maximum line length. + To use a different maximum line length, the --max-line-length=n option + may be added while invoking checkpatch. + + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#breaking-long-lines-and-strings + + **LONG_LINE_COMMENT** + A comment starts before but extends beyond the maximum line length. + To use a different maximum line length, the --max-line-length=n option + may be added while invoking checkpatch. + + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#breaking-long-lines-and-strings + + **TRAILING_STATEMENTS** + Trailing statements (for example after any conditional) should be + on the next line. + Statements, such as:: + + if (x == y) break; + + should be:: + + if (x == y) + break; + + Macros, Attributes and Symbols ------------------------------ @@ -472,7 +753,7 @@ Macros, Attributes and Symbols **BIT_MACRO** Defines like: 1 << could be BIT(digit). - The BIT() macro is defined in include/linux/bitops.h:: + The BIT() macro is defined via include/linux/bits.h:: #define BIT(nr) (1UL << (nr)) @@ -492,6 +773,7 @@ Macros, Attributes and Symbols The kernel does *not* use the ``__DATE__`` and ``__TIME__`` macros, and enables warnings if they are used as they can lead to non-deterministic builds. + See: https://www.kernel.org/doc/html/latest/kbuild/reproducible-builds.html#timestamps **DEFINE_ARCH_HAS** @@ -502,8 +784,12 @@ Macros, Attributes and Symbols want architectures able to override them with optimized ones, we should either use weak functions (appropriate for some cases), or the symbol that protects them should be the same symbol we use. + See: https://lore.kernel.org/lkml/CA+55aFycQ9XJvEOsiM3txHL5bjUc8CeKWJNR_H+MiicaddB42Q@mail.gmail.com/ + **DO_WHILE_MACRO_WITH_TRAILING_SEMICOLON** + do {} while(0) macros should not have a trailing semicolon. + **INIT_ATTRIBUTE** Const init definitions should use __initconst instead of __initdata. @@ -528,6 +814,20 @@ Macros, Attributes and Symbols ... } + **MISPLACED_INIT** + It is possible to use section markers on variables in a way + which gcc doesn't understand (or at least not the way the + developer intended):: + + static struct __initdata samsung_pll_clock exynos4_plls[nr_plls] = { + + does not put exynos4_plls in the .initdata section. The __initdata + marker can be virtually anywhere on the line, except right after + "struct". The preferred location is before the "=" sign if there is + one, or before the trailing ";" otherwise. + + See: https://lore.kernel.org/lkml/1377655732.3619.19.camel@joe-AO722/ + **MULTISTATEMENT_MACRO_USE_DO_WHILE** Macros with multiple statements should be enclosed in a do - while block. Same should also be the case for macros @@ -541,6 +841,10 @@ Macros, Attributes and Symbols See: https://www.kernel.org/doc/html/latest/process/coding-style.html#macros-enums-and-rtl + **PREFER_FALLTHROUGH** + Use the `fallthrough;` pseudo keyword instead of + `/* fallthrough */` like comments. + **WEAK_DECLARATION** Using weak declarations like __attribute__((weak)) or __weak can have unintended link defects. Avoid using them. @@ -551,8 +855,51 @@ Functions and Variables **CAMELCASE** Avoid CamelCase Identifiers. + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#naming + **CONST_CONST** + Using `const const *` is generally meant to be + written `const * const`. + + **CONST_STRUCT** + Using const is generally a good idea. Checkpatch reads + a list of frequently used structs that are always or + almost always constant. + + The existing structs list can be viewed from + `scripts/const_structs.checkpatch`. + + See: https://lore.kernel.org/lkml/alpine.DEB.2.10.1608281509480.3321@hadrien/ + + **EMBEDDED_FUNCTION_NAME** + Embedded function names are less appropriate to use as + refactoring can cause function renaming. Prefer the use of + "%s", __func__ to embedded function names. + + Note that this does not work with -f (--file) checkpatch option + as it depends on patch context providing the function name. + + **FUNCTION_ARGUMENTS** + This warning is emitted due to any of the following reasons: + + 1. Arguments for the function declaration do not follow + the identifier name. Example:: + + void foo + (int bar, int baz) + + This should be corrected to:: + + void foo(int bar, int baz) + + 2. Some arguments for the function definition do not + have an identifier name. Example:: + + void foo(int) + + All arguments should have identifier names. + **FUNCTION_WITHOUT_ARGS** Function declarations without arguments like:: @@ -583,6 +930,34 @@ Functions and Variables return bar; +Permissions +----------- + + **DEVICE_ATTR_PERMS** + The permissions used in DEVICE_ATTR are unusual. + Typically only three permissions are used - 0644 (RW), 0444 (RO) + and 0200 (WO). + + See: https://www.kernel.org/doc/html/latest/filesystems/sysfs.html#attributes + + **EXECUTE_PERMISSIONS** + There is no reason for source files to be executable. The executable + bit can be removed safely. + + **EXPORTED_WORLD_WRITABLE** + Exporting world writable sysfs/debugfs files is usually a bad thing. + When done arbitrarily they can introduce serious security bugs. + In the past, some of the debugfs vulnerabilities would seemingly allow + any local user to write arbitrary values into device registers - a + situation from which little good can be expected to emerge. + + See: https://lore.kernel.org/linux-arm-kernel/cover.1296818921.git.segoon@openwall.com/ + + **NON_OCTAL_PERMISSIONS** + Permission bits should use 4 digit octal permissions (like 0700 or 0444). + Avoid using any other base like decimal. + + Spacing and Brackets -------------------- @@ -616,7 +991,7 @@ Spacing and Brackets 1. With a type on the left:: - ;int [] a; + int [] a; 2. At the beginning of a line for slice initialisers:: @@ -626,12 +1001,6 @@ Spacing and Brackets = { [0...10] = 5 } - **CODE_INDENT** - Code indent should use tabs instead of spaces. - Outside of comments, documentation and Kconfig, - spaces are never used for indentation. - See: https://www.kernel.org/doc/html/latest/process/coding-style.html#indentation - **CONCATENATED_STRING** Concatenated elements should have a space in between. Example:: @@ -644,17 +1013,20 @@ Spacing and Brackets **ELSE_AFTER_BRACE** `else {` should follow the closing block `}` on the same line. + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#placing-braces-and-spaces **LINE_SPACING** Vertical space is wasted given the limited number of lines an editor window can display when multiple blank lines are used. + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#spaces **OPEN_BRACE** The opening brace should be following the function definitions on the next line. For any non-functional block it should be on the same line as the last construct. + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#placing-braces-and-spaces **POINTER_LOCATION** @@ -671,37 +1043,47 @@ Spacing and Brackets **SPACING** Whitespace style used in the kernel sources is described in kernel docs. + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#spaces - **SWITCH_CASE_INDENT_LEVEL** - switch should be at the same indent as case. - Example:: - - switch (suffix) { - case 'G': - case 'g': - mem <<= 30; - break; - case 'M': - case 'm': - mem <<= 20; - break; - case 'K': - case 'k': - mem <<= 10; - /* fall through */ - default: - break; - } - - See: https://www.kernel.org/doc/html/latest/process/coding-style.html#indentation - **TRAILING_WHITESPACE** Trailing whitespace should always be removed. Some editors highlight the trailing whitespace and cause visual distractions when editing files. + See: https://www.kernel.org/doc/html/latest/process/coding-style.html#spaces + **UNNECESSARY_PARENTHESES** + Parentheses are not required in the following cases: + + 1. Function pointer uses:: + + (foo->bar)(); + + could be:: + + foo->bar(); + + 2. Comparisons in if:: + + if ((foo->bar) && (foo->baz)) + if ((foo == bar)) + + could be:: + + if (foo->bar && foo->baz) + if (foo == bar) + + 3. addressof/dereference single Lvalues:: + + &(foo->bar) + *(foo->bar) + + could be:: + + &foo->bar + *foo->bar + **WHILE_AFTER_BRACE** while should follow the closing bracket on the same line:: @@ -723,17 +1105,50 @@ Others The patch seems to be corrupted or lines are wrapped. Please regenerate the patch file before sending it to the maintainer. + **CVS_KEYWORD** + Since linux moved to git, the CVS markers are no longer used. + So, CVS style keywords ($Id$, $Revision$, $Log$) should not be + added. + + **DEFAULT_NO_BREAK** + switch default case is sometimes written as "default:;". This can + cause new cases added below default to be defective. + + A "break;" should be added after empty default statement to avoid + unwanted fallthrough. + **DOS_LINE_ENDINGS** For DOS-formatted patches, there are extra ^M symbols at the end of the line. These should be removed. - **EXECUTE_PERMISSIONS** - There is no reason for source files to be executable. The executable - bit can be removed safely. + **DT_SCHEMA_BINDING_PATCH** + DT bindings moved to a json-schema based format instead of + freeform text. - **NON_OCTAL_PERMISSIONS** - Permission bits should use 4 digit octal permissions (like 0700 or 0444). - Avoid using any other base like decimal. + See: https://www.kernel.org/doc/html/latest/devicetree/bindings/writing-schema.html + + **DT_SPLIT_BINDING_PATCH** + Devicetree bindings should be their own patch. This is because + bindings are logically independent from a driver implementation, + they have a different maintainer (even though they often + are applied via the same tree), and it makes for a cleaner history in the + DT only tree created with git-filter-branch. + + See: https://www.kernel.org/doc/html/latest/devicetree/bindings/submitting-patches.html#i-for-patch-submitters + + **EMBEDDED_FILENAME** + Embedding the complete filename path inside the file isn't particularly + useful as often the path is moved around and becomes incorrect. + + **FILE_PATH_CHANGES** + Whenever files are added, moved, or deleted, the MAINTAINERS file + patterns can be out of sync or outdated. + + So MAINTAINERS might need updating in these cases. + + **MEMSET** + The memset use appears to be incorrect. This may be caused due to + badly ordered parameters. Please recheck the usage. **NOT_UNIFIED_DIFF** The patch file does not appear to be in unified-diff format. Please @@ -742,14 +1157,12 @@ Others **PRINTF_0XDECIMAL** Prefixing 0x with decimal output is defective and should be corrected. - **TRAILING_STATEMENTS** - Trailing statements (for example after any conditional) should be - on the next line. - Like:: + **SPDX_LICENSE_TAG** + The source file is missing or has an improper SPDX identifier tag. + The Linux kernel requires the precise SPDX identifier in all source files, + and it is thoroughly documented in the kernel docs. - if (x == y) break; + See: https://www.kernel.org/doc/html/latest/process/license-rules.html - should be:: - - if (x == y) - break; + **TYPO_SPELLING** + Some words may have been misspelled. Consider reviewing them. diff --git a/Documentation/dev-tools/kunit/api/index.rst b/Documentation/dev-tools/kunit/api/index.rst index 9b9bffe5d41a..b33ad72bcf0b 100644 --- a/Documentation/dev-tools/kunit/api/index.rst +++ b/Documentation/dev-tools/kunit/api/index.rst @@ -10,7 +10,7 @@ API Reference This section documents the KUnit kernel testing API. It is divided into the following sections: -================================= ============================================== -:doc:`test` documents all of the standard testing API - excluding mocking or mocking related features. -================================= ============================================== +Documentation/dev-tools/kunit/api/test.rst + + - documents all of the standard testing API excluding mocking + or mocking related features. diff --git a/Documentation/dev-tools/kunit/faq.rst b/Documentation/dev-tools/kunit/faq.rst index 8d5029ad210a..5c6555d020f3 100644 --- a/Documentation/dev-tools/kunit/faq.rst +++ b/Documentation/dev-tools/kunit/faq.rst @@ -97,7 +97,7 @@ things to try. modules will automatically execute associated tests when loaded. Test results can be collected from ``/sys/kernel/debug/kunit//results``, and can be parsed with ``kunit.py parse``. For more details, see "KUnit on - non-UML architectures" in :doc:`usage`. + non-UML architectures" in Documentation/dev-tools/kunit/usage.rst. If none of the above tricks help, you are always welcome to email any issues to kunit-dev@googlegroups.com. diff --git a/Documentation/dev-tools/kunit/index.rst b/Documentation/dev-tools/kunit/index.rst index 848478838347..25d92a9a05ea 100644 --- a/Documentation/dev-tools/kunit/index.rst +++ b/Documentation/dev-tools/kunit/index.rst @@ -36,7 +36,7 @@ To make running these tests (and reading the results) easier, KUnit offers results. This provides a quick way of running KUnit tests during development, without requiring a virtual machine or separate hardware. -Get started now: :doc:`start` +Get started now: Documentation/dev-tools/kunit/start.rst Why KUnit? ========== @@ -88,9 +88,9 @@ it takes to read their test log? How do I use it? ================ -* :doc:`start` - for new users of KUnit -* :doc:`tips` - for short examples of best practices -* :doc:`usage` - for a more detailed explanation of KUnit features -* :doc:`api/index` - for the list of KUnit APIs used for testing -* :doc:`kunit-tool` - for more information on the kunit_tool helper script -* :doc:`faq` - for answers to some common questions about KUnit +* Documentation/dev-tools/kunit/start.rst - for new users of KUnit +* Documentation/dev-tools/kunit/tips.rst - for short examples of best practices +* Documentation/dev-tools/kunit/usage.rst - for a more detailed explanation of KUnit features +* Documentation/dev-tools/kunit/api/index.rst - for the list of KUnit APIs used for testing +* Documentation/dev-tools/kunit/kunit-tool.rst - for more information on the kunit_tool helper script +* Documentation/dev-tools/kunit/faq.rst - for answers to some common questions about KUnit diff --git a/Documentation/dev-tools/kunit/start.rst b/Documentation/dev-tools/kunit/start.rst index 0e65cabe08eb..63ef7b625c13 100644 --- a/Documentation/dev-tools/kunit/start.rst +++ b/Documentation/dev-tools/kunit/start.rst @@ -21,7 +21,7 @@ The wrapper can be run with: ./tools/testing/kunit/kunit.py run For more information on this wrapper (also called kunit_tool) check out the -:doc:`kunit-tool` page. +Documentation/dev-tools/kunit/kunit-tool.rst page. Creating a .kunitconfig ----------------------- @@ -234,7 +234,7 @@ Congrats! You just wrote your first KUnit test! Next Steps ========== -* Check out the :doc:`tips` page for tips on +* Check out the Documentation/dev-tools/kunit/tips.rst page for tips on writing idiomatic KUnit tests. * Optional: see the :doc:`usage` page for a more in-depth explanation of KUnit. diff --git a/Documentation/dev-tools/kunit/tips.rst b/Documentation/dev-tools/kunit/tips.rst index 8d8c238f7f79..492d2ded2f5a 100644 --- a/Documentation/dev-tools/kunit/tips.rst +++ b/Documentation/dev-tools/kunit/tips.rst @@ -125,7 +125,8 @@ Here's a slightly in-depth example of how one could implement "mocking": Note: here we're able to get away with using ``test->priv``, but if you wanted -something more flexible you could use a named ``kunit_resource``, see :doc:`api/test`. +something more flexible you could use a named ``kunit_resource``, see +Documentation/dev-tools/kunit/api/test.rst. Failing the current test ------------------------ @@ -185,5 +186,5 @@ Alternatively, one can take full control over the error message by using ``KUNIT Next Steps ========== -* Optional: see the :doc:`usage` page for a more +* Optional: see the Documentation/dev-tools/kunit/usage.rst page for a more in-depth explanation of KUnit. diff --git a/Documentation/dev-tools/kunit/usage.rst b/Documentation/dev-tools/kunit/usage.rst index 650f99590df5..3ee7ab91f712 100644 --- a/Documentation/dev-tools/kunit/usage.rst +++ b/Documentation/dev-tools/kunit/usage.rst @@ -10,7 +10,7 @@ understand it. This guide assumes a working knowledge of the Linux kernel and some basic knowledge of testing. For a high level introduction to KUnit, including setting up KUnit for your -project, see :doc:`start`. +project, see Documentation/dev-tools/kunit/start.rst. Organization of this document ============================= @@ -99,7 +99,8 @@ violated; however, the test will continue running, potentially trying other expectations until the test case ends or is otherwise terminated. This is as opposed to *assertions* which are discussed later. -To learn about more expectations supported by KUnit, see :doc:`api/test`. +To learn about more expectations supported by KUnit, see +Documentation/dev-tools/kunit/api/test.rst. .. note:: A single test case should be pretty short, pretty easy to understand, @@ -216,7 +217,8 @@ test suite in a special linker section so that it can be run by KUnit either after late_init, or when the test module is loaded (depending on whether the test was built in or not). -For more information on these types of things see the :doc:`api/test`. +For more information on these types of things see the +Documentation/dev-tools/kunit/api/test.rst. Common Patterns =============== diff --git a/Documentation/dev-tools/testing-overview.rst b/Documentation/dev-tools/testing-overview.rst index b5b46709969c..65feb81edb14 100644 --- a/Documentation/dev-tools/testing-overview.rst +++ b/Documentation/dev-tools/testing-overview.rst @@ -71,15 +71,15 @@ can be used to verify that a test is executing particular functions or lines of code. This is useful for determining how much of the kernel is being tested, and for finding corner-cases which are not covered by the appropriate test. -:doc:`gcov` is GCC's coverage testing tool, which can be used with the kernel -to get global or per-module coverage. Unlike KCOV, it does not record per-task -coverage. Coverage data can be read from debugfs, and interpreted using the -usual gcov tooling. +Documentation/dev-tools/gcov.rst is GCC's coverage testing tool, which can be +used with the kernel to get global or per-module coverage. Unlike KCOV, it +does not record per-task coverage. Coverage data can be read from debugfs, +and interpreted using the usual gcov tooling. -:doc:`kcov` is a feature which can be built in to the kernel to allow -capturing coverage on a per-task level. It's therefore useful for fuzzing and -other situations where information about code executed during, for example, a -single syscall is useful. +Documentation/dev-tools/kcov.rst is a feature which can be built in to the +kernel to allow capturing coverage on a per-task level. It's therefore useful +for fuzzing and other situations where information about code executed during, +for example, a single syscall is useful. Dynamic Analysis Tools diff --git a/Documentation/devicetree/bindings/crypto/cortina,sl3516-crypto.yaml b/Documentation/devicetree/bindings/crypto/cortina,sl3516-crypto.yaml new file mode 100644 index 000000000000..b633b8d0e6f0 --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/cortina,sl3516-crypto.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/crypto/cortina,sl3516-crypto.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: SL3516 cryptographic offloader driver + +maintainers: + - Corentin Labbe + +properties: + compatible: + enum: + - cortina,sl3516-crypto + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + resets: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + - resets + +additionalProperties: false + +examples: + - | + #include + #include + #include + + crypto@62000000 { + compatible = "cortina,sl3516-crypto"; + reg = <0x62000000 0x10000>; + interrupts = <7 IRQ_TYPE_EDGE_RISING>; + resets = <&syscon GEMINI_RESET_SECURITY>; + clocks = <&syscon GEMINI_CLK_GATE_SECURITY>; + }; diff --git a/Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml b/Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml new file mode 100644 index 000000000000..9c53c27bd20a --- /dev/null +++ b/Documentation/devicetree/bindings/crypto/intel,ixp4xx-crypto.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright 2018 Linaro Ltd. +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/crypto/intel,ixp4xx-crypto.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Intel IXP4xx cryptographic engine + +maintainers: + - Linus Walleij + +description: | + The Intel IXP4xx cryptographic engine makes use of the IXP4xx NPE + (Network Processing Engine). Since it is not a device on its own + it is defined as a subnode of the NPE, if crypto support is + available on the platform. + +properties: + compatible: + const: intel,ixp4xx-crypto + + intel,npe-handle: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 1 + description: phandle to the NPE this crypto engine is using, the cell + describing the NPE instance to be used. + + queue-rx: + $ref: /schemas/types.yaml#/definitions/phandle-array + maxItems: 1 + description: phandle to the RX queue on the NPE, the cell describing + the queue instance to be used. + + queue-txready: + $ref: /schemas/types.yaml#/definitions/phandle-array + maxItems: 1 + description: phandle to the TX READY queue on the NPE, the cell describing + the queue instance to be used. + +required: + - compatible + - intel,npe-handle + - queue-rx + - queue-txready + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml b/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml index 1bd2870c3a9c..c435c9f369a4 100644 --- a/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml +++ b/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml @@ -26,9 +26,16 @@ properties: reg: items: - - description: NPE0 register range - - description: NPE1 register range - - description: NPE2 register range + - description: NPE0 (NPE-A) register range + - description: NPE1 (NPE-B) register range + - description: NPE2 (NPE-C) register range + + crypto: + $ref: /schemas/crypto/intel,ixp4xx-crypto.yaml# + type: object + description: Optional node for the embedded crypto engine, the node + should be named with the instance number of the NPE engine used for + the crypto engine. required: - compatible @@ -38,8 +45,15 @@ additionalProperties: false examples: - | - npe@c8006000 { + npe: npe@c8006000 { compatible = "intel,ixp4xx-network-processing-engine"; reg = <0xc8006000 0x1000>, <0xc8007000 0x1000>, <0xc8008000 0x1000>; + + crypto { + compatible = "intel,ixp4xx-crypto"; + intel,npe-handle = <&npe 2>; + queue-rx = <&qmgr 30>; + queue-txready = <&qmgr 29>; + }; }; ... diff --git a/Documentation/devicetree/bindings/media/atmel,isc.yaml b/Documentation/devicetree/bindings/media/atmel,isc.yaml new file mode 100644 index 000000000000..3e4bb8892d94 --- /dev/null +++ b/Documentation/devicetree/bindings/media/atmel,isc.yaml @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: GPL-2.0-only +# Copyright (C) 2016-2021 Microchip Technology, Inc. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/atmel,isc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Atmel Image Sensor Controller (ISC) + +maintainers: + - Eugen Hristev + +description: | + The Image Sensor Controller (ISC) device provides the video input capabilities for the + Atmel/Microchip AT91 SAMA family of devices. + + The ISC has a single parallel input that supports RAW Bayer, RGB or YUV video, + with both external synchronization and BT.656 synchronization for the latter. + +properties: + compatible: + const: atmel,sama5d2-isc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + minItems: 3 + maxItems: 3 + + clock-names: + items: + - const: hclock + - const: iscck + - const: gck + + '#clock-cells': + const: 0 + + clock-output-names: + const: isc-mck + + port: + $ref: /schemas/graph.yaml#/properties/port + description: + Input port node, single endpoint describing the input pad. + + properties: + endpoint: + $ref: video-interfaces.yaml# + + properties: + remote-endpoint: true + + bus-width: + enum: [8, 9, 10, 11, 12] + default: 12 + + hsync-active: + enum: [0, 1] + default: 1 + + vsync-active: + enum: [0, 1] + default: 1 + + pclk-sample: + enum: [0, 1] + default: 1 + + required: + - remote-endpoint + + additionalProperties: false + + additionalProperties: false + +required: + - compatible + - reg + - clocks + - clock-names + - '#clock-cells' + - clock-output-names + - port + +additionalProperties: false + +examples: + - | + #include + + isc: isc@f0008000 { + compatible = "atmel,sama5d2-isc"; + reg = <0xf0008000 0x4000>; + interrupts = <46 IRQ_TYPE_LEVEL_HIGH 5>; + clocks = <&isc_clk>, <&iscck>, <&isc_gclk>; + clock-names = "hclock", "iscck", "gck"; + #clock-cells = <0>; + clock-output-names = "isc-mck"; + + port { + isc_0: endpoint { + remote-endpoint = <&ov7740_0>; + hsync-active = <1>; + vsync-active = <0>; + pclk-sample = <1>; + bus-width = <8>; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/media/atmel-isc.txt b/Documentation/devicetree/bindings/media/atmel-isc.txt deleted file mode 100644 index bbe0e87c6188..000000000000 --- a/Documentation/devicetree/bindings/media/atmel-isc.txt +++ /dev/null @@ -1,65 +0,0 @@ -Atmel Image Sensor Controller (ISC) ----------------------------------------------- - -Required properties for ISC: -- compatible - Must be "atmel,sama5d2-isc". -- reg - Physical base address and length of the registers set for the device. -- interrupts - Should contain IRQ line for the ISC. -- clocks - List of clock specifiers, corresponding to entries in - the clock-names property; - Please refer to clock-bindings.txt. -- clock-names - Required elements: "hclock", "iscck", "gck". -- #clock-cells - Should be 0. -- clock-output-names - Should be "isc-mck". -- pinctrl-names, pinctrl-0 - Please refer to pinctrl-bindings.txt. - -ISC supports a single port node with parallel bus. It should contain one -'port' child node with child 'endpoint' node. Please refer to the bindings -defined in Documentation/devicetree/bindings/media/video-interfaces.txt. - -Example: -isc: isc@f0008000 { - compatible = "atmel,sama5d2-isc"; - reg = <0xf0008000 0x4000>; - interrupts = <46 IRQ_TYPE_LEVEL_HIGH 5>; - clocks = <&isc_clk>, <&iscck>, <&isc_gclk>; - clock-names = "hclock", "iscck", "gck"; - #clock-cells = <0>; - clock-output-names = "isc-mck"; - pinctrl-names = "default"; - pinctrl-0 = <&pinctrl_isc_base &pinctrl_isc_data_8bit &pinctrl_isc_data_9_10 &pinctrl_isc_data_11_12>; - - port { - isc_0: endpoint { - remote-endpoint = <&ov7740_0>; - hsync-active = <1>; - vsync-active = <0>; - pclk-sample = <1>; - }; - }; -}; - -i2c1: i2c@fc028000 { - ov7740: camera@21 { - compatible = "ovti,ov7740"; - reg = <0x21>; - clocks = <&isc>; - clock-names = "xvclk"; - assigned-clocks = <&isc>; - assigned-clock-rates = <24000000>; - - port { - ov7740_0: endpoint { - remote-endpoint = <&isc_0>; - }; - }; - }; -}; diff --git a/Documentation/devicetree/bindings/media/i2c/rda,rda5807.yaml b/Documentation/devicetree/bindings/media/i2c/rda,rda5807.yaml new file mode 100644 index 000000000000..f50e54a722eb --- /dev/null +++ b/Documentation/devicetree/bindings/media/i2c/rda,rda5807.yaml @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/i2c/rda,rda5807.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Unisoc Communications RDA5807 FM radio receiver + +maintainers: + - Paul Cercueil + +properties: + compatible: + enum: + - rda,rda5807 + + reg: + description: I2C address. + maxItems: 1 + + power-supply: true + + rda,lnan: + description: Use LNAN input port. + type: boolean + + rda,lnap: + description: Use LNAP input port. + type: boolean + + rda,analog-out: + description: Enable analog audio output. + type: boolean + + rda,i2s-out: + description: Enable I2S digital audio output. + type: boolean + + rda,lna-microamp: + description: LNA working current, in micro-amperes. + default: 2500 + enum: [1800, 2100, 2500, 3000] + +required: + - compatible + - reg + - power-supply + +additionalProperties: false + +examples: + - | + i2c0 { + #address-cells = <1>; + #size-cells = <0>; + + radio@11 { + compatible = "rda,rda5807"; + reg = <0x11>; + + power-supply = <&ldo6>; + + rda,lnan; + rda,lnap; + rda,analog-out; + }; + }; diff --git a/Documentation/devicetree/bindings/media/mediatek-vcodec.txt b/Documentation/devicetree/bindings/media/mediatek-vcodec.txt index 06db6837cefd..ad1321e5a22d 100644 --- a/Documentation/devicetree/bindings/media/mediatek-vcodec.txt +++ b/Documentation/devicetree/bindings/media/mediatek-vcodec.txt @@ -9,6 +9,7 @@ Required properties: "mediatek,mt8173-vcodec-enc" for mt8173 avc encoder. "mediatek,mt8183-vcodec-enc" for MT8183 encoder. "mediatek,mt8173-vcodec-dec" for MT8173 decoder. + "mediatek,mt8192-vcodec-enc" for MT8192 encoder. - reg : Physical base address of the video codec registers and length of memory mapped region. - interrupts : interrupt number to the cpu. @@ -22,6 +23,7 @@ Required properties: - iommus : should point to the respective IOMMU block with master port as argument, see Documentation/devicetree/bindings/iommu/mediatek,iommu.yaml for details. +- dma-ranges : describes the dma address range space that the codec hw access. One of the two following nodes: - mediatek,vpu : the node of the video processor unit, if using VPU. - mediatek,scp : the node of the SCP unit, if using SCP. diff --git a/Documentation/devicetree/bindings/media/microchip,sama5d4-vdec.yaml b/Documentation/devicetree/bindings/media/microchip,sama5d4-vdec.yaml new file mode 100644 index 000000000000..4b77103ca913 --- /dev/null +++ b/Documentation/devicetree/bindings/media/microchip,sama5d4-vdec.yaml @@ -0,0 +1,47 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) + +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/media/microchip,sama5d4-vdec.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Hantro G1 VPU codec implemented on Microchip SAMA5D4 SoCs + +maintainers: + - Emil Velikov + +description: + Hantro G1 video decode accelerator present on Microchip SAMA5D4 SoCs. + +properties: + compatible: + const: microchip,sama5d4-vdec + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - clocks + +additionalProperties: false + +examples: + - | + #include + #include + + vdec0: vdec@300000 { + compatible = "microchip,sama5d4-vdec"; + reg = <0x00300000 0x100000>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH 4>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 19>; + }; diff --git a/Documentation/devicetree/bindings/media/microchip,xisc.yaml b/Documentation/devicetree/bindings/media/microchip,xisc.yaml new file mode 100644 index 000000000000..41afe2e5f133 --- /dev/null +++ b/Documentation/devicetree/bindings/media/microchip,xisc.yaml @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright (C) 2021 Microchip Technology, Inc. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/microchip,xisc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip eXtended Image Sensor Controller (XISC) + +maintainers: + - Eugen Hristev + +description: | + The eXtended Image Sensor Controller (XISC) device provides the video input capabilities for the + Microchip AT91 SAM family of devices. + + The XISC has a single internal parallel input that supports RAW Bayer, RGB or YUV video. + The source can be either a demuxer from a CSI2 type of bus, or a simple direct bridge to a + parallel sensor. + + The XISC provides one clock output that is used to clock the demuxer/bridge. + +properties: + compatible: + const: microchip,sama7g5-isc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + clock-names: + items: + - const: hclock + + '#clock-cells': + const: 0 + + clock-output-names: + const: isc-mck + + microchip,mipi-mode: + type: boolean + description: + As the XISC is usually connected to a demux/bridge, the XISC receives + the same type of input, however, it should be aware of the type of + signals received. The mipi-mode enables different internal handling + of the data and clock lines. + + port: + $ref: /schemas/graph.yaml#/properties/port + description: + Input port node, single endpoint describing the input pad. + + properties: + endpoint: + $ref: video-interfaces.yaml# + + properties: + bus-type: + enum: [5, 6] + + remote-endpoint: true + + bus-width: + enum: [8, 9, 10, 11, 12] + default: 12 + + hsync-active: + enum: [0, 1] + default: 1 + + vsync-active: + enum: [0, 1] + default: 1 + + pclk-sample: + enum: [0, 1] + default: 1 + + required: + - remote-endpoint + - bus-type + + additionalProperties: false + + additionalProperties: false + +required: + - compatible + - reg + - clocks + - clock-names + - '#clock-cells' + - clock-output-names + - port + +additionalProperties: false + +examples: + - | + #include + #include + #include + + xisc: xisc@e1408000 { + compatible = "microchip,sama7g5-isc"; + reg = <0xe1408000 0x2000>; + interrupts = ; + clocks = <&pmc PMC_TYPE_PERIPHERAL 56>; + clock-names = "hclock"; + #clock-cells = <0>; + clock-output-names = "isc-mck"; + + port { + xisc_in: endpoint { + bus-type = <5>; /* Parallel */ + remote-endpoint = <&csi2dc_out>; + hsync-active = <1>; + vsync-active = <1>; + bus-width = <12>; + }; + }; + }; + diff --git a/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml b/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml index d8ed480482b9..7c09eec78ce5 100644 --- a/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml +++ b/Documentation/devicetree/bindings/media/nxp,imx7-mipi-csi2.yaml @@ -4,15 +4,17 @@ $id: http://devicetree.org/schemas/media/nxp,imx7-mipi-csi2.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: NXP i.MX7 MIPI CSI-2 receiver +title: NXP i.MX7 and i.MX8 MIPI CSI-2 receiver maintainers: - Rui Miguel Silva + - Laurent Pinchart description: |- - The NXP i.MX7 SoC family includes a MIPI CSI-2 receiver IP core, documented - as "CSIS V3.3". The IP core seems to originate from Samsung, and may be - compatible with some of the Exynos4 ad S5P SoCs. + The NXP i.MX7 and i.MX8 families contain SoCs that include a MIPI CSI-2 + receiver IP core named CSIS. The IP core originates from Samsung, and may be + compatible with some of the Exynos4 and S5P SoCs. i.MX7 SoCs use CSIS version + 3.3, and i.MX8 SoCs use CSIS version 3.6.3. While the CSI-2 receiver is separate from the MIPI D-PHY IP core, the PHY is completely wrapped by the CSIS and doesn't expose a control interface of its @@ -20,7 +22,9 @@ description: |- properties: compatible: - const: fsl,imx7-mipi-csi2 + enum: + - fsl,imx7-mipi-csi2 + - fsl,imx8mm-mipi-csi2 reg: maxItems: 1 @@ -29,16 +33,20 @@ properties: maxItems: 1 clocks: + minItems: 3 items: - description: The peripheral clock (a.k.a. APB clock) - description: The external clock (optionally used as the pixel clock) - description: The MIPI D-PHY clock + - description: The AXI clock clock-names: + minItems: 3 items: - const: pclk - const: wrap - const: phy + - const: axi power-domains: maxItems: 1 @@ -71,16 +79,30 @@ properties: properties: data-lanes: - oneOf: - - items: - - const: 1 - - items: - - const: 1 - - const: 2 + items: + minItems: 1 + maxItems: 4 + items: + - const: 1 + - const: 2 + - const: 3 + - const: 4 required: - data-lanes + allOf: + - if: + properties: + compatible: + contains: + const: fsl,imx7-mipi-csi2 + then: + properties: + data-lanes: + items: + maxItems: 2 + port@1: $ref: /schemas/graph.yaml#/properties/port description: @@ -93,12 +115,29 @@ required: - clocks - clock-names - power-domains - - phy-supply - - resets - ports additionalProperties: false +allOf: + - if: + properties: + compatible: + contains: + const: fsl,imx7-mipi-csi2 + then: + required: + - phy-supply + - resets + else: + properties: + clocks: + minItems: 4 + clock-names: + minItems: 4 + phy-supply: false + resets: false + examples: - | #include @@ -106,7 +145,7 @@ examples: #include #include - mipi_csi: mipi-csi@30750000 { + mipi-csi@30750000 { compatible = "fsl,imx7-mipi-csi2"; reg = <0x30750000 0x10000>; interrupts = ; @@ -144,4 +183,46 @@ examples: }; }; + - | + #include + #include + #include + + mipi-csi@32e30000 { + compatible = "fsl,imx8mm-mipi-csi2"; + reg = <0x32e30000 0x1000>; + interrupts = ; + clock-frequency = <333000000>; + clocks = <&clk IMX8MM_CLK_DISP_APB_ROOT>, + <&clk IMX8MM_CLK_CSI1_ROOT>, + <&clk IMX8MM_CLK_CSI1_PHY_REF>, + <&clk IMX8MM_CLK_DISP_AXI_ROOT>; + clock-names = "pclk", "wrap", "phy", "axi"; + power-domains = <&mipi_pd>; + + status = "disabled"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + imx8mm_mipi_csi_in: endpoint { + remote-endpoint = <&imx477_out>; + data-lanes = <1 2 3 4>; + }; + }; + + port@1 { + reg = <1>; + + imx8mm_mipi_csi_out: endpoint { + remote-endpoint = <&csi_in>; + }; + }; + }; + }; + ... diff --git a/Documentation/devicetree/bindings/media/rc.yaml b/Documentation/devicetree/bindings/media/rc.yaml index af9e7e59e5a1..d4c541c4b164 100644 --- a/Documentation/devicetree/bindings/media/rc.yaml +++ b/Documentation/devicetree/bindings/media/rc.yaml @@ -45,6 +45,7 @@ properties: - rc-cec - rc-cinergy - rc-cinergy-1400 + - rc-ct-90405 - rc-d680-dmb - rc-delock-61959 - rc-dib0700-nec @@ -125,7 +126,6 @@ properties: - rc-snapstream-firefly - rc-streamzap - rc-su3000 - - rc-tango - rc-tanix-tx3mini - rc-tanix-tx5max - rc-tbs-nec diff --git a/Documentation/devicetree/bindings/media/renesas,csi2.yaml b/Documentation/devicetree/bindings/media/renesas,csi2.yaml index 20396f1be999..23703b767f5b 100644 --- a/Documentation/devicetree/bindings/media/renesas,csi2.yaml +++ b/Documentation/devicetree/bindings/media/renesas,csi2.yaml @@ -25,6 +25,7 @@ properties: - renesas,r8a774e1-csi2 # RZ/G2H - renesas,r8a7795-csi2 # R-Car H3 - renesas,r8a7796-csi2 # R-Car M3-W + - renesas,r8a77961-csi2 # R-Car M3-W+ - renesas,r8a77965-csi2 # R-Car M3-N - renesas,r8a77970-csi2 # R-Car V3M - renesas,r8a77980-csi2 # R-Car V3H diff --git a/Documentation/devicetree/bindings/media/renesas,isp.yaml b/Documentation/devicetree/bindings/media/renesas,isp.yaml new file mode 100644 index 000000000000..514857d36f6b --- /dev/null +++ b/Documentation/devicetree/bindings/media/renesas,isp.yaml @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +# Copyright (C) 2021 Renesas Electronics Corp. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/media/renesas,isp.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas R-Car ISP Channel Selector + +maintainers: + - Niklas Söderlund + +description: + The R-Car ISP Channel Selector provides MIPI CSI-2 VC and DT filtering + capabilities for the Renesas R-Car family of devices. It is used in + conjunction with the R-Car VIN and CSI-2 modules, which provides the video + capture capabilities. + +properties: + compatible: + items: + - enum: + - renesas,r8a779a0-isp # V3U + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + ports: + $ref: /schemas/graph.yaml#/properties/ports + + properties: + port@0: + $ref: /schemas/graph.yaml#/properties/port + description: + Input port node, multiple endpoints describing the connected R-Car + CSI-2 receivers. + + port@1: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 0. + + port@2: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 1. + + port@3: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 2. + + port@4: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 3. + + port@5: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 4. + + port@6: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 5. + + port@7: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 6. + + port@8: + $ref: /schemas/graph.yaml#/properties/port + description: + Single endpoint describing the R-Car VIN connected to output port 7. + + required: + - port@0 + - port@1 + - port@2 + - port@3 + - port@4 + - port@5 + - port@6 + - port@7 + - port@8 + +required: + - compatible + - reg + - interrupts + - clocks + - power-domains + - resets + - ports + +additionalProperties: false + +examples: + - | + #include + #include + #include + + isp1: isp@fed20000 { + compatible = "renesas,r8a779a0-isp"; + reg = <0xfed20000 0x10000>; + interrupts = ; + clocks = <&cpg CPG_MOD 613>; + power-domains = <&sysc R8A779A0_PD_A3ISP01>; + resets = <&cpg 613>; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + #address-cells = <1>; + #size-cells = <0>; + + reg = <0>; + isp1csi41: endpoint@1 { + reg = <1>; + remote-endpoint = <&csi41isp1>; + }; + }; + + port@1 { + reg = <1>; + isp1vin08: endpoint { + remote-endpoint = <&vin08isp1>; + }; + }; + + port@2 { + reg = <2>; + isp1vin09: endpoint { + remote-endpoint = <&vin09isp1>; + }; + }; + + port@3 { + reg = <3>; + isp1vin10: endpoint { + remote-endpoint = <&vin10isp1>; + }; + }; + + port@4 { + reg = <4>; + isp1vin11: endpoint { + remote-endpoint = <&vin11isp1>; + }; + }; + + port@5 { + reg = <5>; + isp1vin12: endpoint { + remote-endpoint = <&vin12isp1>; + }; + }; + + port@6 { + reg = <6>; + isp1vin13: endpoint { + remote-endpoint = <&vin13isp1>; + }; + }; + + port@7 { + reg = <7>; + isp1vin14: endpoint { + remote-endpoint = <&vin14isp1>; + }; + }; + + port@8 { + reg = <8>; + isp1vin15: endpoint { + remote-endpoint = <&vin15isp1>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/media/renesas,vin.yaml b/Documentation/devicetree/bindings/media/renesas,vin.yaml index dd1a5ce5896c..39bb6db2fb32 100644 --- a/Documentation/devicetree/bindings/media/renesas,vin.yaml +++ b/Documentation/devicetree/bindings/media/renesas,vin.yaml @@ -46,11 +46,13 @@ properties: - renesas,vin-r8a7779 # R-Car H1 - renesas,vin-r8a7795 # R-Car H3 - renesas,vin-r8a7796 # R-Car M3-W + - renesas,vin-r8a77961 # R-Car M3-W+ - renesas,vin-r8a77965 # R-Car M3-N - renesas,vin-r8a77970 # R-Car V3M - renesas,vin-r8a77980 # R-Car V3H - renesas,vin-r8a77990 # R-Car E3 - renesas,vin-r8a77995 # R-Car D3 + - renesas,vin-r8a779a0 # R-Car V3U reg: maxItems: 1 @@ -111,7 +113,7 @@ properties: description: VIN channel number $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 - maximum: 15 + maximum: 31 ports: $ref: /schemas/graph.yaml#/properties/ports @@ -187,6 +189,29 @@ properties: - required: - endpoint@3 + port@2: + $ref: /schemas/graph.yaml#/properties/port + description: + Input port node, multiple endpoints describing all the R-Car ISP + modules connected the VIN. + + properties: + endpoint@0: + $ref: /schemas/graph.yaml#/properties/endpoint + description: Endpoint connected to ISP0. + + endpoint@1: + $ref: /schemas/graph.yaml#/properties/endpoint + description: Endpoint connected to ISP1. + + endpoint@2: + $ref: /schemas/graph.yaml#/properties/endpoint + description: Endpoint connected to ISP2. + + endpoint@3: + $ref: /schemas/graph.yaml#/properties/endpoint + description: Endpoint connected to ISP3. + required: - compatible - reg diff --git a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml index 8d35c327018b..089f11d21b25 100644 --- a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml +++ b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml @@ -15,7 +15,11 @@ description: |- properties: compatible: - const: rockchip,rk3399-vdec + oneOf: + - const: rockchip,rk3399-vdec + - items: + - const: rockchip,rk3228-vdec + - const: rockchip,rk3399-vdec reg: maxItems: 1 @@ -37,6 +41,10 @@ properties: - const: cabac - const: core + assigned-clocks: true + + assigned-clock-rates: true + power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/media/rockchip-vpu.yaml b/Documentation/devicetree/bindings/media/rockchip-vpu.yaml index c81dbc3e8960..b88172a59de7 100644 --- a/Documentation/devicetree/bindings/media/rockchip-vpu.yaml +++ b/Documentation/devicetree/bindings/media/rockchip-vpu.yaml @@ -15,10 +15,19 @@ description: properties: compatible: - enum: - - rockchip,rk3288-vpu - - rockchip,rk3328-vpu - - rockchip,rk3399-vpu + oneOf: + - enum: + - rockchip,rk3036-vpu + - rockchip,rk3066-vpu + - rockchip,rk3288-vpu + - rockchip,rk3328-vpu + - rockchip,rk3399-vpu + - items: + - const: rockchip,rk3188-vpu + - const: rockchip,rk3066-vpu + - items: + - const: rockchip,rk3228-vpu + - const: rockchip,rk3399-vpu reg: maxItems: 1 @@ -35,12 +44,20 @@ properties: - const: vdpu clocks: - maxItems: 2 + oneOf: + - maxItems: 2 + - maxItems: 4 clock-names: - items: - - const: aclk - - const: hclk + oneOf: + - items: + - const: aclk + - const: hclk + - items: + - const: aclk_vdpu + - const: hclk_vdpu + - const: aclk_vepu + - const: hclk_vepu power-domains: maxItems: 1 diff --git a/Documentation/devicetree/bindings/media/tango-ir.txt b/Documentation/devicetree/bindings/media/tango-ir.txt deleted file mode 100644 index a9f00c2bf897..000000000000 --- a/Documentation/devicetree/bindings/media/tango-ir.txt +++ /dev/null @@ -1,21 +0,0 @@ -Sigma Designs Tango IR NEC/RC-5/RC-6 decoder (SMP86xx and SMP87xx) - -Required properties: - -- compatible: "sigma,smp8642-ir" -- reg: address/size of NEC+RC5 area, address/size of RC6 area -- interrupts: spec for IR IRQ -- clocks: spec for IR clock (typically the crystal oscillator) - -Optional properties: - -- linux,rc-map-name: see Documentation/devicetree/bindings/media/rc.txt - -Example: - - ir@10518 { - compatible = "sigma,smp8642-ir"; - reg = <0x10518 0x18>, <0x105e0 0x1c>; - interrupts = <21 IRQ_TYPE_EDGE_RISING>; - clocks = <&xtal>; - }; diff --git a/Documentation/devicetree/bindings/submitting-patches.rst b/Documentation/devicetree/bindings/submitting-patches.rst index 104fa8fb2c17..8087780f1685 100644 --- a/Documentation/devicetree/bindings/submitting-patches.rst +++ b/Documentation/devicetree/bindings/submitting-patches.rst @@ -7,8 +7,8 @@ Submitting Devicetree (DT) binding patches I. For patch submitters ======================= - 0) Normal patch submission rules from Documentation/process/submitting-patches.rst - applies. + 0) Normal patch submission rules from + Documentation/process/submitting-patches.rst applies. 1) The Documentation/ and include/dt-bindings/ portion of the patch should be a separate patch. The preferred subject prefix for binding patches is:: @@ -25,8 +25,8 @@ I. For patch submitters make dt_binding_check - See Documentation/devicetree/bindings/writing-schema.rst for more details about - schema and tools setup. + See Documentation/devicetree/bindings/writing-schema.rst for more details + about schema and tools setup. 3) DT binding files should be dual licensed. The preferred license tag is (GPL-2.0-only OR BSD-2-Clause). @@ -84,7 +84,8 @@ II. For kernel maintainers III. Notes ========== - 0) Please see :doc:`ABI` for details regarding devicetree ABI. + 0) Please see Documentation/devicetree/bindings/ABI.rst for details + regarding devicetree ABI. 1) This document is intended as a general familiarization with the process as decided at the 2013 Kernel Summit. When in doubt, the current word of the diff --git a/Documentation/doc-guide/contributing.rst b/Documentation/doc-guide/contributing.rst index 67ee3691f91f..207fd93d7c80 100644 --- a/Documentation/doc-guide/contributing.rst +++ b/Documentation/doc-guide/contributing.rst @@ -237,10 +237,10 @@ We have been trying to improve the situation through the creation of a set of "books" that group documentation for specific readers. These include: - - :doc:`../admin-guide/index` - - :doc:`../core-api/index` - - :doc:`../driver-api/index` - - :doc:`../userspace-api/index` + - Documentation/admin-guide/index.rst + - Documentation/core-api/index.rst + - Documentation/driver-api/index.rst + - Documentation/userspace-api/index.rst As well as this book on documentation itself. diff --git a/Documentation/driver-api/acpi/linuxized-acpica.rst b/Documentation/driver-api/acpi/linuxized-acpica.rst index 6bee03383225..cc234353d2c4 100644 --- a/Documentation/driver-api/acpi/linuxized-acpica.rst +++ b/Documentation/driver-api/acpi/linuxized-acpica.rst @@ -276,4 +276,4 @@ before they become available from the ACPICA release process. # git clone https://github.com/acpica/acpica # git clone https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git # cd acpica - # generate/linux/divergences.sh -s ../linux + # generate/linux/divergence.sh -s ../linux diff --git a/Documentation/driver-api/gpio/using-gpio.rst b/Documentation/driver-api/gpio/using-gpio.rst index dda069444032..64c8d3f76c3a 100644 --- a/Documentation/driver-api/gpio/using-gpio.rst +++ b/Documentation/driver-api/gpio/using-gpio.rst @@ -9,13 +9,13 @@ with them. For examples of already existing generic drivers that will also be good examples for any other kernel drivers you want to author, refer to -:doc:`drivers-on-gpio` +Documentation/driver-api/gpio/drivers-on-gpio.rst For any kind of mass produced system you want to support, such as servers, laptops, phones, tablets, routers, and any consumer or office or business goods using appropriate kernel drivers is paramount. Submit your code for inclusion in the upstream Linux kernel when you feel it is mature enough and you will get -help to refine it, see :doc:`../../process/submitting-patches`. +help to refine it, see Documentation/process/submitting-patches.rst. In Linux GPIO lines also have a userspace ABI. diff --git a/Documentation/driver-api/ioctl.rst b/Documentation/driver-api/ioctl.rst index c455db0e1627..35795f6a151a 100644 --- a/Documentation/driver-api/ioctl.rst +++ b/Documentation/driver-api/ioctl.rst @@ -25,16 +25,16 @@ ioctl commands that follow modern conventions: ``_IO``, ``_IOR``, with the correct parameters: _IO/_IOR/_IOW/_IOWR - The macro name specifies how the argument will be used.  It may be a + The macro name specifies how the argument will be used. It may be a pointer to data to be passed into the kernel (_IOW), out of the kernel - (_IOR), or both (_IOWR).  _IO can indicate either commands with no + (_IOR), or both (_IOWR). _IO can indicate either commands with no argument or those passing an integer value instead of a pointer. It is recommended to only use _IO for commands without arguments, and use pointers for passing data. type An 8-bit number, often a character literal, specific to a subsystem - or driver, and listed in :doc:`../userspace-api/ioctl/ioctl-number` + or driver, and listed in Documentation/userspace-api/ioctl/ioctl-number.rst nr An 8-bit number identifying the specific command, unique for a give @@ -200,10 +200,10 @@ cause an information leak, which can be used to defeat kernel address space layout randomization (KASLR), helping in an attack. For this reason (and for compat support) it is best to avoid any -implicit padding in data structures.  Where there is implicit padding +implicit padding in data structures. Where there is implicit padding in an existing structure, kernel drivers must be careful to fully initialize an instance of the structure before copying it to user -space.  This is usually done by calling memset() before assigning to +space. This is usually done by calling memset() before assigning to individual members. Subsystem abstractions diff --git a/Documentation/driver-api/media/drivers/bttv-devel.rst b/Documentation/driver-api/media/drivers/bttv-devel.rst index c9aa8b95a5e5..0885a04563a9 100644 --- a/Documentation/driver-api/media/drivers/bttv-devel.rst +++ b/Documentation/driver-api/media/drivers/bttv-devel.rst @@ -21,7 +21,7 @@ log, telling which card type is used. Like this one:: You should verify this is correct. If it isn't, you have to pass the correct board type as insmod argument, ``insmod bttv card=2`` for -example. The file :doc:`/admin-guide/media/bttv-cardlist` has a list +example. The file Documentation/admin-guide/media/bttv-cardlist.rst has a list of valid arguments for card. If your card isn't listed there, you might check the source code for diff --git a/Documentation/driver-api/media/drivers/ccs/ccs-regs.asc b/Documentation/driver-api/media/drivers/ccs/ccs-regs.asc index f2042acc8a45..bbf9213c3388 100644 --- a/Documentation/driver-api/media/drivers/ccs/ccs-regs.asc +++ b/Documentation/driver-api/media/drivers/ccs/ccs-regs.asc @@ -210,7 +210,7 @@ pll_multiplier 0x0306 16 op_pix_clk_div 0x0308 16 op_sys_clk_div 0x030a 16 op_pre_pll_clk_div 0x030c 16 -op_pll_multiplier 0x031e 16 +op_pll_multiplier 0x030e 16 pll_mode 0x0310 8 - f 0 0 - e single 0 diff --git a/Documentation/driver-api/media/drivers/ccs/mk-ccs-regs b/Documentation/driver-api/media/drivers/ccs/mk-ccs-regs index 6668deaf2f19..2a4edc7e051a 100755 --- a/Documentation/driver-api/media/drivers/ccs/mk-ccs-regs +++ b/Documentation/driver-api/media/drivers/ccs/mk-ccs-regs @@ -72,13 +72,14 @@ $uc_header =~ s/[^A-Z0-9]/_/g; my $copyright = "/* Copyright (C) 2019--2020 Intel Corporation */\n"; my $license = "SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause"; +my $note = "/*\n * Generated by $0;\n * do not modify.\n */\n"; for my $fh ($A, $LC) { - print $fh "// $license\n$copyright\n" if defined $fh; + print $fh "// $license\n$copyright$note\n" if defined $fh; } for my $fh ($H, $LH) { - print $fh "/* $license */\n$copyright\n"; + print $fh "/* $license */\n$copyright$note\n"; } sub bit_def($) { diff --git a/Documentation/driver-api/media/drivers/zoran.rst b/Documentation/driver-api/media/drivers/zoran.rst index 83cbae9cedef..b205e10c3154 100644 --- a/Documentation/driver-api/media/drivers/zoran.rst +++ b/Documentation/driver-api/media/drivers/zoran.rst @@ -319,7 +319,7 @@ Conexant bt866 TV encoder ~~~~~~~~~~~~~~~~~~~~~~~~~ - is used in AVS6EYES, and -- can generate: NTSC/PAL, PAL­M, PAL­N +- can generate: NTSC/PAL, PAL-M, PAL-N The adv717x, should be able to produce PAL N. But you find nothing PAL N specific in the registers. Seem that you have to reuse a other standard diff --git a/Documentation/driver-api/media/index.rst b/Documentation/driver-api/media/index.rst index 2ad71dfa8828..813d7db59da7 100644 --- a/Documentation/driver-api/media/index.rst +++ b/Documentation/driver-api/media/index.rst @@ -11,11 +11,13 @@ its supported drivers. Please see: -- :doc:`/admin-guide/media/index` - for usage information about media subsystem and supported drivers; +Documentation/admin-guide/media/index.rst -- :doc:`/userspace-api/media/index` - for the userspace APIs used on media devices. + - for usage information about media subsystem and supported drivers; + +Documentation/userspace-api/media/index.rst + + - for the userspace APIs used on media devices. .. only:: html diff --git a/Documentation/driver-api/pm/devices.rst b/Documentation/driver-api/pm/devices.rst index 6b3bfd29fd84..d448cb57df86 100644 --- a/Documentation/driver-api/pm/devices.rst +++ b/Documentation/driver-api/pm/devices.rst @@ -217,7 +217,7 @@ system-wide transition to a sleep state even though its :c:member:`runtime_auto` flag is clear. For more information about the runtime power management framework, refer to -:file:`Documentation/power/runtime_pm.rst`. +Documentation/power/runtime_pm.rst. Calling Drivers to Enter and Leave System Sleep States @@ -655,7 +655,7 @@ been thawed. Generally speaking, the PM notifiers are suitable for performing actions that either require user space to be available, or at least won't interfere with user space. -For details refer to :doc:`notifiers`. +For details refer to Documentation/driver-api/pm/notifiers.rst. Device Low-Power (suspend) States @@ -726,7 +726,7 @@ it into account in any way. Devices may be defined as IRQ-safe which indicates to the PM core that their runtime PM callbacks may be invoked with disabled interrupts (see -:file:`Documentation/power/runtime_pm.rst` for more information). If an +Documentation/power/runtime_pm.rst for more information). If an IRQ-safe device belongs to a PM domain, the runtime PM of the domain will be disallowed, unless the domain itself is defined as IRQ-safe. However, it makes sense to define a PM domain as IRQ-safe only if all the devices in it @@ -805,7 +805,7 @@ The ``DPM_FLAG_MAY_SKIP_RESUME`` Driver Flag -------------------------------------------- During system-wide resume from a sleep state it's easiest to put devices into -the full-power state, as explained in :file:`Documentation/power/runtime_pm.rst`. +the full-power state, as explained in Documentation/power/runtime_pm.rst. [Refer to that document for more information regarding this particular issue as well as for information on the device runtime power management framework in general.] However, it often is desirable to leave devices in suspend after diff --git a/Documentation/driver-api/surface_aggregator/clients/index.rst b/Documentation/driver-api/surface_aggregator/clients/index.rst index 98ea9946b8a2..30160513afa5 100644 --- a/Documentation/driver-api/surface_aggregator/clients/index.rst +++ b/Documentation/driver-api/surface_aggregator/clients/index.rst @@ -5,7 +5,8 @@ Client Driver Documentation =========================== This is the documentation for client drivers themselves. Refer to -:doc:`../client` for documentation on how to write client drivers. +Documentation/driver-api/surface_aggregator/client.rst for documentation +on how to write client drivers. .. toctree:: :maxdepth: 1 diff --git a/Documentation/driver-api/surface_aggregator/internal.rst b/Documentation/driver-api/surface_aggregator/internal.rst index 72704734982a..8c7c80c9f418 100644 --- a/Documentation/driver-api/surface_aggregator/internal.rst +++ b/Documentation/driver-api/surface_aggregator/internal.rst @@ -87,10 +87,11 @@ native SSAM devices, i.e. devices that are not defined in ACPI and not implemented as platform devices, via |ssam_device| and |ssam_device_driver| simplify management of client devices and client drivers. -Refer to :doc:`client` for documentation regarding the client device/driver -API and interface options for other kernel drivers. It is recommended to -familiarize oneself with that chapter and the :doc:`ssh` before continuing -with the architectural overview below. +Refer to Documentation/driver-api/surface_aggregator/client.rst for +documentation regarding the client device/driver API and interface options +for other kernel drivers. It is recommended to familiarize oneself with +that chapter and the Documentation/driver-api/surface_aggregator/ssh.rst +before continuing with the architectural overview below. Packet Transport Layer @@ -190,9 +191,9 @@ with success on the transmitter thread. Transmission of sequenced packets is limited by the number of concurrently pending packets, i.e. a limit on how many packets may be waiting for an ACK -from the EC in parallel. This limit is currently set to one (see :doc:`ssh` -for the reasoning behind this). Control packets (i.e. ACK and NAK) can -always be transmitted. +from the EC in parallel. This limit is currently set to one (see +Documentation/driver-api/surface_aggregator/ssh.rst for the reasoning behind +this). Control packets (i.e. ACK and NAK) can always be transmitted. Receiver Thread --------------- diff --git a/Documentation/driver-api/surface_aggregator/overview.rst b/Documentation/driver-api/surface_aggregator/overview.rst index 1e9d57e50063..26415e1ab7da 100644 --- a/Documentation/driver-api/surface_aggregator/overview.rst +++ b/Documentation/driver-api/surface_aggregator/overview.rst @@ -73,5 +73,7 @@ being a direct response to a previous request. We may also refer to requests without response as commands. In general, events need to be enabled via one of multiple dedicated requests before they are sent by the EC. -See :doc:`ssh` for a more technical protocol documentation and -:doc:`internal` for an overview of the internal driver architecture. +See Documentation/driver-api/surface_aggregator/ssh.rst for a +more technical protocol documentation and +Documentation/driver-api/surface_aggregator/internal.rst for an +overview of the internal driver architecture. diff --git a/Documentation/driver-api/usb/dma.rst b/Documentation/driver-api/usb/dma.rst index 2b3dbd3265b4..d32c27e11b90 100644 --- a/Documentation/driver-api/usb/dma.rst +++ b/Documentation/driver-api/usb/dma.rst @@ -10,7 +10,7 @@ API overview The big picture is that USB drivers can continue to ignore most DMA issues, though they still must provide DMA-ready buffers (see -:doc:`/core-api/dma-api-howto`). That's how they've worked through +Documentation/core-api/dma-api-howto.rst). That's how they've worked through the 2.4 (and earlier) kernels, or they can now be DMA-aware. DMA-aware usb drivers: @@ -60,7 +60,7 @@ and effects like cache-trashing can impose subtle penalties. force a consistent memory access ordering by using memory barriers. It's not using a streaming DMA mapping, so it's good for small transfers on systems where the I/O would otherwise thrash an IOMMU mapping. (See - :doc:`/core-api/dma-api-howto` for definitions of "coherent" and + Documentation/core-api/dma-api-howto.rst for definitions of "coherent" and "streaming" DMA mappings.) Asking for 1/Nth of a page (as well as asking for N pages) is reasonably @@ -91,7 +91,7 @@ Working with existing buffers Existing buffers aren't usable for DMA without first being mapped into the DMA address space of the device. However, most buffers passed to your driver can safely be used with such DMA mapping. (See the first section -of :doc:`/core-api/dma-api-howto`, titled "What memory is DMA-able?") +of Documentation/core-api/dma-api-howto.rst, titled "What memory is DMA-able?") - When you're using scatterlists, you can map everything at once. On some systems, this kicks in an IOMMU and turns the scatterlists into single diff --git a/Documentation/fault-injection/fault-injection.rst b/Documentation/fault-injection/fault-injection.rst index 31ecfe44e5b4..f47d05ed0d94 100644 --- a/Documentation/fault-injection/fault-injection.rst +++ b/Documentation/fault-injection/fault-injection.rst @@ -78,8 +78,10 @@ configuration of fault-injection capabilities. - /sys/kernel/debug/fail*/times: - specifies how many times failures may happen at most. - A value of -1 means "no limit". + specifies how many times failures may happen at most. A value of -1 + means "no limit". Note, though, that this file only accepts unsigned + values. So, if you want to specify -1, you better use 'printf' instead + of 'echo', e.g.: $ printf %#x -1 > times - /sys/kernel/debug/fail*/space: @@ -167,11 +169,13 @@ configuration of fault-injection capabilities. - ERRNO: retval must be -1 to -MAX_ERRNO (-4096). - ERR_NULL: retval must be 0 or -1 to -MAX_ERRNO (-4096). -- /sys/kernel/debug/fail_function//retval: +- /sys/kernel/debug/fail_function//retval: - specifies the "error" return value to inject to the given - function for given function. This will be created when - user specifies new injection entry. + specifies the "error" return value to inject to the given function. + This will be created when the user specifies a new injection entry. + Note that this file only accepts unsigned values. So, if you want to + use a negative errno, you better use 'printf' instead of 'echo', e.g.: + $ printf %#x -12 > retval Boot option ^^^^^^^^^^^ @@ -255,7 +259,7 @@ Application Examples echo Y > /sys/kernel/debug/$FAILTYPE/task-filter echo 10 > /sys/kernel/debug/$FAILTYPE/probability echo 100 > /sys/kernel/debug/$FAILTYPE/interval - echo -1 > /sys/kernel/debug/$FAILTYPE/times + printf %#x -1 > /sys/kernel/debug/$FAILTYPE/times echo 0 > /sys/kernel/debug/$FAILTYPE/space echo 2 > /sys/kernel/debug/$FAILTYPE/verbose echo 1 > /sys/kernel/debug/$FAILTYPE/ignore-gfp-wait @@ -309,7 +313,7 @@ Application Examples echo N > /sys/kernel/debug/$FAILTYPE/task-filter echo 10 > /sys/kernel/debug/$FAILTYPE/probability echo 100 > /sys/kernel/debug/$FAILTYPE/interval - echo -1 > /sys/kernel/debug/$FAILTYPE/times + printf %#x -1 > /sys/kernel/debug/$FAILTYPE/times echo 0 > /sys/kernel/debug/$FAILTYPE/space echo 2 > /sys/kernel/debug/$FAILTYPE/verbose echo 1 > /sys/kernel/debug/$FAILTYPE/ignore-gfp-wait @@ -336,11 +340,11 @@ Application Examples FAILTYPE=fail_function FAILFUNC=open_ctree echo $FAILFUNC > /sys/kernel/debug/$FAILTYPE/inject - echo -12 > /sys/kernel/debug/$FAILTYPE/$FAILFUNC/retval + printf %#x -12 > /sys/kernel/debug/$FAILTYPE/$FAILFUNC/retval echo N > /sys/kernel/debug/$FAILTYPE/task-filter echo 100 > /sys/kernel/debug/$FAILTYPE/probability echo 0 > /sys/kernel/debug/$FAILTYPE/interval - echo -1 > /sys/kernel/debug/$FAILTYPE/times + printf %#x -1 > /sys/kernel/debug/$FAILTYPE/times echo 0 > /sys/kernel/debug/$FAILTYPE/space echo 1 > /sys/kernel/debug/$FAILTYPE/verbose diff --git a/Documentation/filesystems/dax.rst b/Documentation/filesystems/dax.rst new file mode 100644 index 000000000000..9a1b8fd9e82b --- /dev/null +++ b/Documentation/filesystems/dax.rst @@ -0,0 +1,291 @@ +======================= +Direct Access for files +======================= + +Motivation +---------- + +The page cache is usually used to buffer reads and writes to files. +It is also used to provide the pages which are mapped into userspace +by a call to mmap. + +For block devices that are memory-like, the page cache pages would be +unnecessary copies of the original storage. The `DAX` code removes the +extra copy by performing reads and writes directly to the storage device. +For file mappings, the storage device is mapped directly into userspace. + + +Usage +----- + +If you have a block device which supports `DAX`, you can make a filesystem +on it as usual. The `DAX` code currently only supports files with a block +size equal to your kernel's `PAGE_SIZE`, so you may need to specify a block +size when creating the filesystem. + +Currently 3 filesystems support `DAX`: ext2, ext4 and xfs. Enabling `DAX` on them +is different. + +Enabling DAX on ext2 +-------------------- + +When mounting the filesystem, use the ``-o dax`` option on the command line or +add 'dax' to the options in ``/etc/fstab``. This works to enable `DAX` on all files +within the filesystem. It is equivalent to the ``-o dax=always`` behavior below. + + +Enabling DAX on xfs and ext4 +---------------------------- + +Summary +------- + + 1. There exists an in-kernel file access mode flag `S_DAX` that corresponds to + the statx flag `STATX_ATTR_DAX`. See the manpage for statx(2) for details + about this access mode. + + 2. There exists a persistent flag `FS_XFLAG_DAX` that can be applied to regular + files and directories. This advisory flag can be set or cleared at any + time, but doing so does not immediately affect the `S_DAX` state. + + 3. If the persistent `FS_XFLAG_DAX` flag is set on a directory, this flag will + be inherited by all regular files and subdirectories that are subsequently + created in this directory. Files and subdirectories that exist at the time + this flag is set or cleared on the parent directory are not modified by + this modification of the parent directory. + + 4. There exist dax mount options which can override `FS_XFLAG_DAX` in the + setting of the `S_DAX` flag. Given underlying storage which supports `DAX` the + following hold: + + ``-o dax=inode`` means "follow `FS_XFLAG_DAX`" and is the default. + + ``-o dax=never`` means "never set `S_DAX`, ignore `FS_XFLAG_DAX`." + + ``-o dax=always`` means "always set `S_DAX` ignore `FS_XFLAG_DAX`." + + ``-o dax`` is a legacy option which is an alias for ``dax=always``. + + .. warning:: + + The option ``-o dax`` may be removed in the future so ``-o dax=always`` is + the preferred method for specifying this behavior. + + .. note:: + + Modifications to and the inheritance behavior of `FS_XFLAG_DAX` remain + the same even when the filesystem is mounted with a dax option. However, + in-core inode state (`S_DAX`) will be overridden until the filesystem is + remounted with dax=inode and the inode is evicted from kernel memory. + + 5. The `S_DAX` policy can be changed via: + + a) Setting the parent directory `FS_XFLAG_DAX` as needed before files are + created + + b) Setting the appropriate dax="foo" mount option + + c) Changing the `FS_XFLAG_DAX` flag on existing regular files and + directories. This has runtime constraints and limitations that are + described in 6) below. + + 6. When changing the `S_DAX` policy via toggling the persistent `FS_XFLAG_DAX` + flag, the change to existing regular files won't take effect until the + files are closed by all processes. + + +Details +------- + +There are 2 per-file dax flags. One is a persistent inode setting (`FS_XFLAG_DAX`) +and the other is a volatile flag indicating the active state of the feature +(`S_DAX`). + +`FS_XFLAG_DAX` is preserved within the filesystem. This persistent config +setting can be set, cleared and/or queried using the `FS_IOC_FS`[`GS`]`ETXATTR` ioctl +(see ioctl_xfs_fsgetxattr(2)) or an utility such as 'xfs_io'. + +New files and directories automatically inherit `FS_XFLAG_DAX` from +their parent directory **when created**. Therefore, setting `FS_XFLAG_DAX` at +directory creation time can be used to set a default behavior for an entire +sub-tree. + +To clarify inheritance, here are 3 examples: + +Example A: + +.. code-block:: shell + + mkdir -p a/b/c + xfs_io -c 'chattr +x' a + mkdir a/b/c/d + mkdir a/e + + ------[outcome]------ + + dax: a,e + no dax: b,c,d + +Example B: + +.. code-block:: shell + + mkdir a + xfs_io -c 'chattr +x' a + mkdir -p a/b/c/d + + ------[outcome]------ + + dax: a,b,c,d + no dax: + +Example C: + +.. code-block:: shell + + mkdir -p a/b/c + xfs_io -c 'chattr +x' c + mkdir a/b/c/d + + ------[outcome]------ + + dax: c,d + no dax: a,b + +The current enabled state (`S_DAX`) is set when a file inode is instantiated in +memory by the kernel. It is set based on the underlying media support, the +value of `FS_XFLAG_DAX` and the filesystem's dax mount option. + +statx can be used to query `S_DAX`. + +.. note:: + + That only regular files will ever have `S_DAX` set and therefore statx + will never indicate that `S_DAX` is set on directories. + +Setting the `FS_XFLAG_DAX` flag (specifically or through inheritance) occurs even +if the underlying media does not support dax and/or the filesystem is +overridden with a mount option. + + +Implementation Tips for Block Driver Writers +-------------------------------------------- + +To support `DAX` in your block driver, implement the 'direct_access' +block device operation. It is used to translate the sector number +(expressed in units of 512-byte sectors) to a page frame number (pfn) +that identifies the physical page for the memory. It also returns a +kernel virtual address that can be used to access the memory. + +The direct_access method takes a 'size' parameter that indicates the +number of bytes being requested. The function should return the number +of bytes that can be contiguously accessed at that offset. It may also +return a negative errno if an error occurs. + +In order to support this method, the storage must be byte-accessible by +the CPU at all times. If your device uses paging techniques to expose +a large amount of memory through a smaller window, then you cannot +implement direct_access. Equally, if your device can occasionally +stall the CPU for an extended period, you should also not attempt to +implement direct_access. + +These block devices may be used for inspiration: +- brd: RAM backed block device driver +- dcssblk: s390 dcss block device driver +- pmem: NVDIMM persistent memory driver + + +Implementation Tips for Filesystem Writers +------------------------------------------ + +Filesystem support consists of: + +* Adding support to mark inodes as being `DAX` by setting the `S_DAX` flag in + i_flags +* Implementing ->read_iter and ->write_iter operations which use + :c:func:`dax_iomap_rw()` when inode has `S_DAX` flag set +* Implementing an mmap file operation for `DAX` files which sets the + `VM_MIXEDMAP` and `VM_HUGEPAGE` flags on the `VMA`, and setting the vm_ops to + include handlers for fault, pmd_fault, page_mkwrite, pfn_mkwrite. These + handlers should probably call :c:func:`dax_iomap_fault()` passing the + appropriate fault size and iomap operations. +* Calling :c:func:`iomap_zero_range()` passing appropriate iomap operations + instead of :c:func:`block_truncate_page()` for `DAX` files +* Ensuring that there is sufficient locking between reads, writes, + truncates and page faults + +The iomap handlers for allocating blocks must make sure that allocated blocks +are zeroed out and converted to written extents before being returned to avoid +exposure of uninitialized data through mmap. + +These filesystems may be used for inspiration: + +.. seealso:: + + ext2: see Documentation/filesystems/ext2.rst + +.. seealso:: + + xfs: see Documentation/admin-guide/xfs.rst + +.. seealso:: + + ext4: see Documentation/filesystems/ext4/ + + +Handling Media Errors +--------------------- + +The libnvdimm subsystem stores a record of known media error locations for +each pmem block device (in gendisk->badblocks). If we fault at such location, +or one with a latent error not yet discovered, the application can expect +to receive a `SIGBUS`. Libnvdimm also allows clearing of these errors by simply +writing the affected sectors (through the pmem driver, and if the underlying +NVDIMM supports the clear_poison DSM defined by ACPI). + +Since `DAX` IO normally doesn't go through the ``driver/bio`` path, applications or +sysadmins have an option to restore the lost data from a prior ``backup/inbuilt`` +redundancy in the following ways: + +1. Delete the affected file, and restore from a backup (sysadmin route): + This will free the filesystem blocks that were being used by the file, + and the next time they're allocated, they will be zeroed first, which + happens through the driver, and will clear bad sectors. + +2. Truncate or hole-punch the part of the file that has a bad-block (at least + an entire aligned sector has to be hole-punched, but not necessarily an + entire filesystem block). + +These are the two basic paths that allow `DAX` filesystems to continue operating +in the presence of media errors. More robust error recovery mechanisms can be +built on top of this in the future, for example, involving redundancy/mirroring +provided at the block layer through DM, or additionally, at the filesystem +level. These would have to rely on the above two tenets, that error clearing +can happen either by sending an IO through the driver, or zeroing (also through +the driver). + + +Shortcomings +------------ + +Even if the kernel or its modules are stored on a filesystem that supports +`DAX` on a block device that supports `DAX`, they will still be copied into RAM. + +The DAX code does not work correctly on architectures which have virtually +mapped caches such as ARM, MIPS and SPARC. + +Calling :c:func:`get_user_pages()` on a range of user memory that has been +mmaped from a `DAX` file will fail when there are no 'struct page' to describe +those pages. This problem has been addressed in some device drivers +by adding optional struct page support for pages under the control of +the driver (see `CONFIG_NVDIMM_PFN` in ``drivers/nvdimm`` for an example of +how to do this). In the non struct page cases `O_DIRECT` reads/writes to +those memory ranges from a non-`DAX` file will fail + + +.. note:: + + `O_DIRECT` reads/writes _of a `DAX` file do work, it is the memory that + is being accessed that is key here). Other things that will not work in + the non struct page case include RDMA, :c:func:`sendfile()` and + :c:func:`splice()`. diff --git a/Documentation/filesystems/dax.txt b/Documentation/filesystems/dax.txt deleted file mode 100644 index e03c20564f3a..000000000000 --- a/Documentation/filesystems/dax.txt +++ /dev/null @@ -1,257 +0,0 @@ -Direct Access for files ------------------------ - -Motivation ----------- - -The page cache is usually used to buffer reads and writes to files. -It is also used to provide the pages which are mapped into userspace -by a call to mmap. - -For block devices that are memory-like, the page cache pages would be -unnecessary copies of the original storage. The DAX code removes the -extra copy by performing reads and writes directly to the storage device. -For file mappings, the storage device is mapped directly into userspace. - - -Usage ------ - -If you have a block device which supports DAX, you can make a filesystem -on it as usual. The DAX code currently only supports files with a block -size equal to your kernel's PAGE_SIZE, so you may need to specify a block -size when creating the filesystem. - -Currently 3 filesystems support DAX: ext2, ext4 and xfs. Enabling DAX on them -is different. - -Enabling DAX on ext2 ------------------------------ - -When mounting the filesystem, use the "-o dax" option on the command line or -add 'dax' to the options in /etc/fstab. This works to enable DAX on all files -within the filesystem. It is equivalent to the '-o dax=always' behavior below. - - -Enabling DAX on xfs and ext4 ----------------------------- - -Summary -------- - - 1. There exists an in-kernel file access mode flag S_DAX that corresponds to - the statx flag STATX_ATTR_DAX. See the manpage for statx(2) for details - about this access mode. - - 2. There exists a persistent flag FS_XFLAG_DAX that can be applied to regular - files and directories. This advisory flag can be set or cleared at any - time, but doing so does not immediately affect the S_DAX state. - - 3. If the persistent FS_XFLAG_DAX flag is set on a directory, this flag will - be inherited by all regular files and subdirectories that are subsequently - created in this directory. Files and subdirectories that exist at the time - this flag is set or cleared on the parent directory are not modified by - this modification of the parent directory. - - 4. There exist dax mount options which can override FS_XFLAG_DAX in the - setting of the S_DAX flag. Given underlying storage which supports DAX the - following hold: - - "-o dax=inode" means "follow FS_XFLAG_DAX" and is the default. - - "-o dax=never" means "never set S_DAX, ignore FS_XFLAG_DAX." - - "-o dax=always" means "always set S_DAX ignore FS_XFLAG_DAX." - - "-o dax" is a legacy option which is an alias for "dax=always". - This may be removed in the future so "-o dax=always" is - the preferred method for specifying this behavior. - - NOTE: Modifications to and the inheritance behavior of FS_XFLAG_DAX remain - the same even when the filesystem is mounted with a dax option. However, - in-core inode state (S_DAX) will be overridden until the filesystem is - remounted with dax=inode and the inode is evicted from kernel memory. - - 5. The S_DAX policy can be changed via: - - a) Setting the parent directory FS_XFLAG_DAX as needed before files are - created - - b) Setting the appropriate dax="foo" mount option - - c) Changing the FS_XFLAG_DAX flag on existing regular files and - directories. This has runtime constraints and limitations that are - described in 6) below. - - 6. When changing the S_DAX policy via toggling the persistent FS_XFLAG_DAX - flag, the change to existing regular files won't take effect until the - files are closed by all processes. - - -Details -------- - -There are 2 per-file dax flags. One is a persistent inode setting (FS_XFLAG_DAX) -and the other is a volatile flag indicating the active state of the feature -(S_DAX). - -FS_XFLAG_DAX is preserved within the filesystem. This persistent config -setting can be set, cleared and/or queried using the FS_IOC_FS[GS]ETXATTR ioctl -(see ioctl_xfs_fsgetxattr(2)) or an utility such as 'xfs_io'. - -New files and directories automatically inherit FS_XFLAG_DAX from -their parent directory _when_ _created_. Therefore, setting FS_XFLAG_DAX at -directory creation time can be used to set a default behavior for an entire -sub-tree. - -To clarify inheritance, here are 3 examples: - -Example A: - -mkdir -p a/b/c -xfs_io -c 'chattr +x' a -mkdir a/b/c/d -mkdir a/e - - dax: a,e - no dax: b,c,d - -Example B: - -mkdir a -xfs_io -c 'chattr +x' a -mkdir -p a/b/c/d - - dax: a,b,c,d - no dax: - -Example C: - -mkdir -p a/b/c -xfs_io -c 'chattr +x' c -mkdir a/b/c/d - - dax: c,d - no dax: a,b - - -The current enabled state (S_DAX) is set when a file inode is instantiated in -memory by the kernel. It is set based on the underlying media support, the -value of FS_XFLAG_DAX and the filesystem's dax mount option. - -statx can be used to query S_DAX. NOTE that only regular files will ever have -S_DAX set and therefore statx will never indicate that S_DAX is set on -directories. - -Setting the FS_XFLAG_DAX flag (specifically or through inheritance) occurs even -if the underlying media does not support dax and/or the filesystem is -overridden with a mount option. - - - -Implementation Tips for Block Driver Writers --------------------------------------------- - -To support DAX in your block driver, implement the 'direct_access' -block device operation. It is used to translate the sector number -(expressed in units of 512-byte sectors) to a page frame number (pfn) -that identifies the physical page for the memory. It also returns a -kernel virtual address that can be used to access the memory. - -The direct_access method takes a 'size' parameter that indicates the -number of bytes being requested. The function should return the number -of bytes that can be contiguously accessed at that offset. It may also -return a negative errno if an error occurs. - -In order to support this method, the storage must be byte-accessible by -the CPU at all times. If your device uses paging techniques to expose -a large amount of memory through a smaller window, then you cannot -implement direct_access. Equally, if your device can occasionally -stall the CPU for an extended period, you should also not attempt to -implement direct_access. - -These block devices may be used for inspiration: -- brd: RAM backed block device driver -- dcssblk: s390 dcss block device driver -- pmem: NVDIMM persistent memory driver - - -Implementation Tips for Filesystem Writers ------------------------------------------- - -Filesystem support consists of -- adding support to mark inodes as being DAX by setting the S_DAX flag in - i_flags -- implementing ->read_iter and ->write_iter operations which use dax_iomap_rw() - when inode has S_DAX flag set -- implementing an mmap file operation for DAX files which sets the - VM_MIXEDMAP and VM_HUGEPAGE flags on the VMA, and setting the vm_ops to - include handlers for fault, pmd_fault, page_mkwrite, pfn_mkwrite. These - handlers should probably call dax_iomap_fault() passing the appropriate - fault size and iomap operations. -- calling iomap_zero_range() passing appropriate iomap operations instead of - block_truncate_page() for DAX files -- ensuring that there is sufficient locking between reads, writes, - truncates and page faults - -The iomap handlers for allocating blocks must make sure that allocated blocks -are zeroed out and converted to written extents before being returned to avoid -exposure of uninitialized data through mmap. - -These filesystems may be used for inspiration: -- ext2: see Documentation/filesystems/ext2.rst -- ext4: see Documentation/filesystems/ext4/ -- xfs: see Documentation/admin-guide/xfs.rst - - -Handling Media Errors ---------------------- - -The libnvdimm subsystem stores a record of known media error locations for -each pmem block device (in gendisk->badblocks). If we fault at such location, -or one with a latent error not yet discovered, the application can expect -to receive a SIGBUS. Libnvdimm also allows clearing of these errors by simply -writing the affected sectors (through the pmem driver, and if the underlying -NVDIMM supports the clear_poison DSM defined by ACPI). - -Since DAX IO normally doesn't go through the driver/bio path, applications or -sysadmins have an option to restore the lost data from a prior backup/inbuilt -redundancy in the following ways: - -1. Delete the affected file, and restore from a backup (sysadmin route): - This will free the filesystem blocks that were being used by the file, - and the next time they're allocated, they will be zeroed first, which - happens through the driver, and will clear bad sectors. - -2. Truncate or hole-punch the part of the file that has a bad-block (at least - an entire aligned sector has to be hole-punched, but not necessarily an - entire filesystem block). - -These are the two basic paths that allow DAX filesystems to continue operating -in the presence of media errors. More robust error recovery mechanisms can be -built on top of this in the future, for example, involving redundancy/mirroring -provided at the block layer through DM, or additionally, at the filesystem -level. These would have to rely on the above two tenets, that error clearing -can happen either by sending an IO through the driver, or zeroing (also through -the driver). - - -Shortcomings ------------- - -Even if the kernel or its modules are stored on a filesystem that supports -DAX on a block device that supports DAX, they will still be copied into RAM. - -The DAX code does not work correctly on architectures which have virtually -mapped caches such as ARM, MIPS and SPARC. - -Calling get_user_pages() on a range of user memory that has been mmaped -from a DAX file will fail when there are no 'struct page' to describe -those pages. This problem has been addressed in some device drivers -by adding optional struct page support for pages under the control of -the driver (see CONFIG_NVDIMM_PFN in drivers/nvdimm for an example of -how to do this). In the non struct page cases O_DIRECT reads/writes to -those memory ranges from a non-DAX file will fail (note that O_DIRECT -reads/writes _of a DAX file_ do work, it is the memory that is being -accessed that is key here). Other things that will not work in the -non struct page case include RDMA, sendfile() and splice(). diff --git a/Documentation/filesystems/ext2.rst b/Documentation/filesystems/ext2.rst index c2fce22cfd03..154101cf0e4f 100644 --- a/Documentation/filesystems/ext2.rst +++ b/Documentation/filesystems/ext2.rst @@ -25,7 +25,7 @@ check=none, nocheck (*) Don't do extra checking of bitmaps on mount (check=normal and check=strict options removed) dax Use direct access (no page cache). See - Documentation/filesystems/dax.txt. + Documentation/filesystems/dax.rst. debug Extra debugging information is sent to the kernel syslog. Useful for developers. diff --git a/Documentation/filesystems/ext4/blockgroup.rst b/Documentation/filesystems/ext4/blockgroup.rst index 3da156633339..d5d652addce5 100644 --- a/Documentation/filesystems/ext4/blockgroup.rst +++ b/Documentation/filesystems/ext4/blockgroup.rst @@ -84,7 +84,7 @@ Without the option META\_BG, for safety concerns, all block group descriptors copies are kept in the first block group. Given the default 128MiB(2^27 bytes) block group size and 64-byte group descriptors, ext4 can have at most 2^27/64 = 2^21 block groups. This limits the entire -filesystem size to 2^21 ∗ 2^27 = 2^48bytes or 256TiB. +filesystem size to 2^21 * 2^27 = 2^48bytes or 256TiB. The solution to this problem is to use the metablock group feature (META\_BG), which is already in ext3 for all 2.6 releases. With the diff --git a/Documentation/filesystems/index.rst b/Documentation/filesystems/index.rst index d4853cb919d2..246af51b277a 100644 --- a/Documentation/filesystems/index.rst +++ b/Documentation/filesystems/index.rst @@ -77,6 +77,7 @@ Documentation for filesystem implementations. coda configfs cramfs + dax debugfs dlmfs ecryptfs diff --git a/Documentation/filesystems/path-lookup.rst b/Documentation/filesystems/path-lookup.rst index c482e1619e77..a6fa7619b69e 100644 --- a/Documentation/filesystems/path-lookup.rst +++ b/Documentation/filesystems/path-lookup.rst @@ -448,15 +448,17 @@ described. If it finds a ``LAST_NORM`` component it first calls filesystem to revalidate the result if it is that sort of filesystem. If that doesn't get a good result, it calls "``lookup_slow()``" which takes ``i_rwsem``, rechecks the cache, and then asks the filesystem -to find a definitive answer. Each of these will call -``follow_managed()`` (as described below) to handle any mount points. +to find a definitive answer. -In the absence of symbolic links, ``walk_component()`` creates a new -``struct path`` containing a counted reference to the new dentry and a -reference to the new ``vfsmount`` which is only counted if it is -different from the previous ``vfsmount``. It then calls -``path_to_nameidata()`` to install the new ``struct path`` in the -``struct nameidata`` and drop the unneeded references. +As the last step of walk_component(), step_into() will be called either +directly from walk_component() or from handle_dots(). It calls +handle_mounts(), to check and handle mount points, in which a new +``struct path`` is created containing a counted reference to the new dentry and +a reference to the new ``vfsmount`` which is only counted if it is +different from the previous ``vfsmount``. Then if there is +a symbolic link, step_into() calls pick_link() to deal with it, +otherwise it installs the new ``struct path`` in the ``struct nameidata``, and +drops the unneeded references. This "hand-over-hand" sequencing of getting a reference to the new dentry before dropping the reference to the previous dentry may @@ -470,8 +472,8 @@ Handling the final component ``nd->last_type`` to refer to the final component of the path. It does not call ``walk_component()`` that last time. Handling that final component remains for the caller to sort out. Those callers are -``path_lookupat()``, ``path_parentat()``, ``path_mountpoint()`` and -``path_openat()`` each of which handles the differing requirements of +path_lookupat(), path_parentat() and +path_openat() each of which handles the differing requirements of different system calls. ``path_parentat()`` is clearly the simplest - it just wraps a little bit @@ -486,20 +488,18 @@ perform their operation. object is wanted such as by ``stat()`` or ``chmod()``. It essentially just calls ``walk_component()`` on the final component through a call to ``lookup_last()``. ``path_lookupat()`` returns just the final dentry. - -``path_mountpoint()`` handles the special case of unmounting which must -not try to revalidate the mounted filesystem. It effectively -contains, through a call to ``mountpoint_last()``, an alternate -implementation of ``lookup_slow()`` which skips that step. This is -important when unmounting a filesystem that is inaccessible, such as +It is worth noting that when flag ``LOOKUP_MOUNTPOINT`` is set, +path_lookupat() will unset LOOKUP_JUMPED in nameidata so that in the +subsequent path traversal d_weak_revalidate() won't be called. +This is important when unmounting a filesystem that is inaccessible, such as one provided by a dead NFS server. Finally ``path_openat()`` is used for the ``open()`` system call; it -contains, in support functions starting with "``do_last()``", all the +contains, in support functions starting with "open_last_lookups()", all the complexity needed to handle the different subtleties of O_CREAT (with or without O_EXCL), final "``/``" characters, and trailing symbolic links. We will revisit this in the final part of this series, which -focuses on those symbolic links. "``do_last()``" will sometimes, but +focuses on those symbolic links. "open_last_lookups()" will sometimes, but not always, take ``i_rwsem``, depending on what it finds. Each of these, or the functions which call them, need to be alert to @@ -535,8 +535,7 @@ covered in greater detail in autofs.txt in the Linux documentation tree, but a few notes specifically related to path lookup are in order here. -The Linux VFS has a concept of "managed" dentries which is reflected -in function names such as "``follow_managed()``". There are three +The Linux VFS has a concept of "managed" dentries. There are three potentially interesting things about these dentries corresponding to three different flags that might be set in ``dentry->d_flags``: @@ -652,10 +651,10 @@ RCU-walk finds it cannot stop gracefully, it simply gives up and restarts from the top with REF-walk. This pattern of "try RCU-walk, if that fails try REF-walk" can be -clearly seen in functions like ``filename_lookup()``, -``filename_parentat()``, ``filename_mountpoint()``, -``do_filp_open()``, and ``do_file_open_root()``. These five -correspond roughly to the four ``path_*()`` functions we met earlier, +clearly seen in functions like filename_lookup(), +filename_parentat(), +do_filp_open(), and do_file_open_root(). These four +correspond roughly to the three ``path_*()`` functions we met earlier, each of which calls ``link_path_walk()``. The ``path_*()`` functions are called using different mode flags until a mode is found which works. They are first called with ``LOOKUP_RCU`` set to request "RCU-walk". If @@ -993,8 +992,8 @@ is 4096. There are a number of reasons for this limit; not letting the kernel spend too much time on just one path is one of them. With symbolic links you can effectively generate much longer paths so some sort of limit is needed for the same reason. Linux imposes a limit of -at most 40 symlinks in any one path lookup. It previously imposed a -further limit of eight on the maximum depth of recursion, but that was +at most 40 (MAXSYMLINKS) symlinks in any one path lookup. It previously imposed +a further limit of eight on the maximum depth of recursion, but that was raised to 40 when a separate stack was implemented, so there is now just the one limit. @@ -1061,42 +1060,26 @@ filesystem cannot successfully get a reference in RCU-walk mode, it must return ``-ECHILD`` and ``unlazy_walk()`` will be called to return to REF-walk mode in which the filesystem is allowed to sleep. -The place for all this to happen is the ``i_op->follow_link()`` inode -method. In the present mainline code this is never actually called in -RCU-walk mode as the rewrite is not quite complete. It is likely that -in a future release this method will be passed an ``inode`` pointer when -called in RCU-walk mode so it both (1) knows to be careful, and (2) has the -validated pointer. Much like the ``i_op->permission()`` method we -looked at previously, ``->follow_link()`` would need to be careful that +The place for all this to happen is the ``i_op->get_link()`` inode +method. This is called both in RCU-walk and REF-walk. In RCU-walk the +``dentry*`` argument is NULL, ``->get_link()`` can return -ECHILD to drop out of +RCU-walk. Much like the ``i_op->permission()`` method we +looked at previously, ``->get_link()`` would need to be careful that all the data structures it references are safe to be accessed while -holding no counted reference, only the RCU lock. Though getting a -reference with ``->follow_link()`` is not yet done in RCU-walk mode, the -code is ready to release the reference when that does happen. - -This need to drop the reference to a symlink adds significant -complexity. It requires a reference to the inode so that the -``i_op->put_link()`` inode operation can be called. In REF-walk, that -reference is kept implicitly through a reference to the dentry, so -keeping the ``struct path`` of the symlink is easiest. For RCU-walk, -the pointer to the inode is kept separately. To allow switching from -RCU-walk back to REF-walk in the middle of processing nested symlinks -we also need the seq number for the dentry so we can confirm that -switching back was safe. - -Finally, when providing a reference to a symlink, the filesystem also -provides an opaque "cookie" that must be passed to ``->put_link()`` so that it -knows what to free. This might be the allocated memory area, or a -pointer to the ``struct page`` in the page cache, or something else -completely. Only the filesystem knows what it is. +holding no counted reference, only the RCU lock. A callback +``struct delayed_called`` will be passed to ``->get_link()``: +file systems can set their own put_link function and argument through +set_delayed_call(). Later on, when VFS wants to put link, it will call +do_delayed_call() to invoke that callback function with the argument. In order for the reference to each symlink to be dropped when the walk completes, whether in RCU-walk or REF-walk, the symlink stack needs to contain, along with the path remnants: -- the ``struct path`` to provide a reference to the inode in REF-walk -- the ``struct inode *`` to provide a reference to the inode in RCU-walk +- the ``struct path`` to provide a reference to the previous path +- the ``const char *`` to provide a reference to the to previous name - the ``seq`` to allow the path to be safely switched from RCU-walk to REF-walk -- the ``cookie`` that tells ``->put_path()`` what to put. +- the ``struct delayed_call`` for later invocation. This means that each entry in the symlink stack needs to hold five pointers and an integer instead of just one pointer (the path @@ -1120,12 +1103,10 @@ doesn't need to notice. Getting this ``name`` variable on and off the stack is very straightforward; pushing and popping the references is a little more complex. -When a symlink is found, ``walk_component()`` returns the value ``1`` -(``0`` is returned for any other sort of success, and a negative number -is, as usual, an error indicator). This causes ``get_link()`` to be -called; it then gets the link from the filesystem. Providing that -operation is successful, the old path ``name`` is placed on the stack, -and the new value is used as the ``name`` for a while. When the end of +When a symlink is found, walk_component() calls pick_link() via step_into() +which returns the link from the filesystem. +Providing that operation is successful, the old path ``name`` is placed on the +stack, and the new value is used as the ``name`` for a while. When the end of the path is found (i.e. ``*name`` is ``'\0'``) the old ``name`` is restored off the stack and path walking continues. @@ -1142,23 +1123,23 @@ stack in ``walk_component()`` immediately when the symlink is found; old symlink as it walks that last component. So it is quite convenient for ``walk_component()`` to release the old symlink and pop the references just before pushing the reference information for the -new symlink. It is guided in this by two flags; ``WALK_GET``, which -gives it permission to follow a symlink if it finds one, and -``WALK_PUT``, which tells it to release the current symlink after it has been -followed. ``WALK_PUT`` is tested first, leading to a call to -``put_link()``. ``WALK_GET`` is tested subsequently (by -``should_follow_link()``) leading to a call to ``pick_link()`` which sets -up the stack frame. +new symlink. It is guided in this by three flags: ``WALK_NOFOLLOW`` which +forbids it from following a symlink if it finds one, ``WALK_MORE`` +which indicates that it is yet too early to release the +current symlink, and ``WALK_TRAILING`` which indicates that it is on the final +component of the lookup, so we will check userspace flag ``LOOKUP_FOLLOW`` to +decide whether follow it when it is a symlink and call ``may_follow_link()`` to +check if we have privilege to follow it. Symlinks with no final component ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ A pair of special-case symlinks deserve a little further explanation. Both result in a new ``struct path`` (with mount and dentry) being set -up in the ``nameidata``, and result in ``get_link()`` returning ``NULL``. +up in the ``nameidata``, and result in pick_link() returning ``NULL``. The more obvious case is a symlink to "``/``". All symlinks starting -with "``/``" are detected in ``get_link()`` which resets the ``nameidata`` +with "``/``" are detected in pick_link() which resets the ``nameidata`` to point to the effective filesystem root. If the symlink only contains "``/``" then there is nothing more to do, no components at all, so ``NULL`` is returned to indicate that the symlink can be released and @@ -1175,12 +1156,11 @@ something that looks like a symlink. It is really a reference to the target file, not just the name of it. When you ``readlink`` these objects you get a name that might refer to the same file - unless it has been unlinked or mounted over. When ``walk_component()`` follows -one of these, the ``->follow_link()`` method in "procfs" doesn't return -a string name, but instead calls ``nd_jump_link()`` which updates the -``nameidata`` in place to point to that target. ``->follow_link()`` then -returns ``NULL``. Again there is no final component and ``get_link()`` -reports this by leaving the ``last_type`` field of ``nameidata`` as -``LAST_BIND``. +one of these, the ``->get_link()`` method in "procfs" doesn't return +a string name, but instead calls nd_jump_link() which updates the +``nameidata`` in place to point to that target. ``->get_link()`` then +returns ``NULL``. Again there is no final component and pick_link() +returns ``NULL``. Following the symlink in the final component -------------------------------------------- @@ -1197,42 +1177,38 @@ potentially need to call ``link_path_walk()`` again and again on successive symlinks until one is found that doesn't point to another symlink. -This case is handled by the relevant caller of ``link_path_walk()``, such as -``path_lookupat()`` using a loop that calls ``link_path_walk()``, and then -handles the final component. If the final component is a symlink -that needs to be followed, then ``trailing_symlink()`` is called to set -things up properly and the loop repeats, calling ``link_path_walk()`` -again. This could loop as many as 40 times if the last component of -each symlink is another symlink. +This case is handled by relevant callers of link_path_walk(), such as +path_lookupat(), path_openat() using a loop that calls link_path_walk(), +and then handles the final component by calling open_last_lookups() or +lookup_last(). If it is a symlink that needs to be followed, +open_last_lookups() or lookup_last() will set things up properly and +return the path so that the loop repeats, calling +link_path_walk() again. This could loop as many as 40 times if the last +component of each symlink is another symlink. -The various functions that examine the final component and possibly -report that it is a symlink are ``lookup_last()``, ``mountpoint_last()`` -and ``do_last()``, each of which use the same convention as -``walk_component()`` of returning ``1`` if a symlink was found that needs -to be followed. +Of the various functions that examine the final component, +open_last_lookups() is the most interesting as it works in tandem +with do_open() for opening a file. Part of open_last_lookups() runs +with ``i_rwsem`` held and this part is in a separate function: lookup_open(). -Of these, ``do_last()`` is the most interesting as it is used for -opening a file. Part of ``do_last()`` runs with ``i_rwsem`` held and this -part is in a separate function: ``lookup_open()``. +Explaining open_last_lookups() and do_open() completely is beyond the scope +of this article, but a few highlights should help those interested in exploring +the code. -Explaining ``do_last()`` completely is beyond the scope of this article, -but a few highlights should help those interested in exploring the -code. - -1. Rather than just finding the target file, ``do_last()`` needs to open +1. Rather than just finding the target file, do_open() is used after + open_last_lookup() to open it. If the file was found in the dcache, then ``vfs_open()`` is used for this. If not, then ``lookup_open()`` will either call ``atomic_open()`` (if the filesystem provides it) to combine the final lookup with the open, or - will perform the separate ``lookup_real()`` and ``vfs_create()`` steps + will perform the separate ``i_op->lookup()`` and ``i_op->create()`` steps directly. In the later case the actual "open" of this newly found or - created file will be performed by ``vfs_open()``, just as if the name + created file will be performed by vfs_open(), just as if the name were found in the dcache. -2. ``vfs_open()`` can fail with ``-EOPENSTALE`` if the cached information - wasn't quite current enough. Rather than restarting the lookup from - the top with ``LOOKUP_REVAL`` set, ``lookup_open()`` is called instead, - giving the filesystem a chance to resolve small inconsistencies. - If that doesn't work, only then is the lookup restarted from the top. +2. vfs_open() can fail with ``-EOPENSTALE`` if the cached information + wasn't quite current enough. If it's in RCU-walk ``-ECHILD`` will be returned + otherwise ``-ESTALE`` is returned. When ``-ESTALE`` is returned, the caller may + retry with ``LOOKUP_REVAL`` flag set. 3. An open with O_CREAT **does** follow a symlink in the final component, unlike other creation system calls (like ``mkdir``). So the sequence:: @@ -1242,8 +1218,8 @@ code. will create a file called ``/tmp/bar``. This is not permitted if ``O_EXCL`` is set but otherwise is handled for an O_CREAT open much - like for a non-creating open: ``should_follow_link()`` returns ``1``, and - so does ``do_last()`` so that ``trailing_symlink()`` gets called and the + like for a non-creating open: lookup_last() or open_last_lookup() + returns a non ``NULL`` value, and link_path_walk() gets called and the open process continues on the symlink that was found. Updating the access time diff --git a/Documentation/firmware-guide/acpi/dsd/data-node-references.rst b/Documentation/firmware-guide/acpi/dsd/data-node-references.rst index 9b17dc77d18c..b7ad47df49de 100644 --- a/Documentation/firmware-guide/acpi/dsd/data-node-references.rst +++ b/Documentation/firmware-guide/acpi/dsd/data-node-references.rst @@ -79,7 +79,8 @@ the ANOD object which is also the final target node of the reference. }) } -Please also see a graph example in :doc:`graph`. +Please also see a graph example in +Documentation/firmware-guide/acpi/dsd/graph.rst. References ========== diff --git a/Documentation/firmware-guide/acpi/dsd/graph.rst b/Documentation/firmware-guide/acpi/dsd/graph.rst index 7072db801aeb..4341299aa937 100644 --- a/Documentation/firmware-guide/acpi/dsd/graph.rst +++ b/Documentation/firmware-guide/acpi/dsd/graph.rst @@ -174,4 +174,4 @@ References referenced 2016-10-04. [7] _DSD Device Properties Usage Rules. - :doc:`../DSD-properties-rules` + Documentation/firmware-guide/acpi/DSD-properties-rules.rst diff --git a/Documentation/firmware-guide/acpi/enumeration.rst b/Documentation/firmware-guide/acpi/enumeration.rst index 9f0d5c854fa4..18074eb71860 100644 --- a/Documentation/firmware-guide/acpi/enumeration.rst +++ b/Documentation/firmware-guide/acpi/enumeration.rst @@ -339,8 +339,8 @@ a code like this:: There are also devm_* versions of these functions which release the descriptors once the device is released. -See Documentation/firmware-guide/acpi/gpio-properties.rst for more information about the -_DSD binding related to GPIOs. +See Documentation/firmware-guide/acpi/gpio-properties.rst for more information +about the _DSD binding related to GPIOs. MFD devices =========== @@ -460,7 +460,8 @@ the _DSD of the device object itself or the _DSD of its ancestor in the Otherwise, the _DSD itself is regarded as invalid and therefore the "compatible" property returned by it is meaningless. -Refer to :doc:`DSD-properties-rules` for more information. +Refer to Documentation/firmware-guide/acpi/DSD-properties-rules.rst for more +information. PCI hierarchy representation ============================ diff --git a/Documentation/i2c/instantiating-devices.rst b/Documentation/i2c/instantiating-devices.rst index e558e0a77e0c..890c9360ce19 100644 --- a/Documentation/i2c/instantiating-devices.rst +++ b/Documentation/i2c/instantiating-devices.rst @@ -59,7 +59,7 @@ Declare the I2C devices via ACPI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ACPI can also describe I2C devices. There is special documentation for this -which is currently located at :doc:`../firmware-guide/acpi/enumeration`. +which is currently located at Documentation/firmware-guide/acpi/enumeration.rst. Declare the I2C devices in board files diff --git a/Documentation/i2c/old-module-parameters.rst b/Documentation/i2c/old-module-parameters.rst index 38e55829dee8..b08b6daabce9 100644 --- a/Documentation/i2c/old-module-parameters.rst +++ b/Documentation/i2c/old-module-parameters.rst @@ -17,7 +17,8 @@ address), ``force`` (to forcibly attach the driver to a given device) and With the conversion of the I2C subsystem to the standard device driver binding model, it became clear that these per-module parameters were no longer needed, and that a centralized implementation was possible. The new, -sysfs-based interface is described in :doc:`instantiating-devices`, section +sysfs-based interface is described in +Documentation/i2c/instantiating-devices.rst, section "Method 4: Instantiate from user-space". Below is a mapping from the old module parameters to the new interface. diff --git a/Documentation/i2c/smbus-protocol.rst b/Documentation/i2c/smbus-protocol.rst index 64689d19dd51..9e07e6bbe6a3 100644 --- a/Documentation/i2c/smbus-protocol.rst +++ b/Documentation/i2c/smbus-protocol.rst @@ -27,8 +27,8 @@ a different protocol operation entirely. Each transaction type corresponds to a functionality flag. Before calling a transaction function, a device driver should always check (just once) for the corresponding functionality flag to ensure that the underlying I2C -adapter supports the transaction in question. See :doc:`functionality` for -the details. +adapter supports the transaction in question. See +Documentation/i2c/functionality.rst for the details. Key to symbols diff --git a/Documentation/input/joydev/joystick-api.rst b/Documentation/input/joydev/joystick-api.rst index af5934c10c1c..5db6dc6fe1c5 100644 --- a/Documentation/input/joydev/joystick-api.rst +++ b/Documentation/input/joydev/joystick-api.rst @@ -263,7 +263,7 @@ possible overrun should the name be too long:: char name[128]; if (ioctl(fd, JSIOCGNAME(sizeof(name)), name) < 0) - strncpy(name, "Unknown", sizeof(name)); + strscpy(name, "Unknown", sizeof(name)); printf("Name: %s\n", name); diff --git a/Documentation/kernel-hacking/hacking.rst b/Documentation/kernel-hacking/hacking.rst index 451523424942..df65c19aa7df 100644 --- a/Documentation/kernel-hacking/hacking.rst +++ b/Documentation/kernel-hacking/hacking.rst @@ -601,7 +601,7 @@ Defined in ``include/linux/export.h`` This is the variant of `EXPORT_SYMBOL()` that allows specifying a symbol namespace. Symbol Namespaces are documented in -:doc:`../core-api/symbol-namespaces` +Documentation/core-api/symbol-namespaces.rst :c:func:`EXPORT_SYMBOL_NS_GPL()` -------------------------------- @@ -610,7 +610,7 @@ Defined in ``include/linux/export.h`` This is the variant of `EXPORT_SYMBOL_GPL()` that allows specifying a symbol namespace. Symbol Namespaces are documented in -:doc:`../core-api/symbol-namespaces` +Documentation/core-api/symbol-namespaces.rst Routines and Conventions ======================== diff --git a/Documentation/networking/device_drivers/ethernet/intel/i40e.rst b/Documentation/networking/device_drivers/ethernet/intel/i40e.rst index 2d3f6bd969a2..ac35bd472bdc 100644 --- a/Documentation/networking/device_drivers/ethernet/intel/i40e.rst +++ b/Documentation/networking/device_drivers/ethernet/intel/i40e.rst @@ -466,7 +466,7 @@ network. PTP support varies among Intel devices that support this driver. Use "ethtool -T " to get a definitive list of PTP capabilities supported by the device. -IEEE 802.1ad (QinQ) Support +IEEE 802.1ad (QinQ) Support --------------------------- The IEEE 802.1ad standard, informally known as QinQ, allows for multiple VLAN IDs within a single Ethernet frame. VLAN IDs are sometimes referred to as @@ -523,8 +523,8 @@ of a port's bandwidth (should it be available). The sum of all the values for Maximum Bandwidth is not restricted, because no more than 100% of a port's bandwidth can ever be used. -NOTE: X710/XXV710 devices fail to enable Max VFs (64) when Multiple Functions -per Port (MFP) and SR-IOV are enabled. An error from i40e is logged that says +NOTE: X710/XXV710 devices fail to enable Max VFs (64) when Multiple Functions +per Port (MFP) and SR-IOV are enabled. An error from i40e is logged that says "add vsi failed for VF N, aq_err 16". To workaround the issue, enable less than 64 virtual functions (VFs). diff --git a/Documentation/networking/device_drivers/ethernet/intel/iavf.rst b/Documentation/networking/device_drivers/ethernet/intel/iavf.rst index 25330b7b5168..151af0a8da9c 100644 --- a/Documentation/networking/device_drivers/ethernet/intel/iavf.rst +++ b/Documentation/networking/device_drivers/ethernet/intel/iavf.rst @@ -113,7 +113,7 @@ which the AVF is associated. The following are base mode features: - AVF device ID - HW mailbox is used for VF to PF communications (including on Windows) -IEEE 802.1ad (QinQ) Support +IEEE 802.1ad (QinQ) Support --------------------------- The IEEE 802.1ad standard, informally known as QinQ, allows for multiple VLAN IDs within a single Ethernet frame. VLAN IDs are sometimes referred to as diff --git a/Documentation/networking/devlink/devlink-region.rst b/Documentation/networking/devlink/devlink-region.rst index 3654c3e9658f..58fe95e9a49d 100644 --- a/Documentation/networking/devlink/devlink-region.rst +++ b/Documentation/networking/devlink/devlink-region.rst @@ -22,7 +22,7 @@ The major benefit to creating a region is to provide access to internal address regions that are otherwise inaccessible to the user. Regions may also be used to provide an additional way to debug complex error -states, but see also :doc:`devlink-health` +states, but see also Documentation/networking/devlink/devlink-health.rst Regions may optionally support capturing a snapshot on demand via the ``DEVLINK_CMD_REGION_NEW`` netlink message. A driver wishing to allow diff --git a/Documentation/networking/devlink/devlink-trap.rst b/Documentation/networking/devlink/devlink-trap.rst index 935b6397e8cf..efa5f7f42c88 100644 --- a/Documentation/networking/devlink/devlink-trap.rst +++ b/Documentation/networking/devlink/devlink-trap.rst @@ -495,8 +495,8 @@ help debug packet drops caused by these exceptions. The following list includes links to the description of driver-specific traps registered by various device drivers: - * :doc:`netdevsim` - * :doc:`mlxsw` + * Documentation/networking/devlink/netdevsim.rst + * Documentation/networking/devlink/mlxsw.rst .. _Generic-Packet-Trap-Groups: diff --git a/Documentation/networking/packet_mmap.rst b/Documentation/networking/packet_mmap.rst index 500ef60b1b82..c5da1a5d93de 100644 --- a/Documentation/networking/packet_mmap.rst +++ b/Documentation/networking/packet_mmap.rst @@ -153,7 +153,7 @@ As capture, each frame contains two parts:: struct ifreq s_ifr; ... - strncpy (s_ifr.ifr_name, "eth0", sizeof(s_ifr.ifr_name)); + strscpy_pad (s_ifr.ifr_name, "eth0", sizeof(s_ifr.ifr_name)); /* get interface index of eth0 */ ioctl(this->socket, SIOCGIFINDEX, &s_ifr); diff --git a/Documentation/networking/tuntap.rst b/Documentation/networking/tuntap.rst index a59d1dd6fdcc..4d7087f727be 100644 --- a/Documentation/networking/tuntap.rst +++ b/Documentation/networking/tuntap.rst @@ -107,7 +107,7 @@ Note that the character pointer becomes overwritten with the real device name */ ifr.ifr_flags = IFF_TUN; if( *dev ) - strncpy(ifr.ifr_name, dev, IFNAMSIZ); + strscpy_pad(ifr.ifr_name, dev, IFNAMSIZ); if( (err = ioctl(fd, TUNSETIFF, (void *) &ifr)) < 0 ){ close(fd); diff --git a/Documentation/process/submitting-patches.rst b/Documentation/process/submitting-patches.rst index c66a19201deb..0852bcf73630 100644 --- a/Documentation/process/submitting-patches.rst +++ b/Documentation/process/submitting-patches.rst @@ -10,10 +10,11 @@ can greatly increase the chances of your change being accepted. This document contains a large number of suggestions in a relatively terse format. For detailed information on how the kernel development process -works, see :doc:`development-process`. Also, read :doc:`submit-checklist` +works, see Documentation/process/development-process.rst. Also, read +Documentation/process/submit-checklist.rst for a list of items to check before submitting code. If you are submitting -a driver, also read :doc:`submitting-drivers`; for device tree binding patches, -read :doc:`submitting-patches`. +a driver, also read Documentation/process/submitting-drivers.rst; for device +tree binding patches, read Documentation/process/submitting-patches.rst. This documentation assumes that you're using ``git`` to prepare your patches. If you're unfamiliar with ``git``, you would be well-advised to learn how to @@ -178,8 +179,7 @@ Style-check your changes ------------------------ Check your patch for basic style violations, details of which can be -found in -:ref:`Documentation/process/coding-style.rst `. +found in Documentation/process/coding-style.rst. Failure to do so simply wastes the reviewers time and will get your patch rejected, probably without even being read. @@ -238,7 +238,7 @@ If you have a patch that fixes an exploitable security bug, send that patch to security@kernel.org. For severe bugs, a short embargo may be considered to allow distributors to get the patch out to users; in such cases, obviously, the patch should not be sent to any public lists. See also -:doc:`/admin-guide/security-bugs`. +Documentation/admin-guide/security-bugs.rst. Patches that fix a severe bug in a released kernel should be directed toward the stable maintainers by putting a line like this:: @@ -246,9 +246,8 @@ toward the stable maintainers by putting a line like this:: Cc: stable@vger.kernel.org into the sign-off area of your patch (note, NOT an email recipient). You -should also read -:ref:`Documentation/process/stable-kernel-rules.rst ` -in addition to this file. +should also read Documentation/process/stable-kernel-rules.rst +in addition to this document. If changes affect userland-kernel interfaces, please send the MAN-PAGES maintainer (as listed in the MAINTAINERS file) a man-pages patch, or at @@ -305,8 +304,8 @@ decreasing the likelihood of your MIME-attached change being accepted. Exception: If your mailer is mangling patches then someone may ask you to re-send them using MIME. -See :doc:`/process/email-clients` for hints about configuring your e-mail -client so that it sends your patches untouched. +See Documentation/process/email-clients.rst for hints about configuring +your e-mail client so that it sends your patches untouched. Respond to review comments -------------------------- @@ -324,7 +323,7 @@ for their time. Code review is a tiring and time-consuming process, and reviewers sometimes get grumpy. Even in that case, though, respond politely and address the problems they have pointed out. -See :doc:`email-clients` for recommendations on email +See Documentation/process/email-clients.rst for recommendations on email clients and mailing list etiquette. @@ -562,10 +561,10 @@ method for indicating a bug fixed by the patch. See :ref:`describe_changes` for more details. Note: Attaching a Fixes: tag does not subvert the stable kernel rules -process nor the requirement to Cc: stable@vger.kernel.org on all stable +process nor the requirement to Cc: stable@vger.kernel.org on all stable patch candidates. For more information, please read -:ref:`Documentation/process/stable-kernel-rules.rst ` - +Documentation/process/stable-kernel-rules.rst. + .. _the_canonical_patch_format: The canonical patch format @@ -824,8 +823,7 @@ Greg Kroah-Hartman, "How to piss off a kernel subsystem maintainer". NO!!!! No more huge patch bombs to linux-kernel@vger.kernel.org people! -Kernel Documentation/process/coding-style.rst: - :ref:`Documentation/process/coding-style.rst ` +Kernel Documentation/process/coding-style.rst Linus Torvalds's mail on the canonical patch format: diff --git a/Documentation/scheduler/sched-bwc.rst b/Documentation/scheduler/sched-bwc.rst index 845eee659199..1fc73555f5c4 100644 --- a/Documentation/scheduler/sched-bwc.rst +++ b/Documentation/scheduler/sched-bwc.rst @@ -29,7 +29,7 @@ Quota and period are managed within the cpu subsystem via cgroupfs. .. note:: The cgroupfs files described in this section are only applicable to cgroup v1. For cgroup v2, see - :ref:`Documentation/admin-guide/cgroupv2.rst `. + :ref:`Documentation/admin-guide/cgroup-v2.rst `. - cpu.cfs_quota_us: the total available run-time within a period (in microseconds) diff --git a/Documentation/scheduler/sched-nice-design.rst b/Documentation/scheduler/sched-nice-design.rst index 0571f1b47e64..889bf2b737dc 100644 --- a/Documentation/scheduler/sched-nice-design.rst +++ b/Documentation/scheduler/sched-nice-design.rst @@ -60,7 +60,7 @@ within the constraints of HZ and jiffies and their nasty design level coupling to timeslices and granularity it was not really viable. The second (less frequent but still periodically occurring) complaint -about Linux's nice level support was its assymetry around the origo +about Linux's nice level support was its asymmetry around the origin (which you can see demonstrated in the picture above), or more accurately: the fact that nice level behavior depended on the _absolute_ nice level as well, while the nice API itself is fundamentally diff --git a/Documentation/security/IMA-templates.rst b/Documentation/security/IMA-templates.rst index c5a8432972ef..1a91d92950a7 100644 --- a/Documentation/security/IMA-templates.rst +++ b/Documentation/security/IMA-templates.rst @@ -70,9 +70,18 @@ descriptors by adding their identifier to the format string prefix is shown only if the hash algorithm is not SHA1 or MD5); - 'd-modsig': the digest of the event without the appended modsig; - 'n-ng': the name of the event, without size limitations; - - 'sig': the file signature; + - 'sig': the file signature, or the EVM portable signature if the file + signature is not found; - 'modsig' the appended file signature; - 'buf': the buffer data that was used to generate the hash without size limitations; + - 'evmsig': the EVM portable signature; + - 'iuid': the inode UID; + - 'igid': the inode GID; + - 'imode': the inode mode; + - 'xattrnames': a list of xattr names (separated by ``|``), only if the xattr is + present; + - 'xattrlengths': a list of xattr lengths (u32), only if the xattr is present; + - 'xattrvalues': a list of xattr values; Below, there is the list of defined template descriptors: @@ -82,6 +91,7 @@ Below, there is the list of defined template descriptors: - "ima-sig": its format is ``d-ng|n-ng|sig``; - "ima-buf": its format is ``d-ng|n-ng|buf``; - "ima-modsig": its format is ``d-ng|n-ng|sig|d-modsig|modsig``; + - "evm-sig": its format is ``d-ng|n-ng|evmsig|xattrnames|xattrlengths|xattrvalues|iuid|igid|imode``; Use diff --git a/Documentation/security/landlock.rst b/Documentation/security/landlock.rst index 2e84925ae971..3df68cb1d10f 100644 --- a/Documentation/security/landlock.rst +++ b/Documentation/security/landlock.rst @@ -25,7 +25,8 @@ Any user can enforce Landlock rulesets on their processes. They are merged and evaluated according to the inherited ones in a way that ensures that only more constraints can be added. -User space documentation can be found here: :doc:`/userspace-api/landlock`. +User space documentation can be found here: +Documentation/userspace-api/landlock.rst. Guiding principles for safe access controls =========================================== diff --git a/Documentation/trace/coresight/coresight-etm4x-reference.rst b/Documentation/trace/coresight/coresight-etm4x-reference.rst index b64d9a9c79df..d25dfe86af9b 100644 --- a/Documentation/trace/coresight/coresight-etm4x-reference.rst +++ b/Documentation/trace/coresight/coresight-etm4x-reference.rst @@ -427,7 +427,7 @@ the ‘TRC’ prefix. :Syntax: ``echo idx > vmid_idx`` - Where idx <  numvmidc + Where idx < numvmidc ---- diff --git a/Documentation/trace/coresight/coresight.rst b/Documentation/trace/coresight/coresight.rst index 169749efd8d1..1ec8dc35b1d8 100644 --- a/Documentation/trace/coresight/coresight.rst +++ b/Documentation/trace/coresight/coresight.rst @@ -315,7 +315,8 @@ intermediate links as required. Note: ``cti_sys0`` appears in two of the connections lists above. CTIs can connect to multiple devices and are arranged in a star topology -via the CTM. See (:doc:`coresight-ect`) [#fourth]_ for further details. +via the CTM. See (Documentation/trace/coresight/coresight-ect.rst) +[#fourth]_ for further details. Looking at this device we see 4 connections:: linaro-developer:~# ls -l /sys/bus/coresight/devices/cti_sys0/connections @@ -606,7 +607,8 @@ interface provided for that purpose by the generic STM API:: crw------- 1 root root 10, 61 Jan 3 18:11 /dev/stm0 root@genericarmv8:~# -Details on how to use the generic STM API can be found here:- :doc:`../stm` [#second]_. +Details on how to use the generic STM API can be found here: +- Documentation/trace/stm.rst [#second]_. The CTI & CTM Modules --------------------- @@ -616,7 +618,7 @@ individual CTIs and components, and can propagate these between all CTIs via channels on the CTM (Cross Trigger Matrix). A separate documentation file is provided to explain the use of these devices. -(:doc:`coresight-ect`) [#fourth]_. +(Documentation/trace/coresight/coresight-ect.rst) [#fourth]_. .. [#first] Documentation/ABI/testing/sysfs-bus-coresight-devices-stm diff --git a/Documentation/trace/ftrace.rst b/Documentation/trace/ftrace.rst index 62c98e9bbdd9..cfc81e98e0b8 100644 --- a/Documentation/trace/ftrace.rst +++ b/Documentation/trace/ftrace.rst @@ -40,7 +40,7 @@ See events.rst for more information. Implementation Details ---------------------- -See :doc:`ftrace-design` for details for arch porters and such. +See Documentation/trace/ftrace-design.rst for details for arch porters and such. The File System @@ -354,8 +354,8 @@ of ftrace. Here is a list of some of the key files: is being directly called by the function. If the count is greater than 1 it most likely will be ftrace_ops_list_func(). - If the callback of the function jumps to a trampoline that is - specific to a the callback and not the standard trampoline, + If the callback of a function jumps to a trampoline that is + specific to the callback and which is not the standard trampoline, its address will be printed as well as the function that the trampoline calls. diff --git a/Documentation/translations/index.rst b/Documentation/translations/index.rst index e446e5ed00a6..556b050884fc 100644 --- a/Documentation/translations/index.rst +++ b/Documentation/translations/index.rst @@ -18,6 +18,10 @@ Translations Disclaimer ---------- +.. raw:: latex + + \kerneldocCJKoff + Translation's purpose is to ease reading and understanding in languages other than English. Its aim is to help people who do not understand English or have doubts about its interpretation. Additionally, some people prefer to read diff --git a/Documentation/translations/it_IT/index.rst b/Documentation/translations/it_IT/index.rst index bb8fa7346939..e80a3097aa57 100644 --- a/Documentation/translations/it_IT/index.rst +++ b/Documentation/translations/it_IT/index.rst @@ -4,6 +4,10 @@ Traduzione italiana =================== +.. raw:: latex + + \kerneldocCJKoff + :manutentore: Federico Vaga .. _it_disclaimer: diff --git a/Documentation/translations/it_IT/process/coding-style.rst b/Documentation/translations/it_IT/process/coding-style.rst index 95f2e7c985e2..ecc74ba50d3e 100644 --- a/Documentation/translations/it_IT/process/coding-style.rst +++ b/Documentation/translations/it_IT/process/coding-style.rst @@ -62,7 +62,7 @@ i ``case``. Un esempio.: case 'K': case 'k': mem <<= 10; - /* fall through */ + fallthrough; default: break; } diff --git a/Documentation/translations/ja_JP/index.rst b/Documentation/translations/ja_JP/index.rst index 2f91b895e3c2..f94ba62d41c3 100644 --- a/Documentation/translations/ja_JP/index.rst +++ b/Documentation/translations/ja_JP/index.rst @@ -1,7 +1,8 @@ .. raw:: latex - \renewcommand\thesection* - \renewcommand\thesubsection* + \renewcommand\thesection* + \renewcommand\thesubsection* + \kerneldocCJKon Japanese translations ===================== diff --git a/Documentation/translations/ko_KR/index.rst b/Documentation/translations/ko_KR/index.rst index b9e27d20b039..6ae258118bdf 100644 --- a/Documentation/translations/ko_KR/index.rst +++ b/Documentation/translations/ko_KR/index.rst @@ -1,7 +1,8 @@ .. raw:: latex - \renewcommand\thesection* - \renewcommand\thesubsection* + \renewcommand\thesection* + \renewcommand\thesubsection* + \kerneldocCJKon 한국어 번역 =========== diff --git a/Documentation/translations/zh_CN/admin-guide/index.rst b/Documentation/translations/zh_CN/admin-guide/index.rst index be835ec8e632..460034cbc2ab 100644 --- a/Documentation/translations/zh_CN/admin-guide/index.rst +++ b/Documentation/translations/zh_CN/admin-guide/index.rst @@ -65,6 +65,7 @@ Todolist: clearing-warn-once cpu-load + lockup-watchdogs unicode Todolist: @@ -100,7 +101,6 @@ Todolist: laptops/index lcd-panel-cgram ldm - lockup-watchdogs LSM/index md media/index diff --git a/Documentation/translations/zh_CN/admin-guide/lockup-watchdogs.rst b/Documentation/translations/zh_CN/admin-guide/lockup-watchdogs.rst new file mode 100644 index 000000000000..55ed3f4af442 --- /dev/null +++ b/Documentation/translations/zh_CN/admin-guide/lockup-watchdogs.rst @@ -0,0 +1,66 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/admin-guide/lockup-watchdogs.rst +:Translator: Hailong Liu + +.. _cn_lockup-watchdogs: + + +================================================= +Softlockup与hardlockup检测机制(又名:nmi_watchdog) +================================================= + +Linux中内核实现了一种用以检测系统发生softlockup和hardlockup的看门狗机制。 + +Softlockup是一种会引发系统在内核态中一直循环超过20秒（详见下面“实现”小节）导致 +其他任务没有机会得到运行的BUG。一旦检测到'softlockup'发生，默认情况下系统会打 +印当前堆栈跟踪信息并进入锁定状态。也可配置使其在检测到'softlockup'后进入panic +状态；通过sysctl命令设置“kernel.softlockup_panic”、使用内核启动参数 +“softlockup_panic”（详见Documentation/admin-guide/kernel-parameters.rst）以及使 +能内核编译选项“BOOTPARAM_SOFTLOCKUP_PANIC”都可实现这种配置。 + +而'hardlockup'是一种会引发系统在内核态一直循环超过10秒钟（详见"实现"小节）导致其 +他中断没有机会运行的缺陷。与'softlockup'情况类似，除了使用sysctl命令设置 +'hardlockup_panic'、使能内核选项“BOOTPARAM_HARDLOCKUP_PANIC”以及使用内核参数 +"nmi_watchdog"(详见:”Documentation/admin-guide/kernel-parameters.rst“)外，一旦检 +测到'hardlockup'默认情况下系统打印当前堆栈跟踪信息，然后进入锁定状态。 + +这个panic选项也可以与panic_timeout结合使用（这个panic_timeout是通过稍具迷惑性的 +sysctl命令"kernel.panic"来设置），使系统在panic指定时间后自动重启。 + +实现 +==== + +Softlockup和hardlockup分别建立在hrtimer(高精度定时器)和perf两个子系统上而实现。 +这也就意味着理论上任何架构只要实现了这两个子系统就支持这两种检测机制。 + +Hrtimer用于周期性产生中断并唤醒watchdog线程；NMI perf事件则以”watchdog_thresh“ +(编译时默认初始化为10秒，也可通过”watchdog_thresh“这个sysctl接口来进行配置修改) +为间隔周期产生以检测 hardlockups。如果一个CPU在这个时间段内没有检测到hrtimer中 +断发生，'hardlockup 检测器'(即NMI perf事件处理函数)将会视系统配置而选择产生内核 +警告或者直接panic。 + +而watchdog线程本质上是一个高优先级内核线程，每调度一次就对时间戳进行一次更新。 +如果时间戳在2*watchdog_thresh(这个是softlockup的触发门限)这段时间都未更新,那么 +"softlocup 检测器"(内部hrtimer定时器回调函数)会将相关的调试信息打印到系统日志中， +然后如果系统配置了进入panic流程则进入panic，否则内核继续执行。 + +Hrtimer定时器的周期是2*watchdog_thresh/5，也就是说在hardlockup被触发前hrtimer有 +2~3次机会产生时钟中断。 + +如上所述,内核相当于为系统管理员提供了一个可调节hrtimer定时器和perf事件周期长度 +的调节旋钮。如何通过这个旋钮为特定使用场景配置一个合理的周期值要对lockups检测的 +响应速度和lockups检测开销这二者之间进行权衡。 + +默认情况下所有在线cpu上都会运行一个watchdog线程。不过在内核配置了”NO_HZ_FULL“的 +情况下watchdog线程默认只会运行在管家(housekeeping)cpu上，而”nohz_full“启动参数指 +定的cpu上则不会有watchdog线程运行。试想，如果我们允许watchdog线程在”nohz_full“指 +定的cpu上运行，这些cpu上必须得运行时钟定时器来激发watchdog线程调度；这样一来就会 +使”nohz_full“保护用户程序免受内核干扰的功能失效。当然，副作用就是”nohz_full“指定 +的cpu即使在内核产生了lockup问题我们也无法检测到。不过，至少我们可以允许watchdog +线程在管家(non-tickless)核上继续运行以便我们能继续正常的监测这些cpus上的lockups +事件。 + +不论哪种情况都可以通过sysctl命令kernel.watchdog_cpumask来对没有运行watchdog线程 +的cpu集合进行调节。对于nohz_full而言,如果nohz_full cpu上有异常挂住的情况，通过 +这种方式打开这些cpu上的watchdog进行调试可能会有所作用。 diff --git a/Documentation/translations/zh_CN/core-api/cachetlb.rst b/Documentation/translations/zh_CN/core-api/cachetlb.rst new file mode 100644 index 000000000000..8376485a534d --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/cachetlb.rst @@ -0,0 +1,336 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/cachetlb.rst + +:翻译: + + 司延腾 Yanteng Si + +:校译: + + 吴想成 Wu XiangCheng + +.. _cn_core-api_cachetlb: + +====================== +Linux下的缓存和TLB刷新 +====================== + +:作者: David S. Miller + +*译注：TLB，Translation Lookaside Buffer，页表缓存/变换旁查缓冲器* + +本文描述了由Linux虚拟内存子系统调用的缓存/TLB刷新接口。它列举了每个接 +口，描述了它的预期目的，以及接口被调用后的预期副作用。 + +下面描述的副作用是针对单处理器的实现，以及在单个处理器上发生的情况。若 +为SMP，则只需将定义简单地扩展一下，使发生在某个特定接口的副作用扩展到系 +统的所有处理器上。不要被这句话吓到，以为SMP的缓存/tlb刷新一定是很低 +效的，事实上，这是一个可以进行很多优化的领域。例如，如果可以证明一个用 +户地址空间从未在某个cpu上执行过（见mm_cpumask()），那么就不需要在该 +cpu上对这个地址空间进行刷新。 + +首先是TLB刷新接口，因为它们是最简单的。在Linux下，TLB被抽象为cpu +用来缓存从软件页表获得的虚拟->物理地址转换的东西。这意味着，如果软件页 +表发生变化，这个“TLB”缓存中就有可能出现过时（脏）的翻译。因此，当软件页表 +发生变化时，内核会在页表发生 *变化后* 调用以下一种刷新方法： + +1) ``void flush_tlb_all(void)`` + + 最严格的刷新。在这个接口运行后，任何以前的页表修改都会对cpu可见。 + + 这通常是在内核页表被改变时调用的，因为这种转换在本质上是“全局”的。 + +2) ``void flush_tlb_mm(struct mm_struct *mm)`` + + 这个接口从TLB中刷新整个用户地址空间。在运行后，这个接口必须确保 + 以前对地址空间‘mm’的任何页表修改对cpu来说是可见的。也就是说，在 + 运行后，TLB中不会有‘mm’的页表项。 + + 这个接口被用来处理整个地址空间的页表操作，比如在fork和exec过程 + 中发生的事情。 + +3) ``void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end)`` + + 这里我们要从TLB中刷新一个特定范围的（用户）虚拟地址转换。在运行后， + 这个接口必须确保以前对‘start’到‘end-1’范围内的地址空间‘vma->vm_mm’ + 的任何页表修改对cpu来说是可见的。也就是说，在运行后，TLB中不会有 + ‘mm’的页表项用于‘start’到‘end-1’范围内的虚拟地址。 + + “vma”是用于该区域的备份存储。主要是用于munmap()类型的操作。 + + 提供这个接口是希望端口能够找到一个合适的有效方法来从TLB中删除多 + 个页面大小的转换，而不是让内核为每个可能被修改的页表项调用 + flush_tlb_page(见下文)。 + +4) ``void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)`` + + 这一次我们需要从TLB中删除PAGE_SIZE大小的转换。‘vma’是Linux用来跟 + 踪进程的mmap区域的支持结构体，地址空间可以通过vma->vm_mm获得。另 + 外，可以通过测试（vma->vm_flags & VM_EXEC）来查看这个区域是否是 + 可执行的（因此在split-tlb类型的设置中可能在“指令TLB”中）。 + + 在运行后，这个接口必须确保之前对用户虚拟地址“addr”的地址空间 + “vma->vm_mm”的页表修改对cpu来说是可见的。也就是说，在运行后，TLB + 中不会有虚拟地址‘addr’的‘vma->vm_mm’的页表项。 + + 这主要是在故障处理时使用。 + +5) ``void update_mmu_cache(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep)`` + + 在每个页面故障结束时，这个程序被调用，以告诉体系结构特定的代码，在 + 软件页表中，在地址空间“vma->vm_mm”的虚拟地址“地址”处，现在存在 + 一个翻译。 + + 可以用它所选择的任何方式使用这个信息来进行移植。例如，它可以使用这 + 个事件来为软件管理的TLB配置预装TLB转换。目前sparc64移植就是这么干 + 的。 + +接下来，我们有缓存刷新接口。一般来说，当Linux将现有的虚拟->物理映射 +改变为新的值时，其顺序将是以下形式之一:: + + 1) flush_cache_mm(mm); + change_all_page_tables_of(mm); + flush_tlb_mm(mm); + + 2) flush_cache_range(vma, start, end); + change_range_of_page_tables(mm, start, end); + flush_tlb_range(vma, start, end); + + 3) flush_cache_page(vma, addr, pfn); + set_pte(pte_pointer, new_pte_val); + flush_tlb_page(vma, addr); + +缓存级别的刷新将永远是第一位的，因为这允许我们正确处理那些缓存严格， +且在虚拟地址被从缓存中刷新时要求一个虚拟地址的虚拟->物理转换存在的系统。 +HyperSparc cpu就是这样一个具有这种属性的cpu。 + +下面的缓存刷新程序只需要在特定的cpu需要的范围内处理缓存刷新。大多数 +情况下，这些程序必须为cpu实现，这些cpu有虚拟索引的缓存，当虚拟->物 +理转换被改变或移除时，必须被刷新。因此，例如，IA32处理器的物理索引 +的物理标记的缓存没有必要实现这些接口，因为这些缓存是完全同步的，并 +且不依赖于翻译信息。 + +下面逐个列出这些程序: + +1) ``void flush_cache_mm(struct mm_struct *mm)`` + + 这个接口将整个用户地址空间从高速缓存中刷掉。也就是说，在运行后， + 将没有与‘mm’相关的缓存行。 + + 这个接口被用来处理整个地址空间的页表操作，比如在退出和执行过程 + 中发生的事情。 + +2) ``void flush_cache_dup_mm(struct mm_struct *mm)`` + + 这个接口将整个用户地址空间从高速缓存中刷新掉。也就是说，在运行 + 后，将没有与‘mm’相关的缓存行。 + + 这个接口被用来处理整个地址空间的页表操作，比如在fork过程中发生 + 的事情。 + + 这个选项与flush_cache_mm分开，以允许对VIPT缓存进行一些优化。 + +3) ``void flush_cache_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end)`` + + 在这里，我们要从缓存中刷新一个特定范围的（用户）虚拟地址。运行 + 后，在“start”到“end-1”范围内的虚拟地址的“vma->vm_mm”的缓存中 + 将没有页表项。 + + “vma”是被用于该区域的备份存储。主要是用于munmap()类型的操作。 + + 提供这个接口是希望端口能够找到一个合适的有效方法来从缓存中删 + 除多个页面大小的区域， 而不是让内核为每个可能被修改的页表项调 + 用 flush_cache_page (见下文)。 + +4) ``void flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)`` + + 这一次我们需要从缓存中删除一个PAGE_SIZE大小的区域。“vma”是 + Linux用来跟踪进程的mmap区域的支持结构体，地址空间可以通过 + vma->vm_mm获得。另外，我们可以通过测试（vma->vm_flags & + VM_EXEC）来查看这个区域是否是可执行的（因此在“Harvard”类 + 型的缓存布局中可能是在“指令缓存”中）。 + + “pfn”表示“addr”所对应的物理页框（通过PAGE_SHIFT左移这个 + 值来获得物理地址）。正是这个映射应该从缓存中删除。 + + 在运行之后，对于虚拟地址‘addr’的‘vma->vm_mm’，在缓存中不会 + 有任何页表项，它被翻译成‘pfn’。 + + 这主要是在故障处理过程中使用。 + +5) ``void flush_cache_kmaps(void)`` + + 只有在平台使用高位内存的情况下才需要实现这个程序。它将在所有的 + kmaps失效之前被调用。 + + 运行后，内核虚拟地址范围PKMAP_ADDR(0)到PKMAP_ADDR(LAST_PKMAP) + 的缓存中将没有页表项。 + + 这个程序应该在asm/highmem.h中实现。 + +6) ``void flush_cache_vmap(unsigned long start, unsigned long end)`` + ``void flush_cache_vunmap(unsigned long start, unsigned long end)`` + + 在这里，在这两个接口中，我们从缓存中刷新一个特定范围的（内核） + 虚拟地址。运行后，在“start”到“end-1”范围内的虚拟地址的内核地 + 址空间的缓存中不会有页表项。 + + 这两个程序中的第一个是在vmap_range()安装了页表项之后调用的。 + 第二个是在vunmap_range()删除页表项之前调用的。 + +还有一类cpu缓存问题，目前需要一套完全不同的接口来正确处理。最大 +的问题是处理器的数据缓存中的虚拟别名。 + +.. note:: + + 这段内容有些晦涩，为了减轻中文阅读压力，特作此译注。 + + 别名（alias）属于缓存一致性问题，当不同的虚拟地址映射相同的 + 物理地址，而这些虚拟地址的index不同，此时就发生了别名现象(多 + 个虚拟地址被称为别名)。通俗点来说就是指同一个物理地址的数据被 + 加载到不同的cacheline中就会出现别名现象。 + + 常见的解决方法有两种：第一种是硬件维护一致性，设计特定的cpu电 + 路来解决问题（例如设计为PIPT的cache）；第二种是软件维护一致性， + 就是下面介绍的sparc的解决方案——页面染色，涉及的技术细节太多， + 译者不便展开，请读者自行查阅相关资料。 + +您的移植是否容易在其D-cache中出现虚拟别名？嗯，如果您的D-cache +是虚拟索引的，且cache大于PAGE_SIZE（页大小），并且不能防止同一 +物理地址的多个cache行同时存在，您就会遇到这个问题。 + +如果你的D-cache有这个问题，首先正确定义asm/shmparam.h SHMLBA， +它基本上应该是你的虚拟寻址D-cache的大小（或者如果大小是可变的， +则是最大的可能大小）。这个设置将迫使SYSv IPC层只允许用户进程在 +这个值的倍数的地址上对共享内存进行映射。 + +.. note:: + + 这并不能解决共享mmaps的问题，请查看sparc64移植解决 + 这个问题的一个方法（特别是 SPARC_FLAG_MMAPSHARED）。 + +接下来，你必须解决所有其他情况下的D-cache别名问题。请记住这个事 +实，对于一个给定的页面映射到某个用户地址空间，总是至少还有一个映 +射，那就是内核在其线性映射中从PAGE_OFFSET开始。因此，一旦第一个 +用户将一个给定的物理页映射到它的地址空间，就意味着D-cache的别名 +问题有可能存在，因为内核已经将这个页映射到它的虚拟地址。 + + ``void copy_user_page(void *to, void *from, unsigned long addr, struct page *page)`` + ``void clear_user_page(void *to, unsigned long addr, struct page *page)`` + + 这两个程序在用户匿名或COW页中存储数据。它允许一个端口有效地 + 避免用户空间和内核之间的D-cache别名问题。 + + 例如，一个端口可以在复制过程中把“from”和“to”暂时映射到内核 + 的虚拟地址上。这两个页面的虚拟地址的选择方式是，内核的加载/存 + 储指令发生在虚拟地址上，而这些虚拟地址与用户的页面映射是相同 + 的“颜色”。例如，Sparc64就使用这种技术。 + + “addr”参数告诉了用户最终要映射这个页面的虚拟地址，“page”参 + 数给出了一个指向目标页结构体的指针。 + + 如果D-cache别名不是问题，这两个程序可以简单地直接调用 + memcpy/memset而不做其他事情。 + + ``void flush_dcache_page(struct page *page)`` + + 任何时候，当内核写到一个页面缓存页，或者内核要从一个页面缓存 + 页中读出，并且这个页面的用户空间共享/可写映射可能存在时， + 这个程序就会被调用。 + + .. note:: + + 这个程序只需要为有可能被映射到用户进程的地址空间的 + 页面缓存调用。因此，例如，处理页面缓存中vfs符号链 + 接的VFS层代码根本不需要调用这个接口。 + + “内核写入页面缓存的页面”这句话的意思是，具体来说，内核执行存 + 储指令，在该页面的页面->虚拟映射处弄脏该页面的数据。在这里，通 + 过刷新的手段处理D-cache的别名是很重要的，以确保这些内核存储对 + 该页的用户空间映射是可见的。 + + 推论的情况也同样重要，如果有用户对这个文件有共享+可写的映射， + 我们必须确保内核对这些页面的读取会看到用户所做的最新的存储。 + + 如果D-cache别名不是一个问题，这个程序可以简单地定义为该架构上 + 的nop。 + + 在page->flags (PG_arch_1)中有一个位是“架构私有”。内核保证， + 对于分页缓存的页面，当这样的页面第一次进入分页缓存时，它将清除 + 这个位。 + + 这使得这些接口可以更有效地被实现。如果目前没有用户进程映射这个 + 页面，它允许我们“推迟”（也许是无限期）实际的刷新过程。请看 + sparc64的flush_dcache_page和update_mmu_cache实现，以了解如 + 何做到这一点。 + + 这个想法是，首先在flush_dcache_page()时，如果page->mapping->i_mmap + 是一个空树，只需标记架构私有页标志位。之后，在update_mmu_cache() + 中，会对这个标志位进行检查，如果设置了，就进行刷新，并清除标志位。 + + .. important:: + + 通常很重要的是，如果你推迟刷新，实际的刷新发生在同一个 + CPU上，因为它将cpu存储到页面上，使其变脏。同样，请看 + sparc64关于如何处理这个问题的例子。 + + ``void copy_to_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long user_vaddr, void *dst, void *src, int len)`` + ``void copy_from_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long user_vaddr, void *dst, void *src, int len)`` + + 当内核需要复制任意的数据进出任意的用户页时（比如ptrace()），它将使 + 用这两个程序。 + + 任何必要的缓存刷新或其他需要发生的一致性操作都应该在这里发生。如果 + 处理器的指令缓存没有对cpu存储进行窥探，那么你很可能需要为 + copy_to_user_page()刷新指令缓存。 + + ``void flush_anon_page(struct vm_area_struct *vma, struct page *page, + unsigned long vmaddr)`` + + 当内核需要访问一个匿名页的内容时，它会调用这个函数（目前只有 + get_user_pages()）。注意：flush_dcache_page()故意对匿名页不起作 + 用。默认的实现是nop（对于所有相干的架构应该保持这样）。对于不一致性 + 的架构，它应该刷新vmaddr处的页面缓存。 + + ``void flush_kernel_dcache_page(struct page *page)`` + + 当内核需要修改一个用kmap获得的用户页时，它会在所有修改完成后（但在 + kunmapping之前）调用这个函数，以使底层页面达到最新状态。这里假定用 + 户没有不一致性的缓存副本（即原始页面是从类似get_user_pages()的机制 + 中获得的）。默认的实现是一个nop，在所有相干的架构上都应该如此。在不 + 一致性的架构上，这应该刷新内核缓存中的页面（使用page_address(page)）。 + + + ``void flush_icache_range(unsigned long start, unsigned long end)`` + + 当内核存储到它将执行的地址中时（例如在加载模块时），这个函数被调用。 + + 如果icache不对存储进行窥探，那么这个程序将需要对其进行刷新。 + + ``void flush_icache_page(struct vm_area_struct *vma, struct page *page)`` + + flush_icache_page的所有功能都可以在flush_dcache_page和update_mmu_cache + 中实现。在未来，我们希望能够完全删除这个接口。 + +最后一类API是用于I/O到内核内特意设置的别名地址范围。这种别名是通过使用 +vmap/vmalloc API设置的。由于内核I/O是通过物理页进行的，I/O子系统假定用户 +映射和内核偏移映射是唯一的别名。这对vmap别名来说是不正确的，所以内核中任何 +试图对vmap区域进行I/O的东西都必须手动管理一致性。它必须在做I/O之前刷新vmap +范围，并在I/O返回后使其失效。 + + ``void flush_kernel_vmap_range(void *vaddr, int size)`` + + 刷新vmap区域中指定的虚拟地址范围的内核缓存。这是为了确保内核在vmap范围 + 内修改的任何数据对物理页是可见的。这个设计是为了使这个区域可以安全地执 + 行I/O。注意，这个API并 *没有* 刷新该区域的偏移映射别名。 + + ``void invalidate_kernel_vmap_range(void *vaddr, int size) invalidates`` + + 在vmap区域的一个给定的虚拟地址范围的缓存，这可以防止处理器在物理页的I/O + 发生时通过投机性地读取数据而使缓存变脏。这只对读入vmap区域的数据是必要的。 diff --git a/Documentation/translations/zh_CN/core-api/index.rst b/Documentation/translations/zh_CN/core-api/index.rst index f1fa71e45c77..b4bde9396339 100644 --- a/Documentation/translations/zh_CN/core-api/index.rst +++ b/Documentation/translations/zh_CN/core-api/index.rst @@ -19,12 +19,13 @@ 来的大量 kerneldoc 信息；有朝一日，若有人有动力的话，应当把它们拆分 出来。 -Todolist: +.. toctree:: + :maxdepth: 1 kernel-api - workqueue printk-basics printk-formats + workqueue symbol-namespaces 数据结构和低级实用程序 @@ -32,9 +33,13 @@ Todolist: 在整个内核中使用的函数库。 -Todolist: +.. toctree:: + :maxdepth: 1 kobject + +Todolist: + kref assoc_array xarray @@ -58,12 +63,12 @@ Linux如何让一切同时发生。 详情请参阅 :maxdepth: 1 irq/index - -Todolist: - refcount-vs-atomic local_ops padata + +Todolist: + ../RCU/index 低级硬件管理 @@ -71,9 +76,14 @@ Todolist: 缓存管理，CPU热插拔管理等。 -Todolist: +.. toctree:: + :maxdepth: 1 cachetlb + +Todolist: + + cpu_hotplug memory-hotplug genericirq diff --git a/Documentation/translations/zh_CN/core-api/kernel-api.rst b/Documentation/translations/zh_CN/core-api/kernel-api.rst new file mode 100644 index 000000000000..d6f815ec265b --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/kernel-api.rst @@ -0,0 +1,369 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/kernel-api.rst +:Translator: Yanteng Si + +.. _cn_kernel-api.rst: + + +============ +Linux内核API +============ + + +列表管理函数 +============ + +该API在以下内核代码中: + +include/linux/list.h + +基本的C库函数 +============= + +在编写驱动程序时，一般不能使用C库中的例程。部分函数通常很有用，它们在 +下面被列出。这些函数的行为可能会与ANSI定义的略有不同，这些偏差会在文中 +注明。 + +字符串转换 +---------- + +该API在以下内核代码中: + +lib/vsprintf.c + +include/linux/kernel.h + +include/linux/kernel.h + +lib/kstrtox.c + +lib/string_helpers.c + +字符串处理 +---------- + +该API在以下内核代码中: + +lib/string.c + +include/linux/string.h + +mm/util.c + +基本的内核库函数 +================ + +Linux内核提供了很多实用的基本函数。 + +位运算 +------ + +该API在以下内核代码中: + +include/asm-generic/bitops/instrumented-atomic.h + +include/asm-generic/bitops/instrumented-non-atomic.h + +include/asm-generic/bitops/instrumented-lock.h + +位图运算 +-------- + +该API在以下内核代码中: + +lib/bitmap.c + +include/linux/bitmap.h + +include/linux/bitmap.h + +include/linux/bitmap.h + +lib/bitmap.c + +lib/bitmap.c + +include/linux/bitmap.h + +命令行解析 +---------- + +该API在以下内核代码中: + +lib/cmdline.c + +排序 +---- + +该API在以下内核代码中: + +lib/sort.c + +lib/list_sort.c + +文本检索 +-------- + +该API在以下内核代码中: + +lib/textsearch.c + +lib/textsearch.c + +include/linux/textsearch.h + +Linux中的CRC和数学函数 +====================== + + +CRC函数 +------- + +*译注：CRC，Cyclic Redundancy Check，循环冗余校验* + +该API在以下内核代码中: + +lib/crc4.c + +lib/crc7.c + +lib/crc8.c + +lib/crc16.c + +lib/crc32.c + +lib/crc-ccitt.c + +lib/crc-itu-t.c + +基数为2的对数和幂函数 +--------------------- + +该API在以下内核代码中: + +include/linux/log2.h + +整数幂函数 +---------- + +该API在以下内核代码中: + +lib/math/int_pow.c + +lib/math/int_sqrt.c + +除法函数 +-------- + +该API在以下内核代码中: + +include/asm-generic/div64.h + +include/linux/math64.h + +lib/math/div64.c + +lib/math/gcd.c + +UUID/GUID +--------- + +该API在以下内核代码中: + +lib/uuid.c + +内核IPC设备 +=========== + +IPC实用程序 +----------- + +该API在以下内核代码中: + +ipc/util.c + +FIFO 缓冲区 +=========== + +kfifo接口 +--------- + +该API在以下内核代码中: + +include/linux/kfifo.h + +转发接口支持 +============ + +转发接口支持旨在为工具和设备提供一种有效的机制，将大量数据从内核空间 +转发到用户空间。 + +转发接口 +-------- + +该API在以下内核代码中: + +kernel/relay.c + +kernel/relay.c + +模块支持 +======== + +模块加载 +-------- + +该API在以下内核代码中: + +kernel/kmod.c + +模块接口支持 +------------ + +更多信息请参考文件kernel/module.c。 + +硬件接口 +======== + + +该API在以下内核代码中: + +kernel/dma.c + +资源管理 +-------- + +该API在以下内核代码中: + +kernel/resource.c + +kernel/resource.c + +MTRR处理 +-------- + +该API在以下内核代码中: + +arch/x86/kernel/cpu/mtrr/mtrr.c + +安全框架 +======== + +该API在以下内核代码中: + +security/security.c + +security/inode.c + +审计接口 +======== + +该API在以下内核代码中: + +kernel/audit.c + +kernel/auditsc.c + +kernel/auditfilter.c + +核算框架 +======== + +该API在以下内核代码中: + +kernel/acct.c + +块设备 +====== + +该API在以下内核代码中: + +block/blk-core.c + +block/blk-core.c + +block/blk-map.c + +block/blk-sysfs.c + +block/blk-settings.c + +block/blk-exec.c + +block/blk-flush.c + +block/blk-lib.c + +block/blk-integrity.c + +kernel/trace/blktrace.c + +block/genhd.c + +block/genhd.c + +字符设备 +======== + +该API在以下内核代码中: + +fs/char_dev.c + +时钟框架 +======== + +时钟框架定义了编程接口，以支持系统时钟树的软件管理。该框架广泛用于系统级芯片（SOC）平 +台，以支持电源管理和各种可能需要自定义时钟速率的设备。请注意，这些 “时钟”与计时或实 +时时钟(RTC)无关，它们都有单独的框架。这些:c:type: `struct clk ` 实例可用于管理 +各种时钟信号，例如一个96理例如96MHz的时钟信号，该信号可被用于总线或外设的数据交换，或以 +其他方式触发系统硬件中的同步状态机转换。 + +通过明确的软件时钟门控来支持电源管理：未使用的时钟被禁用，因此系统不会因为改变不在使用 +中的晶体管的状态而浪费电源。在某些系统中，这可能是由硬件时钟门控支持的，其中时钟被门控 +而不在软件中被禁用。芯片的部分，在供电但没有时钟的情况下，可能会保留其最后的状态。这种 +低功耗状态通常被称为*保留模式*。这种模式仍然会产生漏电流，特别是在电路几何结构较细的情 +况下，但对于CMOS电路来说，电能主要是随着时钟翻转而被消耗的。 + +电源感知驱动程序只有在其管理的设备处于活动使用状态时才会启用时钟。此外，系统睡眠状态通 +常根据哪些时钟域处于活动状态而有所不同：“待机”状态可能允许从多个活动域中唤醒，而 +"mem"（暂停到RAM）状态可能需要更全面地关闭来自高速PLL和振荡器的时钟，从而限制了可能 +的唤醒事件源的数量。驱动器的暂停方法可能需要注意目标睡眠状态的系统特定时钟约束。 + +一些平台支持可编程时钟发生器。这些可以被各种外部芯片使用，如其他CPU、多媒体编解码器以 +及对接口时钟有严格要求的设备。 + +该API在以下内核代码中: + +include/linux/clk.h + +同步原语 +======== + +读-复制-更新（RCU） +------------------- + +该API在以下内核代码中: + +include/linux/rcupdate.h + +kernel/rcu/tree.c + +kernel/rcu/tree_exp.h + +kernel/rcu/update.c + +include/linux/srcu.h + +kernel/rcu/srcutree.c + +include/linux/rculist_bl.h + +include/linux/rculist.h + +include/linux/rculist_nulls.h + +include/linux/rcu_sync.h + +kernel/rcu/sync.c diff --git a/Documentation/translations/zh_CN/core-api/kobject.rst b/Documentation/translations/zh_CN/core-api/kobject.rst new file mode 100644 index 000000000000..f0e6a4aeb372 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/kobject.rst @@ -0,0 +1,378 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/kobject.rst +:Translator: Yanteng Si + +.. _cn_core_api_kobject.rst: + +======================================================= +关于kobjects、ksets和ktypes的一切你没想过需要了解的东西 +======================================================= + +:作者: Greg Kroah-Hartman +:最后一次更新: December 19, 2007 + +根据Jon Corbet于2003年10月1日为lwn.net撰写的原创文章改编，网址是： +https://lwn.net/Articles/51437/ + +理解驱动模型和建立在其上的kobject抽象的部分的困难在于，没有明显的切入点。 +处理kobjects需要理解一些不同的类型，所有这些类型都会相互引用。为了使事情 +变得更简单，我们将多路并进，从模糊的术语开始，并逐渐增加细节。那么，先来 +了解一些我们将要使用的术语的简明定义吧。 + + - 一个kobject是一个kobject结构体类型的对象。Kobjects有一个名字和一个 + 引用计数。一个kobject也有一个父指针（允许对象被排列成层次结构），一个 + 特定的类型，并且，通常在sysfs虚拟文件系统中表示。 + + Kobjects本身通常并不引人关注；相反它们常常被嵌入到其他包含真正引人注目 + 的代码的结构体中。 + + 任何结构体都 **不应该** 有一个以上的kobject嵌入其中。如果有的话，对象的引用计 + 数肯定会被打乱，而且不正确，你的代码就会出现错误。所以不要这样做。 + + - ktype是嵌入一个kobject的对象的类型。每个嵌入kobject的结构体都需要一个 + 相应的ktype。ktype控制着kobject在被创建和销毁时的行为。 + + - 一个kset是一组kobjects。这些kobjects可以是相同的ktype或者属于不同的 + ktype。kset是kobjects集合的基本容器类型。Ksets包含它们自己的kobjects， + 但你可以安全地忽略这个实现细节，因为kset的核心代码会自动处理这个kobject。 + + 当你看到一个下面全是其他目录的sysfs目录时，通常这些目录中的每一个都对应 + 于同一个kset中的一个kobject。 + + 我们将研究如何创建和操作所有这些类型。将采取一种自下而上的方法，所以我们 + 将回到kobjects。 + + +嵌入kobjects +============= + +内核代码很少创建孤立的kobject，只有一个主要的例外，下面会解释。相反， +kobjects被用来控制对一个更大的、特定领域的对象的访问。为此，kobjects会被 +嵌入到其他结构中。如果你习惯于用面向对象的术语来思考问题，那么kobjects可 +以被看作是一个顶级的抽象类，其他的类都是从它派生出来的。一个kobject实现了 +一系列的功能，这些功能本身并不特别有用，但在其他对象中却很好用。C语言不允 +许直接表达继承，所以必须使用其他技术——比如结构体嵌入。 + +（对于那些熟悉内核链表实现的人来说，这类似于“list_head”结构本身很少有用， +但总是被嵌入到感兴趣的更大的对象中）。 + +例如， ``drivers/uio/uio.c`` 中的IO代码有一个结构体，定义了与uio设备相 +关的内存区域:: + + struct uio_map { + struct kobject kobj; + struct uio_mem *mem; + }; + +如果你有一个uio_map结构体，找到其嵌入的kobject只是一个使用kobj成员的问题。 +然而，与kobjects一起工作的代码往往会遇到相反的问题：给定一个结构体kobject +的指针，指向包含结构体的指针是什么？你必须避免使用一些技巧（比如假设 +kobject在结构的开头），相反，你得使用container_of()宏，其可以在 ```` +中找到:: + + container_of(ptr, type, member) + +其中: + + * ``ptr`` 是一个指向嵌入kobject的指针， + * ``type`` 是包含结构体的类型， + * ``member`` 是 ``指针`` 所指向的结构体域的名称。 + +container_of()的返回值是一个指向相应容器类型的指针。因此，例如，一个嵌入到 +uio_map结构 **中** 的kobject结构体的指针kp可以被转换为一个指向 **包含** uio_map +结构体的指针，方法是:: + + struct uio_map *u_map = container_of(kp, struct uio_map, kobj); + +为了方便起见，程序员经常定义一个简单的宏，用于将kobject指针 **反推** 到包含 +类型。在早期的 ``drivers/uio/uio.c`` 中正是如此，你可以在这里看到:: + + struct uio_map { + struct kobject kobj; + struct uio_mem *mem; + }; + + #define to_map(map) container_of(map, struct uio_map, kobj) + +其中宏的参数“map”是一个指向有关的kobject结构体的指针。该宏随后被调用:: + + struct uio_map *map = to_map(kobj); + + +kobjects的初始化 +================ + +当然，创建kobject的代码必须初始化该对象。一些内部字段是通过（强制）调用kobject_init() +来设置的:: + + void kobject_init(struct kobject *kobj, struct kobj_type *ktype); + +ktype是正确创建kobject的必要条件，因为每个kobject都必须有一个相关的kobj_type。 +在调用kobject_init()后，为了向sysfs注册kobject，必须调用函数kobject_add():: + + int kobject_add(struct kobject *kobj, struct kobject *parent, + const char *fmt, ...); + +这将正确设置kobject的父级和kobject的名称。如果该kobject要与一个特定的kset相关 +联，在调用kobject_add()之前必须分配kobj->kset。如果kset与kobject相关联，则 +kobject的父级可以在调用kobject_add()时被设置为NULL，则kobject的父级将是kset +本身。 + +由于kobject的名字是在它被添加到内核时设置的，所以kobject的名字不应该被直接操作。 +如果你必须改变kobject的名字，请调用kobject_rename():: + + int kobject_rename(struct kobject *kobj, const char *new_name); + +kobject_rename()函数不会执行任何锁定操作，也不会对name进行可靠性检查，所以调用 +者自己检查和串行化操作是明智的选择 + +有一个叫kobject_set_name()的函数，但那是历史遗产，正在被删除。如果你的代码需 +要调用这个函数，那么它是不正确的，需要被修复。 + +要正确访问kobject的名称，请使用函数kobject_name():: + + const char *kobject_name(const struct kobject * kobj); + +有一个辅助函数可以同时初始化和添加kobject到内核中，令人惊讶的是，该函数被称为 +kobject_init_and_add():: + + int kobject_init_and_add(struct kobject *kobj, struct kobj_type *ktype, + struct kobject *parent, const char *fmt, ...); + +参数与上面描述的单个kobject_init()和kobject_add()函数相同。 + + +Uevents +======= + +当一个kobject被注册到kobject核心后，你需要向全世界宣布它已经被创建了。这可以通 +过调用kobject_uevent()来实现:: + + int kobject_uevent(struct kobject *kobj, enum kobject_action action); + +当kobject第一次被添加到内核时，使用 *KOBJ_ADD* 动作。这应该在该kobject的任 +何属性或子对象被正确初始化后进行，因为当这个调用发生时，用户空间会立即开始寻 +找它们。 + +当kobject从内核中移除时（关于如何做的细节在下面）， **KOBJ_REMOVE** 的uevent +将由kobject核心自动创建，所以调用者不必担心手动操作。 + + +引用计数 +======== + +kobject的关键功能之一是作为它所嵌入的对象的一个引用计数器。只要对该对象的引用 +存在，该对象（以及支持它的代码）就必须继续存在。用于操作kobject的引用计数的低 +级函数是:: + + struct kobject *kobject_get(struct kobject *kobj); + void kobject_put(struct kobject *kobj); + +对kobject_get()的成功调用将增加kobject的引用计数器值并返回kobject的指针。 + +当引用被释放时，对kobject_put()的调用将递减引用计数值，并可能释放该对象。请注 +意，kobject_init()将引用计数设置为1，所以设置kobject的代码最终需要kobject_put() +来释放该引用。 + +因为kobjects是动态的，所以它们不能以静态方式或在堆栈中声明，而总是以动态方式分 +配。未来版本的内核将包含对静态创建的kobjects的运行时检查，并将警告开发者这种不 +当的使用。 + +如果你使用struct kobject只是为了给你的结构体提供一个引用计数器，请使用struct kref +来代替；kobject是多余的。关于如何使用kref结构体的更多信息，请参见Linux内核源代 +码树中的文件Documentation/core-api/kref.rst + + +创建“简单的”kobjects +==================== + +有时，开发者想要的只是在sysfs层次结构中创建一个简单的目录，而不必去搞那些复杂 +的ksets、显示和存储函数，以及其他细节。这是一个应该创建单个kobject的例外。要 +创建这样一个条目（即简单的目录），请使用函数:: + + struct kobject *kobject_create_and_add(const char *name, struct kobject *parent); + +这个函数将创建一个kobject，并将其放在sysfs中指定的父kobject下面的位置。要创 +建与此kobject相关的简单属性，请使用:: + + int sysfs_create_file(struct kobject *kobj, const struct attribute *attr); + +或者:: + + int sysfs_create_group(struct kobject *kobj, const struct attribute_group *grp); + +这里使用的两种类型的属性，与已经用kobject_create_and_add()创建的kobject， +都可以是kobj_attribute类型，所以不需要创建特殊的自定义属性。 + +参见示例模块， ``samples/kobject/kobject-example.c`` ，以了解一个简单的 +kobject和属性的实现。 + + + +ktypes和释放方法 +================ + +以上讨论中还缺少一件重要的事情，那就是当一个kobject的引用次数达到零的时候 +会发生什么。创建kobject的代码通常不知道何时会发生这种情况；首先，如果它知 +道，那么使用kobject就没有什么意义。当sysfs被引入时，即使是可预测的对象生命 +周期也会变得更加复杂，因为内核的其他部分可以获得在系统中注册的任何kobject +的引用。 + +最终的结果是，一个由kobject保护的结构体在其引用计数归零之前不能被释放。引 +用计数不受创建kobject的代码的直接控制。因此，每当它的一个kobjects的最后一 +个引用消失时，必须异步通知该代码。 + +一旦你通过kobject_add()注册了你的kobject，你绝对不能使用kfree()来直接释 +放它。唯一安全的方法是使用kobject_put()。在kobject_init()之后总是使用 +kobject_put()以避免错误的发生是一个很好的做法。 + +这个通知是通过kobject的release()方法完成的。通常这样的方法有如下形式:: + + void my_object_release(struct kobject *kobj) + { + struct my_object *mine = container_of(kobj, struct my_object, kobj); + + /* Perform any additional cleanup on this object, then... */ + kfree(mine); + } + +有一点很重要：每个kobject都必须有一个release()方法，而且这个kobject必 +须持续存在（处于一致的状态），直到这个方法被调用。如果这些约束条件没有 +得到满足，那么代码就是有缺陷的。注意，如果你忘记提供release()方法，内 +核会警告你。不要试图通过提供一个“空”的释放函数来摆脱这个警告。 + +如果你的清理函数只需要调用kfree()，那么你必须创建一个包装函数，该函数 +使用container_of()来向上造型到正确的类型（如上面的例子所示），然后在整个 +结构体上调用kfree()。 + +注意，kobject的名字在release函数中是可用的，但它不能在这个回调中被改 +变。否则，在kobject核心中会出现内存泄漏，这让人很不爽。 + +有趣的是，release()方法并不存储在kobject本身；相反，它与ktype相关。 +因此，让我们引入结构体kobj_type:: + + struct kobj_type { + void (*release)(struct kobject *kobj); + const struct sysfs_ops *sysfs_ops; + struct attribute **default_attrs; + const struct attribute_group **default_groups; + const struct kobj_ns_type_operations *(*child_ns_type)(struct kobject *kobj); + const void *(*namespace)(struct kobject *kobj); + void (*get_ownership)(struct kobject *kobj, kuid_t *uid, kgid_t *gid); + }; + +这个结构提用来描述一个特定类型的kobject（或者更正确地说，包含对象的 +类型）。每个kobject都需要有一个相关的kobj_type结构；当你调用 +kobject_init()或kobject_init_and_add()时必须指定一个指向该结构的 +指针。 + +当然，kobj_type结构中的release字段是指向这种类型的kobject的release() +方法的一个指针。另外两个字段（sysfs_ops 和 default_attrs）控制这种 +类型的对象如何在 sysfs 中被表示；它们超出了本文的范围。 + +default_attrs 指针是一个默认属性的列表，它将为任何用这个 ktype 注册 +的 kobject 自动创建。 + + +ksets +===== + +一个kset仅仅是一个希望相互关联的kobjects的集合。没有限制它们必须是相 +同的ktype，但是如果它们不是相同的，就要非常小心。 + +一个kset有以下功能: + + - 它像是一个包含一组对象的袋子。一个kset可以被内核用来追踪“所有块 + 设备”或“所有PCI设备驱动”。 + + - kset也是sysfs中的一个子目录，与kset相关的kobjects可以在这里显示 + 出来。每个kset都包含一个kobject，它可以被设置为其他kobject的父对象； + sysfs层次结构的顶级目录就是以这种方式构建的。 + + - Ksets可以支持kobjects的 "热插拔"，并影响uevent事件如何被报告给 + 用户空间。 + + 在面向对象的术语中，“kset”是顶级的容器类；ksets包含它们自己的kobject， + 但是这个kobject是由kset代码管理的，不应该被任何其他用户所操纵。 + + kset在一个标准的内核链表中保存它的子对象。Kobjects通过其kset字段指向其 + 包含的kset。在几乎所有的情况下，属于一个kset的kobjects在它们的父 + 对象中都有那个kset（或者，严格地说，它的嵌入kobject）。 + + 由于kset中包含一个kobject，它应该总是被动态地创建，而不是静态地 + 或在堆栈中声明。要创建一个新的kset，请使用:: + + struct kset *kset_create_and_add(const char *name, + const struct kset_uevent_ops *uevent_ops, + struct kobject *parent_kobj); + +当你完成对kset的处理后，调用:: + + void kset_unregister(struct kset *k); + +来销毁它。这将从sysfs中删除该kset并递减其引用计数值。当引用计数 +为零时，该kset将被释放。因为对该kset的其他引用可能仍然存在， +释放可能发生在kset_unregister()返回之后。 + +一个使用kset的例子可以在内核树中的 ``samples/kobject/kset-example.c`` +文件中看到。 + +如果一个kset希望控制与它相关的kobjects的uevent操作，它可以使用 +结构体kset_uevent_ops来处理它:: + + struct kset_uevent_ops { + int (* const filter)(struct kset *kset, struct kobject *kobj); + const char *(* const name)(struct kset *kset, struct kobject *kobj); + int (* const uevent)(struct kset *kset, struct kobject *kobj, + struct kobj_uevent_env *env); + }; + + +过滤器函数允许kset阻止一个特定kobject的uevent被发送到用户空间。 +如果该函数返回0，该uevent将不会被发射出去。 + +name函数将被调用以覆盖uevent发送到用户空间的kset的默认名称。默 +认情况下，该名称将与kset本身相同，但这个函数，如果存在，可以覆盖 +该名称。 + +当uevent即将被发送至用户空间时，uevent函数将被调用，以允许更多 +的环境变量被添加到uevent中。 + +有人可能会问，鉴于没有提出执行该功能的函数，究竟如何将一个kobject +添加到一个kset中。答案是这个任务是由kobject_add()处理的。当一个 +kobject被传递给kobject_add()时，它的kset成员应该指向这个kobject +所属的kset。 kobject_add()将处理剩下的部分。 + +如果属于一个kset的kobject没有父kobject集，它将被添加到kset的目 +录中。并非所有的kset成员都必须住在kset目录中。如果在添加kobject +之前分配了一个明确的父kobject，那么该kobject将被注册到kset中， +但是被添加到父kobject下面。 + + +移除Kobject +=========== + +当一个kobject在kobject核心注册成功后，在代码使用完它时，必须将其 +清理掉。要做到这一点，请调用kobject_put()。通过这样做，kobject核 +心会自动清理这个kobject分配的所有内存。如果为这个对象发送了 ``KOBJ_ADD`` +uevent，那么相应的 ``KOBJ_REMOVE`` uevent也将被发送，任何其他的 +sysfs内务将被正确处理。 + +如果你需要分两次对kobject进行删除（比如说在你要销毁对象时无权睡眠）， +那么调用kobject_del()将从sysfs中取消kobject的注册。这使得kobject +“不可见”，但它并没有被清理掉，而且该对象的引用计数仍然是一样的。在稍 +后的时间调用kobject_put()来完成与该kobject相关的内存的清理。 + +kobject_del()可以用来放弃对父对象的引用，如果循环引用被构建的话。 +在某些情况下，一个父对象引用一个子对象是有效的。循环引用必须通过明 +确调用kobject_del()来打断，这样一个释放函数就会被调用，前一个循环 +中的对象会相互释放。 + + +示例代码出处 +============ + +关于正确使用ksets和kobjects的更完整的例子，请参见示例程序 +``samples/kobject/{kobject-example.c,kset-example.c}`` ，如果 +您选择 ``CONFIG_SAMPLE_KOBJECT`` ，它们将被构建为可加载模块。 diff --git a/Documentation/translations/zh_CN/core-api/local_ops.rst b/Documentation/translations/zh_CN/core-api/local_ops.rst new file mode 100644 index 000000000000..ee67379b6869 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/local_ops.rst @@ -0,0 +1,194 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/local_ops.rst +:Translator: Yanteng Si + +.. _cn_local_ops: + + +======================== +本地原子操作的语义和行为 +======================== + +:作者: Mathieu Desnoyers + + +本文解释了本地原子操作的目的，如何为任何给定的架构实现这些操作，并说明了 +如何正确使用这些操作。它还强调了在内存写入顺序很重要的情况下，跨CPU读取 +这些本地变量时必须采取的预防措施。 + +.. note:: + + 注意，基于 ``local_t`` 的操作不建议用于一般内核操作。请使用 ``this_cpu`` + 操作来代替使用，除非真的有特殊目的。大多数内核中使用的 ``local_t`` 已 + 经被 ``this_cpu`` 操作所取代。 ``this_cpu`` 操作在一条指令中结合了重 + 定位和类似 ``local_t`` 的语义，产生了更紧凑和更快的执行代码。 + + +本地原子操作的目的 +================== + +本地原子操作的目的是提供快速和高度可重入的每CPU计数器。它们通过移除LOCK前 +缀和通常需要在CPU间同步的内存屏障，将标准原子操作的性能成本降到最低。 + +在许多情况下，拥有快速的每CPU原子计数器是很有吸引力的：它不需要禁用中断来保护中 +断处理程序，它允许在NMI（Non Maskable Interrupt）处理程序中使用连贯的计数器。 +它对追踪目的和各种性能监测计数器特别有用。 + +本地原子操作只保证在拥有数据的CPU上的变量修改的原子性。因此，必须注意确保只 +有一个CPU写到 ``local_t`` 的数据。这是通过使用每CPU的数据来实现的，并确 +保我们在一个抢占式安全上下文中修改它。然而，从任何一个CPU读取 ``local_t`` +数据都是允许的：这样它就会显得与所有者CPU的其他内存写入顺序不一致。 + + +针对特定架构的实现 +================== + +这可以通过稍微修改标准的原子操作来实现：只有它们的UP变体必须被保留。这通常 +意味着删除LOCK前缀（在i386和x86_64上）和任何SMP同步屏障。如果架构在SMP和 +UP之间没有不同的行为，在你的架构的 ``local.h`` 中包括 ``asm-generic/local.h`` +就足够了。 + +通过在一个结构体中嵌入一个 ``atomic_long_t`` ， ``local_t`` 类型被定义为 +一个不透明的 ``signed long`` 。这样做的目的是为了使从这个类型到 +``long`` 的转换失败。该定义看起来像:: + + typedef struct { atomic_long_t a; } local_t; + + +使用本地原子操作时应遵循的规则 +============================== + +* 被本地操作触及的变量必须是每cpu的变量。 + +* *只有* 这些变量的CPU所有者才可以写入这些变量。 + +* 这个CPU可以从任何上下文（进程、中断、软中断、nmi...）中使用本地操作来更新 + 它的local_t变量。 + +* 当在进程上下文中使用本地操作时，必须禁用抢占（或中断），以确保进程在获得每 + CPU变量和进行实际的本地操作之间不会被迁移到不同的CPU。 + +* 当在中断上下文中使用本地操作时，在主线内核上不需要特别注意，因为它们将在局 + 部CPU上运行，并且已经禁用了抢占。然而，我建议无论如何都要明确地禁用抢占， + 以确保它在-rt内核上仍能正确工作。 + +* 读取本地cpu变量将提供该变量的当前拷贝。 + +* 对这些变量的读取可以从任何CPU进行，因为对 “ ``long`` ”，对齐的变量的更新 + 总是原子的。由于写入程序的CPU没有进行内存同步，所以在读取 *其他* cpu的变 + 量时，可以读取该变量的过期副本。 + + +如何使用本地原子操作 +==================== + +:: + + #include + #include + + static DEFINE_PER_CPU(local_t, counters) = LOCAL_INIT(0); + + +计数器 +====== + +计数是在一个signed long的所有位上进行的。 + +在可抢占的上下文中，围绕本地原子操作使用 ``get_cpu_var()`` 和 +``put_cpu_var()`` ：它确保在对每个cpu变量进行写访问时，抢占被禁用。比如 +说:: + + local_inc(&get_cpu_var(counters)); + put_cpu_var(counters); + +如果你已经在一个抢占安全上下文中，你可以使用 ``this_cpu_ptr()`` 代替:: + + local_inc(this_cpu_ptr(&counters)); + + + +读取计数器 +========== + +那些本地计数器可以从外部的CPU中读取，以求得计数的总和。请注意，local_read +所看到的跨CPU的数据必须被认为是相对于拥有该数据的CPU上发生的其他内存写入来 +说不符合顺序的:: + + long sum = 0; + for_each_online_cpu(cpu) + sum += local_read(&per_cpu(counters, cpu)); + +如果你想使用远程local_read来同步CPU之间对资源的访问，必须在写入者和读取者 +的CPU上分别使用显式的 ``smp_wmb()`` 和 ``smp_rmb()`` 内存屏障。如果你使 +用 ``local_t`` 变量作为写在缓冲区中的字节的计数器，就会出现这种情况：在缓 +冲区写和计数器增量之间应该有一个 ``smp_wmb()`` ，在计数器读和缓冲区读之间 +也应有一个 ``smp_rmb()`` 。 + +下面是一个使用 ``local.h`` 实现每个cpu基本计数器的示例模块:: + + /* test-local.c + * + * Sample module for local.h usage. + */ + + + #include + #include + #include + + static DEFINE_PER_CPU(local_t, counters) = LOCAL_INIT(0); + + static struct timer_list test_timer; + + /* IPI called on each CPU. */ + static void test_each(void *info) + { + /* Increment the counter from a non preemptible context */ + printk("Increment on cpu %d\n", smp_processor_id()); + local_inc(this_cpu_ptr(&counters)); + + /* This is what incrementing the variable would look like within a + * preemptible context (it disables preemption) : + * + * local_inc(&get_cpu_var(counters)); + * put_cpu_var(counters); + */ + } + + static void do_test_timer(unsigned long data) + { + int cpu; + + /* Increment the counters */ + on_each_cpu(test_each, NULL, 1); + /* Read all the counters */ + printk("Counters read from CPU %d\n", smp_processor_id()); + for_each_online_cpu(cpu) { + printk("Read : CPU %d, count %ld\n", cpu, + local_read(&per_cpu(counters, cpu))); + } + mod_timer(&test_timer, jiffies + 1000); + } + + static int __init test_init(void) + { + /* initialize the timer that will increment the counter */ + timer_setup(&test_timer, do_test_timer, 0); + mod_timer(&test_timer, jiffies + 1); + + return 0; + } + + static void __exit test_exit(void) + { + del_timer_sync(&test_timer); + } + + module_init(test_init); + module_exit(test_exit); + + MODULE_LICENSE("GPL"); + MODULE_AUTHOR("Mathieu Desnoyers"); + MODULE_DESCRIPTION("Local Atomic Ops"); diff --git a/Documentation/translations/zh_CN/core-api/padata.rst b/Documentation/translations/zh_CN/core-api/padata.rst new file mode 100644 index 000000000000..c627f8f131f9 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/padata.rst @@ -0,0 +1,158 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/padata.rst +:Translator: Yanteng Si + +.. _cn_core_api_padata.rst: + +================== +padata并行执行机制 +================== + +:日期: 2020年5月 + +Padata是一种机制，内核可以通过此机制将工作分散到多个CPU上并行完成，同时 +可以选择保持它们的顺序。 + +它最初是为IPsec开发的，它需要在不对这些数据包重新排序的其前提下，为大量的数 +据包进行加密和解密。这是目前padata的序列化作业支持的唯一用途。 + +Padata还支持多线程作业，将作业平均分割，同时在线程之间进行负载均衡和协调。 + +执行序列化作业 +============== + +初始化 +------ + +使用padata执行序列化作业的第一步是建立一个padata_instance结构体，以全面 +控制作业的运行方式:: + + #include + + struct padata_instance *padata_alloc(const char *name); + +'name'即标识了这个实例。 + +然后，通过分配一个padata_shell来完成padata的初始化:: + + struct padata_shell *padata_alloc_shell(struct padata_instance *pinst); + +一个padata_shell用于向padata提交一个作业，并允许一系列这样的作业被独立地 +序列化。一个padata_instance可以有一个或多个padata_shell与之相关联，每个 +都允许一系列独立的作业。 + +修改cpumasks +------------ + +用于运行作业的CPU可以通过两种方式改变，通过padata_set_cpumask()编程或通 +过sysfs。前者的定义是:: + + int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, + cpumask_var_t cpumask); + +这里cpumask_type是PADATA_CPU_PARALLEL（并行）或PADATA_CPU_SERIAL（串行）之一，其中并 +行cpumask描述了哪些处理器将被用来并行执行提交给这个实例的作业，串行cpumask +定义了哪些处理器被允许用作串行化回调处理器。 cpumask指定了要使用的新cpumask。 + +一个实例的cpumasks可能有sysfs文件。例如，pcrypt的文件在 +/sys/kernel/pcrypt/。在一个实例的目录中，有两个文件，parallel_cpumask +和serial_cpumask，任何一个cpumask都可以通过在文件中回显（echo）一个bitmask +来改变，比如说:: + + echo f > /sys/kernel/pcrypt/pencrypt/parallel_cpumask + +读取其中一个文件会显示用户提供的cpumask，它可能与“可用”的cpumask不同。 + +Padata内部维护着两对cpumask，用户提供的cpumask和“可用的”cpumask(每一对由一个 +并行和一个串行cpumask组成)。用户提供的cpumasks在实例分配时默认为所有可能的CPU， +并且可以如上所述进行更改。可用的cpumasks总是用户提供的cpumasks的一个子集，只包 +含用户提供的掩码中的在线CPU；这些是padata实际使用的cpumasks。因此，向padata提 +供一个包含离线CPU的cpumask是合法的。一旦用户提供的cpumask中的一个离线CPU上线， +padata就会使用它。 + +改变CPU掩码的操作代价很高，所以不应频繁更改。 + +运行一个作业 +------------- + +实际上向padata实例提交工作需要创建一个padata_priv结构体，它代表一个作业:: + + struct padata_priv { + /* Other stuff here... */ + void (*parallel)(struct padata_priv *padata); + void (*serial)(struct padata_priv *padata); + }; + +这个结构体几乎肯定会被嵌入到一些针对要做的工作的大结构体中。它的大部分字段对 +padata来说是私有的，但是这个结构在初始化时应该被清零，并且应该提供parallel()和 +serial()函数。在完成工作的过程中，这些函数将被调用，我们马上就会遇到。 + +工作的提交是通过:: + + int padata_do_parallel(struct padata_shell *ps, + struct padata_priv *padata, int *cb_cpu); + +ps和padata结构体必须如上所述进行设置；cb_cpu指向作业完成后用于最终回调的首选CPU； +它必须在当前实例的CPU掩码中（如果不是，cb_cpu指针将被更新为指向实际选择的CPU）。 +padata_do_parallel()的返回值在成功时为0，表示工作正在进行中。-EBUSY意味着有人 +在其他地方正在搞乱实例的CPU掩码，而当cb_cpu不在串行cpumask中、并行或串行cpumasks +中无在线CPU，或实例停止时，则会出现-EINVAL反馈。 + +每个提交给padata_do_parallel()的作业将依次传递给一个CPU上的上述parallel()函数 +的一个调用，所以真正的并行是通过提交多个作业来实现的。parallel()在运行时禁用软 +件中断，因此不能睡眠。parallel()函数把获得的padata_priv结构体指针作为其唯一的参 +数；关于实际要做的工作的信息可能是通过使用container_of()找到封装结构体来获得的。 + +请注意，parallel()没有返回值；padata子系统假定parallel()将从此时开始负责这项工 +作。作业不需要在这次调用中完成，但是，如果parallel()留下了未完成的工作，它应该准 +备在前一个作业完成之前，被以新的作业再次调用 + +序列化作业 +---------- + +当一个作业完成时，parallel()（或任何实际完成该工作的函数）应该通过调用通知padata此 +事:: + + void padata_do_serial(struct padata_priv *padata); + +在未来的某个时刻，padata_do_serial()将触发对padata_priv结构体中serial()函数的调 +用。这个调用将发生在最初要求调用padata_do_parallel()的CPU上；它也是在本地软件中断 +被禁用的情况下运行的。 +请注意，这个调用可能会被推迟一段时间，因为padata代码会努力确保作业按照提交的顺序完 +成。 + +销毁 +---- + +清理一个padata实例时，可以预见的是调用两个free函数，这两个函数对应于分配的逆过程:: + + void padata_free_shell(struct padata_shell *ps); + void padata_free(struct padata_instance *pinst); + +用户有责任确保在调用上述任何一项之前，所有未完成的工作都已完成。 + +运行多线程作业 +============== + +一个多线程作业有一个主线程和零个或多个辅助线程，主线程参与作业，然后等待所有辅助线 +程完成。padata将作业分割成称为chunk的单元，其中chunk是一个线程在一次调用线程函数 +中完成的作业片段。 + +用户必须做三件事来运行一个多线程作业。首先，通过定义一个padata_mt_job结构体来描述 +作业，这在接口部分有解释。这包括一个指向线程函数的指针，padata每次将作业块分配给线 +程时都会调用这个函数。然后，定义线程函数，它接受三个参数： ``start`` 、 ``end`` 和 ``arg`` ， +其中前两个参数限定了线程操作的范围，最后一个是指向作业共享状态的指针，如果有的话。 +准备好共享状态，它通常被分配在主线程的堆栈中。最后，调用padata_do_multithreaded()， +它将在作业完成后返回。 + +接口 +==== + +该API在以下内核代码中: + +include/linux/padata.h + +kernel/padata.c diff --git a/Documentation/translations/zh_CN/core-api/printk-basics.rst b/Documentation/translations/zh_CN/core-api/printk-basics.rst new file mode 100644 index 000000000000..2b20f6303a82 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/printk-basics.rst @@ -0,0 +1,110 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/printk-basics.rst +:Translator: Yanteng Si + +.. _cn_printk-basics.rst: + + +================== +使用printk记录消息 +================== + +printk()是Linux内核中最广为人知的函数之一。它是我们打印消息的标准工具，通常也是追踪和调试 +的最基本方法。如果你熟悉printf(3)，你就能够知道printk()是基于它的，尽管它在功能上有一些不 +同之处: + + - printk() 消息可以指定日志级别。 + + - 格式字符串虽然与C99基本兼容，但并不遵循完全相同的规范。它有一些扩展和一些限制（没 + 有 ``%n`` 或浮点转换指定符）。参见:ref: `如何正确地获得printk格式指定符` 。 + +所有的printk()消息都会被打印到内核日志缓冲区，这是一个通过/dev/kmsg输出到用户空间的环 +形缓冲区。读取它的通常方法是使用 ``dmesg`` 。 + +printk()的用法通常是这样的:: + + printk(KERN_INFO "Message: %s\n", arg); + +其中 ``KERN_INFO`` 是日志级别（注意，它与格式字符串连在一起，日志级别不是一个单独的参数）。 +可用的日志级别是： + + ++----------------+--------+-----------------------------------------------+ +| 名称 | 字符串 | 别名函数 | ++================+========+===============================================+ +| KERN_EMERG | "0" | pr_emerg() | ++----------------+--------+-----------------------------------------------+ +| KERN_ALERT | "1" | pr_alert() | ++----------------+--------+-----------------------------------------------+ +| KERN_CRIT | "2" | pr_crit() | ++----------------+--------+-----------------------------------------------+ +| KERN_ERR | "3" | pr_err() | ++----------------+--------+-----------------------------------------------+ +| KERN_WARNING | "4" | pr_warn() | ++----------------+--------+-----------------------------------------------+ +| KERN_NOTICE | "5" | pr_notice() | ++----------------+--------+-----------------------------------------------+ +| KERN_INFO | "6" | pr_info() | ++----------------+--------+-----------------------------------------------+ +| KERN_DEBUG | "7" | pr_debug() and pr_devel() 若定义了DEBUG | ++----------------+--------+-----------------------------------------------+ +| KERN_DEFAULT | "" | | ++----------------+--------+-----------------------------------------------+ +| KERN_CONT | "c" | pr_cont() | ++----------------+--------+-----------------------------------------------+ + + +日志级别指定了一条消息的重要性。内核根据日志级别和当前 *console_loglevel* （一个内核变量）决 +定是否立即显示消息（将其打印到当前控制台）。如果消息的优先级比 *console_loglevel* 高（日志级 +别值较低），消息将被打印到控制台。 + +如果省略了日志级别，则以 ``KERN_DEFAULT`` 级别打印消息。 + +你可以用以下方法检查当前的 *console_loglevel* :: + + $ cat /proc/sys/kernel/printk + 4 4 1 7 + +结果显示了 *current*, *default*, *minimum* 和 *boot-time-default* 日志级别 + +要改变当前的 console_loglevel，只需在 ``/proc/sys/kernel/printk`` 中写入所需的 +级别。例如，要打印所有的消息到控制台上:: + + # echo 8 > /proc/sys/kernel/printk + +另一种方式，使用 ``dmesg``:: + + # dmesg -n 5 + +设置 console_loglevel 打印 KERN_WARNING (4) 或更严重的消息到控制台。更多消息参 +见 ``dmesg(1)`` 。 + +作为printk()的替代方案，你可以使用 ``pr_*()`` 别名来记录日志。这个系列的宏在宏名中 +嵌入了日志级别。例如:: + + pr_info("Info message no. %d\n", msg_num); + +打印 ``KERN_INFO`` 消息。 + +除了比等效的printk()调用更简洁之外，它们还可以通过pr_fmt()宏为格式字符串使用一个通用 +的定义。例如，在源文件的顶部（在任何 ``#include`` 指令之前）定义这样的内容。:: + + #define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +会在该文件中的每一条 pr_*() 消息前加上发起该消息的模块和函数名称。 + +为了调试，还有两个有条件编译的宏： +pr_debug()和pr_devel()，除非定义了 ``DEBUG`` (或者在pr_debug()的情况下定义了 +``CONFIG_DYNAMIC_DEBUG`` )，否则它们会被编译。 + + +函数接口 +======== + +该API在以下内核代码中: + +kernel/printk/printk.c + +include/linux/printk.h diff --git a/Documentation/translations/zh_CN/core-api/printk-formats.rst b/Documentation/translations/zh_CN/core-api/printk-formats.rst new file mode 100644 index 000000000000..a680c8f164c3 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/printk-formats.rst @@ -0,0 +1,595 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/printk-formats.rst +:Translator: Yanteng Si + +.. _cn_printk-formats.rst: + + +============================== +如何获得正确的printk格式占位符 +============================== + + + +:作者: Randy Dunlap +:作者: Andrew Murray + + +整数类型 +======== + +:: + + 若变量类型是Type，则使用printk格式占位符。 + ------------------------------------------- + char %d 或 %x + unsigned char %u 或 %x + short int %d 或 %x + unsigned short int %u 或 %x + int %d 或 %x + unsigned int %u 或 %x + long %ld 或 %lx + unsigned long %lu 或 %lx + long long %lld 或 %llx + unsigned long long %llu 或 %llx + size_t %zu 或 %zx + ssize_t %zd 或 %zx + s8 %d 或 %x + u8 %u 或 %x + s16 %d 或 %x + u16 %u 或 %x + s32 %d 或 %x + u32 %u 或 %x + s64 %lld 或 %llx + u64 %llu 或 %llx + + +如果 的大小依赖于配置选项 (例如 sector_t, blkcnt_t) 或其大小依赖于架构 +(例如 tcflag_t)，则使用其可能的最大类型的格式占位符并显式强制转换为它。 + +例如 + +:: + + printk("test: sector number/total blocks: %llu/%llu\n", + (unsigned long long)sector, (unsigned long long)blockcount); + +提醒：sizeof()返回类型为size_t。 + +内核的printf不支持%n。显而易见，浮点格式(%e, %f, %g, %a)也不被识别。使用任何不 +支持的占位符或长度限定符都会导致一个WARN并且终止vsnprintf()执行。 + +指针类型 +======== + +一个原始指针值可以用%p打印，它将在打印前对地址进行哈希处理。内核也支持扩展占位符来打印 +不同类型的指针。 + +一些扩展占位符会打印给定地址上的数据，而不是打印地址本身。在这种情况下，以下错误消息可能 +会被打印出来，而不是无法访问的消息:: + + (null) data on plain NULL address + (efault) data on invalid address + (einval) invalid data on a valid address + +普通指针 +---------- + +:: + + %p abcdef12 or 00000000abcdef12 + +没有指定扩展名的指针（即没有修饰符的%p）被哈希（hash），以防止内核内存布局消息的泄露。这 +样还有一个额外的好处，就是提供一个唯一的标识符。在64位机器上，前32位被清零。当没有足够的 +熵进行散列处理时，内核将打印(ptrval)代替 + +如果可能的话，使用专门的修饰符，如%pS或%pB（如下所述），以避免打印一个必须事后解释的非哈 +希地址。如果不可能，而且打印地址的目的是为调试提供更多的消息，使用%p，并在调试过程中 +用 ``no_hash_pointers`` 参数启动内核，这将打印所有未修改的%p地址。如果你 *真的* 想知 +道未修改的地址，请看下面的%px。 + +如果（也只有在）你将地址作为虚拟文件的内容打印出来，例如在procfs或sysfs中（使用 +seq_printf()，而不是printk()）由用户空间进程读取，使用下面描述的%pK修饰符，不 +要用%p或%px。 + + +错误指针 +-------- + +:: + + %pe -ENOSPC + +用于打印错误指针(即IS_ERR()为真的指针)的符号错误名。不知道符号名的错误值会以十进制打印， +而作为%pe参数传递的非ERR_PTR会被视为普通的%p。 + +符号/函数指针 +------------- + +:: + + %pS versatile_init+0x0/0x110 + %ps versatile_init + %pSR versatile_init+0x9/0x110 + (with __builtin_extract_return_addr() translation) + %pB prev_fn_of_versatile_init+0x88/0x88 + + +``S`` 和 ``s`` 占位符用于打印符号格式的指针。它们的结果是符号名称带有(S)或不带有(s)偏移 +量。如果禁用KALLSYMS，则打印符号地址。 + +``B`` 占位符的结果是带有偏移量的符号名，在打印堆栈回溯时应该使用。占位符将考虑编译器优化 +的影响，当使用尾部调用并使用noreturn GCC属性标记时，可能会发生这种优化。 + +如果指针在一个模块内，模块名称和可选的构建ID将被打印在符号名称之后，并在说明符的末尾添加 +一个额外的 ``b`` 。 + +:: + + %pS versatile_init+0x0/0x110 [module_name] + %pSb versatile_init+0x0/0x110 [module_name ed5019fdf5e53be37cb1ba7899292d7e143b259e] + %pSRb versatile_init+0x9/0x110 [module_name ed5019fdf5e53be37cb1ba7899292d7e143b259e] + (with __builtin_extract_return_addr() translation) + %pBb prev_fn_of_versatile_init+0x88/0x88 [module_name ed5019fdf5e53be37cb1ba7899292d7e143b259e] + +来自BPF / tracing追踪的探查指针 +---------------------------------- + +:: + + %pks kernel string + %pus user string + +``k`` 和 ``u`` 指定符用于打印来自内核内存(k)或用户内存(u)的先前探测的内存。后面的 ``s`` 指 +定符的结果是打印一个字符串。对于直接在常规的vsnprintf()中使用时，(k)和(u)注释被忽略，但是，当 +在BPF的bpf_trace_printk()之外使用时，它会读取它所指向的内存，不会出现错误。 + +内核指针 +-------- + +:: + + %pK 01234567 or 0123456789abcdef + +用于打印应该对非特权用户隐藏的内核指针。%pK的行为取决于kptr_restrict sysctl——详见 +Documentation/admin-guide/sysctl/kernel.rst。 + +未经修改的地址 +-------------- + +:: + + %px 01234567 or 0123456789abcdef + +对于打印指针，当你 *真的* 想打印地址时。在用%px打印指针之前，请考虑你是否泄露了内核内 +存布局的敏感消息。%px在功能上等同于%lx（或%lu）。%px是首选，因为它在grep查找时更唯一。 +如果将来我们需要修改内核处理打印指针的方式，我们将能更好地找到调用点。 + +在使用%px之前，请考虑使用%p并在调试过程中启用' ' no_hash_pointer ' '内核参数是否足 +够(参见上面的%p描述)。%px的一个有效场景可能是在panic发生之前立即打印消息，这样无论如何 +都可以防止任何敏感消息被利用，使用%px就不需要用no_hash_pointer来重现panic。 + +指针差异 +-------- + +:: + + %td 2560 + %tx a00 + +为了打印指针的差异，使用ptrdiff_t的%t修饰符。 + +例如:: + + printk("test: difference between pointers: %td\n", ptr2 - ptr1); + +结构体资源（Resources） +----------------------- + +:: + + %pr [mem 0x60000000-0x6fffffff flags 0x2200] or + [mem 0x0000000060000000-0x000000006fffffff flags 0x2200] + %pR [mem 0x60000000-0x6fffffff pref] or + [mem 0x0000000060000000-0x000000006fffffff pref] + +用于打印结构体资源。 ``R`` 和 ``r`` 占位符的结果是打印出的资源带有（R）或不带有（r）解码标志 +成员。 + +通过引用传递。 + +物理地址类型 phys_addr_t +------------------------ + +:: + + %pa[p] 0x01234567 or 0x0123456789abcdef + +用于打印phys_addr_t类型（以及它的衍生物，如resource_size_t），该类型可以根据构建选项而 +变化，无论CPU数据真实物理地址宽度如何。 + +通过引用传递。 + +DMA地址类型dma_addr_t +--------------------- + +:: + + %pad 0x01234567 or 0x0123456789abcdef + +用于打印dma_addr_t类型，该类型可以根据构建选项而变化，而不考虑CPU数据路径的宽度。 + +通过引用传递。 + +原始缓冲区为转义字符串 +---------------------- + +:: + + %*pE[achnops] + +用于将原始缓冲区打印成转义字符串。对于以下缓冲区:: + + 1b 62 20 5c 43 07 22 90 0d 5d + +几个例子展示了如何进行转换（不包括两端的引号）。:: + + %*pE "\eb \C\a"\220\r]" + %*pEhp "\x1bb \C\x07"\x90\x0d]" + %*pEa "\e\142\040\\\103\a\042\220\r\135" + +转换规则是根据可选的标志组合来应用的(详见:c:func:`string_escape_mem` 内核文档): + + - a - ESCAPE_ANY + - c - ESCAPE_SPECIAL + - h - ESCAPE_HEX + - n - ESCAPE_NULL + - o - ESCAPE_OCTAL + - p - ESCAPE_NP + - s - ESCAPE_SPACE + +默认情况下，使用 ESCAPE_ANY_NP。 + +ESCAPE_ANY_NP是许多情况下的明智选择，特别是对于打印SSID。 + +如果字段宽度被省略，那么将只转义1个字节。 + +原始缓冲区为十六进制字符串 +-------------------------- + +:: + + %*ph 00 01 02 ... 3f + %*phC 00:01:02: ... :3f + %*phD 00-01-02- ... -3f + %*phN 000102 ... 3f + +对于打印小的缓冲区（最长64个字节），可以用一定的分隔符作为一个 +十六进制字符串。对于较大的缓冲区，可以考虑使用 +:c:func:`print_hex_dump` 。 + +MAC/FDDI地址 +------------ + +:: + + %pM 00:01:02:03:04:05 + %pMR 05:04:03:02:01:00 + %pMF 00-01-02-03-04-05 + %pm 000102030405 + %pmR 050403020100 + +用于打印以十六进制表示的6字节MAC/FDDI地址。 ``M`` 和 ``m`` 占位符导致打印的 +地址有(M)或没有(m)字节分隔符。默认的字节分隔符是冒号（：）。 + +对于FDDI地址，可以在 ``M`` 占位符之后使用 ``F`` 说明，以使用破折号(——)分隔符 +代替默认的分隔符。 + +对于蓝牙地址， ``R`` 占位符应使用在 ``M`` 占位符之后，以使用反转的字节顺序，适 +合于以小尾端顺序的蓝牙地址的肉眼可见的解析。 + +通过引用传递。 + +IPv4地址 +-------- + +:: + + %pI4 1.2.3.4 + %pi4 001.002.003.004 + %p[Ii]4[hnbl] + +用于打印IPv4点分隔的十进制地址。 ``I4`` 和 ``i4`` 占位符的结果是打印的地址 +有(i4)或没有(I4)前导零。 + +附加的 ``h`` 、 ``n`` 、 ``b`` 和 ``l`` 占位符分别用于指定主机、网络、大 +尾端或小尾端地址。如果没有提供占位符，则使用默认的网络/大尾端顺序。 + +通过引用传递。 + +IPv6 地址 +--------- + +:: + + %pI6 0001:0002:0003:0004:0005:0006:0007:0008 + %pi6 00010002000300040005000600070008 + %pI6c 1:2:3:4:5:6:7:8 + +用于打印IPv6网络顺序的16位十六进制地址。 ``I6`` 和 ``i6`` 占位符的结果是 +打印的地址有(I6)或没有(i6)分号。始终使用前导零。 + +额外的 ``c`` 占位符可与 ``I`` 占位符一起使用，以打印压缩的IPv6地址，如 +https://tools.ietf.org/html/rfc5952 所述 + +通过引用传递。 + +IPv4/IPv6地址(generic, with port, flowinfo, scope) +-------------------------------------------------- + +:: + + %pIS 1.2.3.4 or 0001:0002:0003:0004:0005:0006:0007:0008 + %piS 001.002.003.004 or 00010002000300040005000600070008 + %pISc 1.2.3.4 or 1:2:3:4:5:6:7:8 + %pISpc 1.2.3.4:12345 or [1:2:3:4:5:6:7:8]:12345 + %p[Ii]S[pfschnbl] + +用于打印一个IP地址，不需要区分它的类型是AF_INET还是AF_INET6。一个指向有效结构 +体sockaddr的指针，通过 ``IS`` 或 ``IS`` 指定，可以传递给这个格式占位符。 + +附加的 ``p`` 、 ``f`` 和 ``s`` 占位符用于指定port(IPv4, IPv6)、 +flowinfo (IPv6)和sope(IPv6)。port有一个 ``:`` 前缀，flowinfo是 ``/`` 和 +范围是 ``%`` ，每个后面都跟着实际的值。 + +对于IPv6地址，如果指定了额外的指定符 ``c`` ，则使用 +https://tools.ietf.org/html/rfc5952 描述的压缩IPv6地址。 +如https://tools.ietf.org/html/draft-ietf-6man-text-addr-representation-07 +所建议的，IPv6地址由'['，']'包围，以防止出现额外的占位符 ``p`` ， ``f`` 或 ``s`` 。 + +对于IPv4地址，也可以使用额外的 ``h`` ， ``n`` ， ``b`` 和 ``l`` 说 +明符，但对于IPv6地址则忽略。 + +通过引用传递。 + +更多例子:: + + %pISfc 1.2.3.4 or [1:2:3:4:5:6:7:8]/123456789 + %pISsc 1.2.3.4 or [1:2:3:4:5:6:7:8]%1234567890 + %pISpfc 1.2.3.4:12345 or [1:2:3:4:5:6:7:8]:12345/123456789 + +UUID/GUID地址 +------------- + +:: + + %pUb 00010203-0405-0607-0809-0a0b0c0d0e0f + %pUB 00010203-0405-0607-0809-0A0B0C0D0E0F + %pUl 03020100-0504-0706-0809-0a0b0c0e0e0f + %pUL 03020100-0504-0706-0809-0A0B0C0E0E0F + +用于打印16字节的UUID/GUIDs地址。附加的 ``l`` , ``L`` , ``b`` 和 ``B`` 占位符用 +于指定小写(l)或大写(L)十六进制表示法中的小尾端顺序，以及小写(b)或大写(B)十六进制表 +示法中的大尾端顺序。 + +如果没有使用额外的占位符，则将打印带有小写十六进制表示法的默认大端顺序。 + +通过引用传递。 + +目录项（dentry）的名称 +---------------------- + +:: + + %pd{,2,3,4} + %pD{,2,3,4} + +用于打印dentry名称；如果我们用 :c:func:`d_move` 和它比较，名称可能是新旧混合的，但 +不会oops。 %pd dentry比较安全，其相当于我们以前用的%s dentry->d_name.name，%pd打 +印 ``n`` 最后的组件。 %pD对结构文件做同样的事情。 + + +通过引用传递。 + +块设备（block_device）名称 +-------------------------- + +:: + + %pg sda, sda1 or loop0p1 + +用于打印block_device指针的名称。 + +va_format结构体 +--------------- + +:: + + %pV + +用于打印结构体va_format。这些结构包含一个格式字符串 +和va_list如下 + +:: + + struct va_format { + const char *fmt; + va_list *va; + }; + +实现 "递归vsnprintf"。 + +如果没有一些机制来验证格式字符串和va_list参数的正确性，请不要使用这个功能。 + +通过引用传递。 + +设备树节点 +---------- + +:: + + %pOF[fnpPcCF] + + +用于打印设备树节点结构。默认行为相当于%pOFf。 + + - f - 设备节点全称 + - n - 设备节点名 + - p - 设备节点句柄 + - P - 设备节点路径规范(名称+@单位) + - F - 设备节点标志 + - c - 主要兼容字符串 + - C - 全兼容字符串 + +当使用多个参数时，分隔符是':'。 + +例如 + +:: + + %pOF /foo/bar@0 - Node full name + %pOFf /foo/bar@0 - Same as above + %pOFfp /foo/bar@0:10 - Node full name + phandle + %pOFfcF /foo/bar@0:foo,device:--P- - Node full name + + major compatible string + + node flags + D - dynamic + d - detached + P - Populated + B - Populated bus + +通过引用传递。 + +Fwnode handles +-------------- + +:: + + %pfw[fP] + +用于打印fwnode_handles的消息。默认情况下是打印完整的节点名称，包括路径。 +这些修饰符在功能上等同于上面的%pOF。 + + - f - 节点的全名，包括路径。 + - P - 节点名称，包括地址（如果有的话）。 + +例如 (ACPI) + +:: + + %pfwf \_SB.PCI0.CIO2.port@1.endpoint@0 - Full node name + %pfwP endpoint@0 - Node name + +例如 (OF) + +:: + + %pfwf /ocp@68000000/i2c@48072000/camera@10/port/endpoint - Full name + %pfwP endpoint - Node name + +时间和日期 +---------- + +:: + + %pt[RT] YYYY-mm-ddTHH:MM:SS + %pt[RT]s YYYY-mm-dd HH:MM:SS + %pt[RT]d YYYY-mm-dd + %pt[RT]t HH:MM:SS + %pt[RT][dt][r][s] + +用于打印日期和时间:: + + R struct rtc_time structure + T time64_t type + +以我们（人类）可读的格式。 + +默认情况下，年将以1900为单位递增，月将以1为单位递增。 使用%pt[RT]r (raw) +来抑制这种行为。 + +%pt[RT]s（空格）将覆盖ISO 8601的分隔符，在日期和时间之间使用''（空格）而 +不是'T'（大写T）。当日期或时间被省略时，它不会有任何影响。 + +通过引用传递。 + +clk结构体 +--------- + +:: + + %pC pll1 + %pCn pll1 + +用于打印clk结构。%pC 和 %pCn 打印时钟的名称（通用时钟框架）或唯一的32位 +ID（传统时钟框架）。 + +通过引用传递。 + +位图及其衍生物，如cpumask和nodemask +----------------------------------- + +:: + + %*pb 0779 + %*pbl 0,3-6,8-10 + +对于打印位图（bitmap）及其派生的cpumask和nodemask，%*pb输出以字段宽度为位数的位图， +%*pbl输出以字段宽度为位数的范围列表。 + +字段宽度用值传递，位图用引用传递。可以使用辅助宏cpumask_pr_args()和 +nodemask_pr_args()来方便打印cpumask和nodemask。 + +标志位字段，如页标志、gfp_flags +------------------------------- + +:: + + %pGp referenced|uptodate|lru|active|private|node=0|zone=2|lastcpupid=0x1fffff + %pGg GFP_USER|GFP_DMA32|GFP_NOWARN + %pGv read|exec|mayread|maywrite|mayexec|denywrite + +将flags位字段打印为构造值的符号常量集合。标志的类型由第三个字符给出。目前支持的 +是[p]age flags， [v]ma_flags(都期望 ``unsigned long *`` )和 +[g]fp_flags(期望 ``gfp_t *`` )。标志名称和打印顺序取决于特定的类型。 + +注意，这种格式不应该直接用于跟踪点的:c:func:`TP_printk()` 部分。相反，应使 +用 中的show_*_flags()函数。 + +通过引用传递。 + +网络设备特性 +------------ + +:: + + %pNF 0x000000000000c000 + +用于打印netdev_features_t。 + +通过引用传递。 + +V4L2和DRM FourCC代码(像素格式) +------------------------------ + +:: + + %p4cc + +打印V4L2或DRM使用的FourCC代码，包括格式端序及其十六进制的数值。 + +通过引用传递。 + +例如:: + + %p4cc BG12 little-endian (0x32314742) + %p4cc Y10 little-endian (0x20303159) + %p4cc NV12 big-endian (0xb231564e) + +谢谢 +==== + +如果您添加了其他%p扩展，请在可行的情况下，用一个或多个测试用例扩展。 + +谢谢你的合作和关注。 diff --git a/Documentation/translations/zh_CN/core-api/refcount-vs-atomic.rst b/Documentation/translations/zh_CN/core-api/refcount-vs-atomic.rst new file mode 100644 index 000000000000..ea834e38d2f6 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/refcount-vs-atomic.rst @@ -0,0 +1,154 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/refcount-vs-atomic.rst +:Translator: Yanteng Si + +.. _cn_refcount-vs-atomic: + + +======================================= +与atomic_t相比，refcount_t的API是这样的 +======================================= + +.. contents:: :local: + +简介 +==== + +refcount_t API的目标是为实现对象的引用计数器提供一个最小的API。虽然来自 +lib/refcount.c的独立于架构的通用实现在下面使用了原子操作，但一些 ``refcount_*()`` +和 ``atomic_*()`` 函数在内存顺序保证方面有很多不同。本文档概述了这些差异，并 +提供了相应的例子，以帮助开发者根据这些内存顺序保证的变化来验证他们的代码。 + +本文档中使用的术语尽量遵循tools/memory-model/Documentation/explanation.txt +中定义的正式LKMM。 + +memory-barriers.txt和atomic_t.txt提供了更多关于内存顺序的背景，包括通用的 +和针对原子操作的。 + +内存顺序的相关类型 +================== + +.. note:: 下面的部分只涵盖了本文使用的与原子操作和引用计数器有关的一些内存顺 + 序类型。如果想了解更广泛的情况，请查阅memory-barriers.txt文件。 + +在没有任何内存顺序保证的情况下（即完全无序），atomics和refcounters只提供原 +子性和程序顺序（program order, po）关系（在同一个CPU上）。它保证每个 +``atomic_* ()`` 和 ``refcount_*()`` 操作都是原子性的，指令在单个CPU上按程序 +顺序执行。这是用READ_ONCE()/WRITE_ONCE()和比较并交换原语实现的。 + +强（完全）内存顺序保证在同一CPU上的所有较早加载和存储的指令（所有程序顺序较早 +[po-earlier]指令）在执行任何程序顺序较后指令（po-later）之前完成。它还保证 +同一CPU上储存的程序优先较早的指令和来自其他CPU传播的指令必须在该CPU执行任何 +程序顺序较后指令之前传播到其他CPU（A-累积属性）。这是用smp_mb()实现的。 + +RELEASE内存顺序保证了在同一CPU上所有较早加载和存储的指令（所有程序顺序较早 +指令）在此操作前完成。它还保证同一CPU上储存的程序优先较早的指令和来自其他CPU +传播的指令必须在释放（release）操作之前传播到所有其他CPU（A-累积属性）。这是用 +smp_store_release()实现的。 + +ACQUIRE内存顺序保证了同一CPU上的所有后加载和存储的指令（所有程序顺序较后 +指令）在获取（acquire）操作之后完成。它还保证在获取操作执行后，同一CPU上 +储存的所有程序顺序较后指令必须传播到所有其他CPU。这是用 +smp_acquire__after_ctrl_dep()实现的。 + +对Refcounters的控制依赖（取决于成功）保证了如果一个对象的引用被成功获得（引用计数 +器的增量或增加行为发生了，函数返回true），那么进一步的存储是针对这个操作的命令。对存 +储的控制依赖没有使用任何明确的屏障来实现，而是依赖于CPU不对存储进行猜测。这只是 +一个单一的CPU关系，对其他CPU不提供任何保证。 + + +函数的比较 +========== + +情况1） - 非 “读/修改/写”（RMW）操作 +------------------------------------ + +函数变化: + + * atomic_set() --> refcount_set() + * atomic_read() --> refcount_read() + +内存顺序保证变化: + + * none (两者都是完全无序的) + + +情况2） - 基于增量的操作，不返回任何值 +-------------------------------------- + +函数变化: + + * atomic_inc() --> refcount_inc() + * atomic_add() --> refcount_add() + +内存顺序保证变化: + + * none (两者都是完全无序的) + +情况3） - 基于递减的RMW操作，没有返回值 +--------------------------------------- + +函数变化: + + * atomic_dec() --> refcount_dec() + +内存顺序保证变化: + + * 完全无序的 --> RELEASE顺序 + + +情况4） - 基于增量的RMW操作，返回一个值 +--------------------------------------- + +函数变化: + + * atomic_inc_not_zero() --> refcount_inc_not_zero() + * 无原子性对应函数 --> refcount_add_not_zero() + +内存顺序保证变化: + + * 完全有序的 --> 控制依赖于存储的成功 + +.. note:: 此处 **假设** 了，必要的顺序是作为获得对象指针的结果而提供的。 + + +情况 5） - 基于Dec/Sub递减的通用RMW操作，返回一个值 +--------------------------------------------------- + +函数变化: + + * atomic_dec_and_test() --> refcount_dec_and_test() + * atomic_sub_and_test() --> refcount_sub_and_test() + +内存顺序保证变化: + + * 完全有序的 --> RELEASE顺序 + 成功后ACQUIRE顺序 + + +情况6）其他基于递减的RMW操作，返回一个值 +---------------------------------------- + +函数变化: + + * 无原子性对应函数 --> refcount_dec_if_one() + * ``atomic_add_unless(&var, -1, 1)`` --> ``refcount_dec_not_one(&var)`` + +内存顺序保证变化: + + * 完全有序的 --> RELEASE顺序 + 控制依赖 + +.. note:: atomic_add_unless()只在执行成功时提供完整的顺序。 + + +情况7）--基于锁的RMW +-------------------- + +函数变化: + + * atomic_dec_and_lock() --> refcount_dec_and_lock() + * atomic_dec_and_mutex_lock() --> refcount_dec_and_mutex_lock() + +内存顺序保证变化: + + * 完全有序 --> RELEASE顺序 + 控制依赖 + 持有 diff --git a/Documentation/translations/zh_CN/core-api/symbol-namespaces.rst b/Documentation/translations/zh_CN/core-api/symbol-namespaces.rst new file mode 100644 index 000000000000..ce05c29c7697 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/symbol-namespaces.rst @@ -0,0 +1,142 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/symbol-namespaces.rst +:Translator: Yanteng Si + +.. _cn_symbol-namespaces.rst: + + +================================= +符号命名空间（Symbol Namespaces） +================================= + +本文档描述了如何使用符号命名空间来构造通过EXPORT_SYMBOL()系列宏导出的内核内符号的导出面。 + +.. 目录 + + === 1 简介 + === 2 如何定义符号命名空间 + --- 2.1 使用EXPORT_SYMBOL宏 + --- 2.2 使用DEFAULT_SYMBOL_NAMESPACE定义 + === 3 如何使用命名空间中导出的符号 + === 4 加载使用命名空间符号的模块 + === 5 自动创建MODULE_IMPORT_NS声明 + +1. 简介 +======= + +符号命名空间已经被引入，作为构造内核内API的导出面的一种手段。它允许子系统维护者将 +他们导出的符号划分进独立的命名空间。这对于文档的编写非常有用（想想SUBSYSTEM_DEBUG +命名空间），也可以限制一组符号在内核其他部分的使用。今后，使用导出到命名空间的符号 +的模块必须导入命名空间。否则，内核将根据其配置，拒绝加载该模块或警告说缺少 +导入。 + +2. 如何定义符号命名空间 +======================= + +符号可以用不同的方法导出到命名空间。所有这些都在改变 EXPORT_SYMBOL 和与之类似的那些宏 +被检测到的方式，以创建 ksymtab 条目。 + +2.1 使用EXPORT_SYMBOL宏 +======================= + +除了允许将内核符号导出到内核符号表的宏EXPORT_SYMBOL()和EXPORT_SYMBOL_GPL()之外， +这些宏的变体还可以将符号导出到某个命名空间：EXPORT_SYMBOL_NS() 和 EXPORT_SYMBOL_NS_GPL()。 +它们需要一个额外的参数：命名空间（the namespace）。请注意，由于宏扩展，该参数需 +要是一个预处理器符号。例如，要把符号 ``usb_stor_suspend`` 导出到命名空间 ``USB_STORAGE``， +请使用:: + + EXPORT_SYMBOL_NS(usb_stor_suspend, USB_STORAGE); + +相应的 ksymtab 条目结构体 ``kernel_symbol`` 将有相应的成员 ``命名空间`` 集。 +导出时未指明命名空间的符号将指向 ``NULL`` 。如果没有定义命名空间，则默认没有。 +``modpost`` 和kernel/module.c分别在构建时或模块加载时使用名称空间。 + +2.2 使用DEFAULT_SYMBOL_NAMESPACE定义 +==================================== + +为一个子系统的所有符号定义命名空间可能会非常冗长，并可能变得难以维护。因此，我 +们提供了一个默认定义（DEFAULT_SYMBOL_NAMESPACE），如果设置了这个定义， 它将成 +为所有没有指定命名空间的 EXPORT_SYMBOL() 和 EXPORT_SYMBOL_GPL() 宏扩展的默认 +定义。 + +有多种方法来指定这个定义，使用哪种方法取决于子系统和维护者的喜好。第一种方法是在 +子系统的 ``Makefile`` 中定义默认命名空间。例如，如果要将usb-common中定义的所有符号导 +出到USB_COMMON命名空间，可以在drivers/usb/common/Makefile中添加这样一行:: + + ccflags-y += -DDEFAULT_SYMBOL_NAMESPACE=USB_COMMON + +这将影响所有 EXPORT_SYMBOL() 和 EXPORT_SYMBOL_GPL() 语句。当这个定义存在时， +用EXPORT_SYMBOL_NS()导出的符号仍然会被导出到作为命名空间参数传递的命名空间中， +因为这个参数优先于默认的符号命名空间。 + +定义默认命名空间的第二个选项是直接在编译单元中作为预处理声明。上面的例子就会变 +成:: + + #undef DEFAULT_SYMBOL_NAMESPACE + #define DEFAULT_SYMBOL_NAMESPACE USB_COMMON + +应置于相关编译单元中任何 EXPORT_SYMBOL 宏之前 + +3. 如何使用命名空间中导出的符号 +=============================== + +为了使用被导出到命名空间的符号，内核模块需要明确地导入这些命名空间。 +否则内核可能会拒绝加载该模块。模块代码需要使用宏MODULE_IMPORT_NS来 +表示它所使用的命名空间的符号。例如，一个使用usb_stor_suspend符号的 +模块，需要使用如下语句导入命名空间USB_STORAGE:: + + MODULE_IMPORT_NS(USB_STORAGE); + +这将在模块中为每个导入的命名空间创建一个 ``modinfo`` 标签。这也顺带 +使得可以用modinfo检查模块已导入的命名空间:: + + $ modinfo drivers/usb/storage/ums-karma.ko + [...] + import_ns: USB_STORAGE + [...] + + +建议将 MODULE_IMPORT_NS() 语句添加到靠近其他模块元数据定义的地方， +如 MODULE_AUTHOR() 或 MODULE_LICENSE() 。关于自动创建缺失的导入 +语句的方法，请参考第5节。 + +4. 加载使用命名空间符号的模块 +============================= + +在模块加载时（比如 ``insmod`` ），内核将检查每个从模块中引用的符号是否可 +用，以及它可能被导出到的名字空间是否被模块导入。内核的默认行为是拒绝 +加载那些没有指明足以导入的模块。此错误会被记录下来，并且加载将以 +EINVAL方式失败。要允许加载不满足这个前提条件的模块，可以使用此配置选项： +设置 MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS=y 将使加载不受影响，但会 +发出警告。 + +5. 自动创建MODULE_IMPORT_NS声明 +=============================== + +缺少命名空间的导入可以在构建时很容易被检测到。事实上，如果一个模块 +使用了一个命名空间的符号而没有导入它，modpost会发出警告。 +MODULE_IMPORT_NS()语句通常会被添加到一个明确的位置（和其他模块元 +数据一起）。为了使模块作者（和子系统维护者）的生活更加轻松，我们提 +供了一个脚本和make目标来修复丢失的导入。修复丢失的导入可以用:: + + $ make nsdeps + +对模块作者来说，以下情况可能很典型:: + + - 编写依赖未导入命名空间的符号的代码 + - ``make`` + - 注意 ``modpost`` 的警告，提醒你有一个丢失的导入。 + - 运行 ``make nsdeps``将导入添加到正确的代码位置。 + +对于引入命名空间的子系统维护者来说，其步骤非常相似。同样，make nsdeps最终将 +为树内模块添加缺失的命名空间导入:: + + - 向命名空间转移或添加符号（例如，使用EXPORT_SYMBOL_NS()）。 + - `make e`（最好是用allmodconfig来覆盖所有的内核模块）。 + - 注意 ``modpost`` 的警告，提醒你有一个丢失的导入。 + - 运行 ``maknsdeps``将导入添加到正确的代码位置。 + +你也可以为外部模块的构建运行nsdeps。典型的用法是:: + + $ make -C M=$PWD nsdeps diff --git a/Documentation/translations/zh_CN/core-api/workqueue.rst b/Documentation/translations/zh_CN/core-api/workqueue.rst new file mode 100644 index 000000000000..0b8f730db6c0 --- /dev/null +++ b/Documentation/translations/zh_CN/core-api/workqueue.rst @@ -0,0 +1,337 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/core-api/workqueue.rst +:Translator: Yanteng Si + +.. _cn_workqueue.rst: + + +========================= +并发管理的工作队列 (cmwq) +========================= + +:日期: September, 2010 +:作者: Tejun Heo +:作者: Florian Mickler + + +简介 +==== + +在很多情况下，需要一个异步进程的执行环境，工作队列（wq）API是这种情况下 +最常用的机制。 + +当需要这样一个异步执行上下文时，一个描述将要执行的函数的工作项（work， +即一个待执行的任务）被放在队列中。一个独立的线程作为异步执行环境。该队 +列被称为workqueue，线程被称为工作者（worker，即执行这一队列的线程）。 + +当工作队列上有工作项时，工作者会一个接一个地执行与工作项相关的函数。当 +工作队列中没有任何工作项时，工作者就会变得空闲。当一个新的工作项被排入 +队列时，工作者又开始执行。 + + +为什么要cmwq? +============= + +在最初的wq实现中，多线程（MT）wq在每个CPU上有一个工作者线程，而单线程 +（ST）wq在全系统有一个工作者线程。一个MT wq需要保持与CPU数量相同的工 +作者数量。这些年来，内核增加了很多MT wq的用户，随着CPU核心数量的不断 +增加，一些系统刚启动就达到了默认的32k PID的饱和空间。 + +尽管MT wq浪费了大量的资源，但所提供的并发性水平却不能令人满意。这个限 +制在ST和MT wq中都有，只是在MT中没有那么严重。每个wq都保持着自己独立的 +工作者池。一个MT wq只能为每个CPU提供一个执行环境，而一个ST wq则为整个 +系统提供一个。工作项必须竞争这些非常有限的执行上下文，从而导致各种问题， +包括在单一执行上下文周围容易发生死锁。 + +(MT wq)所提供的并发性水平和资源使用之间的矛盾也迫使其用户做出不必要的权衡，比 +如libata选择使用ST wq来轮询PIO，并接受一个不必要的限制，即没有两个轮 +询PIO可以同时进行。由于MT wq并没有提供更好的并发性，需要更高层次的并 +发性的用户，如async或fscache，不得不实现他们自己的线程池。 + +并发管理工作队列（cmwq）是对wq的重新实现，重点是以下目标。 + +* 保持与原始工作队列API的兼容性。 + +* 使用由所有wq共享的每CPU统一的工作者池，在不浪费大量资源的情况下按 +* 需提供灵活的并发水平。 + +* 自动调节工作者池和并发水平，使API用户不需要担心这些细节。 + + +设计 +==== + +为了简化函数的异步执行，引入了一个新的抽象概念，即工作项。 + +一个工作项是一个简单的结构，它持有一个指向将被异步执行的函数的指针。 +每当一个驱动程序或子系统希望一个函数被异步执行时，它必须建立一个指 +向该函数的工作项，并在工作队列中排队等待该工作项。（就是挂到workqueue +队列里面去） + +特定目的线程，称为工作线程（工作者），一个接一个地执行队列中的功能。 +如果没有工作项排队，工作者线程就会闲置。这些工作者线程被管理在所谓 +的工作者池中。 + +cmwq设计区分了面向用户的工作队列，子系统和驱动程序在上面排队工作， +以及管理工作者池和处理排队工作项的后端机制。 + +每个可能的CPU都有两个工作者池，一个用于正常的工作项，另一个用于高 +优先级的工作项，还有一些额外的工作者池，用于服务未绑定工作队列的工 +作项目——这些后备池的数量是动态的。 + +当他们认为合适的时候，子系统和驱动程序可以通过特殊的 +``workqueue API`` 函数创建和排队工作项。他们可以通过在工作队列上 +设置标志来影响工作项执行方式的某些方面，他们把工作项放在那里。这些 +标志包括诸如CPU定位、并发限制、优先级等等。要获得详细的概述，请参 +考下面的 ``alloc_workqueue()`` 的 API 描述。 + +当一个工作项被排入一个工作队列时，目标工作池将根据队列参数和工作队 +列属性确定，并被附加到工作池的共享工作列表上。例如，除非特别重写， +否则一个绑定的工作队列的工作项将被排在与发起线程运行的CPU相关的普 +通或高级工作工作者池的工作项列表中。 + +对于任何工作者池的实施，管理并发水平（有多少执行上下文处于活动状 +态）是一个重要问题。最低水平是为了节省资源，而饱和水平是指系统被 +充分使用。 + +每个与实际CPU绑定的worker-pool通过钩住调度器来实现并发管理。每当 +一个活动的工作者被唤醒或睡眠时，工作者池就会得到通知，并跟踪当前可 +运行的工作者的数量。一般来说，工作项不会占用CPU并消耗很多周期。这 +意味着保持足够的并发性以防止工作处理停滞应该是最优的。只要CPU上有 +一个或多个可运行的工作者，工作者池就不会开始执行新的工作，但是，当 +最后一个运行的工作者进入睡眠状态时，它会立即安排一个新的工作者，这 +样CPU就不会在有待处理的工作项目时闲置。这允许在不损失执行带宽的情 +况下使用最少的工作者。 + +除了kthreads的内存空间外，保留空闲的工作者并没有其他成本，所以cmwq +在杀死它们之前会保留一段时间的空闲。 + +对于非绑定的工作队列，后备池的数量是动态的。可以使用 +``apply_workqueue_attrs()`` 为非绑定工作队列分配自定义属性， +workqueue将自动创建与属性相匹配的后备工作者池。调节并发水平的责任在 +用户身上。也有一个标志可以将绑定的wq标记为忽略并发管理。 +详情请参考API部分。 + +前进进度的保证依赖于当需要更多的执行上下文时可以创建工作者，这也是 +通过使用救援工作者来保证的。所有可能在处理内存回收的代码路径上使用 +的工作项都需要在wq上排队，wq上保留了一个救援工作者，以便在内存有压 +力的情况下下执行。否则，工作者池就有可能出现死锁，等待执行上下文释 +放出来。 + + +应用程序编程接口 (API) +====================== + +``alloc_workqueue()`` 分配了一个wq。原来的 ``create_*workqueue()`` +函数已被废弃，并计划删除。 ``alloc_workqueue()`` 需要三个 +参数 - ``@name`` , ``@flags`` 和 ``@max_active`` 。 +``@name`` 是wq的名称，如果有的话，也用作救援线程的名称。 + +一个wq不再管理执行资源，而是作为前进进度保证、刷新(flush)和 +工作项属性的域。 ``@flags`` 和 ``@max_active`` 控制着工作 +项如何被分配执行资源、安排和执行。 + + +``flags`` +--------- + +``WQ_UNBOUND`` + 排队到非绑定wq的工作项由特殊的工作者池提供服务，这些工作者不 + 绑定在任何特定的CPU上。这使得wq表现得像一个简单的执行环境提 + 供者，没有并发管理。非绑定工作者池试图尽快开始执行工作项。非 + 绑定的wq牺牲了局部性，但在以下情况下是有用的。 + + * 预计并发水平要求会有很大的波动，使用绑定的wq最终可能会在不 + 同的CPU上产生大量大部分未使用的工作者，因为发起线程在不同 + 的CPU上跳转。 + + * 长期运行的CPU密集型工作负载，可以由系统调度器更好地管理。 + +``WQ_FREEZABLE`` + 一个可冻结的wq参与了系统暂停操作的冻结阶段。wq上的工作项被 + 排空，在解冻之前没有新的工作项开始执行。 + +``WQ_MEM_RECLAIM`` + 所有可能在内存回收路径中使用的wq都必须设置这个标志。无论内 + 存压力如何，wq都能保证至少有一个执行上下文。 + +``WQ_HIGHPRI`` + 高优先级wq的工作项目被排到目标cpu的高优先级工作者池中。高 + 优先级的工作者池由具有较高级别的工作者线程提供服务。 + + 请注意，普通工作者池和高优先级工作者池之间并不相互影响。他 + 们各自维护其独立的工作者池，并在其工作者之间实现并发管理。 + +``WQ_CPU_INTENSIVE`` + CPU密集型wq的工作项对并发水平没有贡献。换句话说，可运行的 + CPU密集型工作项不会阻止同一工作者池中的其他工作项开始执行。 + 这对于那些预计会占用CPU周期的绑定工作项很有用，这样它们的 + 执行就会受到系统调度器的监管。 + + 尽管CPU密集型工作项不会对并发水平做出贡献，但它们的执行开 + 始仍然受到并发管理的管制，可运行的非CPU密集型工作项会延迟 + CPU密集型工作项的执行。 + + 这个标志对于未绑定的wq来说是没有意义的。 + +请注意，标志 ``WQ_NON_REENTRANT`` 不再存在，因为现在所有的工作 +队列都是不可逆的——任何工作项都保证在任何时间内最多被整个系统的一 +个工作者执行。 + + +``max_active`` +-------------- + +``@max_active`` 决定了每个CPU可以分配给wq的工作项的最大执行上 +下文数量。例如，如果 ``@max_active为16`` ，每个CPU最多可以同 +时执行16个wq的工作项。 + +目前，对于一个绑定的wq， ``@max_active`` 的最大限制是512，当指 +定为0时使用的默认值是256。对于非绑定的wq，其限制是512和 +4 * ``num_possible_cpus()`` 中的较高值。这些值被选得足够高，所 +以它们不是限制性因素，同时会在失控情况下提供保护。 + +一个wq的活动工作项的数量通常由wq的用户来调节，更具体地说，是由用 +户在同一时间可以排列多少个工作项来调节。除非有特定的需求来控制活动 +工作项的数量，否则建议指定 为"0"。 + +一些用户依赖于ST wq的严格执行顺序。 ``@max_active`` 为1和 ``WQ_UNBOUND`` +的组合用来实现这种行为。这种wq上的工作项目总是被排到未绑定的工作池 +中，并且在任何时候都只有一个工作项目处于活动状态，从而实现与ST wq相 +同的排序属性。 + +在目前的实现中，上述配置只保证了特定NUMA节点内的ST行为。相反， +``alloc_ordered_queue()`` 应该被用来实现全系统的ST行为。 + + +执行场景示例 +============ + +下面的示例执行场景试图说明cmwq在不同配置下的行为。 + + 工作项w0、w1、w2被排到同一个CPU上的一个绑定的wq q0上。w0 + 消耗CPU 5ms，然后睡眠10ms，然后在完成之前再次消耗CPU 5ms。 + +忽略所有其他的任务、工作和处理开销，并假设简单的FIFO调度， +下面是一个高度简化的原始wq的可能事件序列的版本。:: + + TIME IN MSECS EVENT + 0 w0 starts and burns CPU + 5 w0 sleeps + 15 w0 wakes up and burns CPU + 20 w0 finishes + 20 w1 starts and burns CPU + 25 w1 sleeps + 35 w1 wakes up and finishes + 35 w2 starts and burns CPU + 40 w2 sleeps + 50 w2 wakes up and finishes + +And with cmwq with ``@max_active`` >= 3, :: + + TIME IN MSECS EVENT + 0 w0 starts and burns CPU + 5 w0 sleeps + 5 w1 starts and burns CPU + 10 w1 sleeps + 10 w2 starts and burns CPU + 15 w2 sleeps + 15 w0 wakes up and burns CPU + 20 w0 finishes + 20 w1 wakes up and finishes + 25 w2 wakes up and finishes + +如果 ``@max_active`` == 2, :: + + TIME IN MSECS EVENT + 0 w0 starts and burns CPU + 5 w0 sleeps + 5 w1 starts and burns CPU + 10 w1 sleeps + 15 w0 wakes up and burns CPU + 20 w0 finishes + 20 w1 wakes up and finishes + 20 w2 starts and burns CPU + 25 w2 sleeps + 35 w2 wakes up and finishes + +现在，我们假设w1和w2被排到了不同的wq q1上，这个wq q1 +有 ``WQ_CPU_INTENSIVE`` 设置:: + + TIME IN MSECS EVENT + 0 w0 starts and burns CPU + 5 w0 sleeps + 5 w1 and w2 start and burn CPU + 10 w1 sleeps + 15 w2 sleeps + 15 w0 wakes up and burns CPU + 20 w0 finishes + 20 w1 wakes up and finishes + 25 w2 wakes up and finishes + + +指南 +==== + +* 如果一个wq可能处理在内存回收期间使用的工作项目，请不 + 要忘记使用 ``WQ_MEM_RECLAIM`` 。每个设置了 + ``WQ_MEM_RECLAIM`` 的wq都有一个为其保留的执行环境。 + 如果在内存回收过程中使用的多个工作项之间存在依赖关系， + 它们应该被排在不同的wq中，每个wq都有 ``WQ_MEM_RECLAIM`` 。 + +* 除非需要严格排序，否则没有必要使用ST wq。 + +* 除非有特殊需要，建议使用0作为@max_active。在大多数使用情 + 况下，并发水平通常保持在默认限制之下。 + +* 一个wq作为前进进度保证（WQ_MEM_RECLAIM，冲洗（flush）和工 + 作项属性的域。不涉及内存回收的工作项，不需要作为工作项组的一 + 部分被刷新，也不需要任何特殊属性，可以使用系统中的一个wq。使 + 用专用wq和系统wq在执行特性上没有区别。 + +* 除非工作项预计会消耗大量的CPU周期，否则使用绑定的wq通常是有 + 益的，因为wq操作和工作项执行中的定位水平提高了。 + + +调试 +==== + +因为工作函数是由通用的工作者线程执行的，所以需要一些手段来揭示一些行为不端的工作队列用户。 + +工作者线程在进程列表中显示为: :: + + root 5671 0.0 0.0 0 0 ? S 12:07 0:00 [kworker/0:1] + root 5672 0.0 0.0 0 0 ? S 12:07 0:00 [kworker/1:2] + root 5673 0.0 0.0 0 0 ? S 12:12 0:00 [kworker/0:0] + root 5674 0.0 0.0 0 0 ? S 12:13 0:00 [kworker/1:0] + +如果kworkers失控了（使用了太多的cpu），有两类可能的问题: + + 1. 正在迅速调度的事情 + 2. 一个消耗大量cpu周期的工作项。 + +第一个可以用追踪的方式进行跟踪: :: + + $ echo workqueue:workqueue_queue_work > /sys/kernel/debug/tracing/set_event + $ cat /sys/kernel/debug/tracing/trace_pipe > out.txt + (wait a few secs) + +如果有什么东西在工作队列上忙着做循环，它就会主导输出，可以用工作项函数确定违规者。 + +对于第二类问题，应该可以只检查违规工作者线程的堆栈跟踪。 :: + + $ cat /proc/THE_OFFENDING_KWORKER/stack + +工作项函数在堆栈追踪中应该是微不足道的。 + + +内核内联文档参考 +================ + +该API在以下内核代码中: + +include/linux/workqueue.h + +kernel/workqueue.c diff --git a/Documentation/translations/zh_CN/dev-tools/index.rst b/Documentation/translations/zh_CN/dev-tools/index.rst index fd73c479917b..e6c99f2f543f 100644 --- a/Documentation/translations/zh_CN/dev-tools/index.rst +++ b/Documentation/translations/zh_CN/dev-tools/index.rst @@ -19,13 +19,13 @@ :maxdepth: 2 gcov + kasan Todolist: - coccinelle - sparse - kcov - - kasan - ubsan - kmemleak - kcsan diff --git a/Documentation/translations/zh_CN/dev-tools/kasan.rst b/Documentation/translations/zh_CN/dev-tools/kasan.rst new file mode 100644 index 000000000000..23db9d419047 --- /dev/null +++ b/Documentation/translations/zh_CN/dev-tools/kasan.rst @@ -0,0 +1,417 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/dev-tools/kasan.rst +:Translator: 万家兵 Wan Jiabing + +内核地址消毒剂(KASAN) +===================== + +概述 +---- + +KernelAddressSANitizer(KASAN)是一种动态内存安全错误检测工具，主要功能是 +检查内存越界访问和使用已释放内存的问题。KASAN有三种模式: + +1. 通用KASAN（与用户空间的ASan类似） +2. 基于软件标签的KASAN（与用户空间的HWASan类似） +3. 基于硬件标签的KASAN（基于硬件内存标签） + +由于通用KASAN的内存开销较大，通用KASAN主要用于调试。基于软件标签的KASAN +可用于dogfood测试，因为它具有较低的内存开销，并允许将其用于实际工作量。 +基于硬件标签的KASAN具有较低的内存和性能开销，因此可用于生产。同时可用于 +检测现场内存问题或作为安全缓解措施。 + +软件KASAN模式（#1和#2）使用编译时工具在每次内存访问之前插入有效性检查， +因此需要一个支持它的编译器版本。 + +通用KASAN在GCC和Clang受支持。GCC需要8.3.0或更高版本。任何受支持的Clang +版本都是兼容的，但从Clang 11才开始支持检测全局变量的越界访问。 + +基于软件标签的KASAN模式仅在Clang中受支持。 + +硬件KASAN模式（#3）依赖硬件来执行检查，但仍需要支持内存标签指令的编译器 +版本。GCC 10+和Clang 11+支持此模式。 + +两种软件KASAN模式都适用于SLUB和SLAB内存分配器，而基于硬件标签的KASAN目前 +仅支持SLUB。 + +目前x86_64、arm、arm64、xtensa、s390、riscv架构支持通用KASAN模式，仅 +arm64架构支持基于标签的KASAN模式。 + +用法 +---- + +要启用KASAN，请使用以下命令配置内核:: + + CONFIG_KASAN=y + +同时在 ``CONFIG_KASAN_GENERIC`` (启用通用KASAN模式)， ``CONFIG_KASAN_SW_TAGS`` +(启用基于硬件标签的KASAN模式)，和 ``CONFIG_KASAN_HW_TAGS`` (启用基于硬件标签 +的KASAN模式)之间进行选择。 + +对于软件模式，还可以在 ``CONFIG_KASAN_OUTLINE`` 和 ``CONFIG_KASAN_INLINE`` +之间进行选择。outline和inline是编译器插桩类型。前者产生较小的二进制文件， +而后者快1.1-2倍。 + +要将受影响的slab对象的alloc和free堆栈跟踪包含到报告中，请启用 +``CONFIG_STACKTRACE`` 。要包括受影响物理页面的分配和释放堆栈跟踪的话， +请启用 ``CONFIG_PAGE_OWNER`` 并使用 ``page_owner=on`` 进行引导。 + +错误报告 +~~~~~~~~ + +典型的KASAN报告如下所示:: + + ================================================================== + BUG: KASAN: slab-out-of-bounds in kmalloc_oob_right+0xa8/0xbc [test_kasan] + Write of size 1 at addr ffff8801f44ec37b by task insmod/2760 + + CPU: 1 PID: 2760 Comm: insmod Not tainted 4.19.0-rc3+ #698 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.10.2-1 04/01/2014 + Call Trace: + dump_stack+0x94/0xd8 + print_address_description+0x73/0x280 + kasan_report+0x144/0x187 + __asan_report_store1_noabort+0x17/0x20 + kmalloc_oob_right+0xa8/0xbc [test_kasan] + kmalloc_tests_init+0x16/0x700 [test_kasan] + do_one_initcall+0xa5/0x3ae + do_init_module+0x1b6/0x547 + load_module+0x75df/0x8070 + __do_sys_init_module+0x1c6/0x200 + __x64_sys_init_module+0x6e/0xb0 + do_syscall_64+0x9f/0x2c0 + entry_SYSCALL_64_after_hwframe+0x44/0xa9 + RIP: 0033:0x7f96443109da + RSP: 002b:00007ffcf0b51b08 EFLAGS: 00000202 ORIG_RAX: 00000000000000af + RAX: ffffffffffffffda RBX: 000055dc3ee521a0 RCX: 00007f96443109da + RDX: 00007f96445cff88 RSI: 0000000000057a50 RDI: 00007f9644992000 + RBP: 000055dc3ee510b0 R08: 0000000000000003 R09: 0000000000000000 + R10: 00007f964430cd0a R11: 0000000000000202 R12: 00007f96445cff88 + R13: 000055dc3ee51090 R14: 0000000000000000 R15: 0000000000000000 + + Allocated by task 2760: + save_stack+0x43/0xd0 + kasan_kmalloc+0xa7/0xd0 + kmem_cache_alloc_trace+0xe1/0x1b0 + kmalloc_oob_right+0x56/0xbc [test_kasan] + kmalloc_tests_init+0x16/0x700 [test_kasan] + do_one_initcall+0xa5/0x3ae + do_init_module+0x1b6/0x547 + load_module+0x75df/0x8070 + __do_sys_init_module+0x1c6/0x200 + __x64_sys_init_module+0x6e/0xb0 + do_syscall_64+0x9f/0x2c0 + entry_SYSCALL_64_after_hwframe+0x44/0xa9 + + Freed by task 815: + save_stack+0x43/0xd0 + __kasan_slab_free+0x135/0x190 + kasan_slab_free+0xe/0x10 + kfree+0x93/0x1a0 + umh_complete+0x6a/0xa0 + call_usermodehelper_exec_async+0x4c3/0x640 + ret_from_fork+0x35/0x40 + + The buggy address belongs to the object at ffff8801f44ec300 + which belongs to the cache kmalloc-128 of size 128 + The buggy address is located 123 bytes inside of + 128-byte region [ffff8801f44ec300, ffff8801f44ec380) + The buggy address belongs to the page: + page:ffffea0007d13b00 count:1 mapcount:0 mapping:ffff8801f7001640 index:0x0 + flags: 0x200000000000100(slab) + raw: 0200000000000100 ffffea0007d11dc0 0000001a0000001a ffff8801f7001640 + raw: 0000000000000000 0000000080150015 00000001ffffffff 0000000000000000 + page dumped because: kasan: bad access detected + + Memory state around the buggy address: + ffff8801f44ec200: fc fc fc fc fc fc fc fc fb fb fb fb fb fb fb fb + ffff8801f44ec280: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc + >ffff8801f44ec300: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 03 + ^ + ffff8801f44ec380: fc fc fc fc fc fc fc fc fb fb fb fb fb fb fb fb + ffff8801f44ec400: fb fb fb fb fb fb fb fb fc fc fc fc fc fc fc fc + ================================================================== + +报告标题总结了发生的错误类型以及导致该错误的访问类型。紧随其后的是错误访问的 +堆栈跟踪、所访问内存分配位置的堆栈跟踪（对于访问了slab对象的情况）以及对象 +被释放的位置的堆栈跟踪（对于访问已释放内存的问题报告）。接下来是对访问的 +slab对象的描述以及关于访问的内存页的信息。 + +最后，报告展示了访问地址周围的内存状态。在内部，KASAN单独跟踪每个内存颗粒的 +内存状态，根据KASAN模式分为8或16个对齐字节。报告的内存状态部分中的每个数字 +都显示了围绕访问地址的其中一个内存颗粒的状态。 + +对于通用KASAN，每个内存颗粒的大小为8个字节。每个颗粒的状态被编码在一个影子字节 +中。这8个字节可以是可访问的，部分访问的，已释放的或成为Redzone的一部分。KASAN +对每个影子字节使用以下编码:00表示对应内存区域的所有8个字节都可以访问；数字N +(1 <= N <= 7)表示前N个字节可访问，其他(8 - N)个字节不可访问；任何负值都表示 +无法访问整个8字节。KASAN使用不同的负值来区分不同类型的不可访问内存，如redzones +或已释放的内存（参见 mm/kasan/kasan.h）。 + +在上面的报告中，箭头指向影子字节 ``03`` ，表示访问的地址是部分可访问的。 + +对于基于标签的KASAN模式，报告最后的部分显示了访问地址周围的内存标签 +(参考 `实施细则`_ 章节)。 + +请注意，KASAN错误标题（如 ``slab-out-of-bounds`` 或 ``use-after-free`` ） +是尽量接近的:KASAN根据其拥有的有限信息打印出最可能的错误类型。错误的实际类型 +可能会有所不同。 + +通用KASAN还报告两个辅助调用堆栈跟踪。这些堆栈跟踪指向代码中与对象交互但不直接 +出现在错误访问堆栈跟踪中的位置。目前，这包括 call_rcu() 和排队的工作队列。 + +启动参数 +~~~~~~~~ + +KASAN受通用 ``panic_on_warn`` 命令行参数的影响。启用该功能后，KASAN在打印错误 +报告后会引起内核恐慌。 + +默认情况下，KASAN只为第一次无效内存访问打印错误报告。使用 ``kasan_multi_shot`` ， +KASAN会针对每个无效访问打印报告。这有效地禁用了KASAN报告的 ``panic_on_warn`` 。 + +基于硬件标签的KASAN模式（请参阅下面有关各种模式的部分）旨在在生产中用作安全缓解 +措施。因此，它支持允许禁用KASAN或控制其功能的引导参数。 + +- ``kasan=off`` 或 ``=on`` 控制KASAN是否启用 (默认: ``on`` )。 + +- ``kasan.mode=sync`` 或 ``=async`` 控制KASAN是否配置为同步或异步执行模式(默认: + ``sync`` )。同步模式：当标签检查错误发生时，立即检测到错误访问。异步模式： + 延迟错误访问检测。当标签检查错误发生时，信息存储在硬件中（在arm64的 + TFSR_EL1寄存器中）。内核会定期检查硬件，并且仅在这些检查期间报告标签错误。 + +- ``kasan.stacktrace=off`` 或 ``=on`` 禁用或启用alloc和free堆栈跟踪收集 + (默认: ``on`` )。 + +- ``kasan.fault=report`` 或 ``=panic`` 控制是只打印KASAN报告还是同时使内核恐慌 + (默认: ``report`` )。即使启用了 ``kasan_multi_shot`` ，也会发生内核恐慌。 + +实施细则 +-------- + +通用KASAN +~~~~~~~~~ + +软件KASAN模式使用影子内存来记录每个内存字节是否可以安全访问，并使用编译时工具 +在每次内存访问之前插入影子内存检查。 + +通用KASAN将1/8的内核内存专用于其影子内存（16TB以覆盖x86_64上的128TB），并使用 +具有比例和偏移量的直接映射将内存地址转换为其相应的影子地址。 + +这是将地址转换为其相应影子地址的函数:: + + static inline void *kasan_mem_to_shadow(const void *addr) + { + return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT) + + KASAN_SHADOW_OFFSET; + } + +在这里 ``KASAN_SHADOW_SCALE_SHIFT = 3`` 。 + +编译时工具用于插入内存访问检查。编译器在每次访问大小为1、2、4、8或16的内存之前 +插入函数调用( ``__asan_load*(addr)`` , ``__asan_store*(addr)``)。这些函数通过 +检查相应的影子内存来检查内存访问是否有效。 + +使用inline插桩，编译器不进行函数调用，而是直接插入代码来检查影子内存。此选项 +显著地增大了内核体积，但与outline插桩内核相比，它提供了x1.1-x2的性能提升。 + +通用KASAN是唯一一种通过隔离延迟重新使用已释放对象的模式 +（参见 mm/kasan/quarantine.c 以了解实现）。 + +基于软件标签的KASAN模式 +~~~~~~~~~~~~~~~~~~~~~~~ + +基于软件标签的KASAN使用软件内存标签方法来检查访问有效性。目前仅针对arm64架构实现。 + +基于软件标签的KASAN使用arm64 CPU的顶部字节忽略(TBI)特性在内核指针的顶部字节中 +存储一个指针标签。它使用影子内存来存储与每个16字节内存单元相关的内存标签(因此， +它将内核内存的1/16专用于影子内存)。 + +在每次内存分配时，基于软件标签的KASAN都会生成一个随机标签，用这个标签标记分配 +的内存，并将相同的标签嵌入到返回的指针中。 + +基于软件标签的KASAN使用编译时工具在每次内存访问之前插入检查。这些检查确保正在 +访问的内存的标签等于用于访问该内存的指针的标签。如果标签不匹配，基于软件标签 +的KASAN会打印错误报告。 + +基于软件标签的KASAN也有两种插桩模式（outline，发出回调来检查内存访问；inline， +执行内联的影子内存检查）。使用outline插桩模式，会从执行访问检查的函数打印错误 +报告。使用inline插桩，编译器会发出 ``brk`` 指令，并使用专用的 ``brk`` 处理程序 +来打印错误报告。 + +基于软件标签的KASAN使用0xFF作为匹配所有指针标签（不检查通过带有0xFF指针标签 +的指针进行的访问）。值0xFE当前保留用于标记已释放的内存区域。 + +基于软件标签的KASAN目前仅支持对Slab和page_alloc内存进行标记。 + +基于硬件标签的KASAN模式 +~~~~~~~~~~~~~~~~~~~~~~~ + +基于硬件标签的KASAN在概念上类似于软件模式，但它是使用硬件内存标签作为支持而 +不是编译器插桩和影子内存。 + +基于硬件标签的KASAN目前仅针对arm64架构实现，并且基于ARMv8.5指令集架构中引入 +的arm64内存标记扩展(MTE)和最高字节忽略(TBI)。 + +特殊的arm64指令用于为每次内存分配指定内存标签。相同的标签被指定给指向这些分配 +的指针。在每次内存访问时，硬件确保正在访问的内存的标签等于用于访问该内存的指针 +的标签。如果标签不匹配，则会生成故障并打印报告。 + +基于硬件标签的KASAN使用0xFF作为匹配所有指针标签（不检查通过带有0xFF指针标签的 +指针进行的访问）。值0xFE当前保留用于标记已释放的内存区域。 + +基于硬件标签的KASAN目前仅支持对Slab和page_alloc内存进行标记。 + +如果硬件不支持MTE（ARMv8.5之前），则不会启用基于硬件标签的KASAN。在这种情况下， +所有KASAN引导参数都将被忽略。 + +请注意，启用CONFIG_KASAN_HW_TAGS始终会导致启用内核中的TBI。即使提供了 +``kasan.mode=off`` 或硬件不支持MTE（但支持TBI）。 + +基于硬件标签的KASAN只报告第一个发现的错误。之后，MTE标签检查将被禁用。 + +影子内存 +-------- + +内核将内存映射到地址空间的几个不同部分。内核虚拟地址的范围很大：没有足够的真实 +内存来支持内核可以访问的每个地址的真实影子区域。因此，KASAN只为地址空间的某些 +部分映射真实的影子。 + +默认行为 +~~~~~~~~ + +默认情况下，体系结构仅将实际内存映射到用于线性映射的阴影区域（以及可能的其他 +小区域）。对于所有其他区域 —— 例如vmalloc和vmemmap空间 —— 一个只读页面被映射 +到阴影区域上。这个只读的影子页面声明所有内存访问都是允许的。 + +这给模块带来了一个问题：它们不存在于线性映射中，而是存在于专用的模块空间中。 +通过连接模块分配器，KASAN临时映射真实的影子内存以覆盖它们。例如，这允许检测 +对模块全局变量的无效访问。 + +这也造成了与 ``VMAP_STACK`` 的不兼容：如果堆栈位于vmalloc空间中，它将被分配 +只读页面的影子内存，并且内核在尝试为堆栈变量设置影子数据时会出错。 + +CONFIG_KASAN_VMALLOC +~~~~~~~~~~~~~~~~~~~~ + +使用 ``CONFIG_KASAN_VMALLOC`` ，KASAN可以以更大的内存使用为代价覆盖vmalloc +空间。目前，这在x86、riscv、s390和powerpc上受支持。 + +这通过连接到vmalloc和vmap并动态分配真实的影子内存来支持映射。 + +vmalloc空间中的大多数映射都很小，需要不到一整页的阴影空间。因此，为每个映射 +分配一个完整的影子页面将是一种浪费。此外，为了确保不同的映射使用不同的影子 +页面，映射必须与 ``KASAN_GRANULE_SIZE * PAGE_SIZE`` 对齐。 + +相反，KASAN跨多个映射共享后备空间。当vmalloc空间中的映射使用影子区域的特定 +页面时，它会分配一个后备页面。此页面稍后可以由其他vmalloc映射共享。 + +KASAN连接到vmap基础架构以懒清理未使用的影子内存。 + +为了避免交换映射的困难，KASAN预测覆盖vmalloc空间的阴影区域部分将不会被早期 +的阴影页面覆盖，但是将不会被映射。这将需要更改特定于arch的代码。 + +这允许在x86上支持 ``VMAP_STACK`` ，并且可以简化对没有固定模块区域的架构的支持。 + +对于开发者 +---------- + +忽略访问 +~~~~~~~~ + +软件KASAN模式使用编译器插桩来插入有效性检查。此类检测可能与内核的某些部分 +不兼容，因此需要禁用。 + +内核的其他部分可能会访问已分配对象的元数据。通常，KASAN会检测并报告此类访问， +但在某些情况下（例如，在内存分配器中），这些访问是有效的。 + +对于软件KASAN模式，要禁用特定文件或目录的检测，请将 ``KASAN_SANITIZE`` 添加 +到相应的内核Makefile中: + +- 对于单个文件(例如，main.o):: + + KASAN_SANITIZE_main.o := n + +- 对于一个目录下的所有文件:: + + KASAN_SANITIZE := n + +对于软件KASAN模式，要在每个函数的基础上禁用检测，请使用KASAN特定的 +``__no_sanitize_address`` 函数属性或通用的 ``noinstr`` 。 + +请注意，禁用编译器插桩（基于每个文件或每个函数）会使KASAN忽略在软件KASAN模式 +的代码中直接发生的访问。当访问是间接发生的（通过调用检测函数）或使用没有编译器 +插桩的基于硬件标签的模式时，它没有帮助。 + +对于软件KASAN模式，要在当前任务的一部分内核代码中禁用KASAN报告，请使用 +``kasan_disable_current()``/``kasan_enable_current()`` 部分注释这部分代码。 +这也会禁用通过函数调用发生的间接访问的报告。 + +对于基于标签的KASAN模式（包括硬件模式），要禁用访问检查，请使用 +``kasan_reset_tag()`` 或 ``page_kasan_tag_reset()`` 。请注意，通过 +``page_kasan_tag_reset()`` 临时禁用访问检查需要通过 ``page_kasan_tag`` +/ ``page_kasan_tag_set`` 保存和恢复每页KASAN标签。 + +测试 +~~~~ + +有一些KASAN测试可以验证KASAN是否正常工作并可以检测某些类型的内存损坏。 +测试由两部分组成: + +1. 与KUnit测试框架集成的测试。使用 ``CONFIG_KASAN_KUNIT_TEST`` 启用。 +这些测试可以通过几种不同的方式自动运行和部分验证；请参阅下面的说明。 + +2. 与KUnit不兼容的测试。使用 ``CONFIG_KASAN_MODULE_TEST`` 启用并且只能作为模块 +运行。这些测试只能通过加载内核模块并检查内核日志以获取KASAN报告来手动验证。 + +如果检测到错误，每个KUnit兼容的KASAN测试都会打印多个KASAN报告之一，然后测试打印 +其编号和状态。 + +当测试通过:: + + ok 28 - kmalloc_double_kzfree + +当由于 ``kmalloc`` 失败而导致测试失败时:: + + # kmalloc_large_oob_right: ASSERTION FAILED at lib/test_kasan.c:163 + Expected ptr is not null, but is + not ok 4 - kmalloc_large_oob_right + +当由于缺少KASAN报告而导致测试失败时:: + + # kmalloc_double_kzfree: EXPECTATION FAILED at lib/test_kasan.c:629 + Expected kasan_data->report_expected == kasan_data->report_found, but + kasan_data->report_expected == 1 + kasan_data->report_found == 0 + not ok 28 - kmalloc_double_kzfree + +最后打印所有KASAN测试的累积状态。成功:: + + ok 1 - kasan + +或者，如果其中一项测试失败:: + + not ok 1 - kasan + +有几种方法可以运行与KUnit兼容的KASAN测试。 + +1. 可加载模块 + + 启用 ``CONFIG_KUNIT`` 后，KASAN-KUnit测试可以构建为可加载模块，并通过使用 + ``insmod`` 或 ``modprobe`` 加载 ``test_kasan.ko`` 来运行。 + +2. 内置 + + 通过内置 ``CONFIG_KUNIT`` ，也可以内置KASAN-KUnit测试。在这种情况下， + 测试将在启动时作为后期初始化调用运行。 + +3. 使用kunit_tool + + 通过内置 ``CONFIG_KUNIT`` 和 ``CONFIG_KASAN_KUNIT_TEST`` ，还可以使用 + ``kunit_tool`` 以更易读的方式查看KUnit测试结果。这不会打印通过测试 + 的KASAN报告。有关 ``kunit_tool`` 更多最新信息，请参阅 + `KUnit文档 `_ 。 + +.. _KUnit: https://www.kernel.org/doc/html/latest/dev-tools/kunit/index.html diff --git a/Documentation/translations/zh_CN/index.rst b/Documentation/translations/zh_CN/index.rst index d56d6b7092e6..1f953d3439a5 100644 --- a/Documentation/translations/zh_CN/index.rst +++ b/Documentation/translations/zh_CN/index.rst @@ -4,6 +4,7 @@ \renewcommand\thesection* \renewcommand\thesubsection* + \kerneldocCJKon .. _linux_doc_zh: @@ -72,11 +73,11 @@ TODOlist: dev-tools/index doc-guide/index kernel-hacking/index + maintainer/index TODOList: * trace/index -* maintainer/index * fault-injection/index * livepatch/index * rust/index @@ -153,6 +154,7 @@ TODOList: arm64/index riscv/index openrisc/index + parisc/index TODOList: @@ -160,7 +162,6 @@ TODOList: * ia64/index * m68k/index * nios2/index -* parisc/index * powerpc/index * s390/index * sh/index diff --git a/Documentation/translations/zh_CN/maintainer/configure-git.rst b/Documentation/translations/zh_CN/maintainer/configure-git.rst new file mode 100644 index 000000000000..a45ea736f73b --- /dev/null +++ b/Documentation/translations/zh_CN/maintainer/configure-git.rst @@ -0,0 +1,62 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/maintainer/configure-git.rst + +:译者: + + 吴想成 Wu XiangCheng + +.. _configuregit_zh: + +Git配置 +======= + +本章讲述了维护者级别的git配置。 + +Documentation/maintainer/pull-requests.rst 中使用的标记分支应使用开发人员的 +GPG公钥进行签名。可以通过将 ``-u`` 标志传递给 ``git tag`` 来创建签名标记。 +但是，由于 *通常* 对同一项目使用同一个密钥，因此可以设置:: + + git config user.signingkey "keyname" + +或者手动编辑你的 ``.git/config`` 或 ``~/.gitconfig`` 文件:: + + [user] + name = Jane Developer + email = jd@domain.org + signingkey = jd@domain.org + +你可能需要告诉 ``git`` 去使用 ``gpg2``:: + + [gpg] + program = /path/to/gpg2 + +你可能也需要告诉 ``gpg`` 去使用哪个 ``tty`` （添加到你的shell rc文件中）:: + + export GPG_TTY=$(tty) + + +创建链接到lore.kernel.org的提交 +------------------------------- + +http://lore.kernel.org 网站是所有涉及或影响内核开发的邮件列表的总存档。在这里 +存储补丁存档是推荐的做法，当维护人员将补丁应用到子系统树时，最好提供一个指向 +lore存档链接的标签，以便浏览提交历史的人可以找到某个更改背后的相关讨论和基本 +原理。链接标签如下所示： + + Link: https://lore.kernel.org/r/ + +通过在git中添加以下钩子，可以将此配置为在发布 ``git am`` 时自动执行： + +.. code-block:: none + + $ git config am.messageid true + $ cat >.git/hooks/applypatch-msg <<'EOF' + #!/bin/sh + . git-sh-setup + perl -pi -e 's|^Message-Id:\s*]+)>?$|Link: https://lore.kernel.org/r/$1|g;' "$1" + test -x "$GIT_DIR/hooks/commit-msg" && + exec "$GIT_DIR/hooks/commit-msg" ${1+"$@"} + : + EOF + $ chmod a+x .git/hooks/applypatch-msg diff --git a/Documentation/translations/zh_CN/maintainer/index.rst b/Documentation/translations/zh_CN/maintainer/index.rst new file mode 100644 index 000000000000..eb75ccea9a21 --- /dev/null +++ b/Documentation/translations/zh_CN/maintainer/index.rst @@ -0,0 +1,21 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/maintainer/index.rst + +============== +内核维护者手册 +============== + +本文档本是内核维护者手册的首页。 +本手册还需要大量完善！请自由提出（和编写）本手册的补充内容。 +*译注：指英文原版* + +.. toctree:: + :maxdepth: 2 + + configure-git + rebasing-and-merging + pull-requests + maintainer-entry-profile + modifying-patches + diff --git a/Documentation/translations/zh_CN/maintainer/maintainer-entry-profile.rst b/Documentation/translations/zh_CN/maintainer/maintainer-entry-profile.rst new file mode 100644 index 000000000000..a1ee99c4786e --- /dev/null +++ b/Documentation/translations/zh_CN/maintainer/maintainer-entry-profile.rst @@ -0,0 +1,92 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/maintainer/maintainer-entry-profile.rst + +:译者: + + 吴想成 Wu XiangCheng + +.. _maintainerentryprofile_zh: + +维护者条目概要 +============== + +维护人员条目概要补充了顶层过程文档（提交补丁，提交驱动程序……），增加了子系 +统/设备驱动程序本地习惯以及有关补丁提交生命周期的相关内容。贡献者使用此文档 +来调整他们的期望和避免常见错误；维护人员可以使用这些信息超越子系统层面查看 +是否有机会汇聚到通用实践中。 + + +总览 +---- + +提供了子系统如何操作的介绍。MAINTAINERS文件告诉了贡献者应发送某文件的补丁到哪， +但它没有传达其他子系统的本地基础设施和机制以协助开发。 + +请考虑以下问题： + +- 当补丁被本地树接纳或合并到上游时是否有通知？ +- 子系统是否使用patchwork实例？Patchwork状态变更是否有通知？ +- 是否有任何机器人或CI基础设施监视列表，或子系统是否使用自动测试反馈以便把 + 控接纳补丁？ +- 被拉入-next的Git分支是哪个？ +- 贡献者应针对哪个分支提交？ +- 是否链接到其他维护者条目概要？例如一个设备驱动可能指向其父子系统的条目。 + 这使得贡献者意识到某维护者可能对提交链中其他维护者负有的义务。 + + +提交检查单补遗 +-------------- + +列出强制性和咨询性标准，超出通用标准“提交检查表，以便维护者检查一个补丁是否 +足够健康。例如：“通过checkpatch.pl，没有错误、没有警告。通过单元测试详见某处”。 + +提交检查单补遗还可以包括有关硬件规格状态的详细信息。例如，子系统接受补丁之前 +是否需要考虑在某个修订版上发布的规范。 + + +开发周期的关键日期 +------------------ + +提交者常常会误以为补丁可以在合并窗口关闭之前的任何时间发送，且下一个-rc1时仍 +可以。事实上，大多数补丁都需要在下一个合并窗口打开之前提前进入linux-next中。 +向提交者澄清关键日期（以-rc发布周为标志）以明确什么时候补丁会被考虑合并以及 +何时需要等待下一个-rc。 + +至少需要讲明： + +- 最后一个可以提交新功能的-rc： + 针对下一个合并窗口的新功能提交应该在此点之前首次发布以供考虑。在此时间点 + 之后提交的补丁应该明确他们的目标为下下个合并窗口，或者给出应加快进度被接受 + 的充足理由。通常新特性贡献者的提交应出现在-rc5之前。 + +- 最后合并-rc：合并决策的最后期限。 + 向贡献者指出尚未接受的补丁集需要等待下下个合并窗口。当然，维护者没有义务 + 接受所有给定的补丁集，但是如果审阅在此时间点尚未结束，那么希望贡献者应该 + 等待并在下一个合并窗口重新提交。 + +可选项： + +- 开发基线分支的首个-rc，列在概述部分，视为已为新提交做好准备。 + + +审阅节奏 +-------- + +贡献者最担心的问题之一是：补丁集已发布却未收到反馈，应在多久后发送提醒。除了 +指定在重新提交之前要等待多长时间，还可以指示更新的首选样式；例如，重新发送 +整个系列，或私下发送提醒邮件。本节也可以列出本区域的代码审阅方式，以及获取 +不能直接从维护者那里得到的反馈的方法。 + + +现有概要 +-------- + +这里列出了现有的维护人员条目概要；我们可能会想要在不久的将来做一些不同的事情。 + +.. toctree:: + :maxdepth: 1 + + ../doc-guide/maintainer-profile + ../../../nvdimm/maintainer-entry-profile + ../../../riscv/patch-acceptance diff --git a/Documentation/translations/zh_CN/maintainer/modifying-patches.rst b/Documentation/translations/zh_CN/maintainer/modifying-patches.rst new file mode 100644 index 000000000000..7f6326137f6b --- /dev/null +++ b/Documentation/translations/zh_CN/maintainer/modifying-patches.rst @@ -0,0 +1,51 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/maintainer/modifying-patches.rst + +:译者: + + 吴想成 Wu XiangCheng + +.. _modifyingpatches_zh: + +修改补丁 +======== + +如果你是子系统或者分支的维护者，由于代码在你的和提交者的树中并不完全相同， +有时你需要稍微修改一下收到的补丁以合并它们。 + +如果你严格遵守开发者来源证书的规则（c），你应该要求提交者重做，但这完全是会 +适得其反的时间、精力浪费。规则（b）允许你调整代码，但这样修改提交者的代码并 +让他背书你的错误是非常不礼貌的。为解决此问题，建议在你之前最后一个 +Signed-off-by标签和你的之间添加一行，以指示更改的性质。这没有强制性要求，最 +好在描述前面加上你的邮件和/或姓名，用方括号括住整行，以明显指出你对最后一刻 +的更改负责。例如:: + + Signed-off-by: Random J Developer + [lucky@maintainer.example.org: struct foo moved from foo.c to foo.h] + Signed-off-by: Lucky K Maintainer + +如果您维护着一个稳定的分支，并希望同时明确贡献、跟踪更改、合并修复，并保护 +提交者免受责难，这种做法尤其有用。请注意，在任何情况下都不得更改作者的身份 +（From头），因为它会在变更日志中显示。 + +向后移植（back-port）人员特别要注意：为了便于跟踪，请在提交消息的顶部（即主题行 +之后）插入补丁的来源，这是一种常见而有用的做法。例如，我们可以在3.x稳定版本 +中看到以下内容:: + + Date: Tue Oct 7 07:26:38 2014 -0400 + + libata: Un-break ATA blacklist + + commit 1c40279960bcd7d52dbdf1d466b20d24b99176c8 upstream. + +下面是一个旧的内核在某补丁被向后移植后会出现的:: + + Date: Tue May 13 22:12:27 2008 +0200 + + wireless, airo: waitbusy() won't delay + + [backport of 2.6 commit b7acbdfbd1f277c1eb23f344f899cfa4cd0bf36a] + +不管什么格式，这些信息都为人们跟踪你的树，以及试图解决你树中的错误的人提供了 +有价值的帮助。 diff --git a/Documentation/translations/zh_CN/maintainer/pull-requests.rst b/Documentation/translations/zh_CN/maintainer/pull-requests.rst new file mode 100644 index 000000000000..f46d6f3f2498 --- /dev/null +++ b/Documentation/translations/zh_CN/maintainer/pull-requests.rst @@ -0,0 +1,148 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/maintainer/pull-requests.rst + +:译者: + + 吴想成 Wu XiangCheng + +.. _pullrequests_zh: + +如何创建拉取请求 +================ + +本章描述维护人员如何创建并向其他维护人员提交拉取请求。这对将更改从一个维护者 +树转移到另一个维护者树非常有用。 + +本文档由Tobin C. Harding（当时他尚不是一名经验丰富的维护人员）编写，内容主要 +来自Greg Kroah Hartman和Linus Torvalds在LKML上的评论。Jonathan Corbet和Mauro +Carvalho Chehab提出了一些建议和修改。错误不可避免，如有问题，请找Tobin C. +Harding 。 + +原始邮件线程:: + + http://lkml.kernel.org/r/20171114110500.GA21175@kroah.com + + +创建分支 +-------- + +首先，您需要将希望包含拉取请求里的所有更改都放在单独分支中。通常您将基于某开发 +人员树的一个分支，一般是打算向其发送拉取请求的开发人员。 + +为了创建拉取请求，您必须首先标记刚刚创建的分支。建议您选择一个有意义的标记名， +以即使过了一段时间您和他人仍能理解的方式。在名称中包含源子系统和目标内核版本 +的指示也是一个好的做法。 + +Greg提供了以下内容。对于一个含有drivers/char中混杂事项、将应用于4.15-rc1内核的 +拉取请求，可以命名为 ``char-misc-4.15-rc1`` 。如果要在 ``char-misc-next`` 分支 +上打上此标记，您可以使用以下命令:: + + git tag -s char-misc-4.15-rc1 char-misc-next + +这将在 ``char-misc-next`` 分支的最后一个提交上创建一个名为 ``char-misc-4.15-rc1`` +的标记，并用您的gpg密钥签名（参见 Documentation/maintainer/configure-git.rst ）。 + +Linus只接受基于签名过的标记的拉取请求。其他维护者可能会有所不同。 + +当您运行上述命令时 ``git`` 会打开编辑器要求你描述一下这个标记。在本例中您需要 +描述拉取请求，所以请概述一下包含的内容，为什么要合并，是否完成任何测试。所有 +这些信息都将留在标记中，然后在维护者合并拉取请求时保留在合并提交中。所以把它 +写好，它将永远留在内核中。 + +正如Linus所说:: + + 不管怎么样，至少对我来说，重要的是 *信息* 。我需要知道我在拉取什么、 + 为什么我要拉取。我也希望将此消息用于合并消息，因此它不仅应该对我有 + 意义，也应该可以成为一个有意义的历史记录。 + + 注意，如果拉取请求有一些不寻常的地方，请详细说明。如果你修改了并非 + 由你维护的文件，请解释 **为什么** 。我总会在差异中看到的，如果你不 + 提的话，我只会觉得分外可疑。当你在合并窗口后给我发新东西的时候， + （甚至是比较重大的错误修复），不仅需要解释做了什么、为什么这么做， + 还请解释一下 **时间问题** 。为什么错过了合并窗口…… + + 我会看你写在拉取请求邮件和签名标记里面的内容，所以根据你的工作流， + 你可以在签名标记里面描述工作内容（也会自动放进拉取请求邮件），也 + 可以只在标记里面放个占位符，稍后在你实际发给我拉取请求时描述工作内容。 + + 是的，我会编辑这些消息。部分因为我需要做一些琐碎的格式调整（整体缩进、 + 括号等），也因为此消息可能对我有意义（描述了冲突或一些个人问题）而对 + 合并提交信息上下文没啥意义，因此我需要尽力让它有意义起来。我也会 + 修复一些拼写和语法错误，特别是非母语者（母语者也是;^）。但我也会删掉 + 或增加一些内容。 + + Linus + +Greg给出了一个拉取请求的例子:: + + Char/Misc patches for 4.15-rc1 + + Here is the big char/misc patch set for the 4.15-rc1 merge window. + Contained in here is the normal set of new functions added to all + of these crazy drivers, as well as the following brand new + subsystems: + - time_travel_controller: Finally a set of drivers for the + latest time travel bus architecture that provides i/o to + the CPU before it asked for it, allowing uninterrupted + processing + - relativity_shifters: due to the affect that the + time_travel_controllers have on the overall system, there + was a need for a new set of relativity shifter drivers to + accommodate the newly formed black holes that would + threaten to suck CPUs into them. This subsystem handles + this in a way to successfully neutralize the problems. + There is a Kconfig option to force these to be enabled + when needed, so problems should not occur. + + All of these patches have been successfully tested in the latest + linux-next releases, and the original problems that it found have + all been resolved (apologies to anyone living near Canberra for the + lack of the Kconfig options in the earlier versions of the + linux-next tree creations.) + + Signed-off-by: Your-name-here + + +此标记消息格式就像一个git提交。顶部有一行“总结标题”， 一定要在下面sign-off。 + +现在您已经有了一个本地签名标记，您需要将它推送到可以被拉取的位置:: + + git push origin char-misc-4.15-rc1 + + +创建拉取请求 +------------ + +最后要做的是创建拉取请求消息。可以使用 ``git request-pull`` 命令让 ``git`` +为你做这件事，但它需要确定你想拉取什么，以及拉取针对的基础（显示正确的拉取 +更改和变更状态）。以下命令将生成一个拉取请求:: + + git request-pull master git://git.kernel.org/pub/scm/linux/kernel/git/gregkh/char-misc.git/ char-misc-4.15-rc1 + +引用Greg的话:: + + 此命令要求git比较从“char-misc-4.15-rc1”标记位置到“master”分支头（上述 + 例子中指向了我从Linus的树分叉的地方，通常是-rc发布）的差异，并去使用 + git:// 协议拉取。如果你希望使用 https:// 协议，也可以用在这里（但是请 + 注意，部分人由于防火墙问题没法用https协议拉取）。 + + 如果char-misc-4.15-rc1标记没有出现在我要求拉取的仓库中，git会提醒 + 它不在那里，所以记得推送到公开地方。 + + “git request-pull”会包含git树的地址和需要拉取的特定标记，以及标记 + 描述全文（详尽描述标记）。同时它也会创建此拉取请求的差异状态和单个 + 提交的缩短日志。 + +Linus回复说他倾向于 ``git://`` 协议。其他维护者可能有不同的偏好。另外，请注意 +如果你创建的拉取请求没有签名标记， ``https://`` 可能是更好的选择。完整的讨论 +请看原邮件。 + + +提交拉取请求 +------------ + +拉取请求的提交方式与普通补丁相同。向维护人员发送内联电子邮件并抄送LKML以及 +任何必要特定子系统的列表。对Linus的拉取请求通常有如下主题行:: + + [GIT PULL] changes for v4.15-rc1 diff --git a/Documentation/translations/zh_CN/maintainer/rebasing-and-merging.rst b/Documentation/translations/zh_CN/maintainer/rebasing-and-merging.rst new file mode 100644 index 000000000000..83b7dabfe88b --- /dev/null +++ b/Documentation/translations/zh_CN/maintainer/rebasing-and-merging.rst @@ -0,0 +1,165 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/maintainer/rebasing-and-merging.rst + +:译者: + + 吴想成 Wu XiangCheng + +========== +变基与合并 +========== + +一般来说，维护子系统需要熟悉Git源代码管理系统。Git是一个功能强大的工具，有 +很多功能；就像这类工具常出现的情况一样，使用这些功能的方法有对有错。本文档 +特别介绍了变基与合并的用法。维护者经常在错误使用这些工具时遇到麻烦，但避免 +问题实际上并不那么困难。 + +总的来说，需要注意的一点是：与许多其他项目不同，内核社区并不害怕在其开发历史 +中看到合并提交。事实上，考虑到该项目的规模，避免合并几乎是不可能的。维护者会 +在希望避免合并时遇到一些问题，而过于频繁的合并也会带来另一些问题。 + +变基 +==== + +“变基（Rebase）”是更改存储库中一系列提交的历史记录的过程。有两种不同型的操作 +都被称为变基，因为这两种操作都使用 ``git rebase`` 命令，但它们之间存在显著 +差异： + + - 更改一系列补丁的父提交（起始提交）。例如，变基操作可以将基于上一内核版本 + 的一个补丁集重建到当前版本上。在下面的讨论中，我们将此操作称为“变根”。 + + - 通过修复（或删除）损坏的提交、添加补丁、添加标记以更改一系列补丁的历史， + 来提交变更日志或更改已应用提交的顺序。在下文中，这种类型的操作称为“历史 + 修改” + +术语“变基”将用于指代上述两种操作。如果使用得当，变基可以产生更清晰、更整洁的 +开发历史；如果使用不当，它可能会模糊历史并引入错误。 + +以下一些经验法则可以帮助开发者避免最糟糕的变基风险： + + - 已经发布到你私人系统之外世界的历史通常不应更改。其他人可能会拉取你的树 + 的副本，然后基于它进行工作；修改你的树会给他们带来麻烦。如果工作需要变基， + 这通常是表明它还没有准备好提交到公共存储库的信号。 + + 但是，总有例外。有些树（linux-next是一个典型的例子）由于它们的需要经常 + 变基，开发人员知道不要基于它们来工作。开发人员有时会公开一个不稳定的分支， + 供其他人或自动测试服务进行测试。如果您确实以这种方式公开了一个可能不稳定 + 的分支，请确保潜在使用者知道不要基于它来工作。 + + - 不要在包含由他人创建的历史的分支上变基。如果你从别的开发者的仓库拉取了变更， + 那你现在就成了他们历史记录的保管人。你不应该改变它，除了少数例外情况。例如 + 树中有问题的提交必须显式恢复（即通过另一个补丁修复），而不是通过修改历史而 + 消失。 + + - 没有合理理由，不要对树变根。仅为了切换到更新的基或避免与上游储存库的合并 + 通常不是合理理由。 + + - 如果你必须对储存库进行变根，请不要随机选取一个提交作为新基。在发布节点之间 + 内核通常处于一个相对不稳定的状态；基于其中某点进行开发会增加遇到意外错误的 + 几率。当一系列补丁必须移动到新基时，请选择移动到一个稳定节点（例如-rc版本 + 节点）。 + + - 请知悉对补丁系列进行变根（或做明显的历史修改）会改变它们的开发环境，且很 + 可能使做过的大部分测试失效。一般来说，变基后的补丁系列应当像新代码一样对 + 待，并重新测试。 + +合并窗口麻烦的一个常见原因是，Linus收到了一个明显在拉取请求发送之前不久才变根 +（通常是变根到随机的提交上）的补丁系列。这样一个系列被充分测试的可能性相对较 +低，拉取请求被接受的几率也同样较低。 + +相反，如果变基仅限于私有树、提交基于一个通用的起点、且经过充分测试，则引起 +麻烦的可能性就很低。 + +合并 +==== + +内核开发过程中，合并是一个很常见的操作；5.1版本开发周期中有超过1126个合并 +——差不多占了整体的9%。内核开发工作积累在100多个不同的子系统树中，每个 +子系统树都可能包含多个主题分支；每个分支通常独立于其他分支进行开发。因此 +在任何给定分支进入上游储存库之前，至少需要一次合并。 + +许多项目要求拉取请求中的分支基于当前主干，这样历史记录中就不会出现合并提交。 +内核并不是这样；任何为了避免合并而重新对分支变基都很可能导致麻烦。 + +子系统维护人员发现他们必须进行两种类型的合并：从较低层级的子系统树和从其他 +子系统树（同级树或主线）进行合并。这两种情况下要遵循的最佳实践是不同的。 + +合并较低层级树 +-------------- + +较大的子系统往往有多个级别的维护人员，较低级别的维护人员向较高级别发送拉取 +请求。合并这样的请求执行几乎肯定会生成一个合并提交；这也是应该的。实际上， +子系统维护人员可能希望在极少数快进合并情况下使用 ``-–no-ff`` 标志来强制添加 +合并提交，以便记录合并的原因。 **任何** 类型的合并的变更日志必须说明 +*为什么* 合并。对于较低级别的树，“为什么”通常是对该取所带来的变化的总结。 + +各级维护人员都应在他们的拉取请求上使用经签名的标签，上游维护人员应在拉取分支 +时验证标签。不这样做会威胁整个开发过程的安全。 + +根据上面列出的规则，一旦您将其他人的历史记录合并到树中，您就不得对该分支进行 +变基，即使您能够这样做。 + +合并同级树或上游树 +------------------ + +虽然来自下游的合并是常见且不起眼的，但当需要将一个分支推向上游时，其中来自 +其他树的合并往往是一个危险信号。这种合并需要仔细考虑并加以充分证明，否则后续 +的拉取请求很可能会被拒绝。 + +想要将主分支合并到存储库中是很自然的；这种类型的合并通常被称为“反向合并” +。反向合并有助于确保与并行的开发没有冲突，并且通常会给人一种温暖、舒服的 +感觉，即处于最新。但这种诱惑几乎总是应该避免的。 + +为什么呢？反向合并将搅乱你自己分支的开发历史。它们会大大增加你遇到来自社区 +其他地方的错误的机会，且使你很难确保你所管理的工作稳定并准备好合入上游。 +频繁的合并还可以掩盖树中开发过程中的问题；它们会隐藏与其他树的交互，而这些 +交互不应该（经常）发生在管理良好的分支中。 + +也就是说，偶尔需要进行反向合并；当这种情况发生时，一定要在提交信息中记录 +*为什么* 。同样，在一个众所周知的稳定点进行合并，而不是随机提交。即使这样， +你也不应该反向合并一棵比你的直接上游树更高层级的树；如果确实需要更高级别的 +反向合并，应首先在上游树进行。 + +导致合并相关问题最常见的原因之一是：在发送拉取请求之前维护者合并上游以解决 +合并冲突。同样，这种诱惑很容易理解，但绝对应该避免。对于最终拉取请求来说 +尤其如此：Linus坚信他更愿意看到合并冲突，而不是不必要的反向合并。看到冲突 +可以让他了解潜在的问题所在。他做过很多合并（在5.1版本开发周期中是382次）， +而且在解决冲突方面也很在行——通常比参与的开发人员要强。 + +那么，当他们的子系统分支和主线之间发生冲突时，维护人员应该怎么做呢？最重要 +的一步是在拉取请求中提示Linus会发生冲突；如果啥都没说则表明您的分支可以正常 +合入。对于特别困难的冲突，创建并推送一个 *独立* 分支来展示你将如何解决问题。 +在拉取请求中提到该分支，但是请求本身应该针对未合并的分支。 + +即使不存在已知冲突，在发送拉取请求之前进行合并测试也是个好主意。它可能会提醒 +您一些在linux-next树中没有发现的问题，并帮助您准确地理解您正在要求上游做什么。 + +合并上游树或另一个子系统树的另一个原因是解决依赖关系。这些依赖性问题有时确实 +会发生，而且有时与另一棵树交叉合并是解决这些问题的最佳方法；同样，在这种情况 +下，合并提交应该解释为什么要进行合并。花点时间把它做好；会有人阅读这些变更 +日志。 + +然而依赖性问题通常表明需要改变方法。合并另一个子系统树以解决依赖性风险会带来 +其他缺陷，几乎永远不应这样做。如果该子系统树无法被合到上游，那么它的任何问题 +也都会阻碍你的树合并。更可取的选择包括与维护人员达成一致意见，在其中一个树中 +同时进行两组更改；或者创建一个主题分支专门处理可以合并到两个树中的先决条件提交。 +如果依赖关系与主要的基础结构更改相关，正确的解决方案可能是将依赖提交保留一个 +开发周期，以便这些更改有时间在主线上稳定。 + +最后 +==== + +在开发周期的开头合并主线是比较常见的，可以获取树中其他地方的更改和修复。同样， +这样的合并应该选择一个众所周知的发布点，而不是一些随机点。如果在合并窗口期间 +上游分支已完全清空到主线中，则可以使用以下命令向前拉取它:: + + git merge v5.2-rc1^0 + +“^0”使Git执行快进合并（在这种情况下这应该可以），从而避免多余的虚假合并提交。 + +上面列出的就是指导方针了。总是会有一些情况需要不同的解决方案，这些指导原则 +不应阻止开发人员在需要时做正确的事情。但是，我们应该时刻考虑是否真的出现了 +这样的需求，并准备好解释为什么需要做一些不寻常的事情。 diff --git a/Documentation/translations/zh_CN/parisc/debugging.rst b/Documentation/translations/zh_CN/parisc/debugging.rst new file mode 100644 index 000000000000..c21beb986e15 --- /dev/null +++ b/Documentation/translations/zh_CN/parisc/debugging.rst @@ -0,0 +1,42 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/parisc/debugging.rst +:Translator: Yanteng Si + +.. _cn_parisc_debugging: + +================= +调试PA-RISC +================= + +好吧，这里有一些关于调试linux/parisc的较底层部分的信息。 + + +1. 绝对地址 +===================== + +很多汇编代码目前运行在实模式下，这意味着会使用绝对地址，而不是像内核其他 +部分那样使用虚拟地址。要将绝对地址转换为虚拟地址，你可以在System.map中查 +找，添加__PAGE_OFFSET（目前是0x10000000）。 + + +2. HPMCs +======== + +当实模式的代码试图访问不存在的内存时，会出现HPMC（high priority machine +check）而不是内核oops。若要调试HPMC，请尝试找到系统响应程序/请求程序地址。 +系统请求程序地址应该与（某）处理器的HPA（I/O范围内的高地址）相匹配；系统响应程 +序地址是实模式代码试图访问的地址。 + +系统响应程序地址的典型值是大于__PAGE_OFFSET （0x10000000）的地址，这意味着 +在实模式试图访问它之前，虚拟地址没有被翻译成物理地址。 + + +3. 有趣的Q位 +============ + +某些非常关键的代码必须清除PSW中的Q位。当Q位被清除时，CPU不会更新中断处理 +程序所读取的寄存器，以找出机器被中断的位置——所以如果你在清除Q位的指令和再 +次设置Q位的RFI之间遇到中断，你不知道它到底发生在哪里。如果你幸运的话，IAOQ +会指向清除Q位的指令，如果你不幸运的话，它会指向任何地方。通常Q位的问题会 +表现为无法解释的系统挂起或物理内存越界。 diff --git a/Documentation/translations/zh_CN/parisc/index.rst b/Documentation/translations/zh_CN/parisc/index.rst new file mode 100644 index 000000000000..a47454ebe32e --- /dev/null +++ b/Documentation/translations/zh_CN/parisc/index.rst @@ -0,0 +1,28 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/parisc/index.rst +:Translator: Yanteng Si + +.. _cn_parisc_index: + +==================== +PA-RISC体系架构 +==================== + +.. toctree:: + :maxdepth: 2 + + debugging + registers + +Todolist: + + features + +.. only:: subproject and html + + Indices + ======= + + * :ref:`genindex` diff --git a/Documentation/translations/zh_CN/parisc/registers.rst b/Documentation/translations/zh_CN/parisc/registers.rst new file mode 100644 index 000000000000..71e2404cd103 --- /dev/null +++ b/Documentation/translations/zh_CN/parisc/registers.rst @@ -0,0 +1,153 @@ +.. include:: ../disclaimer-zh_CN.rst + +:Original: Documentation/parisc/registers.rst +:Translator: Yanteng Si + +.. _cn_parisc_registers: + +========================= +Linux/PA-RISC的寄存器用法 +========================= + +[ 用星号表示目前尚未实现的计划用途。 ] + +ABI约定的通用寄存器 +=================== + +控制寄存器 +---------- + +============================ ================================= +CR 0 (恢复计数器) 用于ptrace +CR 1-CR 7(无定义) 未使用 +CR 8 (Protection ID) 每进程值* +CR 9, 12, 13 (PIDS) 未使用 +CR10 (CCR) FPU延迟保存* +CR11 按照ABI的规定（SAR） +CR14 (中断向量) 初始化为 fault_vector +CR15 (EIEM) 所有位初始化为1* +CR16 (间隔计时器) 读取周期数/写入开始时间间隔计时器 +CR17-CR22 中断参数 +CR19 中断指令寄存器 +CR20 中断空间寄存器 +CR21 中断偏移量寄存器 +CR22 中断 PSW +CR23 (EIRR) 读取未决中断/写入清除位 +CR24 (TR 0) 内核空间页目录指针 +CR25 (TR 1) 用户空间页目录指针 +CR26 (TR 2) 不使用 +CR27 (TR 3) 线程描述符指针 +CR28 (TR 4) 不使用 +CR29 (TR 5) 不使用 +CR30 (TR 6) 当前 / 0 +CR31 (TR 7) 临时寄存器，在不同地方使用 +============================ ================================= + +空间寄存器（内核模式） +---------------------- + +======== ============================== +SR0 临时空间寄存器 +SR4-SR7 设置为0 +SR1 临时空间寄存器 +SR2 内核不应该破坏它 +SR3 用于用户空间访问（当前进程） +======== ============================== + +空间寄存器（用户模式） +---------------------- + +======== ============================ +SR0 临时空间寄存器 +SR1 临时空间寄存器 +SR2 保存Linux gateway page的空间 +SR3 在内核中保存用户地址空间的值 +SR4-SR7 定义了用户/内核的短地址空间 +======== ============================ + + +处理器状态字 +------------ + +====================== ================================================ +W （64位地址） 0 +E （小尾端） 0 +S （安全间隔计时器） 0 +T （产生分支陷阱） 0 +H （高特权级陷阱） 0 +L （低特权级陷阱） 0 +N （撤销下一条指令） 被C代码使用 +X （数据存储中断禁用） 0 +B （产生分支） 被C代码使用 +C （代码地址转译） 1, 在执行实模式代码时为0 +V （除法步长校正） 被C代码使用 +M （HPMC 掩码） 0, 在执行HPMC操作*时为1 +C/B （进/借 位） 被C代码使用 +O （有序引用） 1* +F （性能监视器） 0 +R （回收计数器陷阱） 0 +Q （收集中断状态） 1 （在rfi之前的代码中为0） +P （保护标识符） 1* +D （数据地址转译） 1, 在执行实模式代码时为0 +I （外部中断掩码） 由cli()/sti()宏使用。 +====================== ================================================ + +“隐形”寄存器（影子寄存器） +--------------------------- + +============= =================== +PSW W 默认值 0 +PSW E 默认值 0 +影子寄存器 被中断处理代码使用 +TOC启用位 1 +============= =================== + +---------------------------------------------------------- + +PA-RISC架构定义了7个寄存器作为“影子寄存器”。这些寄存器在 +RETURN FROM INTERRUPTION AND RESTORE指令中使用，通过消 +除中断处理程序中对一般寄存器（GR）的保存和恢复的需要来减 +少状态保存和恢复时间。影子寄存器是GRs 1, 8, 9, 16, 17, +24和25。 + +------------------------------------------------------------------------- + +寄存器使用说明，最初由John Marvin提供，并由Randolph Chung提供一些补充说明。 + +对于通用寄存器: + +r1,r2,r19-r26,r28,r29 & r31可以在不保存它们的情况下被使用。当然，如果你 +关心它们，在调用另一个程序之前，你也需要保存它们。上面的一些寄存器确实 +有特殊的含义，你应该注意一下: + + r1: + addil指令是硬性规定将其结果放在r1中，所以如果你使用这条指令要 + 注意这点。 + + r2: + 这就是返回指针。一般来说，你不想使用它，因为你需要这个指针来返 + 回给你的调用者。然而，它与这组寄存器组合在一起，因为调用者不能 + 依赖你返回时的值是相同的，也就是说，你可以将r2复制到另一个寄存 + 器，并在作废r2后通过该寄存器返回，这应该不会给调用程序带来问题。 + + r19-r22: + 这些通常被认为是临时寄存器。 + 请注意，在64位中它们是arg7-arg4。 + + r23-r26: + 这些是arg3-arg0，也就是说，如果你不再关心传入的值，你可以使用 + 它们。 + + r28,r29: + 这俩是ret0和ret1。它们是你传入返回值的地方。r28是主返回值。当返回 + 小结构体时，r29也可以用来将数据传回给调用程序。 + + r30: + 栈指针 + + r31: + ble指令将返回指针放在这里。 + + + r3-r18,r27,r30需要被保存和恢复。r3-r18只是一般用途的寄存器。 + r27是数据指针，用来使对全局变量的引用更容易。r30是栈指针。 diff --git a/Documentation/translations/zh_CN/process/8.Conclusion.rst b/Documentation/translations/zh_CN/process/8.Conclusion.rst index 71c3e30efc6f..4707f0101964 100644 --- a/Documentation/translations/zh_CN/process/8.Conclusion.rst +++ b/Documentation/translations/zh_CN/process/8.Conclusion.rst @@ -19,7 +19,7 @@ :ref:`Documentation/translations/zh_CN/process/howto.rst ` 文件是一个重要的起点； :ref:`Documentation/translations/zh_CN/process/submitting-patches.rst ` -和 :ref:`Documentation/transaltions/zh_CN/process/submitting-drivers.rst ` +和 :ref:`Documentation/translations/zh_CN/process/submitting-drivers.rst ` 也是所有内核开发人员都应该阅读的内容。许多内部内核API都是使用kerneldoc机制 记录的；“make htmldocs”或“make pdfdocs”可用于以HTML或PDF格式生成这些文档 （尽管某些发行版提供的tex版本会遇到内部限制，无法正确处理文档）。 diff --git a/Documentation/translations/zh_CN/process/coding-style.rst b/Documentation/translations/zh_CN/process/coding-style.rst index 406d43a02c02..b8c484a84d10 100644 --- a/Documentation/translations/zh_CN/process/coding-style.rst +++ b/Documentation/translations/zh_CN/process/coding-style.rst @@ -61,7 +61,7 @@ Linux 内核代码风格 case 'K': case 'k': mem <<= 10; - /* fall through */ + fallthrough; default: break; } diff --git a/Documentation/usb/ehci.rst b/Documentation/usb/ehci.rst index 31f650e7c1b4..76190501907a 100644 --- a/Documentation/usb/ehci.rst +++ b/Documentation/usb/ehci.rst @@ -1,4 +1,4 @@ -=========== +=========== EHCI driver =========== diff --git a/Documentation/usb/gadget_printer.rst b/Documentation/usb/gadget_printer.rst index 5e5516c69075..e611a6d91093 100644 --- a/Documentation/usb/gadget_printer.rst +++ b/Documentation/usb/gadget_printer.rst @@ -1,4 +1,4 @@ -=============================== +=============================== Linux USB Printer Gadget Driver =============================== diff --git a/Documentation/userspace-api/landlock.rst b/Documentation/userspace-api/landlock.rst index 62c9361a3c7f..f35552ff19ba 100644 --- a/Documentation/userspace-api/landlock.rst +++ b/Documentation/userspace-api/landlock.rst @@ -145,7 +145,8 @@ Bind mounts and OverlayFS Landlock enables to restrict access to file hierarchies, which means that these access rights can be propagated with bind mounts (cf. -:doc:`/filesystems/sharedsubtree`) but not with :doc:`/filesystems/overlayfs`. +Documentation/filesystems/sharedsubtree.rst) but not with +Documentation/filesystems/overlayfs.rst. A bind mount mirrors a source file hierarchy to a destination. The destination hierarchy is then composed of the exact same files, on which Landlock rules can @@ -170,8 +171,8 @@ Inheritance Every new thread resulting from a :manpage:`clone(2)` inherits Landlock domain restrictions from its parent. This is similar to the seccomp inheritance (cf. -:doc:`/userspace-api/seccomp_filter`) or any other LSM dealing with task's -:manpage:`credentials(7)`. For instance, one process's thread may apply +Documentation/userspace-api/seccomp_filter.rst) or any other LSM dealing with +task's :manpage:`credentials(7)`. For instance, one process's thread may apply Landlock rules to itself, but they will not be automatically applied to other sibling threads (unlike POSIX thread credential changes, cf. :manpage:`nptl(7)`). @@ -278,7 +279,7 @@ Memory usage ------------ Kernel memory allocated to create rulesets is accounted and can be restricted -by the :doc:`/admin-guide/cgroup-v1/memory`. +by the Documentation/admin-guide/cgroup-v1/memory.rst. Questions and answers ===================== @@ -303,7 +304,7 @@ issues, especially when untrusted processes can manipulate them (cf. Additional documentation ======================== -* :doc:`/security/landlock` +* Documentation/security/landlock.rst * https://landlock.io .. Links diff --git a/Documentation/userspace-api/media/Makefile b/Documentation/userspace-api/media/Makefile index 81a4a1a53bce..00922aa7efde 100644 --- a/Documentation/userspace-api/media/Makefile +++ b/Documentation/userspace-api/media/Makefile @@ -7,8 +7,8 @@ PARSER = $(srctree)/Documentation/sphinx/parse-headers.pl UAPI = $(srctree)/include/uapi/linux KAPI = $(srctree)/include/linux -FILES = audio.h.rst ca.h.rst dmx.h.rst frontend.h.rst net.h.rst video.h.rst \ - videodev2.h.rst media.h.rst cec.h.rst lirc.h.rst +FILES = ca.h.rst dmx.h.rst frontend.h.rst net.h.rst \ + videodev2.h.rst media.h.rst cec.h.rst lirc.h.rst TARGETS := $(addprefix $(BUILDDIR)/, $(FILES)) @@ -21,9 +21,6 @@ quiet_gen_rst = echo ' PARSE $(patsubst $(srctree)/%,%,$<)'; \ silent_gen_rst = ${gen_rst} -$(BUILDDIR)/audio.h.rst: ${UAPI}/dvb/audio.h ${PARSER} $(SRC_DIR)/audio.h.rst.exceptions - @$($(quiet)gen_rst) - $(BUILDDIR)/ca.h.rst: ${UAPI}/dvb/ca.h ${PARSER} $(SRC_DIR)/ca.h.rst.exceptions @$($(quiet)gen_rst) @@ -36,9 +33,6 @@ $(BUILDDIR)/frontend.h.rst: ${UAPI}/dvb/frontend.h ${PARSER} $(SRC_DIR)/frontend $(BUILDDIR)/net.h.rst: ${UAPI}/dvb/net.h ${PARSER} $(SRC_DIR)/net.h.rst.exceptions @$($(quiet)gen_rst) -$(BUILDDIR)/video.h.rst: ${UAPI}/dvb/video.h ${PARSER} $(SRC_DIR)/video.h.rst.exceptions - @$($(quiet)gen_rst) - $(BUILDDIR)/videodev2.h.rst: ${UAPI}/videodev2.h ${PARSER} $(SRC_DIR)/videodev2.h.rst.exceptions @$($(quiet)gen_rst) diff --git a/Documentation/userspace-api/media/audio.h.rst.exceptions b/Documentation/userspace-api/media/audio.h.rst.exceptions deleted file mode 100644 index cf6620477f73..000000000000 --- a/Documentation/userspace-api/media/audio.h.rst.exceptions +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -# Ignore header name -ignore define _DVBAUDIO_H_ - -# Undocumented audio caps, as this is a deprecated API anyway -ignore define AUDIO_CAP_DTS -ignore define AUDIO_CAP_LPCM -ignore define AUDIO_CAP_MP1 -ignore define AUDIO_CAP_MP2 -ignore define AUDIO_CAP_MP3 -ignore define AUDIO_CAP_AAC -ignore define AUDIO_CAP_OGG -ignore define AUDIO_CAP_SDDS -ignore define AUDIO_CAP_AC3 - -# some typedefs should point to struct/enums -replace typedef audio_mixer_t :c:type:`audio_mixer` -replace typedef audio_status_t :c:type:`audio_status` diff --git a/Documentation/userspace-api/media/drivers/hantro.rst b/Documentation/userspace-api/media/drivers/hantro.rst new file mode 100644 index 000000000000..cd9754b4e005 --- /dev/null +++ b/Documentation/userspace-api/media/drivers/hantro.rst @@ -0,0 +1,19 @@ +.. SPDX-License-Identifier: GPL-2.0 + +Hantro video decoder driver +=========================== + +The Hantro video decoder driver implements the following driver-specific controls: + +``V4L2_CID_HANTRO_HEVC_SLICE_HEADER_SKIP (integer)`` + Specifies to Hantro HEVC video decoder driver the number of data (in bits) to + skip in the slice segment header. + If non-IDR, the bits to be skipped go from syntax element "pic_output_flag" + to before syntax element "slice_temporal_mvp_enabled_flag". + If IDR, the skipped bits are just "pic_output_flag" + (separate_colour_plane_flag is not supported). + +.. note:: + + This control is not yet part of the public kernel API and + it is expected to change. diff --git a/Documentation/userspace-api/media/drivers/index.rst b/Documentation/userspace-api/media/drivers/index.rst index 1a9038f5f9fa..12e3c512d718 100644 --- a/Documentation/userspace-api/media/drivers/index.rst +++ b/Documentation/userspace-api/media/drivers/index.rst @@ -33,6 +33,7 @@ For more details see the file COPYING in the source distribution of Linux. ccs cx2341x-uapi + hantro imx-uapi max2175 meye-uapi diff --git a/Documentation/userspace-api/media/dvb/dmx-fopen.rst b/Documentation/userspace-api/media/dvb/dmx-fopen.rst index 8f0a2b831d4a..50b36eb4371e 100644 --- a/Documentation/userspace-api/media/dvb/dmx-fopen.rst +++ b/Documentation/userspace-api/media/dvb/dmx-fopen.rst @@ -82,7 +82,7 @@ appropriately. :widths: 1 16 - - ``EMFILE`` - - “Too many open files”, i.e. no more filters available. + - "Too many open files", i.e. no more filters available. The generic error codes are described at the :ref:`Generic Error Codes ` chapter. diff --git a/Documentation/userspace-api/media/dvb/dmx-fread.rst b/Documentation/userspace-api/media/dvb/dmx-fread.rst index 78e9daef595a..88c4cddf7c30 100644 --- a/Documentation/userspace-api/media/dvb/dmx-fread.rst +++ b/Documentation/userspace-api/media/dvb/dmx-fread.rst @@ -34,7 +34,7 @@ Description This system call returns filtered data, which might be section or Packetized Elementary Stream (PES) data. The filtered data is transferred from -the driver’s internal circular buffer to ``buf``. The maximum amount of data +the driver's internal circular buffer to ``buf``. The maximum amount of data to be transferred is implied by count. .. note:: diff --git a/Documentation/userspace-api/media/dvb/dmx-set-filter.rst b/Documentation/userspace-api/media/dvb/dmx-set-filter.rst index f43455b7adae..1b8c8071b14f 100644 --- a/Documentation/userspace-api/media/dvb/dmx-set-filter.rst +++ b/Documentation/userspace-api/media/dvb/dmx-set-filter.rst @@ -37,7 +37,7 @@ parameters provided. A timeout may be defined stating number of seconds to wait for a section to be loaded. A value of 0 means that no timeout should be applied. Finally there is a flag field where it is possible to state whether a section should be CRC-checked, whether the filter should -be a ”one-shot” filter, i.e. if the filtering operation should be +be a "one-shot" filter, i.e. if the filtering operation should be stopped after the first section is received, and whether the filtering operation should be started immediately (without waiting for a :ref:`DMX_START` ioctl call). If a filter was previously set-up, this diff --git a/Documentation/userspace-api/media/dvb/headers.rst b/Documentation/userspace-api/media/dvb/headers.rst index 9743ffc35096..88c3eb33a89e 100644 --- a/Documentation/userspace-api/media/dvb/headers.rst +++ b/Documentation/userspace-api/media/dvb/headers.rst @@ -14,10 +14,3 @@ Digital TV uAPI headers .. kernel-include:: $BUILDDIR/ca.h.rst .. kernel-include:: $BUILDDIR/net.h.rst - -Legacy uAPI -*********** - -.. kernel-include:: $BUILDDIR/audio.h.rst - -.. kernel-include:: $BUILDDIR/video.h.rst diff --git a/Documentation/userspace-api/media/dvb/intro.rst b/Documentation/userspace-api/media/dvb/intro.rst index a935f3914e56..6784ae79657c 100644 --- a/Documentation/userspace-api/media/dvb/intro.rst +++ b/Documentation/userspace-api/media/dvb/intro.rst @@ -107,7 +107,7 @@ Audio and video decoder a Systems on a Chip (SoC) integrated circuit. It may also not be needed for certain usages (e.g. for data-only - uses like “internet over satellite”). + uses like "internet over satellite"). :ref:`stb_components` shows a crude schematic of the control and data flow between those components. @@ -148,9 +148,9 @@ individual devices are called: - ``/dev/dvb/adapterN/caM``, -where ``N`` enumerates the Digital TV cards in a system starting from 0, and +where ``N`` enumerates the Digital TV cards in a system starting from 0, and ``M`` enumerates the devices of each type within each adapter, starting -from 0, too. We will omit the “``/dev/dvb/adapterN/``\ ” in the further +from 0, too. We will omit the "``/dev/dvb/adapterN/``\ " in the further discussion of these devices. More details about the data structures and function calls of all the diff --git a/Documentation/userspace-api/media/dvb/legacy_dvb_apis.rst b/Documentation/userspace-api/media/dvb/legacy_dvb_apis.rst index 6104879d728a..b97d56ee543c 100644 --- a/Documentation/userspace-api/media/dvb/legacy_dvb_apis.rst +++ b/Documentation/userspace-api/media/dvb/legacy_dvb_apis.rst @@ -11,11 +11,6 @@ The APIs described here **should not** be used on new drivers or applications. The DVBv3 frontend API has issues with new delivery systems, including DVB-S2, DVB-T2, ISDB, etc. -There's just one driver for a very legacy hardware using the Digital TV -audio and video APIs. No modern drivers should use it. Instead, audio and -video should be using the V4L2 and ALSA APIs, and the pipelines should -be set via the Media Controller API. - .. attention:: The APIs described here doesn't necessarily reflect the current @@ -28,5 +23,3 @@ be set via the Media Controller API. :maxdepth: 1 frontend_legacy_dvbv3_api - video - audio diff --git a/Documentation/userspace-api/media/fdl-appendix.rst b/Documentation/userspace-api/media/fdl-appendix.rst index 683ebed87017..b1bc725b4ec7 100644 --- a/Documentation/userspace-api/media/fdl-appendix.rst +++ b/Documentation/userspace-api/media/fdl-appendix.rst @@ -13,14 +13,14 @@ GNU Free Documentation License =========== The purpose of this License is to make a manual, textbook, or other -written document “free” in the sense of freedom: to assure everyone the +written document "free" in the sense of freedom: to assure everyone the effective freedom to copy and redistribute it, with or without modifying it, either commercially or noncommercially. Secondarily, this License preserves for the author and publisher a way to get credit for their work, while not being considered responsible for modifications made by others. -This License is a kind of “copyleft”, which means that derivative works +This License is a kind of "copyleft", which means that derivative works of the document must themselves be free in the same sense. It complements the GNU General Public License, which is a copyleft license designed for free software. @@ -44,21 +44,21 @@ works whose purpose is instruction or reference. This License applies to any manual or other work that contains a notice placed by the copyright holder saying it can be distributed under the -terms of this License. The “Document”, below, refers to any such manual +terms of this License. The "Document", below, refers to any such manual or work. Any member of the public is a licensee, and is addressed as -“you”. +"you". .. _fdl-modified: -A “Modified Version” of the Document means any work containing the +A "Modified Version" of the Document means any work containing the Document or a portion of it, either copied verbatim, or with modifications and/or translated into another language. .. _fdl-secondary: -A “Secondary Section” is a named appendix or a front-matter section of +A "Secondary Section" is a named appendix or a front-matter section of the :ref:`Document ` that deals exclusively with the relationship of the publishers or authors of the Document to the Document's overall subject (or to related matters) and contains nothing @@ -72,7 +72,7 @@ regarding them. .. _fdl-invariant: -The “Invariant Sections” are certain +The "Invariant Sections" are certain :ref:`Secondary Sections ` whose titles are designated, as being those of Invariant Sections, in the notice that says that the :ref:`Document ` is released under this License. @@ -80,14 +80,14 @@ as being those of Invariant Sections, in the notice that says that the .. _fdl-cover-texts: -The “Cover Texts” are certain short passages of text that are listed, as +The "Cover Texts" are certain short passages of text that are listed, as Front-Cover Texts or Back-Cover Texts, in the notice that says that the :ref:`Document ` is released under this License. .. _fdl-transparent: -A “Transparent” copy of the :ref:`Document ` means a +A "Transparent" copy of the :ref:`Document ` means a machine-readable copy, represented in a format whose specification is available to the general public, whose contents can be viewed and edited directly and straightforwardly with generic text editors or (for images @@ -97,7 +97,7 @@ formatters or for automatic translation to a variety of formats suitable for input to text formatters. A copy made in an otherwise Transparent file format whose markup has been designed to thwart or discourage subsequent modification by readers is not Transparent. A copy that is -not “Transparent” is called “Opaque”. +not "Transparent" is called "Opaque". Examples of suitable formats for Transparent copies include plain ASCII without markup, Texinfo input format, LaTeX input format, SGML or XML @@ -111,10 +111,10 @@ word processors for output purposes only. .. _fdl-title-page: -The “Title Page” means, for a printed book, the title page itself, plus +The "Title Page" means, for a printed book, the title page itself, plus such following pages as are needed to hold, legibly, the material this License requires to appear in the title page. For works in formats which -do not have any title page as such, “Title Page” means the text near the +do not have any title page as such, "Title Page" means the text near the most prominent appearance of the work's title, preceding the beginning of the body of the text. @@ -242,11 +242,11 @@ Modified Version: Include an unaltered copy of this License. - **I.** - Preserve the section entitled “History”, and its title, and add to it + Preserve the section entitled "History", and its title, and add to it an item stating at least the title, year, new authors, and publisher of the :ref:`Modified Version ` as given on the :ref:`Title Page `. If there is no section entitled - “History” in the :ref:`Document `, create one stating + "History" in the :ref:`Document `, create one stating the title, year, authors, and publisher of the Document as given on its Title Page, then add an item describing the Modified Version as stated in the previous sentence. @@ -256,13 +256,13 @@ Modified Version: :ref:`Document ` for public access to a :ref:`Transparent ` copy of the Document, and likewise the network locations given in the Document for previous - versions it was based on. These may be placed in the “History” + versions it was based on. These may be placed in the "History" section. You may omit a network location for a work that was published at least four years before the Document itself, or if the original publisher of the version it refers to gives permission. - **K.** - In any section entitled “Acknowledgements” or “Dedications”, preserve + In any section entitled "Acknowledgements" or "Dedications", preserve the section's title, and preserve in the section all the substance and tone of each of the contributor acknowledgements and/or dedications given therein. @@ -274,11 +274,11 @@ Modified Version: part of the section titles. - **M.** - Delete any section entitled “Endorsements”. Such a section may not be + Delete any section entitled "Endorsements". Such a section may not be included in the :ref:`Modified Version `. - **N.** - Do not retitle any existing section as “Endorsements” or to conflict + Do not retitle any existing section as "Endorsements" or to conflict in title with any :ref:`Invariant Section `. If the :ref:`Modified Version ` includes new @@ -290,7 +290,7 @@ of :ref:`Invariant Sections ` in the Modified Version's license notice. These titles must be distinct from any other section titles. -You may add a section entitled “Endorsements”, provided it contains +You may add a section entitled "Endorsements", provided it contains nothing but endorsements of your :ref:`Modified Version ` by various parties--for example, statements of peer review or that the text has been approved by @@ -337,11 +337,11 @@ the original author or publisher of that section if known, or else a unique number. Make the same adjustment to the section titles in the list of Invariant Sections in the license notice of the combined work. -In the combination, you must combine any sections entitled “History” in -the various original documents, forming one section entitled “History”; -likewise combine any sections entitled “Acknowledgements”, and any -sections entitled “Dedications”. You must delete all sections entitled -“Endorsements.” +In the combination, you must combine any sections entitled "History" in +the various original documents, forming one section entitled "History"; +likewise combine any sections entitled "Acknowledgements", and any +sections entitled "Dedications". You must delete all sections entitled +"Endorsements." .. _fdl-section6: @@ -372,7 +372,7 @@ with other separate and independent documents or works, in or on a volume of a storage or distribution medium, does not as a whole count as a :ref:`Modified Version ` of the Document, provided no compilation copyright is claimed for the compilation. Such a compilation -is called an “aggregate”, and this License does not apply to the other +is called an "aggregate", and this License does not apply to the other self-contained works thus compiled with the Document , on account of their being thus compiled, if they are not themselves derivative works of the Document. If the :ref:`Cover Text ` @@ -429,7 +429,7 @@ concerns. See Each version of the License is given a distinguishing version number. If the :ref:`Document ` specifies that a particular -numbered version of this License “or any later version” applies to it, +numbered version of this License "or any later version" applies to it, you have the option of following the terms and conditions either of that specified version or of any later version that has been published (not as a draft) by the Free Software Foundation. If the Document does not @@ -455,13 +455,13 @@ notices just after the title page: being LIST THEIR TITLES, with the :ref:`Front-Cover Texts ` being LIST, and with the :ref:`Back-Cover Texts ` being LIST. A copy - of the license is included in the section entitled “GNU Free - Documentation License”. + of the license is included in the section entitled "GNU Free + Documentation License". -If you have no :ref:`Invariant Sections `, write “with -no Invariant Sections” instead of saying which ones are invariant. If -you have no :ref:`Front-Cover Texts `, write “no -Front-Cover Texts” instead of “Front-Cover Texts being LIST”; likewise +If you have no :ref:`Invariant Sections `, write "with +no Invariant Sections" instead of saying which ones are invariant. If +you have no :ref:`Front-Cover Texts `, write "no +Front-Cover Texts" instead of "Front-Cover Texts being LIST"; likewise for :ref:`Back-Cover Texts `. If your document contains nontrivial examples of program code, we diff --git a/Documentation/userspace-api/media/glossary.rst b/Documentation/userspace-api/media/glossary.rst index cb165d7176b7..96a360edbf3b 100644 --- a/Documentation/userspace-api/media/glossary.rst +++ b/Documentation/userspace-api/media/glossary.rst @@ -116,7 +116,7 @@ Glossary - :term:`RC API`; and - :term:`V4L2 API`. - See :doc:`index`. + See Documentation/userspace-api/media/index.rst. MC API **Media Controller API** diff --git a/Documentation/userspace-api/media/index.rst b/Documentation/userspace-api/media/index.rst index 7f42f83b9f59..d839904be085 100644 --- a/Documentation/userspace-api/media/index.rst +++ b/Documentation/userspace-api/media/index.rst @@ -11,12 +11,14 @@ used by media devices. Please see: -- :doc:`/admin-guide/media/index` - for usage information about media subsystem and supported drivers; +Documentation/admin-guide/media/index.rst -- :doc:`/driver-api/media/index` - for driver development information and Kernel APIs used by - media devices; + - for usage information about media subsystem and supported drivers; + +Documentation/driver-api/media/index.rst + + - for driver development information and Kernel APIs used by + media devices; .. only:: html diff --git a/Documentation/userspace-api/media/v4l/biblio.rst b/Documentation/userspace-api/media/v4l/biblio.rst index 64d241daf63c..7b8e6738ff9e 100644 --- a/Documentation/userspace-api/media/v4l/biblio.rst +++ b/Documentation/userspace-api/media/v4l/biblio.rst @@ -51,7 +51,7 @@ ISO 13818-1 =========== -:title: ITU-T Rec. H.222.0 | ISO/IEC 13818-1 "Information technology — Generic coding of moving pictures and associated audio information: Systems" +:title: ITU-T Rec. H.222.0 | ISO/IEC 13818-1 "Information technology --- Generic coding of moving pictures and associated audio information: Systems" :author: International Telecommunication Union (http://www.itu.ch), International Organisation for Standardisation (http://www.iso.ch) @@ -61,7 +61,7 @@ ISO 13818-2 =========== -:title: ITU-T Rec. H.262 | ISO/IEC 13818-2 "Information technology — Generic coding of moving pictures and associated audio information: Video" +:title: ITU-T Rec. H.262 | ISO/IEC 13818-2 "Information technology --- Generic coding of moving pictures and associated audio information: Video" :author: International Telecommunication Union (http://www.itu.ch), International Organisation for Standardisation (http://www.iso.ch) @@ -150,7 +150,7 @@ ITU-T.81 ======== -:title: ITU-T Recommendation T.81 "Information Technology — Digital Compression and Coding of Continous-Tone Still Images — Requirements and Guidelines" +:title: ITU-T Recommendation T.81 "Information Technology --- Digital Compression and Coding of Continous-Tone Still Images --- Requirements and Guidelines" :author: International Telecommunication Union (http://www.itu.int) @@ -310,7 +310,7 @@ ISO 12232:2006 ============== -:title: Photography — Digital still cameras — Determination of exposure index, ISO speed ratings, standard output sensitivity, and recommended exposure index +:title: Photography --- Digital still cameras --- Determination of exposure index, ISO speed ratings, standard output sensitivity, and recommended exposure index :author: International Organization for Standardization (http://www.iso.org) diff --git a/Documentation/userspace-api/media/v4l/dev-decoder.rst b/Documentation/userspace-api/media/v4l/dev-decoder.rst index 3d4138a4ba69..5b9b83feeceb 100644 --- a/Documentation/userspace-api/media/v4l/dev-decoder.rst +++ b/Documentation/userspace-api/media/v4l/dev-decoder.rst @@ -38,7 +38,7 @@ Conventions and Notations Used in This Document 6. i = [a..b]: sequence of integers from a to b, inclusive, i.e. i = [0..2]: i = 0, 1, 2. -7. Given an ``OUTPUT`` buffer A, then A’ represents a buffer on the ``CAPTURE`` +7. Given an ``OUTPUT`` buffer A, then A' represents a buffer on the ``CAPTURE`` queue containing data that resulted from processing buffer A. .. _decoder-glossary: @@ -288,7 +288,7 @@ Initialization Changing the ``OUTPUT`` format may change the currently set ``CAPTURE`` format. How the new ``CAPTURE`` format is determined is up to the decoder - and the client must ensure it matches its needs afterwards. + and the client must ensure it matches its needs afterwards. 2. Allocate source (bytestream) buffers via :c:func:`VIDIOC_REQBUFS` on ``OUTPUT``. @@ -874,7 +874,7 @@ it may be affected as per normal decoder operation. any of the following results on the ``CAPTURE`` queue is allowed: - {A’, B’, G’, H’}, {A’, G’, H’}, {G’, H’}. + {A', B', G', H'}, {A', G', H'}, {G', H'}. To determine the CAPTURE buffer containing the first decoded frame after the seek, the client may observe the timestamps to match the CAPTURE and OUTPUT diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst index 3fc04daa9ffb..72f5e85b4f34 100644 --- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec-stateless.rst @@ -1244,3 +1244,217 @@ FWHT Flags * - __u8 - ``padding[3]`` - Applications and drivers must set this to zero. + +.. _v4l2-codec-stateless-mpeg2: + +``V4L2_CID_STATELESS_MPEG2_SEQUENCE (struct)`` + Specifies the sequence parameters (as extracted from the bitstream) for the + associated MPEG-2 slice data. This includes fields matching the syntax + elements from the sequence header and sequence extension parts of the + bitstream as specified by :ref:`mpeg2part2`. + +.. c:type:: v4l2_ctrl_mpeg2_sequence + +.. raw:: latex + + \small + +.. cssclass:: longtable + +.. tabularcolumns:: |p{1.4cm}|p{6.5cm}|p{9.4cm}| + +.. flat-table:: struct v4l2_ctrl_mpeg2_sequence + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u16 + - ``horizontal_size`` + - The width of the displayable part of the frame's luminance component. + * - __u16 + - ``vertical_size`` + - The height of the displayable part of the frame's luminance component. + * - __u32 + - ``vbv_buffer_size`` + - Used to calculate the required size of the video buffering verifier, + defined (in bits) as: 16 * 1024 * vbv_buffer_size. + * - __u16 + - ``profile_and_level_indication`` + - The current profile and level indication as extracted from the + bitstream. + * - __u8 + - ``chroma_format`` + - The chrominance sub-sampling format (1: 4:2:0, 2: 4:2:2, 3: 4:4:4). + * - __u8 + - ``flags`` + - See :ref:`MPEG-2 Sequence Flags `. + +.. _mpeg2_sequence_flags: + +``MPEG-2 Sequence Flags`` + +.. cssclass:: longtable + +.. flat-table:: + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - ``V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE`` + - 0x01 + - Indication that all the frames for the sequence are progressive instead + of interlaced. + +.. raw:: latex + + \normalsize + +``V4L2_CID_STATELESS_MPEG2_PICTURE (struct)`` + Specifies the picture parameters (as extracted from the bitstream) for the + associated MPEG-2 slice data. This includes fields matching the syntax + elements from the picture header and picture coding extension parts of the + bitstream as specified by :ref:`mpeg2part2`. + +.. c:type:: v4l2_ctrl_mpeg2_picture + +.. raw:: latex + + \small + +.. cssclass:: longtable + +.. tabularcolumns:: |p{1.0cm}|p{5.6cm}|p{10.7cm}| + +.. flat-table:: struct v4l2_ctrl_mpeg2_picture + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u64 + - ``backward_ref_ts`` + - Timestamp of the V4L2 capture buffer to use as backward reference, used + with B-coded and P-coded frames. The timestamp refers to the + ``timestamp`` field in struct :c:type:`v4l2_buffer`. Use the + :c:func:`v4l2_timeval_to_ns()` function to convert the struct + :c:type:`timeval` in struct :c:type:`v4l2_buffer` to a __u64. + * - __u64 + - ``forward_ref_ts`` + - Timestamp for the V4L2 capture buffer to use as forward reference, used + with B-coded frames. The timestamp refers to the ``timestamp`` field in + struct :c:type:`v4l2_buffer`. Use the :c:func:`v4l2_timeval_to_ns()` + function to convert the struct :c:type:`timeval` in struct + :c:type:`v4l2_buffer` to a __u64. + * - __u32 + - ``flags`` + - See :ref:`MPEG-2 Picture Flags `. + * - __u8 + - ``f_code[2][2]`` + - Motion vector codes. + * - __u8 + - ``picture_coding_type`` + - Picture coding type for the frame covered by the current slice + (V4L2_MPEG2_PIC_CODING_TYPE_I, V4L2_MPEG2_PIC_CODING_TYPE_P or + V4L2_MPEG2_PIC_CODING_TYPE_B). + * - __u8 + - ``picture_structure`` + - Picture structure (1: interlaced top field, 2: interlaced bottom field, + 3: progressive frame). + * - __u8 + - ``intra_dc_precision`` + - Precision of Discrete Cosine transform (0: 8 bits precision, + 1: 9 bits precision, 2: 10 bits precision, 3: 11 bits precision). + * - __u8 + - ``reserved[5]`` + - Applications and drivers must set this to zero. + +.. _mpeg2_picture_flags: + +``MPEG-2 Picture Flags`` + +.. cssclass:: longtable + +.. flat-table:: + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - ``V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST`` + - 0x00000001 + - If set and it's an interlaced stream, top field is output first. + * - ``V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT`` + - 0x00000002 + - If set only frame-DCT and frame prediction are used. + * - ``V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV`` + - 0x00000004 + - If set motion vectors are coded for intra macroblocks. + * - ``V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE`` + - 0x00000008 + - This flag affects the inverse quantization process. + * - ``V4L2_MPEG2_PIC_FLAG_INTRA_VLC`` + - 0x00000010 + - This flag affects the decoding of transform coefficient data. + * - ``V4L2_MPEG2_PIC_FLAG_ALT_SCAN`` + - 0x00000020 + - This flag affects the decoding of transform coefficient data. + * - ``V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST`` + - 0x00000040 + - This flag affects the decoding process of progressive frames. + * - ``V4L2_MPEG2_PIC_FLAG_PROGRESSIVE`` + - 0x00000080 + - Indicates whether the current frame is progressive. + +.. raw:: latex + + \normalsize + +``V4L2_CID_STATELESS_MPEG2_QUANTISATION (struct)`` + Specifies quantisation matrices, in zigzag scanning order, for the + associated MPEG-2 slice data. This control is initialized by the kernel + to the matrices default values. If a bitstream transmits a user-defined + quantisation matrices load, applications are expected to use this control. + Applications are also expected to set the control loading the default + values, if the quantisation matrices need to be reset, for instance on a + sequence header. This process is specified by section 6.3.7. + "Quant matrix extension" of the specification. + +.. c:type:: v4l2_ctrl_mpeg2_quantisation + +.. tabularcolumns:: |p{0.8cm}|p{8.0cm}|p{8.5cm}| + +.. cssclass:: longtable + +.. raw:: latex + + \small + +.. flat-table:: struct v4l2_ctrl_mpeg2_quantisation + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __u8 + - ``intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for intra-coded frames, in zigzag + scanning order. It is relevant for both luma and chroma components, + although it can be superseded by the chroma-specific matrix for + non-4:2:0 YUV formats. + * - __u8 + - ``non_intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for non-intra-coded frames, in + zigzag scanning order. It is relevant for both luma and chroma + components, although it can be superseded by the chroma-specific matrix + for non-4:2:0 YUV formats. + * - __u8 + - ``chroma_intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for the chominance component of + intra-coded frames, in zigzag scanning order. Only relevant for + non-4:2:0 YUV formats. + * - __u8 + - ``chroma_non_intra_quantiser_matrix[64]`` + - The quantisation matrix coefficients for the chrominance component of + non-intra-coded frames, in zigzag scanning order. Only relevant for + non-4:2:0 YUV formats. + +.. raw:: latex + + \normalsize diff --git a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst index b0de4e6e7ebd..8c6e2a11ed95 100644 --- a/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst +++ b/Documentation/userspace-api/media/v4l/ext-ctrls-codec.rst @@ -1606,223 +1606,6 @@ enum v4l2_mpeg_video_h264_hierarchical_coding_type - ``V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L6_BR (integer)`` Indicates bit rate (bps) for hierarchical coding layer 6 for H264 encoder. -.. _v4l2-mpeg-mpeg2: - -``V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS (struct)`` - Specifies the slice parameters (as extracted from the bitstream) for the - associated MPEG-2 slice data. This includes the necessary parameters for - configuring a stateless hardware decoding pipeline for MPEG-2. - The bitstream parameters are defined according to :ref:`mpeg2part2`. - - .. note:: - - This compound control is not yet part of the public kernel API and - it is expected to change. - -.. c:type:: v4l2_ctrl_mpeg2_slice_params - -.. tabularcolumns:: |p{5.6cm}|p{4.6cm}|p{7.1cm}| - -.. cssclass:: longtable - -.. flat-table:: struct v4l2_ctrl_mpeg2_slice_params - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - __u32 - - ``bit_size`` - - Size (in bits) of the current slice data. - * - __u32 - - ``data_bit_offset`` - - Offset (in bits) to the video data in the current slice data. - * - struct :c:type:`v4l2_mpeg2_sequence` - - ``sequence`` - - Structure with MPEG-2 sequence metadata, merging relevant fields from - the sequence header and sequence extension parts of the bitstream. - * - struct :c:type:`v4l2_mpeg2_picture` - - ``picture`` - - Structure with MPEG-2 picture metadata, merging relevant fields from - the picture header and picture coding extension parts of the bitstream. - * - __u64 - - ``backward_ref_ts`` - - Timestamp of the V4L2 capture buffer to use as backward reference, used - with B-coded and P-coded frames. The timestamp refers to the - ``timestamp`` field in struct :c:type:`v4l2_buffer`. Use the - :c:func:`v4l2_timeval_to_ns()` function to convert the struct - :c:type:`timeval` in struct :c:type:`v4l2_buffer` to a __u64. - * - __u64 - - ``forward_ref_ts`` - - Timestamp for the V4L2 capture buffer to use as forward reference, used - with B-coded frames. The timestamp refers to the ``timestamp`` field in - struct :c:type:`v4l2_buffer`. Use the :c:func:`v4l2_timeval_to_ns()` - function to convert the struct :c:type:`timeval` in struct - :c:type:`v4l2_buffer` to a __u64. - * - __u32 - - ``quantiser_scale_code`` - - Code used to determine the quantization scale to use for the IDCT. - -.. c:type:: v4l2_mpeg2_sequence - -.. cssclass:: longtable - -.. tabularcolumns:: |p{1.4cm}|p{6.5cm}|p{9.4cm}| - -.. flat-table:: struct v4l2_mpeg2_sequence - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - __u16 - - ``horizontal_size`` - - The width of the displayable part of the frame's luminance component. - * - __u16 - - ``vertical_size`` - - The height of the displayable part of the frame's luminance component. - * - __u32 - - ``vbv_buffer_size`` - - Used to calculate the required size of the video buffering verifier, - defined (in bits) as: 16 * 1024 * vbv_buffer_size. - * - __u16 - - ``profile_and_level_indication`` - - The current profile and level indication as extracted from the - bitstream. - * - __u8 - - ``progressive_sequence`` - - Indication that all the frames for the sequence are progressive instead - of interlaced. - * - __u8 - - ``chroma_format`` - - The chrominance sub-sampling format (1: 4:2:0, 2: 4:2:2, 3: 4:4:4). - -.. c:type:: v4l2_mpeg2_picture - -.. raw:: latex - - \small - -.. cssclass:: longtable - -.. tabularcolumns:: |p{1.0cm}|p{5.6cm}|p{10.7cm}| - -.. flat-table:: struct v4l2_mpeg2_picture - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - __u8 - - ``picture_coding_type`` - - Picture coding type for the frame covered by the current slice - (V4L2_MPEG2_PICTURE_CODING_TYPE_I, V4L2_MPEG2_PICTURE_CODING_TYPE_P or - V4L2_MPEG2_PICTURE_CODING_TYPE_B). - * - __u8 - - ``f_code[2][2]`` - - Motion vector codes. - * - __u8 - - ``intra_dc_precision`` - - Precision of Discrete Cosine transform (0: 8 bits precision, - 1: 9 bits precision, 2: 10 bits precision, 3: 11 bits precision). - * - __u8 - - ``picture_structure`` - - Picture structure (1: interlaced top field, 2: interlaced bottom field, - 3: progressive frame). - * - __u8 - - ``top_field_first`` - - If set to 1 and interlaced stream, top field is output first. - * - __u8 - - ``frame_pred_frame_dct`` - - If set to 1, only frame-DCT and frame prediction are used. - * - __u8 - - ``concealment_motion_vectors`` - - If set to 1, motion vectors are coded for intra macroblocks. - * - __u8 - - ``q_scale_type`` - - This flag affects the inverse quantization process. - * - __u8 - - ``intra_vlc_format`` - - This flag affects the decoding of transform coefficient data. - * - __u8 - - ``alternate_scan`` - - This flag affects the decoding of transform coefficient data. - * - __u8 - - ``repeat_first_field`` - - This flag affects the decoding process of progressive frames. - * - __u16 - - ``progressive_frame`` - - Indicates whether the current frame is progressive. - -.. raw:: latex - - \normalsize - -``V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION (struct)`` - Specifies quantization matrices (as extracted from the bitstream) for the - associated MPEG-2 slice data. - - .. note:: - - This compound control is not yet part of the public kernel API and - it is expected to change. - -.. c:type:: v4l2_ctrl_mpeg2_quantization - -.. tabularcolumns:: |p{0.8cm}|p{8.0cm}|p{8.5cm}| - -.. cssclass:: longtable - -.. raw:: latex - - \small - -.. flat-table:: struct v4l2_ctrl_mpeg2_quantization - :header-rows: 0 - :stub-columns: 0 - :widths: 1 1 2 - - * - __u8 - - ``load_intra_quantiser_matrix`` - - One bit to indicate whether to load the ``intra_quantiser_matrix`` data. - * - __u8 - - ``load_non_intra_quantiser_matrix`` - - One bit to indicate whether to load the ``non_intra_quantiser_matrix`` - data. - * - __u8 - - ``load_chroma_intra_quantiser_matrix`` - - One bit to indicate whether to load the - ``chroma_intra_quantiser_matrix`` data, only relevant for non-4:2:0 YUV - formats. - * - __u8 - - ``load_chroma_non_intra_quantiser_matrix`` - - One bit to indicate whether to load the - ``chroma_non_intra_quantiser_matrix`` data, only relevant for non-4:2:0 - YUV formats. - * - __u8 - - ``intra_quantiser_matrix[64]`` - - The quantization matrix coefficients for intra-coded frames, in zigzag - scanning order. It is relevant for both luma and chroma components, - although it can be superseded by the chroma-specific matrix for - non-4:2:0 YUV formats. - * - __u8 - - ``non_intra_quantiser_matrix[64]`` - - The quantization matrix coefficients for non-intra-coded frames, in - zigzag scanning order. It is relevant for both luma and chroma - components, although it can be superseded by the chroma-specific matrix - for non-4:2:0 YUV formats. - * - __u8 - - ``chroma_intra_quantiser_matrix[64]`` - - The quantization matrix coefficients for the chominance component of - intra-coded frames, in zigzag scanning order. Only relevant for - non-4:2:0 YUV formats. - * - __u8 - - ``chroma_non_intra_quantiser_matrix[64]`` - - The quantization matrix coefficients for the chrominance component of - non-intra-coded frames, in zigzag scanning order. Only relevant for - non-4:2:0 YUV formats. - -.. raw:: latex - - \normalsize - ``V4L2_CID_FWHT_I_FRAME_QP (integer)`` Quantization parameter for an I frame for FWHT. Valid range: from 1 to 31. @@ -2924,6 +2707,9 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - * - __u8 - ``chroma_format_idc`` - + * - __u8 + - ``sps_max_sub_layers_minus1`` + - * - __u64 - ``flags`` - See :ref:`Sequence Parameter Set Flags ` @@ -3000,6 +2786,12 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - * - __u8 - ``num_extra_slice_header_bits`` - + * - __u8 + - ``num_ref_idx_l0_default_active_minus1`` + - Specifies the inferred value of num_ref_idx_l0_active_minus1 + * - __u8 + - ``num_ref_idx_l1_default_active_minus1`` + - Specifies the inferred value of num_ref_idx_l1_active_minus1 * - __s8 - ``init_qp_minus26`` - @@ -3053,7 +2845,7 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - :stub-columns: 0 :widths: 1 1 2 - * - ``V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT`` + * - ``V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT_ENABLED`` - 0x00000001 - * - ``V4L2_HEVC_PPS_FLAG_OUTPUT_FLAG_PRESENT`` @@ -3110,6 +2902,14 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - * - ``V4L2_HEVC_PPS_FLAG_SLICE_SEGMENT_HEADER_EXTENSION_PRESENT`` - 0x00040000 - + * - ``V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT`` + - 0x00080000 + - Specifies the presence of deblocking filter control syntax elements in + the PPS + * - ``V4L2_HEVC_PPS_FLAG_UNIFORM_SPACING`` + - 0x00100000 + - Specifies that tile column boundaries and likewise tile row boundaries + are distributed uniformly across the picture .. raw:: latex @@ -3200,9 +3000,6 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - * - __u8 - ``pic_struct`` - - * - __u8 - - ``num_active_dpb_entries`` - - The number of entries in ``dpb``. * - __u8 - ``ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` - The list of L0 reference elements as indices in the DPB. @@ -3210,22 +3007,8 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - - ``ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` - The list of L1 reference elements as indices in the DPB. * - __u8 - - ``num_rps_poc_st_curr_before`` - - The number of reference pictures in the short-term set that come before - the current frame. - * - __u8 - - ``num_rps_poc_st_curr_after`` - - The number of reference pictures in the short-term set that come after - the current frame. - * - __u8 - - ``num_rps_poc_lt_curr`` - - The number of reference pictures in the long-term set. - * - __u8 - - ``padding[7]`` + - ``padding`` - Applications and drivers must set this to zero. - * - struct :c:type:`v4l2_hevc_dpb_entry` - - ``dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` - - The decoded picture buffer, for meta-data about reference frames. * - struct :c:type:`v4l2_hevc_pred_weight_table` - ``pred_weight_table`` - The prediction weight coefficients for inter-picture prediction. @@ -3277,6 +3060,9 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - * - ``V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED`` - 0x00000100 - + * - ``V4L2_HEVC_SLICE_PARAMS_FLAG_DEPENDENT_SLICE_SEGMENT`` + - 0x00000200 + - .. raw:: latex @@ -3478,3 +3264,78 @@ enum v4l2_mpeg_video_hevc_size_of_length_field - encoding the next frame queued after setting this control. This provides a bitmask which consists of bits [0, LTR_COUNT-1]. This is applicable to the H264 and HEVC encoders. + +``V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS (struct)`` + Specifies various decode parameters, especially the references picture order + count (POC) for all the lists (short, long, before, current, after) and the + number of entries for each of them. + These parameters are defined according to :ref:`hevc`. + They are described in section 8.3 "Slice decoding process" of the + specification. + +.. c:type:: v4l2_ctrl_hevc_decode_params + +.. cssclass:: longtable + +.. flat-table:: struct v4l2_ctrl_hevc_decode_params + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - __s32 + - ``pic_order_cnt_val`` + - PicOrderCntVal as described in section 8.3.1 "Decoding process + for picture order count" of the specification. + * - __u8 + - ``num_active_dpb_entries`` + - The number of entries in ``dpb``. + * - struct :c:type:`v4l2_hevc_dpb_entry` + - ``dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - The decoded picture buffer, for meta-data about reference frames. + * - __u8 + - ``num_poc_st_curr_before`` + - The number of reference pictures in the short-term set that come before + the current frame. + * - __u8 + - ``num_poc_st_curr_after`` + - The number of reference pictures in the short-term set that come after + the current frame. + * - __u8 + - ``num_poc_lt_curr`` + - The number of reference pictures in the long-term set. + * - __u8 + - ``poc_st_curr_before[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - PocStCurrBefore as described in section 8.3.2 "Decoding process for reference + picture set. + * - __u8 + - ``poc_st_curr_after[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - PocStCurrAfter as described in section 8.3.2 "Decoding process for reference + picture set. + * - __u8 + - ``poc_lt_curr[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]`` + - PocLtCurr as described in section 8.3.2 "Decoding process for reference + picture set. + * - __u64 + - ``flags`` + - See :ref:`Decode Parameters Flags ` + +.. _hevc_decode_params_flags: + +``Decode Parameters Flags`` + +.. cssclass:: longtable + +.. flat-table:: + :header-rows: 0 + :stub-columns: 0 + :widths: 1 1 2 + + * - ``V4L2_HEVC_DECODE_PARAM_FLAG_IRAP_PIC`` + - 0x00000001 + - + * - ``V4L2_HEVC_DECODE_PARAM_FLAG_IDR_PIC`` + - 0x00000002 + - + * - ``V4L2_HEVC_DECODE_PARAM_FLAG_NO_OUTPUT_OF_PRIOR`` + - 0x00000004 + - diff --git a/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst b/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst index 6dba70da822b..0ede39907ee2 100644 --- a/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst +++ b/Documentation/userspace-api/media/v4l/pixfmt-compressed.rst @@ -112,12 +112,13 @@ Compressed Formats - 'MG2S' - MPEG-2 parsed slice data, as extracted from the MPEG-2 bitstream. This format is adapted for stateless video decoders that implement a - MPEG-2 pipeline (using the :ref:`mem2mem` and :ref:`media-request-api`). + MPEG-2 pipeline with the :ref:`stateless_decoder`. Metadata associated with the frame to decode is required to be passed - through the ``V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS`` control and - quantization matrices can optionally be specified through the - ``V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION`` control. - See the :ref:`associated Codec Control IDs `. + through the ``V4L2_CID_STATELESS_MPEG2_SEQUENCE`` and + ``V4L2_CID_STATELESS_MPEG2_PICTURE`` controls. + Quantisation matrices can optionally be specified through the + ``V4L2_CID_STATELESS_MPEG2_QUANTISATION`` control. + See the :ref:`associated Codec Control IDs `. Exactly one output and one capture buffer must be provided for use with this pixel format. The output buffer must contain the appropriate number of macroblocks to decode a full corresponding frame to the matching diff --git a/Documentation/userspace-api/media/v4l/pixfmt-meta-intel-ipu3.rst b/Documentation/userspace-api/media/v4l/pixfmt-meta-intel-ipu3.rst index 5f33d35532ef..84d81dd7a7b5 100644 --- a/Documentation/userspace-api/media/v4l/pixfmt-meta-intel-ipu3.rst +++ b/Documentation/userspace-api/media/v4l/pixfmt-meta-intel-ipu3.rst @@ -78,4 +78,4 @@ hardware and algorithm details. Intel IPU3 ImgU uAPI data types =============================== -.. kernel-doc:: drivers/staging/media/ipu3/include/intel-ipu3.h +.. kernel-doc:: drivers/staging/media/ipu3/include/uapi/intel-ipu3.h diff --git a/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst b/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst index 3ba22983d21f..2d6bc8d94380 100644 --- a/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst +++ b/Documentation/userspace-api/media/v4l/vidioc-g-ext-ctrls.rst @@ -221,6 +221,18 @@ still cause this situation. - ``p_vp8_frame`` - A pointer to a struct :c:type:`v4l2_ctrl_vp8_frame`. Valid if this control is of type ``V4L2_CTRL_TYPE_VP8_FRAME``. + * - struct :c:type:`v4l2_ctrl_mpeg2_sequence` * + - ``p_mpeg2_sequence`` + - A pointer to a struct :c:type:`v4l2_ctrl_mpeg2_sequence`. Valid if this control is + of type ``V4L2_CTRL_TYPE_MPEG2_SEQUENCE``. + * - struct :c:type:`v4l2_ctrl_mpeg2_picture` * + - ``p_mpeg2_picture`` + - A pointer to a struct :c:type:`v4l2_ctrl_mpeg2_picture`. Valid if this control is + of type ``V4L2_CTRL_TYPE_MPEG2_PICTURE``. + * - struct :c:type:`v4l2_ctrl_mpeg2_quantisation` * + - ``p_mpeg2_quantisation`` + - A pointer to a struct :c:type:`v4l2_ctrl_mpeg2_quantisation`. Valid if this control is + of type ``V4L2_CTRL_TYPE_MPEG2_QUANTISATION``. * - struct :c:type:`v4l2_ctrl_hdr10_cll_info` * - ``p_hdr10_cll`` - A pointer to a struct :c:type:`v4l2_ctrl_hdr10_cll_info`. Valid if this control is diff --git a/Documentation/userspace-api/media/v4l/vidioc-queryctrl.rst b/Documentation/userspace-api/media/v4l/vidioc-queryctrl.rst index 8a285daedc6a..f9ecf6276129 100644 --- a/Documentation/userspace-api/media/v4l/vidioc-queryctrl.rst +++ b/Documentation/userspace-api/media/v4l/vidioc-queryctrl.rst @@ -417,18 +417,24 @@ See also the examples in :ref:`control`. - any - An unsigned 32-bit valued control ranging from minimum to maximum inclusive. The step value indicates the increment between values. - * - ``V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS`` + * - ``V4L2_CTRL_TYPE_MPEG2_QUANTISATION`` - n/a - n/a - n/a - - A struct :c:type:`v4l2_ctrl_mpeg2_slice_params`, containing MPEG-2 - slice parameters for stateless video decoders. - * - ``V4L2_CTRL_TYPE_MPEG2_QUANTIZATION`` + - A struct :c:type:`v4l2_ctrl_mpeg2_quantisation`, containing MPEG-2 + quantisation matrices for stateless video decoders. + * - ``V4L2_CTRL_TYPE_MPEG2_SEQUENCE`` - n/a - n/a - n/a - - A struct :c:type:`v4l2_ctrl_mpeg2_quantization`, containing MPEG-2 - quantization matrices for stateless video decoders. + - A struct :c:type:`v4l2_ctrl_mpeg2_sequence`, containing MPEG-2 + sequence parameters for stateless video decoders. + * - ``V4L2_CTRL_TYPE_MPEG2_PICTURE`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_mpeg2_picture`, containing MPEG-2 + picture parameters for stateless video decoders. * - ``V4L2_CTRL_TYPE_AREA`` - n/a - n/a @@ -495,6 +501,12 @@ See also the examples in :ref:`control`. - n/a - A struct :c:type:`v4l2_ctrl_vp8_frame`, containing VP8 frame parameters for stateless video decoders. + * - ``V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS`` + - n/a + - n/a + - n/a + - A struct :c:type:`v4l2_ctrl_hevc_decode_params`, containing HEVC + decoding parameters for stateless video decoders. .. raw:: latex diff --git a/Documentation/userspace-api/media/video.h.rst.exceptions b/Documentation/userspace-api/media/video.h.rst.exceptions deleted file mode 100644 index ea9de59ad8b7..000000000000 --- a/Documentation/userspace-api/media/video.h.rst.exceptions +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 - -# Ignore header name -ignore define _UAPI_DVBVIDEO_H_ - -# This is a deprecated obscure API. Just ignore things we don't know -ignore define VIDEO_CMD_PLAY -ignore define VIDEO_CMD_STOP -ignore define VIDEO_CMD_FREEZE -ignore define VIDEO_CMD_CONTINUE -ignore define VIDEO_CMD_FREEZE_TO_BLACK -ignore define VIDEO_CMD_STOP_TO_BLACK -ignore define VIDEO_CMD_STOP_IMMEDIATELY -ignore define VIDEO_PLAY_FMT_NONE -ignore define VIDEO_PLAY_FMT_GOP -ignore define VIDEO_VSYNC_FIELD_UNKNOWN -ignore define VIDEO_VSYNC_FIELD_ODD -ignore define VIDEO_VSYNC_FIELD_EVEN -ignore define VIDEO_VSYNC_FIELD_PROGRESSIVE -ignore define VIDEO_EVENT_SIZE_CHANGED -ignore define VIDEO_EVENT_FRAME_RATE_CHANGED -ignore define VIDEO_EVENT_DECODER_STOPPED -ignore define VIDEO_EVENT_VSYNC -ignore define VIDEO_CAP_MPEG1 -ignore define VIDEO_CAP_MPEG2 -ignore define VIDEO_CAP_SYS -ignore define VIDEO_CAP_PROG -ignore define VIDEO_CAP_SPU -ignore define VIDEO_CAP_NAVI -ignore define VIDEO_CAP_CSS - -# some typedefs should point to struct/enums -replace typedef video_format_t :c:type:`video_format` -replace typedef video_system_t :c:type:`video_system` -replace typedef video_displayformat_t :c:type:`video_displayformat` -replace typedef video_size_t :c:type:`video_size` -replace typedef video_stream_source_t :c:type:`video_stream_source` -replace typedef video_play_state_t :c:type:`video_play_state` -replace typedef video_navi_pack_t :c:type:`video_navi_pack` diff --git a/Documentation/userspace-api/media/videodev2.h.rst.exceptions b/Documentation/userspace-api/media/videodev2.h.rst.exceptions index f59940352faa..2217b56c2686 100644 --- a/Documentation/userspace-api/media/videodev2.h.rst.exceptions +++ b/Documentation/userspace-api/media/videodev2.h.rst.exceptions @@ -134,8 +134,9 @@ replace symbol V4L2_CTRL_TYPE_STRING :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_U16 :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_U32 :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_U8 :c:type:`v4l2_ctrl_type` -replace symbol V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS :c:type:`v4l2_ctrl_type` -replace symbol V4L2_CTRL_TYPE_MPEG2_QUANTIZATION :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_MPEG2_SEQUENCE :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_MPEG2_PICTURE :c:type:`v4l2_ctrl_type` +replace symbol V4L2_CTRL_TYPE_MPEG2_QUANTISATION :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_H264_SPS :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_H264_PPS :c:type:`v4l2_ctrl_type` replace symbol V4L2_CTRL_TYPE_H264_SCALING_MATRIX :c:type:`v4l2_ctrl_type` diff --git a/Documentation/userspace-api/seccomp_filter.rst b/Documentation/userspace-api/seccomp_filter.rst index 6efb41cc8072..d61219889e49 100644 --- a/Documentation/userspace-api/seccomp_filter.rst +++ b/Documentation/userspace-api/seccomp_filter.rst @@ -259,6 +259,18 @@ and ``ioctl(SECCOMP_IOCTL_NOTIF_SEND)`` a response, indicating what should be returned to userspace. The ``id`` member of ``struct seccomp_notif_resp`` should be the same ``id`` as in ``struct seccomp_notif``. +Userspace can also add file descriptors to the notifying process via +``ioctl(SECCOMP_IOCTL_NOTIF_ADDFD)``. The ``id`` member of +``struct seccomp_notif_addfd`` should be the same ``id`` as in +``struct seccomp_notif``. The ``newfd_flags`` flag may be used to set flags +like O_EXEC on the file descriptor in the notifying process. If the supervisor +wants to inject the file descriptor with a specific number, the +``SECCOMP_ADDFD_FLAG_SETFD`` flag can be used, and set the ``newfd`` member to +the specific number to use. If that file descriptor is already open in the +notifying process it will be replaced. The supervisor can also add an FD, and +respond atomically by using the ``SECCOMP_ADDFD_FLAG_SEND`` flag and the return +value will be the injected file descriptor number. + It is worth noting that ``struct seccomp_data`` contains the values of register arguments to the syscall, but does not contain pointers to memory. The task's memory is accessible to suitably privileged traces via ``ptrace()`` or diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index 7fcb2fd38f42..c7b165ca70b6 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -688,9 +688,14 @@ MSRs that have been set successfully. Defines the vcpu responses to the cpuid instruction. Applications should use the KVM_SET_CPUID2 ioctl if available. -Note, when this IOCTL fails, KVM gives no guarantees that previous valid CPUID -configuration (if there is) is not corrupted. Userspace can get a copy of the -resulting CPUID configuration through KVM_GET_CPUID2 in case. +Caveat emptor: + - If this IOCTL fails, KVM gives no guarantees that previous valid CPUID + configuration (if there is) is not corrupted. Userspace can get a copy + of the resulting CPUID configuration through KVM_GET_CPUID2 in case. + - Using KVM_SET_CPUID{,2} after KVM_RUN, i.e. changing the guest vCPU model + after running the guest, may cause guest instability. + - Using heterogeneous CPUID configurations, modulo APIC IDs, topology, etc... + may cause guest instability. :: @@ -5034,6 +5039,260 @@ see KVM_XEN_VCPU_SET_ATTR above. The KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST type may not be used with the KVM_XEN_VCPU_GET_ATTR ioctl. +4.130 KVM_ARM_MTE_COPY_TAGS +--------------------------- + +:Capability: KVM_CAP_ARM_MTE +:Architectures: arm64 +:Type: vm ioctl +:Parameters: struct kvm_arm_copy_mte_tags +:Returns: number of bytes copied, < 0 on error (-EINVAL for incorrect + arguments, -EFAULT if memory cannot be accessed). + +:: + + struct kvm_arm_copy_mte_tags { + __u64 guest_ipa; + __u64 length; + void __user *addr; + __u64 flags; + __u64 reserved[2]; + }; + +Copies Memory Tagging Extension (MTE) tags to/from guest tag memory. The +``guest_ipa`` and ``length`` fields must be ``PAGE_SIZE`` aligned. The ``addr`` +field must point to a buffer which the tags will be copied to or from. + +``flags`` specifies the direction of copy, either ``KVM_ARM_TAGS_TO_GUEST`` or +``KVM_ARM_TAGS_FROM_GUEST``. + +The size of the buffer to store the tags is ``(length / 16)`` bytes +(granules in MTE are 16 bytes long). Each byte contains a single tag +value. This matches the format of ``PTRACE_PEEKMTETAGS`` and +``PTRACE_POKEMTETAGS``. + +If an error occurs before any data is copied then a negative error code is +returned. If some tags have been copied before an error occurs then the number +of bytes successfully copied is returned. If the call completes successfully +then ``length`` is returned. + +4.131 KVM_GET_SREGS2 +------------------ + +:Capability: KVM_CAP_SREGS2 +:Architectures: x86 +:Type: vcpu ioctl +:Parameters: struct kvm_sregs2 (out) +:Returns: 0 on success, -1 on error + +Reads special registers from the vcpu. +This ioctl (when supported) replaces the KVM_GET_SREGS. + +:: + +struct kvm_sregs2 { + /* out (KVM_GET_SREGS2) / in (KVM_SET_SREGS2) */ + struct kvm_segment cs, ds, es, fs, gs, ss; + struct kvm_segment tr, ldt; + struct kvm_dtable gdt, idt; + __u64 cr0, cr2, cr3, cr4, cr8; + __u64 efer; + __u64 apic_base; + __u64 flags; + __u64 pdptrs[4]; +}; + +flags values for ``kvm_sregs2``: + +``KVM_SREGS2_FLAGS_PDPTRS_VALID`` + + Indicates thats the struct contain valid PDPTR values. + + +4.132 KVM_SET_SREGS2 +------------------ + +:Capability: KVM_CAP_SREGS2 +:Architectures: x86 +:Type: vcpu ioctl +:Parameters: struct kvm_sregs2 (in) +:Returns: 0 on success, -1 on error + +Writes special registers into the vcpu. +See KVM_GET_SREGS2 for the data structures. +This ioctl (when supported) replaces the KVM_SET_SREGS. + +4.133 KVM_GET_STATS_FD +---------------------- + +:Capability: KVM_CAP_STATS_BINARY_FD +:Architectures: all +:Type: vm ioctl, vcpu ioctl +:Parameters: none +:Returns: statistics file descriptor on success, < 0 on error + +Errors: + + ====== ====================================================== + ENOMEM if the fd could not be created due to lack of memory + EMFILE if the number of opened files exceeds the limit + ====== ====================================================== + +The returned file descriptor can be used to read VM/vCPU statistics data in +binary format. The data in the file descriptor consists of four blocks +organized as follows: + ++-------------+ +| Header | ++-------------+ +| id string | ++-------------+ +| Descriptors | ++-------------+ +| Stats Data | ++-------------+ + +Apart from the header starting at offset 0, please be aware that it is +not guaranteed that the four blocks are adjacent or in the above order; +the offsets of the id, descriptors and data blocks are found in the +header. However, all four blocks are aligned to 64 bit offsets in the +file and they do not overlap. + +All blocks except the data block are immutable. Userspace can read them +only one time after retrieving the file descriptor, and then use ``pread`` or +``lseek`` to read the statistics repeatedly. + +All data is in system endianness. + +The format of the header is as follows:: + + struct kvm_stats_header { + __u32 flags; + __u32 name_size; + __u32 num_desc; + __u32 id_offset; + __u32 desc_offset; + __u32 data_offset; + }; + +The ``flags`` field is not used at the moment. It is always read as 0. + +The ``name_size`` field is the size (in byte) of the statistics name string +(including trailing '\0') which is contained in the "id string" block and +appended at the end of every descriptor. + +The ``num_desc`` field is the number of descriptors that are included in the +descriptor block. (The actual number of values in the data block may be +larger, since each descriptor may comprise more than one value). + +The ``id_offset`` field is the offset of the id string from the start of the +file indicated by the file descriptor. It is a multiple of 8. + +The ``desc_offset`` field is the offset of the Descriptors block from the start +of the file indicated by the file descriptor. It is a multiple of 8. + +The ``data_offset`` field is the offset of the Stats Data block from the start +of the file indicated by the file descriptor. It is a multiple of 8. + +The id string block contains a string which identifies the file descriptor on +which KVM_GET_STATS_FD was invoked. The size of the block, including the +trailing ``'\0'``, is indicated by the ``name_size`` field in the header. + +The descriptors block is only needed to be read once for the lifetime of the +file descriptor contains a sequence of ``struct kvm_stats_desc``, each followed +by a string of size ``name_size``. + + #define KVM_STATS_TYPE_SHIFT 0 + #define KVM_STATS_TYPE_MASK (0xF << KVM_STATS_TYPE_SHIFT) + #define KVM_STATS_TYPE_CUMULATIVE (0x0 << KVM_STATS_TYPE_SHIFT) + #define KVM_STATS_TYPE_INSTANT (0x1 << KVM_STATS_TYPE_SHIFT) + #define KVM_STATS_TYPE_PEAK (0x2 << KVM_STATS_TYPE_SHIFT) + + #define KVM_STATS_UNIT_SHIFT 4 + #define KVM_STATS_UNIT_MASK (0xF << KVM_STATS_UNIT_SHIFT) + #define KVM_STATS_UNIT_NONE (0x0 << KVM_STATS_UNIT_SHIFT) + #define KVM_STATS_UNIT_BYTES (0x1 << KVM_STATS_UNIT_SHIFT) + #define KVM_STATS_UNIT_SECONDS (0x2 << KVM_STATS_UNIT_SHIFT) + #define KVM_STATS_UNIT_CYCLES (0x3 << KVM_STATS_UNIT_SHIFT) + + #define KVM_STATS_BASE_SHIFT 8 + #define KVM_STATS_BASE_MASK (0xF << KVM_STATS_BASE_SHIFT) + #define KVM_STATS_BASE_POW10 (0x0 << KVM_STATS_BASE_SHIFT) + #define KVM_STATS_BASE_POW2 (0x1 << KVM_STATS_BASE_SHIFT) + + struct kvm_stats_desc { + __u32 flags; + __s16 exponent; + __u16 size; + __u32 offset; + __u32 unused; + char name[]; + }; + +The ``flags`` field contains the type and unit of the statistics data described +by this descriptor. Its endianness is CPU native. +The following flags are supported: + +Bits 0-3 of ``flags`` encode the type: + * ``KVM_STATS_TYPE_CUMULATIVE`` + The statistics data is cumulative. The value of data can only be increased. + Most of the counters used in KVM are of this type. + The corresponding ``size`` field for this type is always 1. + All cumulative statistics data are read/write. + * ``KVM_STATS_TYPE_INSTANT`` + The statistics data is instantaneous. Its value can be increased or + decreased. This type is usually used as a measurement of some resources, + like the number of dirty pages, the number of large pages, etc. + All instant statistics are read only. + The corresponding ``size`` field for this type is always 1. + * ``KVM_STATS_TYPE_PEAK`` + The statistics data is peak. The value of data can only be increased, and + represents a peak value for a measurement, for example the maximum number + of items in a hash table bucket, the longest time waited and so on. + The corresponding ``size`` field for this type is always 1. + +Bits 4-7 of ``flags`` encode the unit: + * ``KVM_STATS_UNIT_NONE`` + There is no unit for the value of statistics data. This usually means that + the value is a simple counter of an event. + * ``KVM_STATS_UNIT_BYTES`` + It indicates that the statistics data is used to measure memory size, in the + unit of Byte, KiByte, MiByte, GiByte, etc. The unit of the data is + determined by the ``exponent`` field in the descriptor. + * ``KVM_STATS_UNIT_SECONDS`` + It indicates that the statistics data is used to measure time or latency. + * ``KVM_STATS_UNIT_CYCLES`` + It indicates that the statistics data is used to measure CPU clock cycles. + +Bits 8-11 of ``flags``, together with ``exponent``, encode the scale of the +unit: + * ``KVM_STATS_BASE_POW10`` + The scale is based on power of 10. It is used for measurement of time and + CPU clock cycles. For example, an exponent of -9 can be used with + ``KVM_STATS_UNIT_SECONDS`` to express that the unit is nanoseconds. + * ``KVM_STATS_BASE_POW2`` + The scale is based on power of 2. It is used for measurement of memory size. + For example, an exponent of 20 can be used with ``KVM_STATS_UNIT_BYTES`` to + express that the unit is MiB. + +The ``size`` field is the number of values of this statistics data. Its +value is usually 1 for most of simple statistics. 1 means it contains an +unsigned 64bit data. + +The ``offset`` field is the offset from the start of Data Block to the start of +the corresponding statistics data. + +The ``unused`` field is reserved for future support for other types of +statistics data, like log/linear histogram. Its value is always 0 for the types +defined above. + +The ``name`` field is the name string of the statistics data. The name string +starts at the end of ``struct kvm_stats_desc``. The maximum length including +the trailing ``'\0'``, is indicated by ``name_size`` in the header. + +The Stats Data block contains an array of 64-bit values in the same order +as the descriptors in Descriptors block. + 5. The kvm_run structure ======================== @@ -6323,6 +6582,7 @@ KVM_RUN_BUS_LOCK flag is used to distinguish between them. This capability can be used to check / enable 2nd DAWR feature provided by POWER10 processor. + 7.24 KVM_CAP_VM_COPY_ENC_CONTEXT_FROM ------------------------------------- @@ -6360,7 +6620,67 @@ system fingerprint. To prevent userspace from circumventing such restrictions by running an enclave in a VM, KVM prevents access to privileged attributes by default. -See Documentation/x86/sgx/2.Kernel-internals.rst for more details. +See Documentation/x86/sgx.rst for more details. + +7.26 KVM_CAP_PPC_RPT_INVALIDATE +------------------------------- + +:Capability: KVM_CAP_PPC_RPT_INVALIDATE +:Architectures: ppc +:Type: vm + +This capability indicates that the kernel is capable of handling +H_RPT_INVALIDATE hcall. + +In order to enable the use of H_RPT_INVALIDATE in the guest, +user space might have to advertise it for the guest. For example, +IBM pSeries (sPAPR) guest starts using it if "hcall-rpt-invalidate" is +present in the "ibm,hypertas-functions" device-tree property. + +This capability is enabled for hypervisors on platforms like POWER9 +that support radix MMU. + +7.27 KVM_CAP_EXIT_ON_EMULATION_FAILURE +-------------------------------------- + +:Architectures: x86 +:Parameters: args[0] whether the feature should be enabled or not + +When this capability is enabled, an emulation failure will result in an exit +to userspace with KVM_INTERNAL_ERROR (except when the emulator was invoked +to handle a VMware backdoor instruction). Furthermore, KVM will now provide up +to 15 instruction bytes for any exit to userspace resulting from an emulation +failure. When these exits to userspace occur use the emulation_failure struct +instead of the internal struct. They both have the same layout, but the +emulation_failure struct matches the content better. It also explicitly +defines the 'flags' field which is used to describe the fields in the struct +that are valid (ie: if KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES is +set in the 'flags' field then both 'insn_size' and 'insn_bytes' have valid data +in them.) + +7.28 KVM_CAP_ARM_MTE +-------------------- + +:Architectures: arm64 +:Parameters: none + +This capability indicates that KVM (and the hardware) supports exposing the +Memory Tagging Extensions (MTE) to the guest. It must also be enabled by the +VMM before creating any VCPUs to allow the guest access. Note that MTE is only +available to a guest running in AArch64 mode and enabling this capability will +cause attempts to create AArch32 VCPUs to fail. + +When enabled the guest is able to access tags associated with any memory given +to the guest. KVM will ensure that the tags are maintained during swap or +hibernation of the host; however the VMM needs to manually save/restore the +tags as appropriate if the VM is migrated. + +When this capability is enabled all memory in memslots must be mapped as +not-shareable (no MAP_SHARED), attempts to create a memslot with a +MAP_SHARED mmap will result in an -EINVAL return. + +When enabled the VMM may make use of the ``KVM_ARM_MTE_COPY_TAGS`` ioctl to +perform a bulk copy of tags to/from the guest. 8. Other capabilities. ====================== @@ -6891,3 +7211,33 @@ This capability is always enabled. This capability indicates that the KVM virtual PTP service is supported in the host. A VMM can check whether the service is available to the guest on migration. + +8.33 KVM_CAP_HYPERV_ENFORCE_CPUID +----------------------------- + +Architectures: x86 + +When enabled, KVM will disable emulated Hyper-V features provided to the +guest according to the bits Hyper-V CPUID feature leaves. Otherwise, all +currently implmented Hyper-V features are provided unconditionally when +Hyper-V identification is set in the HYPERV_CPUID_INTERFACE (0x40000001) +leaf. + +8.34 KVM_CAP_EXIT_HYPERCALL +--------------------------- + +:Capability: KVM_CAP_EXIT_HYPERCALL +:Architectures: x86 +:Type: vm + +This capability, if enabled, will cause KVM to exit to userspace +with KVM_EXIT_HYPERCALL exit reason to process some hypercalls. + +Calling KVM_CHECK_EXTENSION for this capability will return a bitmask +of hypercalls that can be configured to exit to userspace. +Right now, the only such hypercall is KVM_HC_MAP_GPA_RANGE. + +The argument to KVM_ENABLE_CAP is also a bitmask, and must be a subset +of the result of KVM_CHECK_EXTENSION. KVM will forward to userspace +the hypercalls whose corresponding bit is in the argument, and return +ENOSYS for the others. diff --git a/Documentation/virt/kvm/cpuid.rst b/Documentation/virt/kvm/cpuid.rst index cf62162d4be2..bda3e3e737d7 100644 --- a/Documentation/virt/kvm/cpuid.rst +++ b/Documentation/virt/kvm/cpuid.rst @@ -96,6 +96,13 @@ KVM_FEATURE_MSI_EXT_DEST_ID 15 guest checks this feature bit before using extended destination ID bits in MSI address bits 11-5. +KVM_FEATURE_HC_MAP_GPA_RANGE 16 guest checks this feature bit before + using the map gpa range hypercall + to notify the page state change + +KVM_FEATURE_MIGRATION_CONTROL 17 guest checks this feature bit before + using MSR_KVM_MIGRATION_CONTROL + KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 24 host will warn if no guest-side per-cpu warps are expected in kvmclock diff --git a/Documentation/virt/kvm/hypercalls.rst b/Documentation/virt/kvm/hypercalls.rst index ed4fddd364ea..e56fa8b9cfca 100644 --- a/Documentation/virt/kvm/hypercalls.rst +++ b/Documentation/virt/kvm/hypercalls.rst @@ -169,3 +169,24 @@ a0: destination APIC ID :Usage example: When sending a call-function IPI-many to vCPUs, yield if any of the IPI target vCPUs was preempted. + +8. KVM_HC_MAP_GPA_RANGE +------------------------- +:Architecture: x86 +:Status: active +:Purpose: Request KVM to map a GPA range with the specified attributes. + +a0: the guest physical address of the start page +a1: the number of (4kb) pages (must be contiguous in GPA space) +a2: attributes + + Where 'attributes' : + * bits 3:0 - preferred page size encoding 0 = 4kb, 1 = 2mb, 2 = 1gb, etc... + * bit 4 - plaintext = 0, encrypted = 1 + * bits 63:5 - reserved (must be zero) + +**Implementation note**: this hypercall is implemented in userspace via +the KVM_CAP_EXIT_HYPERCALL capability. Userspace must enable that capability +before advertising KVM_FEATURE_HC_MAP_GPA_RANGE in the guest CPUID. In +addition, if the guest supports KVM_FEATURE_MIGRATION_CONTROL, userspace +must also set up an MSR filter to process writes to MSR_KVM_MIGRATION_CONTROL. diff --git a/Documentation/virt/kvm/locking.rst b/Documentation/virt/kvm/locking.rst index 1fc860c007a3..35eca377543d 100644 --- a/Documentation/virt/kvm/locking.rst +++ b/Documentation/virt/kvm/locking.rst @@ -16,6 +16,11 @@ The acquisition orders for mutexes are as follows: - kvm->slots_lock is taken outside kvm->irq_lock, though acquiring them together is quite rare. +- Unlike kvm->slots_lock, kvm->slots_arch_lock is released before + synchronize_srcu(&kvm->srcu). Therefore kvm->slots_arch_lock + can be taken inside a kvm->srcu read-side critical section, + while kvm->slots_lock cannot. + On x86: - vcpu->mutex is taken outside kvm->arch.hyperv.hv_lock diff --git a/Documentation/virt/kvm/mmu.rst b/Documentation/virt/kvm/mmu.rst index 20d85daed395..f60f5488e121 100644 --- a/Documentation/virt/kvm/mmu.rst +++ b/Documentation/virt/kvm/mmu.rst @@ -180,8 +180,8 @@ Shadow pages contain the following information: role.gpte_is_8_bytes: Reflects the size of the guest PTE for which the page is valid, i.e. '1' if 64-bit gptes are in use, '0' if 32-bit gptes are in use. - role.nxe: - Contains the value of efer.nxe for which the page is valid. + role.efer_nx: + Contains the value of efer.nx for which the page is valid. role.cr0_wp: Contains the value of cr0.wp for which the page is valid. role.smep_andnot_wp: @@ -192,9 +192,6 @@ Shadow pages contain the following information: Contains the value of cr4.smap && !cr0.wp for which the page is valid (pages for which this is true are different from other pages; see the treatment of cr0.wp=0 below). - role.ept_sp: - This is a virtual flag to denote a shadowed nested EPT page. ept_sp - is true if "cr0_wp && smap_andnot_wp", an otherwise invalid combination. role.smm: Is 1 if the page is valid in system management mode. This field determines which of the kvm_memslots array was used to build this diff --git a/Documentation/virt/kvm/msr.rst b/Documentation/virt/kvm/msr.rst index e37a14c323d2..9315fc385fb0 100644 --- a/Documentation/virt/kvm/msr.rst +++ b/Documentation/virt/kvm/msr.rst @@ -376,3 +376,16 @@ data: write '1' to bit 0 of the MSR, this causes the host to re-scan its queue and check if there are more notifications pending. The MSR is available if KVM_FEATURE_ASYNC_PF_INT is present in CPUID. + +MSR_KVM_MIGRATION_CONTROL: + 0x4b564d08 + +data: + This MSR is available if KVM_FEATURE_MIGRATION_CONTROL is present in + CPUID. Bit 0 represents whether live migration of the guest is allowed. + + When a guest is started, bit 0 will be 0 if the guest has encrypted + memory and 1 if the guest does not have encrypted memory. If the + guest is communicating page encryption status to the host using the + ``KVM_HC_MAP_GPA_RANGE`` hypercall, it can set bit 0 in this MSR to + allow live migration of the guest. diff --git a/Documentation/virt/kvm/s390-pv-boot.rst b/Documentation/virt/kvm/s390-pv-boot.rst index ad1f7866c001..73a6083cb5e7 100644 --- a/Documentation/virt/kvm/s390-pv-boot.rst +++ b/Documentation/virt/kvm/s390-pv-boot.rst @@ -10,7 +10,7 @@ The memory of Protected Virtual Machines (PVMs) is not accessible to I/O or the hypervisor. In those cases where the hypervisor needs to access the memory of a PVM, that memory must be made accessible. Memory made accessible to the hypervisor will be encrypted. See -:doc:`s390-pv` for details." +Documentation/virt/kvm/s390-pv.rst for details." On IPL (boot) a small plaintext bootloader is started, which provides information about the encrypted components and necessary metadata to diff --git a/Documentation/virt/kvm/vcpu-requests.rst b/Documentation/virt/kvm/vcpu-requests.rst index af1b37441e0a..ad2915ef7020 100644 --- a/Documentation/virt/kvm/vcpu-requests.rst +++ b/Documentation/virt/kvm/vcpu-requests.rst @@ -304,6 +304,6 @@ VCPU returns from the call. References ========== -.. [atomic-ops] Documentation/core-api/atomic_ops.rst +.. [atomic-ops] Documentation/atomic_bitops.txt and Documentation/atomic_t.txt .. [memory-barriers] Documentation/memory-barriers.txt .. [lwn-mb] https://lwn.net/Articles/573436/ diff --git a/Documentation/vm/zswap.rst b/Documentation/vm/zswap.rst index d8d9fa4a1f0d..8edb8d578caf 100644 --- a/Documentation/vm/zswap.rst +++ b/Documentation/vm/zswap.rst @@ -10,7 +10,7 @@ Overview Zswap is a lightweight compressed cache for swap pages. It takes pages that are in the process of being swapped out and attempts to compress them into a dynamically allocated RAM-based memory pool. zswap basically trades CPU cycles -for potentially reduced swap I/O.  This trade-off can also result in a +for potentially reduced swap I/O. This trade-off can also result in a significant performance improvement if reads from the compressed cache are faster than reads from a swap device. @@ -26,7 +26,7 @@ faster than reads from a swap device. performance impact of swapping. * Overcommitted guests that share a common I/O resource can dramatically reduce their swap I/O pressure, avoiding heavy handed I/O - throttling by the hypervisor. This allows more work to get done with less + throttling by the hypervisor. This allows more work to get done with less impact to the guest workload and guests sharing the I/O subsystem * Users with SSDs as swap devices can extend the life of the device by drastically reducing life-shortening writes. diff --git a/Documentation/x86/boot.rst b/Documentation/x86/boot.rst index fc844913dece..894a19897005 100644 --- a/Documentation/x86/boot.rst +++ b/Documentation/x86/boot.rst @@ -1343,7 +1343,7 @@ follow:: In addition to read/modify/write the setup header of the struct boot_params as that of 16-bit boot protocol, the boot loader should also fill the additional fields of the struct boot_params as -described in chapter :doc:`zero-page`. +described in chapter Documentation/x86/zero-page.rst. After setting up the struct boot_params, the boot loader can load the 32/64-bit kernel in the same way as that of 16-bit boot protocol. @@ -1379,7 +1379,7 @@ can be calculated as follows:: In addition to read/modify/write the setup header of the struct boot_params as that of 16-bit boot protocol, the boot loader should also fill the additional fields of the struct boot_params as described -in chapter :doc:`zero-page`. +in chapter Documentation/x86/zero-page.rst. After setting up the struct boot_params, the boot loader can load 64-bit kernel in the same way as that of 16-bit boot protocol, but diff --git a/Documentation/x86/buslock.rst b/Documentation/x86/buslock.rst new file mode 100644 index 000000000000..7c051e714943 --- /dev/null +++ b/Documentation/x86/buslock.rst @@ -0,0 +1,126 @@ +.. SPDX-License-Identifier: GPL-2.0 + +.. include:: + +=============================== +Bus lock detection and handling +=============================== + +:Copyright: |copy| 2021 Intel Corporation +:Authors: - Fenghua Yu + - Tony Luck + +Problem +======= + +A split lock is any atomic operation whose operand crosses two cache lines. +Since the operand spans two cache lines and the operation must be atomic, +the system locks the bus while the CPU accesses the two cache lines. + +A bus lock is acquired through either split locked access to writeback (WB) +memory or any locked access to non-WB memory. This is typically thousands of +cycles slower than an atomic operation within a cache line. It also disrupts +performance on other cores and brings the whole system to its knees. + +Detection +========= + +Intel processors may support either or both of the following hardware +mechanisms to detect split locks and bus locks. + +#AC exception for split lock detection +-------------------------------------- + +Beginning with the Tremont Atom CPU split lock operations may raise an +Alignment Check (#AC) exception when a split lock operation is attemped. + +#DB exception for bus lock detection +------------------------------------ + +Some CPUs have the ability to notify the kernel by an #DB trap after a user +instruction acquires a bus lock and is executed. This allows the kernel to +terminate the application or to enforce throttling. + +Software handling +================= + +The kernel #AC and #DB handlers handle bus lock based on the kernel +parameter "split_lock_detect". Here is a summary of different options: + ++------------------+----------------------------+-----------------------+ +|split_lock_detect=|#AC for split lock |#DB for bus lock | ++------------------+----------------------------+-----------------------+ +|off |Do nothing |Do nothing | ++------------------+----------------------------+-----------------------+ +|warn |Kernel OOPs |Warn once per task and | +|(default) |Warn once per task and |and continues to run. | +| |disable future checking | | +| |When both features are | | +| |supported, warn in #AC | | ++------------------+----------------------------+-----------------------+ +|fatal |Kernel OOPs |Send SIGBUS to user. | +| |Send SIGBUS to user | | +| |When both features are | | +| |supported, fatal in #AC | | ++------------------+----------------------------+-----------------------+ +|ratelimit:N |Do nothing |Limit bus lock rate to | +|(0 < N <= 1000) | |N bus locks per second | +| | |system wide and warn on| +| | |bus locks. | ++------------------+----------------------------+-----------------------+ + +Usages +====== + +Detecting and handling bus lock may find usages in various areas: + +It is critical for real time system designers who build consolidated real +time systems. These systems run hard real time code on some cores and run +"untrusted" user processes on other cores. The hard real time cannot afford +to have any bus lock from the untrusted processes to hurt real time +performance. To date the designers have been unable to deploy these +solutions as they have no way to prevent the "untrusted" user code from +generating split lock and bus lock to block the hard real time code to +access memory during bus locking. + +It's also useful for general computing to prevent guests or user +applications from slowing down the overall system by executing instructions +with bus lock. + + +Guidance +======== +off +--- + +Disable checking for split lock and bus lock. This option can be useful if +there are legacy applications that trigger these events at a low rate so +that mitigation is not needed. + +warn +---- + +A warning is emitted when a bus lock is detected which allows to identify +the offending application. This is the default behavior. + +fatal +----- + +In this case, the bus lock is not tolerated and the process is killed. + +ratelimit +--------- + +A system wide bus lock rate limit N is specified where 0 < N <= 1000. This +allows a bus lock rate up to N bus locks per second. When the bus lock rate +is exceeded then any task which is caught via the buslock #DB exception is +throttled by enforced sleeps until the rate goes under the limit again. + +This is an effective mitigation in cases where a minimal impact can be +tolerated, but an eventual Denial of Service attack has to be prevented. It +allows to identify the offending processes and analyze whether they are +malicious or just badly written. + +Selecting a rate limit of 1000 allows the bus to be locked for up to about +seven million cycles each second (assuming 7000 cycles for each bus +lock). On a 2 GHz processor that would be about 0.35% system slowdown. diff --git a/Documentation/x86/index.rst b/Documentation/x86/index.rst index 4693e192b447..0004f5d2283e 100644 --- a/Documentation/x86/index.rst +++ b/Documentation/x86/index.rst @@ -29,6 +29,7 @@ x86-specific Documentation microcode resctrl tsx_async_abort + buslock usb-legacy-support i386/index x86_64/index diff --git a/Documentation/x86/mtrr.rst b/Documentation/x86/mtrr.rst index c5b695d75349..9f0b1851771a 100644 --- a/Documentation/x86/mtrr.rst +++ b/Documentation/x86/mtrr.rst @@ -28,7 +28,7 @@ are aligned with platform MTRR setup. If MTRRs are only set up by the platform firmware code though and the OS does not make any specific MTRR mapping requests mtrr_type_lookup() should always return MTRR_TYPE_INVALID. -For details refer to :doc:`pat`. +For details refer to Documentation/x86/pat.rst. .. tip:: On Intel P6 family processors (Pentium Pro, Pentium II and later) diff --git a/Documentation/x86/x86_64/boot-options.rst b/Documentation/x86/x86_64/boot-options.rst index 324cefff92e7..5f62b3b86357 100644 --- a/Documentation/x86/x86_64/boot-options.rst +++ b/Documentation/x86/x86_64/boot-options.rst @@ -247,16 +247,11 @@ Multiple x86-64 PCI-DMA mapping implementations exist, for example: Kernel boot message: "PCI-DMA: Using software bounce buffering for IO (SWIOTLB)" - 4. : IBM Calgary hardware IOMMU. Used in IBM - pSeries and xSeries servers. This hardware IOMMU supports DMA address - mapping with memory protection, etc. - Kernel boot message: "PCI-DMA: Using Calgary IOMMU" - :: iommu=[][,noagp][,off][,force][,noforce] [,memaper[=]][,merge][,fullflush][,nomerge] - [,noaperture][,calgary] + [,noaperture] General iommu options: @@ -295,8 +290,6 @@ iommu options only relevant to the AMD GART hardware IOMMU: Don't initialize the AGP driver and use full aperture. panic Always panic when IOMMU overflows. - calgary - Use the Calgary IOMMU if it is available iommu options only relevant to the software bounce buffering (SWIOTLB) IOMMU implementation: @@ -307,28 +300,6 @@ implementation: force Force all IO through the software TLB. -Settings for the IBM Calgary hardware IOMMU currently found in IBM -pSeries and xSeries machines - - calgary=[64k,128k,256k,512k,1M,2M,4M,8M] - Set the size of each PCI slot's translation table when using the - Calgary IOMMU. This is the size of the translation table itself - in main memory. The smallest table, 64k, covers an IO space of - 32MB; the largest, 8MB table, can cover an IO space of 4GB. - Normally the kernel will make the right choice by itself. - calgary=[translate_empty_slots] - Enable translation even on slots that have no devices attached to - them, in case a device will be hotplugged in the future. - calgary=[disable=] - Disable translation on a given PHB. For - example, the built-in graphics adapter resides on the first bridge - (PCI bus number 0); if translation (isolation) is enabled on this - bridge, X servers that access the hardware directly from user - space might stop working. Use this option if you have devices that - are accessed from userspace directly on some PCI host bridge. - panic - Always panic when IOMMU overflows - Miscellaneous ============= diff --git a/MAINTAINERS b/MAINTAINERS index 662b45053e15..bbd4c729dd7e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1811,6 +1811,7 @@ F: Documentation/devicetree/bindings/net/cortina,gemini-ethernet.txt F: Documentation/devicetree/bindings/pinctrl/cortina,gemini-pinctrl.txt F: Documentation/devicetree/bindings/rtc/faraday,ftrtc010.txt F: arch/arm/mach-gemini/ +F: drivers/crypto/gemini/ F: drivers/net/ethernet/cortina/ F: drivers/pinctrl/pinctrl-gemini.c F: drivers/rtc/rtc-ftrtc010.c @@ -1972,6 +1973,7 @@ F: Documentation/devicetree/bindings/interrupt-controller/intel,ixp4xx-interrupt F: Documentation/devicetree/bindings/timer/intel,ixp4xx-timer.yaml F: arch/arm/mach-ixp4xx/ F: drivers/clocksource/timer-ixp4xx.c +F: drivers/crypto/ixp4xx_crypto.c F: drivers/gpio/gpio-ixp4xx.c F: drivers/irqchip/irq-ixp4xx.c F: include/linux/irqchip/irq-ixp4xx.h @@ -4610,6 +4612,12 @@ S: Supported F: drivers/video/console/ F: include/linux/console* +CONTEXT TRACKING +M: Frederic Weisbecker +S: Maintained +F: kernel/context_tracking.c +F: include/linux/context_tracking* + CONTROL GROUP (CGROUP) M: Tejun Heo M: Zefan Li @@ -6778,7 +6786,7 @@ F: include/video/s1d13xxxfb.h EROFS FILE SYSTEM M: Gao Xiang -M: Chao Yu +M: Chao Yu L: linux-erofs@lists.ozlabs.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/xiang/erofs.git @@ -7173,7 +7181,7 @@ F: include/video/ FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER M: Horia Geantă -M: Aymen Sghaier +M: Pankaj Gupta L: linux-crypto@vger.kernel.org S: Maintained F: Documentation/devicetree/bindings/crypto/fsl-sec4.txt @@ -7563,6 +7571,12 @@ M: Kieran Bingham S: Supported F: scripts/gdb/ +GEMINI CRYPTO DRIVER +M: Corentin Labbe +L: linux-crypto@vger.kernel.org +S: Maintained +F: drivers/crypto/gemini/ + GEMTEK FM RADIO RECEIVER DRIVER M: Hans Verkuil L: linux-media@vger.kernel.org @@ -9241,6 +9255,12 @@ F: Documentation/admin-guide/media/ipu3_rcb.svg F: Documentation/userspace-api/media/v4l/pixfmt-meta-intel-ipu3.rst F: drivers/staging/media/ipu3/ +INTEL IXP4XX CRYPTO SUPPORT +M: Corentin Labbe +L: linux-crypto@vger.kernel.org +S: Maintained +F: drivers/crypto/ixp4xx_crypto.c + INTEL IXP4XX QMGR, NPE, ETHERNET and HSS SUPPORT M: Krzysztof Halasa S: Maintained @@ -9992,6 +10012,8 @@ F: arch/arm64/include/asm/kvm* F: arch/arm64/include/uapi/asm/kvm* F: arch/arm64/kvm/ F: include/kvm/arm_* +F: tools/testing/selftests/kvm/*/aarch64/ +F: tools/testing/selftests/kvm/aarch64/ KERNEL VIRTUAL MACHINE FOR MIPS (KVM/mips) M: Huacai Chen @@ -11289,6 +11311,7 @@ F: include/media/imx.h MEDIA DRIVERS FOR FREESCALE IMX7 M: Rui Miguel Silva +M: Laurent Pinchart L: linux-media@vger.kernel.org S: Maintained T: git git://linuxtv.org/media_tree.git @@ -11398,6 +11421,7 @@ L: linux-renesas-soc@vger.kernel.org S: Supported T: git git://linuxtv.org/media_tree.git F: Documentation/devicetree/bindings/media/renesas,csi2.yaml +F: Documentation/devicetree/bindings/media/renesas,isp.yaml F: Documentation/devicetree/bindings/media/renesas,vin.yaml F: drivers/media/platform/rcar-vin/ @@ -11987,11 +12011,13 @@ MICROCHIP ISC DRIVER M: Eugen Hristev L: linux-media@vger.kernel.org S: Supported -F: Documentation/devicetree/bindings/media/atmel-isc.txt +F: Documentation/devicetree/bindings/media/atmel,isc.yaml +F: Documentation/devicetree/bindings/media/microchip,xisc.yaml F: drivers/media/platform/atmel/atmel-isc-base.c F: drivers/media/platform/atmel/atmel-isc-regs.h F: drivers/media/platform/atmel/atmel-isc.h F: drivers/media/platform/atmel/atmel-sama5d2-isc.c +F: drivers/media/platform/atmel/atmel-sama7g5-isc.c F: include/linux/atmel-isc-media.h MICROCHIP ISI DRIVER @@ -12599,7 +12625,7 @@ S: Orphan F: drivers/net/ethernet/natsemi/natsemi.c NCR 5380 SCSI DRIVERS -M: Finn Thain +M: Finn Thain M: Michael Schmitz L: linux-scsi@vger.kernel.org S: Maintained @@ -13077,7 +13103,7 @@ F: Documentation/filesystems/ntfs.rst F: fs/ntfs/ NUBUS SUBSYSTEM -M: Finn Thain +M: Finn Thain L: linux-m68k@lists.linux-m68k.org S: Maintained F: arch/*/include/asm/nubus.h @@ -15144,6 +15170,13 @@ S: Maintained F: Documentation/devicetree/bindings/opp/qcom-nvmem-cpufreq.txt F: drivers/cpufreq/qcom-cpufreq-nvmem.c +QUALCOMM CRYPTO DRIVERS +M: Thara Gopinath +L: linux-crypto@vger.kernel.org +L: linux-arm-msm@vger.kernel.org +S: Maintained +F: drivers/crypto/qce/ + QUALCOMM EMAC GIGABIT ETHERNET DRIVER M: Timur Tabi L: netdev@vger.kernel.org @@ -17009,6 +17042,13 @@ S: Maintained F: drivers/ssb/ F: include/linux/ssb/ +SONY IMX208 SENSOR DRIVER +M: Sakari Ailus +L: linux-media@vger.kernel.org +S: Maintained +T: git git://linuxtv.org/media_tree.git +F: drivers/media/i2c/imx208.c + SONY IMX214 SENSOR DRIVER M: Ricardo Ribalda L: linux-media@vger.kernel.org diff --git a/arch/alpha/include/asm/page.h b/arch/alpha/include/asm/page.h index 268f99b4602b..18f48a6f2ff6 100644 --- a/arch/alpha/include/asm/page.h +++ b/arch/alpha/include/asm/page.h @@ -17,9 +17,9 @@ extern void clear_page(void *page); #define clear_user_page(page, vaddr, pg) clear_page(page) -#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ - alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vmaddr) -#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vmaddr) +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE extern void copy_page(void * _to, void * _from); #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) diff --git a/arch/arm/boot/dts/sama5d4.dtsi b/arch/arm/boot/dts/sama5d4.dtsi index 05c55875835d..e47e1ca63043 100644 --- a/arch/arm/boot/dts/sama5d4.dtsi +++ b/arch/arm/boot/dts/sama5d4.dtsi @@ -101,6 +101,13 @@ nfc_sram: sram@100000 { ranges = <0 0x100000 0x2400>; }; + vdec0: vdec@300000 { + compatible = "microchip,sama5d4-vdec"; + reg = <0x00300000 0x100000>; + interrupts = <19 IRQ_TYPE_LEVEL_HIGH 4>; + clocks = <&pmc PMC_TYPE_PERIPHERAL 19>; + }; + usb0: gadget@400000 { compatible = "atmel,sama5d3-udc"; reg = <0x00400000 0x100000 diff --git a/arch/arm/crypto/Makefile b/arch/arm/crypto/Makefile index 8f26c454ea12..eafa898ba6a7 100644 --- a/arch/arm/crypto/Makefile +++ b/arch/arm/crypto/Makefile @@ -45,20 +45,12 @@ poly1305-arm-y := poly1305-core.o poly1305-glue.o nhpoly1305-neon-y := nh-neon-core.o nhpoly1305-neon-glue.o curve25519-neon-y := curve25519-core.o curve25519-glue.o -ifdef REGENERATE_ARM_CRYPTO quiet_cmd_perl = PERL $@ cmd_perl = $(PERL) $(<) > $(@) -$(src)/poly1305-core.S_shipped: $(src)/poly1305-armv4.pl +$(obj)/%-core.S: $(src)/%-armv4.pl $(call cmd,perl) -$(src)/sha256-core.S_shipped: $(src)/sha256-armv4.pl - $(call cmd,perl) - -$(src)/sha512-core.S_shipped: $(src)/sha512-armv4.pl - $(call cmd,perl) -endif - clean-files += poly1305-core.S sha256-core.S sha512-core.S # massage the perlasm code a bit so we only get the NEON routine if we need it diff --git a/arch/arm/crypto/poly1305-core.S_shipped b/arch/arm/crypto/poly1305-core.S_shipped deleted file mode 100644 index 37b71d990293..000000000000 --- a/arch/arm/crypto/poly1305-core.S_shipped +++ /dev/null @@ -1,1158 +0,0 @@ -#ifndef __KERNEL__ -# include "arm_arch.h" -#else -# define __ARM_ARCH__ __LINUX_ARM_ARCH__ -# define __ARM_MAX_ARCH__ __LINUX_ARM_ARCH__ -# define poly1305_init poly1305_init_arm -# define poly1305_blocks poly1305_blocks_arm -# define poly1305_emit poly1305_emit_arm -.globl poly1305_blocks_neon -#endif - -#if defined(__thumb2__) -.syntax unified -.thumb -#else -.code 32 -#endif - -.text - -.globl poly1305_emit -.globl poly1305_blocks -.globl poly1305_init -.type poly1305_init,%function -.align 5 -poly1305_init: -.Lpoly1305_init: - stmdb sp!,{r4-r11} - - eor r3,r3,r3 - cmp r1,#0 - str r3,[r0,#0] @ zero hash value - str r3,[r0,#4] - str r3,[r0,#8] - str r3,[r0,#12] - str r3,[r0,#16] - str r3,[r0,#36] @ clear is_base2_26 - add r0,r0,#20 - -#ifdef __thumb2__ - it eq -#endif - moveq r0,#0 - beq .Lno_key - -#if __ARM_MAX_ARCH__>=7 - mov r3,#-1 - str r3,[r0,#28] @ impossible key power value -# ifndef __KERNEL__ - adr r11,.Lpoly1305_init - ldr r12,.LOPENSSL_armcap -# endif -#endif - ldrb r4,[r1,#0] - mov r10,#0x0fffffff - ldrb r5,[r1,#1] - and r3,r10,#-4 @ 0x0ffffffc - ldrb r6,[r1,#2] - ldrb r7,[r1,#3] - orr r4,r4,r5,lsl#8 - ldrb r5,[r1,#4] - orr r4,r4,r6,lsl#16 - ldrb r6,[r1,#5] - orr r4,r4,r7,lsl#24 - ldrb r7,[r1,#6] - and r4,r4,r10 - -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -# if !defined(_WIN32) - ldr r12,[r11,r12] @ OPENSSL_armcap_P -# endif -# if defined(__APPLE__) || defined(_WIN32) - ldr r12,[r12] -# endif -#endif - ldrb r8,[r1,#7] - orr r5,r5,r6,lsl#8 - ldrb r6,[r1,#8] - orr r5,r5,r7,lsl#16 - ldrb r7,[r1,#9] - orr r5,r5,r8,lsl#24 - ldrb r8,[r1,#10] - and r5,r5,r3 - -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) - tst r12,#ARMV7_NEON @ check for NEON -# ifdef __thumb2__ - adr r9,.Lpoly1305_blocks_neon - adr r11,.Lpoly1305_blocks - it ne - movne r11,r9 - adr r12,.Lpoly1305_emit - orr r11,r11,#1 @ thumb-ify addresses - orr r12,r12,#1 -# else - add r12,r11,#(.Lpoly1305_emit-.Lpoly1305_init) - ite eq - addeq r11,r11,#(.Lpoly1305_blocks-.Lpoly1305_init) - addne r11,r11,#(.Lpoly1305_blocks_neon-.Lpoly1305_init) -# endif -#endif - ldrb r9,[r1,#11] - orr r6,r6,r7,lsl#8 - ldrb r7,[r1,#12] - orr r6,r6,r8,lsl#16 - ldrb r8,[r1,#13] - orr r6,r6,r9,lsl#24 - ldrb r9,[r1,#14] - and r6,r6,r3 - - ldrb r10,[r1,#15] - orr r7,r7,r8,lsl#8 - str r4,[r0,#0] - orr r7,r7,r9,lsl#16 - str r5,[r0,#4] - orr r7,r7,r10,lsl#24 - str r6,[r0,#8] - and r7,r7,r3 - str r7,[r0,#12] -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) - stmia r2,{r11,r12} @ fill functions table - mov r0,#1 -#else - mov r0,#0 -#endif -.Lno_key: - ldmia sp!,{r4-r11} -#if __ARM_ARCH__>=5 - bx lr @ bx lr -#else - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size poly1305_init,.-poly1305_init -.type poly1305_blocks,%function -.align 5 -poly1305_blocks: -.Lpoly1305_blocks: - stmdb sp!,{r3-r11,lr} - - ands r2,r2,#-16 - beq .Lno_data - - add r2,r2,r1 @ end pointer - sub sp,sp,#32 - -#if __ARM_ARCH__<7 - ldmia r0,{r4-r12} @ load context - add r0,r0,#20 - str r2,[sp,#16] @ offload stuff - str r0,[sp,#12] -#else - ldr lr,[r0,#36] @ is_base2_26 - ldmia r0!,{r4-r8} @ load hash value - str r2,[sp,#16] @ offload stuff - str r0,[sp,#12] - - adds r9,r4,r5,lsl#26 @ base 2^26 -> base 2^32 - mov r10,r5,lsr#6 - adcs r10,r10,r6,lsl#20 - mov r11,r6,lsr#12 - adcs r11,r11,r7,lsl#14 - mov r12,r7,lsr#18 - adcs r12,r12,r8,lsl#8 - mov r2,#0 - teq lr,#0 - str r2,[r0,#16] @ clear is_base2_26 - adc r2,r2,r8,lsr#24 - - itttt ne - movne r4,r9 @ choose between radixes - movne r5,r10 - movne r6,r11 - movne r7,r12 - ldmia r0,{r9-r12} @ load key - it ne - movne r8,r2 -#endif - - mov lr,r1 - cmp r3,#0 - str r10,[sp,#20] - str r11,[sp,#24] - str r12,[sp,#28] - b .Loop - -.align 4 -.Loop: -#if __ARM_ARCH__<7 - ldrb r0,[lr],#16 @ load input -# ifdef __thumb2__ - it hi -# endif - addhi r8,r8,#1 @ 1<<128 - ldrb r1,[lr,#-15] - ldrb r2,[lr,#-14] - ldrb r3,[lr,#-13] - orr r1,r0,r1,lsl#8 - ldrb r0,[lr,#-12] - orr r2,r1,r2,lsl#16 - ldrb r1,[lr,#-11] - orr r3,r2,r3,lsl#24 - ldrb r2,[lr,#-10] - adds r4,r4,r3 @ accumulate input - - ldrb r3,[lr,#-9] - orr r1,r0,r1,lsl#8 - ldrb r0,[lr,#-8] - orr r2,r1,r2,lsl#16 - ldrb r1,[lr,#-7] - orr r3,r2,r3,lsl#24 - ldrb r2,[lr,#-6] - adcs r5,r5,r3 - - ldrb r3,[lr,#-5] - orr r1,r0,r1,lsl#8 - ldrb r0,[lr,#-4] - orr r2,r1,r2,lsl#16 - ldrb r1,[lr,#-3] - orr r3,r2,r3,lsl#24 - ldrb r2,[lr,#-2] - adcs r6,r6,r3 - - ldrb r3,[lr,#-1] - orr r1,r0,r1,lsl#8 - str lr,[sp,#8] @ offload input pointer - orr r2,r1,r2,lsl#16 - add r10,r10,r10,lsr#2 - orr r3,r2,r3,lsl#24 -#else - ldr r0,[lr],#16 @ load input - it hi - addhi r8,r8,#1 @ padbit - ldr r1,[lr,#-12] - ldr r2,[lr,#-8] - ldr r3,[lr,#-4] -# ifdef __ARMEB__ - rev r0,r0 - rev r1,r1 - rev r2,r2 - rev r3,r3 -# endif - adds r4,r4,r0 @ accumulate input - str lr,[sp,#8] @ offload input pointer - adcs r5,r5,r1 - add r10,r10,r10,lsr#2 - adcs r6,r6,r2 -#endif - add r11,r11,r11,lsr#2 - adcs r7,r7,r3 - add r12,r12,r12,lsr#2 - - umull r2,r3,r5,r9 - adc r8,r8,#0 - umull r0,r1,r4,r9 - umlal r2,r3,r8,r10 - umlal r0,r1,r7,r10 - ldr r10,[sp,#20] @ reload r10 - umlal r2,r3,r6,r12 - umlal r0,r1,r5,r12 - umlal r2,r3,r7,r11 - umlal r0,r1,r6,r11 - umlal r2,r3,r4,r10 - str r0,[sp,#0] @ future r4 - mul r0,r11,r8 - ldr r11,[sp,#24] @ reload r11 - adds r2,r2,r1 @ d1+=d0>>32 - eor r1,r1,r1 - adc lr,r3,#0 @ future r6 - str r2,[sp,#4] @ future r5 - - mul r2,r12,r8 - eor r3,r3,r3 - umlal r0,r1,r7,r12 - ldr r12,[sp,#28] @ reload r12 - umlal r2,r3,r7,r9 - umlal r0,r1,r6,r9 - umlal r2,r3,r6,r10 - umlal r0,r1,r5,r10 - umlal r2,r3,r5,r11 - umlal r0,r1,r4,r11 - umlal r2,r3,r4,r12 - ldr r4,[sp,#0] - mul r8,r9,r8 - ldr r5,[sp,#4] - - adds r6,lr,r0 @ d2+=d1>>32 - ldr lr,[sp,#8] @ reload input pointer - adc r1,r1,#0 - adds r7,r2,r1 @ d3+=d2>>32 - ldr r0,[sp,#16] @ reload end pointer - adc r3,r3,#0 - add r8,r8,r3 @ h4+=d3>>32 - - and r1,r8,#-4 - and r8,r8,#3 - add r1,r1,r1,lsr#2 @ *=5 - adds r4,r4,r1 - adcs r5,r5,#0 - adcs r6,r6,#0 - adcs r7,r7,#0 - adc r8,r8,#0 - - cmp r0,lr @ done yet? - bhi .Loop - - ldr r0,[sp,#12] - add sp,sp,#32 - stmdb r0,{r4-r8} @ store the result - -.Lno_data: -#if __ARM_ARCH__>=5 - ldmia sp!,{r3-r11,pc} -#else - ldmia sp!,{r3-r11,lr} - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size poly1305_blocks,.-poly1305_blocks -.type poly1305_emit,%function -.align 5 -poly1305_emit: -.Lpoly1305_emit: - stmdb sp!,{r4-r11} - - ldmia r0,{r3-r7} - -#if __ARM_ARCH__>=7 - ldr ip,[r0,#36] @ is_base2_26 - - adds r8,r3,r4,lsl#26 @ base 2^26 -> base 2^32 - mov r9,r4,lsr#6 - adcs r9,r9,r5,lsl#20 - mov r10,r5,lsr#12 - adcs r10,r10,r6,lsl#14 - mov r11,r6,lsr#18 - adcs r11,r11,r7,lsl#8 - mov r0,#0 - adc r0,r0,r7,lsr#24 - - tst ip,ip - itttt ne - movne r3,r8 - movne r4,r9 - movne r5,r10 - movne r6,r11 - it ne - movne r7,r0 -#endif - - adds r8,r3,#5 @ compare to modulus - adcs r9,r4,#0 - adcs r10,r5,#0 - adcs r11,r6,#0 - adc r0,r7,#0 - tst r0,#4 @ did it carry/borrow? - -#ifdef __thumb2__ - it ne -#endif - movne r3,r8 - ldr r8,[r2,#0] -#ifdef __thumb2__ - it ne -#endif - movne r4,r9 - ldr r9,[r2,#4] -#ifdef __thumb2__ - it ne -#endif - movne r5,r10 - ldr r10,[r2,#8] -#ifdef __thumb2__ - it ne -#endif - movne r6,r11 - ldr r11,[r2,#12] - - adds r3,r3,r8 - adcs r4,r4,r9 - adcs r5,r5,r10 - adc r6,r6,r11 - -#if __ARM_ARCH__>=7 -# ifdef __ARMEB__ - rev r3,r3 - rev r4,r4 - rev r5,r5 - rev r6,r6 -# endif - str r3,[r1,#0] - str r4,[r1,#4] - str r5,[r1,#8] - str r6,[r1,#12] -#else - strb r3,[r1,#0] - mov r3,r3,lsr#8 - strb r4,[r1,#4] - mov r4,r4,lsr#8 - strb r5,[r1,#8] - mov r5,r5,lsr#8 - strb r6,[r1,#12] - mov r6,r6,lsr#8 - - strb r3,[r1,#1] - mov r3,r3,lsr#8 - strb r4,[r1,#5] - mov r4,r4,lsr#8 - strb r5,[r1,#9] - mov r5,r5,lsr#8 - strb r6,[r1,#13] - mov r6,r6,lsr#8 - - strb r3,[r1,#2] - mov r3,r3,lsr#8 - strb r4,[r1,#6] - mov r4,r4,lsr#8 - strb r5,[r1,#10] - mov r5,r5,lsr#8 - strb r6,[r1,#14] - mov r6,r6,lsr#8 - - strb r3,[r1,#3] - strb r4,[r1,#7] - strb r5,[r1,#11] - strb r6,[r1,#15] -#endif - ldmia sp!,{r4-r11} -#if __ARM_ARCH__>=5 - bx lr @ bx lr -#else - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size poly1305_emit,.-poly1305_emit -#if __ARM_MAX_ARCH__>=7 -.fpu neon - -.type poly1305_init_neon,%function -.align 5 -poly1305_init_neon: -.Lpoly1305_init_neon: - ldr r3,[r0,#48] @ first table element - cmp r3,#-1 @ is value impossible? - bne .Lno_init_neon - - ldr r4,[r0,#20] @ load key base 2^32 - ldr r5,[r0,#24] - ldr r6,[r0,#28] - ldr r7,[r0,#32] - - and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 - mov r3,r4,lsr#26 - mov r4,r5,lsr#20 - orr r3,r3,r5,lsl#6 - mov r5,r6,lsr#14 - orr r4,r4,r6,lsl#12 - mov r6,r7,lsr#8 - orr r5,r5,r7,lsl#18 - and r3,r3,#0x03ffffff - and r4,r4,#0x03ffffff - and r5,r5,#0x03ffffff - - vdup.32 d0,r2 @ r^1 in both lanes - add r2,r3,r3,lsl#2 @ *5 - vdup.32 d1,r3 - add r3,r4,r4,lsl#2 - vdup.32 d2,r2 - vdup.32 d3,r4 - add r4,r5,r5,lsl#2 - vdup.32 d4,r3 - vdup.32 d5,r5 - add r5,r6,r6,lsl#2 - vdup.32 d6,r4 - vdup.32 d7,r6 - vdup.32 d8,r5 - - mov r5,#2 @ counter - -.Lsquare_neon: - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 - @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 - @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 - @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 - - vmull.u32 q5,d0,d0[1] - vmull.u32 q6,d1,d0[1] - vmull.u32 q7,d3,d0[1] - vmull.u32 q8,d5,d0[1] - vmull.u32 q9,d7,d0[1] - - vmlal.u32 q5,d7,d2[1] - vmlal.u32 q6,d0,d1[1] - vmlal.u32 q7,d1,d1[1] - vmlal.u32 q8,d3,d1[1] - vmlal.u32 q9,d5,d1[1] - - vmlal.u32 q5,d5,d4[1] - vmlal.u32 q6,d7,d4[1] - vmlal.u32 q8,d1,d3[1] - vmlal.u32 q7,d0,d3[1] - vmlal.u32 q9,d3,d3[1] - - vmlal.u32 q5,d3,d6[1] - vmlal.u32 q8,d0,d5[1] - vmlal.u32 q6,d5,d6[1] - vmlal.u32 q7,d7,d6[1] - vmlal.u32 q9,d1,d5[1] - - vmlal.u32 q8,d7,d8[1] - vmlal.u32 q5,d1,d8[1] - vmlal.u32 q6,d3,d8[1] - vmlal.u32 q7,d5,d8[1] - vmlal.u32 q9,d0,d7[1] - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ lazy reduction as discussed in "NEON crypto" by D.J. Bernstein - @ and P. Schwabe - @ - @ H0>>+H1>>+H2>>+H3>>+H4 - @ H3>>+H4>>*5+H0>>+H1 - @ - @ Trivia. - @ - @ Result of multiplication of n-bit number by m-bit number is - @ n+m bits wide. However! Even though 2^n is a n+1-bit number, - @ m-bit number multiplied by 2^n is still n+m bits wide. - @ - @ Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2, - @ and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit - @ one is n+1 bits wide. - @ - @ >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that - @ H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4 - @ can be 27. However! In cases when their width exceeds 26 bits - @ they are limited by 2^26+2^6. This in turn means that *sum* - @ of the products with these values can still be viewed as sum - @ of 52-bit numbers as long as the amount of addends is not a - @ power of 2. For example, - @ - @ H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4, - @ - @ which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or - @ 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than - @ 8 * (2^52) or 2^55. However, the value is then multiplied by - @ by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12), - @ which is less than 32 * (2^52) or 2^57. And when processing - @ data we are looking at triple as many addends... - @ - @ In key setup procedure pre-reduced H0 is limited by 5*4+1 and - @ 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the - @ input H0 is limited by (5*4+1)*3 addends, or 58 bits, while - @ 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32 - @ instruction accepts 2x32-bit input and writes 2x64-bit result. - @ This means that result of reduction have to be compressed upon - @ loop wrap-around. This can be done in the process of reduction - @ to minimize amount of instructions [as well as amount of - @ 128-bit instructions, which benefits low-end processors], but - @ one has to watch for H2 (which is narrower than H0) and 5*H4 - @ not being wider than 58 bits, so that result of right shift - @ by 26 bits fits in 32 bits. This is also useful on x86, - @ because it allows to use paddd in place for paddq, which - @ benefits Atom, where paddq is ridiculously slow. - - vshr.u64 q15,q8,#26 - vmovn.i64 d16,q8 - vshr.u64 q4,q5,#26 - vmovn.i64 d10,q5 - vadd.i64 q9,q9,q15 @ h3 -> h4 - vbic.i32 d16,#0xfc000000 @ &=0x03ffffff - vadd.i64 q6,q6,q4 @ h0 -> h1 - vbic.i32 d10,#0xfc000000 - - vshrn.u64 d30,q9,#26 - vmovn.i64 d18,q9 - vshr.u64 q4,q6,#26 - vmovn.i64 d12,q6 - vadd.i64 q7,q7,q4 @ h1 -> h2 - vbic.i32 d18,#0xfc000000 - vbic.i32 d12,#0xfc000000 - - vadd.i32 d10,d10,d30 - vshl.u32 d30,d30,#2 - vshrn.u64 d8,q7,#26 - vmovn.i64 d14,q7 - vadd.i32 d10,d10,d30 @ h4 -> h0 - vadd.i32 d16,d16,d8 @ h2 -> h3 - vbic.i32 d14,#0xfc000000 - - vshr.u32 d30,d10,#26 - vbic.i32 d10,#0xfc000000 - vshr.u32 d8,d16,#26 - vbic.i32 d16,#0xfc000000 - vadd.i32 d12,d12,d30 @ h0 -> h1 - vadd.i32 d18,d18,d8 @ h3 -> h4 - - subs r5,r5,#1 - beq .Lsquare_break_neon - - add r6,r0,#(48+0*9*4) - add r7,r0,#(48+1*9*4) - - vtrn.32 d0,d10 @ r^2:r^1 - vtrn.32 d3,d14 - vtrn.32 d5,d16 - vtrn.32 d1,d12 - vtrn.32 d7,d18 - - vshl.u32 d4,d3,#2 @ *5 - vshl.u32 d6,d5,#2 - vshl.u32 d2,d1,#2 - vshl.u32 d8,d7,#2 - vadd.i32 d4,d4,d3 - vadd.i32 d2,d2,d1 - vadd.i32 d6,d6,d5 - vadd.i32 d8,d8,d7 - - vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! - vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! - vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! - vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! - vst1.32 {d8[0]},[r6,:32] - vst1.32 {d8[1]},[r7,:32] - - b .Lsquare_neon - -.align 4 -.Lsquare_break_neon: - add r6,r0,#(48+2*4*9) - add r7,r0,#(48+3*4*9) - - vmov d0,d10 @ r^4:r^3 - vshl.u32 d2,d12,#2 @ *5 - vmov d1,d12 - vshl.u32 d4,d14,#2 - vmov d3,d14 - vshl.u32 d6,d16,#2 - vmov d5,d16 - vshl.u32 d8,d18,#2 - vmov d7,d18 - vadd.i32 d2,d2,d12 - vadd.i32 d4,d4,d14 - vadd.i32 d6,d6,d16 - vadd.i32 d8,d8,d18 - - vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! - vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! - vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! - vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! - vst1.32 {d8[0]},[r6] - vst1.32 {d8[1]},[r7] - -.Lno_init_neon: - bx lr @ bx lr -.size poly1305_init_neon,.-poly1305_init_neon - -.type poly1305_blocks_neon,%function -.align 5 -poly1305_blocks_neon: -.Lpoly1305_blocks_neon: - ldr ip,[r0,#36] @ is_base2_26 - - cmp r2,#64 - blo .Lpoly1305_blocks - - stmdb sp!,{r4-r7} - vstmdb sp!,{d8-d15} @ ABI specification says so - - tst ip,ip @ is_base2_26? - bne .Lbase2_26_neon - - stmdb sp!,{r1-r3,lr} - bl .Lpoly1305_init_neon - - ldr r4,[r0,#0] @ load hash value base 2^32 - ldr r5,[r0,#4] - ldr r6,[r0,#8] - ldr r7,[r0,#12] - ldr ip,[r0,#16] - - and r2,r4,#0x03ffffff @ base 2^32 -> base 2^26 - mov r3,r4,lsr#26 - veor d10,d10,d10 - mov r4,r5,lsr#20 - orr r3,r3,r5,lsl#6 - veor d12,d12,d12 - mov r5,r6,lsr#14 - orr r4,r4,r6,lsl#12 - veor d14,d14,d14 - mov r6,r7,lsr#8 - orr r5,r5,r7,lsl#18 - veor d16,d16,d16 - and r3,r3,#0x03ffffff - orr r6,r6,ip,lsl#24 - veor d18,d18,d18 - and r4,r4,#0x03ffffff - mov r1,#1 - and r5,r5,#0x03ffffff - str r1,[r0,#36] @ set is_base2_26 - - vmov.32 d10[0],r2 - vmov.32 d12[0],r3 - vmov.32 d14[0],r4 - vmov.32 d16[0],r5 - vmov.32 d18[0],r6 - adr r5,.Lzeros - - ldmia sp!,{r1-r3,lr} - b .Lhash_loaded - -.align 4 -.Lbase2_26_neon: - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ load hash value - - veor d10,d10,d10 - veor d12,d12,d12 - veor d14,d14,d14 - veor d16,d16,d16 - veor d18,d18,d18 - vld4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]! - adr r5,.Lzeros - vld1.32 {d18[0]},[r0] - sub r0,r0,#16 @ rewind - -.Lhash_loaded: - add r4,r1,#32 - mov r3,r3,lsl#24 - tst r2,#31 - beq .Leven - - vld4.32 {d20[0],d22[0],d24[0],d26[0]},[r1]! - vmov.32 d28[0],r3 - sub r2,r2,#16 - add r4,r1,#32 - -# ifdef __ARMEB__ - vrev32.8 q10,q10 - vrev32.8 q13,q13 - vrev32.8 q11,q11 - vrev32.8 q12,q12 -# endif - vsri.u32 d28,d26,#8 @ base 2^32 -> base 2^26 - vshl.u32 d26,d26,#18 - - vsri.u32 d26,d24,#14 - vshl.u32 d24,d24,#12 - vadd.i32 d29,d28,d18 @ add hash value and move to #hi - - vbic.i32 d26,#0xfc000000 - vsri.u32 d24,d22,#20 - vshl.u32 d22,d22,#6 - - vbic.i32 d24,#0xfc000000 - vsri.u32 d22,d20,#26 - vadd.i32 d27,d26,d16 - - vbic.i32 d20,#0xfc000000 - vbic.i32 d22,#0xfc000000 - vadd.i32 d25,d24,d14 - - vadd.i32 d21,d20,d10 - vadd.i32 d23,d22,d12 - - mov r7,r5 - add r6,r0,#48 - - cmp r2,r2 - b .Long_tail - -.align 4 -.Leven: - subs r2,r2,#64 - it lo - movlo r4,r5 - - vmov.i32 q14,#1<<24 @ padbit, yes, always - vld4.32 {d20,d22,d24,d26},[r1] @ inp[0:1] - add r1,r1,#64 - vld4.32 {d21,d23,d25,d27},[r4] @ inp[2:3] (or 0) - add r4,r4,#64 - itt hi - addhi r7,r0,#(48+1*9*4) - addhi r6,r0,#(48+3*9*4) - -# ifdef __ARMEB__ - vrev32.8 q10,q10 - vrev32.8 q13,q13 - vrev32.8 q11,q11 - vrev32.8 q12,q12 -# endif - vsri.u32 q14,q13,#8 @ base 2^32 -> base 2^26 - vshl.u32 q13,q13,#18 - - vsri.u32 q13,q12,#14 - vshl.u32 q12,q12,#12 - - vbic.i32 q13,#0xfc000000 - vsri.u32 q12,q11,#20 - vshl.u32 q11,q11,#6 - - vbic.i32 q12,#0xfc000000 - vsri.u32 q11,q10,#26 - - vbic.i32 q10,#0xfc000000 - vbic.i32 q11,#0xfc000000 - - bls .Lskip_loop - - vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! @ load r^2 - vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! @ load r^4 - vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! - vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! - b .Loop_neon - -.align 5 -.Loop_neon: - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 - @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r - @ ___________________/ - @ ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 - @ ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r - @ ___________________/ ____________________/ - @ - @ Note that we start with inp[2:3]*r^2. This is because it - @ doesn't depend on reduction in previous iteration. - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4 - @ d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4 - @ d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4 - @ d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4 - @ d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4 - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ inp[2:3]*r^2 - - vadd.i32 d24,d24,d14 @ accumulate inp[0:1] - vmull.u32 q7,d25,d0[1] - vadd.i32 d20,d20,d10 - vmull.u32 q5,d21,d0[1] - vadd.i32 d26,d26,d16 - vmull.u32 q8,d27,d0[1] - vmlal.u32 q7,d23,d1[1] - vadd.i32 d22,d22,d12 - vmull.u32 q6,d23,d0[1] - - vadd.i32 d28,d28,d18 - vmull.u32 q9,d29,d0[1] - subs r2,r2,#64 - vmlal.u32 q5,d29,d2[1] - it lo - movlo r4,r5 - vmlal.u32 q8,d25,d1[1] - vld1.32 d8[1],[r7,:32] - vmlal.u32 q6,d21,d1[1] - vmlal.u32 q9,d27,d1[1] - - vmlal.u32 q5,d27,d4[1] - vmlal.u32 q8,d23,d3[1] - vmlal.u32 q9,d25,d3[1] - vmlal.u32 q6,d29,d4[1] - vmlal.u32 q7,d21,d3[1] - - vmlal.u32 q8,d21,d5[1] - vmlal.u32 q5,d25,d6[1] - vmlal.u32 q9,d23,d5[1] - vmlal.u32 q6,d27,d6[1] - vmlal.u32 q7,d29,d6[1] - - vmlal.u32 q8,d29,d8[1] - vmlal.u32 q5,d23,d8[1] - vmlal.u32 q9,d21,d7[1] - vmlal.u32 q6,d25,d8[1] - vmlal.u32 q7,d27,d8[1] - - vld4.32 {d21,d23,d25,d27},[r4] @ inp[2:3] (or 0) - add r4,r4,#64 - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ (hash+inp[0:1])*r^4 and accumulate - - vmlal.u32 q8,d26,d0[0] - vmlal.u32 q5,d20,d0[0] - vmlal.u32 q9,d28,d0[0] - vmlal.u32 q6,d22,d0[0] - vmlal.u32 q7,d24,d0[0] - vld1.32 d8[0],[r6,:32] - - vmlal.u32 q8,d24,d1[0] - vmlal.u32 q5,d28,d2[0] - vmlal.u32 q9,d26,d1[0] - vmlal.u32 q6,d20,d1[0] - vmlal.u32 q7,d22,d1[0] - - vmlal.u32 q8,d22,d3[0] - vmlal.u32 q5,d26,d4[0] - vmlal.u32 q9,d24,d3[0] - vmlal.u32 q6,d28,d4[0] - vmlal.u32 q7,d20,d3[0] - - vmlal.u32 q8,d20,d5[0] - vmlal.u32 q5,d24,d6[0] - vmlal.u32 q9,d22,d5[0] - vmlal.u32 q6,d26,d6[0] - vmlal.u32 q8,d28,d8[0] - - vmlal.u32 q7,d28,d6[0] - vmlal.u32 q5,d22,d8[0] - vmlal.u32 q9,d20,d7[0] - vmov.i32 q14,#1<<24 @ padbit, yes, always - vmlal.u32 q6,d24,d8[0] - vmlal.u32 q7,d26,d8[0] - - vld4.32 {d20,d22,d24,d26},[r1] @ inp[0:1] - add r1,r1,#64 -# ifdef __ARMEB__ - vrev32.8 q10,q10 - vrev32.8 q11,q11 - vrev32.8 q12,q12 - vrev32.8 q13,q13 -# endif - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ lazy reduction interleaved with base 2^32 -> base 2^26 of - @ inp[0:3] previously loaded to q10-q13 and smashed to q10-q14. - - vshr.u64 q15,q8,#26 - vmovn.i64 d16,q8 - vshr.u64 q4,q5,#26 - vmovn.i64 d10,q5 - vadd.i64 q9,q9,q15 @ h3 -> h4 - vbic.i32 d16,#0xfc000000 - vsri.u32 q14,q13,#8 @ base 2^32 -> base 2^26 - vadd.i64 q6,q6,q4 @ h0 -> h1 - vshl.u32 q13,q13,#18 - vbic.i32 d10,#0xfc000000 - - vshrn.u64 d30,q9,#26 - vmovn.i64 d18,q9 - vshr.u64 q4,q6,#26 - vmovn.i64 d12,q6 - vadd.i64 q7,q7,q4 @ h1 -> h2 - vsri.u32 q13,q12,#14 - vbic.i32 d18,#0xfc000000 - vshl.u32 q12,q12,#12 - vbic.i32 d12,#0xfc000000 - - vadd.i32 d10,d10,d30 - vshl.u32 d30,d30,#2 - vbic.i32 q13,#0xfc000000 - vshrn.u64 d8,q7,#26 - vmovn.i64 d14,q7 - vaddl.u32 q5,d10,d30 @ h4 -> h0 [widen for a sec] - vsri.u32 q12,q11,#20 - vadd.i32 d16,d16,d8 @ h2 -> h3 - vshl.u32 q11,q11,#6 - vbic.i32 d14,#0xfc000000 - vbic.i32 q12,#0xfc000000 - - vshrn.u64 d30,q5,#26 @ re-narrow - vmovn.i64 d10,q5 - vsri.u32 q11,q10,#26 - vbic.i32 q10,#0xfc000000 - vshr.u32 d8,d16,#26 - vbic.i32 d16,#0xfc000000 - vbic.i32 d10,#0xfc000000 - vadd.i32 d12,d12,d30 @ h0 -> h1 - vadd.i32 d18,d18,d8 @ h3 -> h4 - vbic.i32 q11,#0xfc000000 - - bhi .Loop_neon - -.Lskip_loop: - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 - - add r7,r0,#(48+0*9*4) - add r6,r0,#(48+1*9*4) - adds r2,r2,#32 - it ne - movne r2,#0 - bne .Long_tail - - vadd.i32 d25,d24,d14 @ add hash value and move to #hi - vadd.i32 d21,d20,d10 - vadd.i32 d27,d26,d16 - vadd.i32 d23,d22,d12 - vadd.i32 d29,d28,d18 - -.Long_tail: - vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! @ load r^1 - vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! @ load r^2 - - vadd.i32 d24,d24,d14 @ can be redundant - vmull.u32 q7,d25,d0 - vadd.i32 d20,d20,d10 - vmull.u32 q5,d21,d0 - vadd.i32 d26,d26,d16 - vmull.u32 q8,d27,d0 - vadd.i32 d22,d22,d12 - vmull.u32 q6,d23,d0 - vadd.i32 d28,d28,d18 - vmull.u32 q9,d29,d0 - - vmlal.u32 q5,d29,d2 - vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! - vmlal.u32 q8,d25,d1 - vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! - vmlal.u32 q6,d21,d1 - vmlal.u32 q9,d27,d1 - vmlal.u32 q7,d23,d1 - - vmlal.u32 q8,d23,d3 - vld1.32 d8[1],[r7,:32] - vmlal.u32 q5,d27,d4 - vld1.32 d8[0],[r6,:32] - vmlal.u32 q9,d25,d3 - vmlal.u32 q6,d29,d4 - vmlal.u32 q7,d21,d3 - - vmlal.u32 q8,d21,d5 - it ne - addne r7,r0,#(48+2*9*4) - vmlal.u32 q5,d25,d6 - it ne - addne r6,r0,#(48+3*9*4) - vmlal.u32 q9,d23,d5 - vmlal.u32 q6,d27,d6 - vmlal.u32 q7,d29,d6 - - vmlal.u32 q8,d29,d8 - vorn q0,q0,q0 @ all-ones, can be redundant - vmlal.u32 q5,d23,d8 - vshr.u64 q0,q0,#38 - vmlal.u32 q9,d21,d7 - vmlal.u32 q6,d25,d8 - vmlal.u32 q7,d27,d8 - - beq .Lshort_tail - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ (hash+inp[0:1])*r^4:r^3 and accumulate - - vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]! @ load r^3 - vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]! @ load r^4 - - vmlal.u32 q7,d24,d0 - vmlal.u32 q5,d20,d0 - vmlal.u32 q8,d26,d0 - vmlal.u32 q6,d22,d0 - vmlal.u32 q9,d28,d0 - - vmlal.u32 q5,d28,d2 - vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]! - vmlal.u32 q8,d24,d1 - vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]! - vmlal.u32 q6,d20,d1 - vmlal.u32 q9,d26,d1 - vmlal.u32 q7,d22,d1 - - vmlal.u32 q8,d22,d3 - vld1.32 d8[1],[r7,:32] - vmlal.u32 q5,d26,d4 - vld1.32 d8[0],[r6,:32] - vmlal.u32 q9,d24,d3 - vmlal.u32 q6,d28,d4 - vmlal.u32 q7,d20,d3 - - vmlal.u32 q8,d20,d5 - vmlal.u32 q5,d24,d6 - vmlal.u32 q9,d22,d5 - vmlal.u32 q6,d26,d6 - vmlal.u32 q7,d28,d6 - - vmlal.u32 q8,d28,d8 - vorn q0,q0,q0 @ all-ones - vmlal.u32 q5,d22,d8 - vshr.u64 q0,q0,#38 - vmlal.u32 q9,d20,d7 - vmlal.u32 q6,d24,d8 - vmlal.u32 q7,d26,d8 - -.Lshort_tail: - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ horizontal addition - - vadd.i64 d16,d16,d17 - vadd.i64 d10,d10,d11 - vadd.i64 d18,d18,d19 - vadd.i64 d12,d12,d13 - vadd.i64 d14,d14,d15 - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ lazy reduction, but without narrowing - - vshr.u64 q15,q8,#26 - vand.i64 q8,q8,q0 - vshr.u64 q4,q5,#26 - vand.i64 q5,q5,q0 - vadd.i64 q9,q9,q15 @ h3 -> h4 - vadd.i64 q6,q6,q4 @ h0 -> h1 - - vshr.u64 q15,q9,#26 - vand.i64 q9,q9,q0 - vshr.u64 q4,q6,#26 - vand.i64 q6,q6,q0 - vadd.i64 q7,q7,q4 @ h1 -> h2 - - vadd.i64 q5,q5,q15 - vshl.u64 q15,q15,#2 - vshr.u64 q4,q7,#26 - vand.i64 q7,q7,q0 - vadd.i64 q5,q5,q15 @ h4 -> h0 - vadd.i64 q8,q8,q4 @ h2 -> h3 - - vshr.u64 q15,q5,#26 - vand.i64 q5,q5,q0 - vshr.u64 q4,q8,#26 - vand.i64 q8,q8,q0 - vadd.i64 q6,q6,q15 @ h0 -> h1 - vadd.i64 q9,q9,q4 @ h3 -> h4 - - cmp r2,#0 - bne .Leven - - @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - @ store hash value - - vst4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]! - vst1.32 {d18[0]},[r0] - - vldmia sp!,{d8-d15} @ epilogue - ldmia sp!,{r4-r7} - bx lr @ bx lr -.size poly1305_blocks_neon,.-poly1305_blocks_neon - -.align 5 -.Lzeros: -.long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -#ifndef __KERNEL__ -.LOPENSSL_armcap: -# ifdef _WIN32 -.word OPENSSL_armcap_P -# else -.word OPENSSL_armcap_P-.Lpoly1305_init -# endif -.comm OPENSSL_armcap_P,4,4 -.hidden OPENSSL_armcap_P -#endif -#endif -.asciz "Poly1305 for ARMv4/NEON, CRYPTOGAMS by @dot-asm" -.align 2 diff --git a/arch/arm/crypto/sha256-core.S_shipped b/arch/arm/crypto/sha256-core.S_shipped deleted file mode 100644 index 6363014a50d7..000000000000 --- a/arch/arm/crypto/sha256-core.S_shipped +++ /dev/null @@ -1,2816 +0,0 @@ -@ SPDX-License-Identifier: GPL-2.0 - -@ This code is taken from the OpenSSL project but the author (Andy Polyakov) -@ has relicensed it under the GPLv2. Therefore this program is free software; -@ you can redistribute it and/or modify it under the terms of the GNU General -@ Public License version 2 as published by the Free Software Foundation. -@ -@ The original headers, including the original license headers, are -@ included below for completeness. - -@ ==================================================================== -@ Written by Andy Polyakov for the OpenSSL -@ project. The module is, however, dual licensed under OpenSSL and -@ CRYPTOGAMS licenses depending on where you obtain it. For further -@ details see https://www.openssl.org/~appro/cryptogams/. -@ ==================================================================== - -@ SHA256 block procedure for ARMv4. May 2007. - -@ Performance is ~2x better than gcc 3.4 generated code and in "abso- -@ lute" terms is ~2250 cycles per 64-byte block or ~35 cycles per -@ byte [on single-issue Xscale PXA250 core]. - -@ July 2010. -@ -@ Rescheduling for dual-issue pipeline resulted in 22% improvement on -@ Cortex A8 core and ~20 cycles per processed byte. - -@ February 2011. -@ -@ Profiler-assisted and platform-specific optimization resulted in 16% -@ improvement on Cortex A8 core and ~15.4 cycles per processed byte. - -@ September 2013. -@ -@ Add NEON implementation. On Cortex A8 it was measured to process one -@ byte in 12.5 cycles or 23% faster than integer-only code. Snapdragon -@ S4 does it in 12.5 cycles too, but it's 50% faster than integer-only -@ code (meaning that latter performs sub-optimally, nothing was done -@ about it). - -@ May 2014. -@ -@ Add ARMv8 code path performing at 2.0 cpb on Apple A7. - -#ifndef __KERNEL__ -# include "arm_arch.h" -#else -# define __ARM_ARCH__ __LINUX_ARM_ARCH__ -# define __ARM_MAX_ARCH__ 7 -#endif - -.text -#if __ARM_ARCH__<7 -.code 32 -#else -.syntax unified -# ifdef __thumb2__ -.thumb -# else -.code 32 -# endif -#endif - -.type K256,%object -.align 5 -K256: -.word 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 -.word 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 -.word 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 -.word 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 -.word 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc -.word 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da -.word 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 -.word 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 -.word 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 -.word 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 -.word 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 -.word 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 -.word 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 -.word 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 -.word 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 -.word 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 -.size K256,.-K256 -.word 0 @ terminator -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.LOPENSSL_armcap: -.word OPENSSL_armcap_P-sha256_block_data_order -#endif -.align 5 - -.global sha256_block_data_order -.type sha256_block_data_order,%function -sha256_block_data_order: -.Lsha256_block_data_order: -#if __ARM_ARCH__<7 - sub r3,pc,#8 @ sha256_block_data_order -#else - adr r3,.Lsha256_block_data_order -#endif -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) - ldr r12,.LOPENSSL_armcap - ldr r12,[r3,r12] @ OPENSSL_armcap_P - tst r12,#ARMV8_SHA256 - bne .LARMv8 - tst r12,#ARMV7_NEON - bne .LNEON -#endif - add r2,r1,r2,lsl#6 @ len to point at the end of inp - stmdb sp!,{r0,r1,r2,r4-r11,lr} - ldmia r0,{r4,r5,r6,r7,r8,r9,r10,r11} - sub r14,r3,#256+32 @ K256 - sub sp,sp,#16*4 @ alloca(X[16]) -.Loop: -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 -# else - ldrb r2,[r1,#3] -# endif - eor r3,r5,r6 @ magic - eor r12,r12,r12 -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 0 -# if 0==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r8,r8,ror#5 - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r8,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 0 - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 0==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r8,r8,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r8,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r11,r11,r2 @ h+=X[i] - str r2,[sp,#0*4] - eor r2,r9,r10 - add r11,r11,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r8 - add r11,r11,r12 @ h+=K256[i] - eor r2,r2,r10 @ Ch(e,f,g) - eor r0,r4,r4,ror#11 - add r11,r11,r2 @ h+=Ch(e,f,g) -#if 0==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 0<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r4,r5 @ a^b, b^c in next round -#else - ldr r2,[sp,#2*4] @ from future BODY_16_xx - eor r12,r4,r5 @ a^b, b^c in next round - ldr r1,[sp,#15*4] @ from future BODY_16_xx -#endif - eor r0,r0,r4,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r7,r7,r11 @ d+=h - eor r3,r3,r5 @ Maj(a,b,c) - add r11,r11,r0,ror#2 @ h+=Sigma0(a) - @ add r11,r11,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 1 -# if 1==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r7,r7,ror#5 - add r11,r11,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r7,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 1 - add r11,r11,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 1==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r7,r7,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r7,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r10,r10,r2 @ h+=X[i] - str r2,[sp,#1*4] - eor r2,r8,r9 - add r10,r10,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r7 - add r10,r10,r3 @ h+=K256[i] - eor r2,r2,r9 @ Ch(e,f,g) - eor r0,r11,r11,ror#11 - add r10,r10,r2 @ h+=Ch(e,f,g) -#if 1==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 1<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r11,r4 @ a^b, b^c in next round -#else - ldr r2,[sp,#3*4] @ from future BODY_16_xx - eor r3,r11,r4 @ a^b, b^c in next round - ldr r1,[sp,#0*4] @ from future BODY_16_xx -#endif - eor r0,r0,r11,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r6,r6,r10 @ d+=h - eor r12,r12,r4 @ Maj(a,b,c) - add r10,r10,r0,ror#2 @ h+=Sigma0(a) - @ add r10,r10,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 2 -# if 2==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r6,r6,ror#5 - add r10,r10,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r6,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 2 - add r10,r10,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 2==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r6,r6,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r6,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r9,r9,r2 @ h+=X[i] - str r2,[sp,#2*4] - eor r2,r7,r8 - add r9,r9,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r6 - add r9,r9,r12 @ h+=K256[i] - eor r2,r2,r8 @ Ch(e,f,g) - eor r0,r10,r10,ror#11 - add r9,r9,r2 @ h+=Ch(e,f,g) -#if 2==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 2<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r10,r11 @ a^b, b^c in next round -#else - ldr r2,[sp,#4*4] @ from future BODY_16_xx - eor r12,r10,r11 @ a^b, b^c in next round - ldr r1,[sp,#1*4] @ from future BODY_16_xx -#endif - eor r0,r0,r10,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r5,r5,r9 @ d+=h - eor r3,r3,r11 @ Maj(a,b,c) - add r9,r9,r0,ror#2 @ h+=Sigma0(a) - @ add r9,r9,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 3 -# if 3==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r5,r5,ror#5 - add r9,r9,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r5,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 3 - add r9,r9,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 3==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r5,r5,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r5,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r8,r8,r2 @ h+=X[i] - str r2,[sp,#3*4] - eor r2,r6,r7 - add r8,r8,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r5 - add r8,r8,r3 @ h+=K256[i] - eor r2,r2,r7 @ Ch(e,f,g) - eor r0,r9,r9,ror#11 - add r8,r8,r2 @ h+=Ch(e,f,g) -#if 3==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 3<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r9,r10 @ a^b, b^c in next round -#else - ldr r2,[sp,#5*4] @ from future BODY_16_xx - eor r3,r9,r10 @ a^b, b^c in next round - ldr r1,[sp,#2*4] @ from future BODY_16_xx -#endif - eor r0,r0,r9,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r4,r4,r8 @ d+=h - eor r12,r12,r10 @ Maj(a,b,c) - add r8,r8,r0,ror#2 @ h+=Sigma0(a) - @ add r8,r8,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 4 -# if 4==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r4,r4,ror#5 - add r8,r8,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r4,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 4 - add r8,r8,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 4==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r4,r4,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r4,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r7,r7,r2 @ h+=X[i] - str r2,[sp,#4*4] - eor r2,r5,r6 - add r7,r7,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r4 - add r7,r7,r12 @ h+=K256[i] - eor r2,r2,r6 @ Ch(e,f,g) - eor r0,r8,r8,ror#11 - add r7,r7,r2 @ h+=Ch(e,f,g) -#if 4==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 4<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r8,r9 @ a^b, b^c in next round -#else - ldr r2,[sp,#6*4] @ from future BODY_16_xx - eor r12,r8,r9 @ a^b, b^c in next round - ldr r1,[sp,#3*4] @ from future BODY_16_xx -#endif - eor r0,r0,r8,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r11,r11,r7 @ d+=h - eor r3,r3,r9 @ Maj(a,b,c) - add r7,r7,r0,ror#2 @ h+=Sigma0(a) - @ add r7,r7,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 5 -# if 5==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r11,r11,ror#5 - add r7,r7,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r11,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 5 - add r7,r7,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 5==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r11,r11,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r11,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r6,r6,r2 @ h+=X[i] - str r2,[sp,#5*4] - eor r2,r4,r5 - add r6,r6,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r11 - add r6,r6,r3 @ h+=K256[i] - eor r2,r2,r5 @ Ch(e,f,g) - eor r0,r7,r7,ror#11 - add r6,r6,r2 @ h+=Ch(e,f,g) -#if 5==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 5<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r7,r8 @ a^b, b^c in next round -#else - ldr r2,[sp,#7*4] @ from future BODY_16_xx - eor r3,r7,r8 @ a^b, b^c in next round - ldr r1,[sp,#4*4] @ from future BODY_16_xx -#endif - eor r0,r0,r7,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r10,r10,r6 @ d+=h - eor r12,r12,r8 @ Maj(a,b,c) - add r6,r6,r0,ror#2 @ h+=Sigma0(a) - @ add r6,r6,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 6 -# if 6==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r10,r10,ror#5 - add r6,r6,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r10,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 6 - add r6,r6,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 6==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r10,r10,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r10,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r5,r5,r2 @ h+=X[i] - str r2,[sp,#6*4] - eor r2,r11,r4 - add r5,r5,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r10 - add r5,r5,r12 @ h+=K256[i] - eor r2,r2,r4 @ Ch(e,f,g) - eor r0,r6,r6,ror#11 - add r5,r5,r2 @ h+=Ch(e,f,g) -#if 6==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 6<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r6,r7 @ a^b, b^c in next round -#else - ldr r2,[sp,#8*4] @ from future BODY_16_xx - eor r12,r6,r7 @ a^b, b^c in next round - ldr r1,[sp,#5*4] @ from future BODY_16_xx -#endif - eor r0,r0,r6,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r9,r9,r5 @ d+=h - eor r3,r3,r7 @ Maj(a,b,c) - add r5,r5,r0,ror#2 @ h+=Sigma0(a) - @ add r5,r5,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 7 -# if 7==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r9,r9,ror#5 - add r5,r5,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r9,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 7 - add r5,r5,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 7==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r9,r9,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r9,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r4,r4,r2 @ h+=X[i] - str r2,[sp,#7*4] - eor r2,r10,r11 - add r4,r4,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r9 - add r4,r4,r3 @ h+=K256[i] - eor r2,r2,r11 @ Ch(e,f,g) - eor r0,r5,r5,ror#11 - add r4,r4,r2 @ h+=Ch(e,f,g) -#if 7==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 7<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r5,r6 @ a^b, b^c in next round -#else - ldr r2,[sp,#9*4] @ from future BODY_16_xx - eor r3,r5,r6 @ a^b, b^c in next round - ldr r1,[sp,#6*4] @ from future BODY_16_xx -#endif - eor r0,r0,r5,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r8,r8,r4 @ d+=h - eor r12,r12,r6 @ Maj(a,b,c) - add r4,r4,r0,ror#2 @ h+=Sigma0(a) - @ add r4,r4,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 8 -# if 8==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r8,r8,ror#5 - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r8,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 8 - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 8==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r8,r8,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r8,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r11,r11,r2 @ h+=X[i] - str r2,[sp,#8*4] - eor r2,r9,r10 - add r11,r11,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r8 - add r11,r11,r12 @ h+=K256[i] - eor r2,r2,r10 @ Ch(e,f,g) - eor r0,r4,r4,ror#11 - add r11,r11,r2 @ h+=Ch(e,f,g) -#if 8==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 8<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r4,r5 @ a^b, b^c in next round -#else - ldr r2,[sp,#10*4] @ from future BODY_16_xx - eor r12,r4,r5 @ a^b, b^c in next round - ldr r1,[sp,#7*4] @ from future BODY_16_xx -#endif - eor r0,r0,r4,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r7,r7,r11 @ d+=h - eor r3,r3,r5 @ Maj(a,b,c) - add r11,r11,r0,ror#2 @ h+=Sigma0(a) - @ add r11,r11,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 9 -# if 9==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r7,r7,ror#5 - add r11,r11,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r7,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 9 - add r11,r11,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 9==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r7,r7,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r7,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r10,r10,r2 @ h+=X[i] - str r2,[sp,#9*4] - eor r2,r8,r9 - add r10,r10,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r7 - add r10,r10,r3 @ h+=K256[i] - eor r2,r2,r9 @ Ch(e,f,g) - eor r0,r11,r11,ror#11 - add r10,r10,r2 @ h+=Ch(e,f,g) -#if 9==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 9<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r11,r4 @ a^b, b^c in next round -#else - ldr r2,[sp,#11*4] @ from future BODY_16_xx - eor r3,r11,r4 @ a^b, b^c in next round - ldr r1,[sp,#8*4] @ from future BODY_16_xx -#endif - eor r0,r0,r11,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r6,r6,r10 @ d+=h - eor r12,r12,r4 @ Maj(a,b,c) - add r10,r10,r0,ror#2 @ h+=Sigma0(a) - @ add r10,r10,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 10 -# if 10==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r6,r6,ror#5 - add r10,r10,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r6,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 10 - add r10,r10,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 10==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r6,r6,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r6,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r9,r9,r2 @ h+=X[i] - str r2,[sp,#10*4] - eor r2,r7,r8 - add r9,r9,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r6 - add r9,r9,r12 @ h+=K256[i] - eor r2,r2,r8 @ Ch(e,f,g) - eor r0,r10,r10,ror#11 - add r9,r9,r2 @ h+=Ch(e,f,g) -#if 10==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 10<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r10,r11 @ a^b, b^c in next round -#else - ldr r2,[sp,#12*4] @ from future BODY_16_xx - eor r12,r10,r11 @ a^b, b^c in next round - ldr r1,[sp,#9*4] @ from future BODY_16_xx -#endif - eor r0,r0,r10,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r5,r5,r9 @ d+=h - eor r3,r3,r11 @ Maj(a,b,c) - add r9,r9,r0,ror#2 @ h+=Sigma0(a) - @ add r9,r9,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 11 -# if 11==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r5,r5,ror#5 - add r9,r9,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r5,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 11 - add r9,r9,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 11==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r5,r5,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r5,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r8,r8,r2 @ h+=X[i] - str r2,[sp,#11*4] - eor r2,r6,r7 - add r8,r8,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r5 - add r8,r8,r3 @ h+=K256[i] - eor r2,r2,r7 @ Ch(e,f,g) - eor r0,r9,r9,ror#11 - add r8,r8,r2 @ h+=Ch(e,f,g) -#if 11==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 11<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r9,r10 @ a^b, b^c in next round -#else - ldr r2,[sp,#13*4] @ from future BODY_16_xx - eor r3,r9,r10 @ a^b, b^c in next round - ldr r1,[sp,#10*4] @ from future BODY_16_xx -#endif - eor r0,r0,r9,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r4,r4,r8 @ d+=h - eor r12,r12,r10 @ Maj(a,b,c) - add r8,r8,r0,ror#2 @ h+=Sigma0(a) - @ add r8,r8,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 12 -# if 12==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r4,r4,ror#5 - add r8,r8,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r4,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 12 - add r8,r8,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 12==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r4,r4,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r4,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r7,r7,r2 @ h+=X[i] - str r2,[sp,#12*4] - eor r2,r5,r6 - add r7,r7,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r4 - add r7,r7,r12 @ h+=K256[i] - eor r2,r2,r6 @ Ch(e,f,g) - eor r0,r8,r8,ror#11 - add r7,r7,r2 @ h+=Ch(e,f,g) -#if 12==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 12<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r8,r9 @ a^b, b^c in next round -#else - ldr r2,[sp,#14*4] @ from future BODY_16_xx - eor r12,r8,r9 @ a^b, b^c in next round - ldr r1,[sp,#11*4] @ from future BODY_16_xx -#endif - eor r0,r0,r8,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r11,r11,r7 @ d+=h - eor r3,r3,r9 @ Maj(a,b,c) - add r7,r7,r0,ror#2 @ h+=Sigma0(a) - @ add r7,r7,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 13 -# if 13==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r11,r11,ror#5 - add r7,r7,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r11,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 13 - add r7,r7,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 13==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r11,r11,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r11,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r6,r6,r2 @ h+=X[i] - str r2,[sp,#13*4] - eor r2,r4,r5 - add r6,r6,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r11 - add r6,r6,r3 @ h+=K256[i] - eor r2,r2,r5 @ Ch(e,f,g) - eor r0,r7,r7,ror#11 - add r6,r6,r2 @ h+=Ch(e,f,g) -#if 13==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 13<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r7,r8 @ a^b, b^c in next round -#else - ldr r2,[sp,#15*4] @ from future BODY_16_xx - eor r3,r7,r8 @ a^b, b^c in next round - ldr r1,[sp,#12*4] @ from future BODY_16_xx -#endif - eor r0,r0,r7,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r10,r10,r6 @ d+=h - eor r12,r12,r8 @ Maj(a,b,c) - add r6,r6,r0,ror#2 @ h+=Sigma0(a) - @ add r6,r6,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 14 -# if 14==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r10,r10,ror#5 - add r6,r6,r12 @ h+=Maj(a,b,c) from the past - eor r0,r0,r10,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 14 - add r6,r6,r12 @ h+=Maj(a,b,c) from the past - ldrb r12,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r12,lsl#8 - ldrb r12,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 14==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r10,r10,ror#5 - orr r2,r2,r12,lsl#24 - eor r0,r0,r10,ror#19 @ Sigma1(e) -#endif - ldr r12,[r14],#4 @ *K256++ - add r5,r5,r2 @ h+=X[i] - str r2,[sp,#14*4] - eor r2,r11,r4 - add r5,r5,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r10 - add r5,r5,r12 @ h+=K256[i] - eor r2,r2,r4 @ Ch(e,f,g) - eor r0,r6,r6,ror#11 - add r5,r5,r2 @ h+=Ch(e,f,g) -#if 14==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 14<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r6,r7 @ a^b, b^c in next round -#else - ldr r2,[sp,#0*4] @ from future BODY_16_xx - eor r12,r6,r7 @ a^b, b^c in next round - ldr r1,[sp,#13*4] @ from future BODY_16_xx -#endif - eor r0,r0,r6,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r9,r9,r5 @ d+=h - eor r3,r3,r7 @ Maj(a,b,c) - add r5,r5,r0,ror#2 @ h+=Sigma0(a) - @ add r5,r5,r3 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - @ ldr r2,[r1],#4 @ 15 -# if 15==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r9,r9,ror#5 - add r5,r5,r3 @ h+=Maj(a,b,c) from the past - eor r0,r0,r9,ror#19 @ Sigma1(e) -# ifndef __ARMEB__ - rev r2,r2 -# endif -#else - @ ldrb r2,[r1,#3] @ 15 - add r5,r5,r3 @ h+=Maj(a,b,c) from the past - ldrb r3,[r1,#2] - ldrb r0,[r1,#1] - orr r2,r2,r3,lsl#8 - ldrb r3,[r1],#4 - orr r2,r2,r0,lsl#16 -# if 15==15 - str r1,[sp,#17*4] @ make room for r1 -# endif - eor r0,r9,r9,ror#5 - orr r2,r2,r3,lsl#24 - eor r0,r0,r9,ror#19 @ Sigma1(e) -#endif - ldr r3,[r14],#4 @ *K256++ - add r4,r4,r2 @ h+=X[i] - str r2,[sp,#15*4] - eor r2,r10,r11 - add r4,r4,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r9 - add r4,r4,r3 @ h+=K256[i] - eor r2,r2,r11 @ Ch(e,f,g) - eor r0,r5,r5,ror#11 - add r4,r4,r2 @ h+=Ch(e,f,g) -#if 15==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 15<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r5,r6 @ a^b, b^c in next round -#else - ldr r2,[sp,#1*4] @ from future BODY_16_xx - eor r3,r5,r6 @ a^b, b^c in next round - ldr r1,[sp,#14*4] @ from future BODY_16_xx -#endif - eor r0,r0,r5,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r8,r8,r4 @ d+=h - eor r12,r12,r6 @ Maj(a,b,c) - add r4,r4,r0,ror#2 @ h+=Sigma0(a) - @ add r4,r4,r12 @ h+=Maj(a,b,c) -.Lrounds_16_xx: - @ ldr r2,[sp,#1*4] @ 16 - @ ldr r1,[sp,#14*4] - mov r0,r2,ror#7 - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#0*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#9*4] - - add r12,r12,r0 - eor r0,r8,r8,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r8,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r11,r11,r2 @ h+=X[i] - str r2,[sp,#0*4] - eor r2,r9,r10 - add r11,r11,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r8 - add r11,r11,r12 @ h+=K256[i] - eor r2,r2,r10 @ Ch(e,f,g) - eor r0,r4,r4,ror#11 - add r11,r11,r2 @ h+=Ch(e,f,g) -#if 16==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 16<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r4,r5 @ a^b, b^c in next round -#else - ldr r2,[sp,#2*4] @ from future BODY_16_xx - eor r12,r4,r5 @ a^b, b^c in next round - ldr r1,[sp,#15*4] @ from future BODY_16_xx -#endif - eor r0,r0,r4,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r7,r7,r11 @ d+=h - eor r3,r3,r5 @ Maj(a,b,c) - add r11,r11,r0,ror#2 @ h+=Sigma0(a) - @ add r11,r11,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#2*4] @ 17 - @ ldr r1,[sp,#15*4] - mov r0,r2,ror#7 - add r11,r11,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#1*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#10*4] - - add r3,r3,r0 - eor r0,r7,r7,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r7,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r10,r10,r2 @ h+=X[i] - str r2,[sp,#1*4] - eor r2,r8,r9 - add r10,r10,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r7 - add r10,r10,r3 @ h+=K256[i] - eor r2,r2,r9 @ Ch(e,f,g) - eor r0,r11,r11,ror#11 - add r10,r10,r2 @ h+=Ch(e,f,g) -#if 17==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 17<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r11,r4 @ a^b, b^c in next round -#else - ldr r2,[sp,#3*4] @ from future BODY_16_xx - eor r3,r11,r4 @ a^b, b^c in next round - ldr r1,[sp,#0*4] @ from future BODY_16_xx -#endif - eor r0,r0,r11,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r6,r6,r10 @ d+=h - eor r12,r12,r4 @ Maj(a,b,c) - add r10,r10,r0,ror#2 @ h+=Sigma0(a) - @ add r10,r10,r12 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#3*4] @ 18 - @ ldr r1,[sp,#0*4] - mov r0,r2,ror#7 - add r10,r10,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#2*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#11*4] - - add r12,r12,r0 - eor r0,r6,r6,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r6,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r9,r9,r2 @ h+=X[i] - str r2,[sp,#2*4] - eor r2,r7,r8 - add r9,r9,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r6 - add r9,r9,r12 @ h+=K256[i] - eor r2,r2,r8 @ Ch(e,f,g) - eor r0,r10,r10,ror#11 - add r9,r9,r2 @ h+=Ch(e,f,g) -#if 18==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 18<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r10,r11 @ a^b, b^c in next round -#else - ldr r2,[sp,#4*4] @ from future BODY_16_xx - eor r12,r10,r11 @ a^b, b^c in next round - ldr r1,[sp,#1*4] @ from future BODY_16_xx -#endif - eor r0,r0,r10,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r5,r5,r9 @ d+=h - eor r3,r3,r11 @ Maj(a,b,c) - add r9,r9,r0,ror#2 @ h+=Sigma0(a) - @ add r9,r9,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#4*4] @ 19 - @ ldr r1,[sp,#1*4] - mov r0,r2,ror#7 - add r9,r9,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#3*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#12*4] - - add r3,r3,r0 - eor r0,r5,r5,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r5,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r8,r8,r2 @ h+=X[i] - str r2,[sp,#3*4] - eor r2,r6,r7 - add r8,r8,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r5 - add r8,r8,r3 @ h+=K256[i] - eor r2,r2,r7 @ Ch(e,f,g) - eor r0,r9,r9,ror#11 - add r8,r8,r2 @ h+=Ch(e,f,g) -#if 19==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 19<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r9,r10 @ a^b, b^c in next round -#else - ldr r2,[sp,#5*4] @ from future BODY_16_xx - eor r3,r9,r10 @ a^b, b^c in next round - ldr r1,[sp,#2*4] @ from future BODY_16_xx -#endif - eor r0,r0,r9,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r4,r4,r8 @ d+=h - eor r12,r12,r10 @ Maj(a,b,c) - add r8,r8,r0,ror#2 @ h+=Sigma0(a) - @ add r8,r8,r12 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#5*4] @ 20 - @ ldr r1,[sp,#2*4] - mov r0,r2,ror#7 - add r8,r8,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#4*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#13*4] - - add r12,r12,r0 - eor r0,r4,r4,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r4,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r7,r7,r2 @ h+=X[i] - str r2,[sp,#4*4] - eor r2,r5,r6 - add r7,r7,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r4 - add r7,r7,r12 @ h+=K256[i] - eor r2,r2,r6 @ Ch(e,f,g) - eor r0,r8,r8,ror#11 - add r7,r7,r2 @ h+=Ch(e,f,g) -#if 20==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 20<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r8,r9 @ a^b, b^c in next round -#else - ldr r2,[sp,#6*4] @ from future BODY_16_xx - eor r12,r8,r9 @ a^b, b^c in next round - ldr r1,[sp,#3*4] @ from future BODY_16_xx -#endif - eor r0,r0,r8,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r11,r11,r7 @ d+=h - eor r3,r3,r9 @ Maj(a,b,c) - add r7,r7,r0,ror#2 @ h+=Sigma0(a) - @ add r7,r7,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#6*4] @ 21 - @ ldr r1,[sp,#3*4] - mov r0,r2,ror#7 - add r7,r7,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#5*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#14*4] - - add r3,r3,r0 - eor r0,r11,r11,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r11,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r6,r6,r2 @ h+=X[i] - str r2,[sp,#5*4] - eor r2,r4,r5 - add r6,r6,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r11 - add r6,r6,r3 @ h+=K256[i] - eor r2,r2,r5 @ Ch(e,f,g) - eor r0,r7,r7,ror#11 - add r6,r6,r2 @ h+=Ch(e,f,g) -#if 21==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 21<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r7,r8 @ a^b, b^c in next round -#else - ldr r2,[sp,#7*4] @ from future BODY_16_xx - eor r3,r7,r8 @ a^b, b^c in next round - ldr r1,[sp,#4*4] @ from future BODY_16_xx -#endif - eor r0,r0,r7,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r10,r10,r6 @ d+=h - eor r12,r12,r8 @ Maj(a,b,c) - add r6,r6,r0,ror#2 @ h+=Sigma0(a) - @ add r6,r6,r12 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#7*4] @ 22 - @ ldr r1,[sp,#4*4] - mov r0,r2,ror#7 - add r6,r6,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#6*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#15*4] - - add r12,r12,r0 - eor r0,r10,r10,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r10,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r5,r5,r2 @ h+=X[i] - str r2,[sp,#6*4] - eor r2,r11,r4 - add r5,r5,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r10 - add r5,r5,r12 @ h+=K256[i] - eor r2,r2,r4 @ Ch(e,f,g) - eor r0,r6,r6,ror#11 - add r5,r5,r2 @ h+=Ch(e,f,g) -#if 22==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 22<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r6,r7 @ a^b, b^c in next round -#else - ldr r2,[sp,#8*4] @ from future BODY_16_xx - eor r12,r6,r7 @ a^b, b^c in next round - ldr r1,[sp,#5*4] @ from future BODY_16_xx -#endif - eor r0,r0,r6,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r9,r9,r5 @ d+=h - eor r3,r3,r7 @ Maj(a,b,c) - add r5,r5,r0,ror#2 @ h+=Sigma0(a) - @ add r5,r5,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#8*4] @ 23 - @ ldr r1,[sp,#5*4] - mov r0,r2,ror#7 - add r5,r5,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#7*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#0*4] - - add r3,r3,r0 - eor r0,r9,r9,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r9,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r4,r4,r2 @ h+=X[i] - str r2,[sp,#7*4] - eor r2,r10,r11 - add r4,r4,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r9 - add r4,r4,r3 @ h+=K256[i] - eor r2,r2,r11 @ Ch(e,f,g) - eor r0,r5,r5,ror#11 - add r4,r4,r2 @ h+=Ch(e,f,g) -#if 23==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 23<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r5,r6 @ a^b, b^c in next round -#else - ldr r2,[sp,#9*4] @ from future BODY_16_xx - eor r3,r5,r6 @ a^b, b^c in next round - ldr r1,[sp,#6*4] @ from future BODY_16_xx -#endif - eor r0,r0,r5,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r8,r8,r4 @ d+=h - eor r12,r12,r6 @ Maj(a,b,c) - add r4,r4,r0,ror#2 @ h+=Sigma0(a) - @ add r4,r4,r12 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#9*4] @ 24 - @ ldr r1,[sp,#6*4] - mov r0,r2,ror#7 - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#8*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#1*4] - - add r12,r12,r0 - eor r0,r8,r8,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r8,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r11,r11,r2 @ h+=X[i] - str r2,[sp,#8*4] - eor r2,r9,r10 - add r11,r11,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r8 - add r11,r11,r12 @ h+=K256[i] - eor r2,r2,r10 @ Ch(e,f,g) - eor r0,r4,r4,ror#11 - add r11,r11,r2 @ h+=Ch(e,f,g) -#if 24==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 24<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r4,r5 @ a^b, b^c in next round -#else - ldr r2,[sp,#10*4] @ from future BODY_16_xx - eor r12,r4,r5 @ a^b, b^c in next round - ldr r1,[sp,#7*4] @ from future BODY_16_xx -#endif - eor r0,r0,r4,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r7,r7,r11 @ d+=h - eor r3,r3,r5 @ Maj(a,b,c) - add r11,r11,r0,ror#2 @ h+=Sigma0(a) - @ add r11,r11,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#10*4] @ 25 - @ ldr r1,[sp,#7*4] - mov r0,r2,ror#7 - add r11,r11,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#9*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#2*4] - - add r3,r3,r0 - eor r0,r7,r7,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r7,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r10,r10,r2 @ h+=X[i] - str r2,[sp,#9*4] - eor r2,r8,r9 - add r10,r10,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r7 - add r10,r10,r3 @ h+=K256[i] - eor r2,r2,r9 @ Ch(e,f,g) - eor r0,r11,r11,ror#11 - add r10,r10,r2 @ h+=Ch(e,f,g) -#if 25==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 25<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r11,r4 @ a^b, b^c in next round -#else - ldr r2,[sp,#11*4] @ from future BODY_16_xx - eor r3,r11,r4 @ a^b, b^c in next round - ldr r1,[sp,#8*4] @ from future BODY_16_xx -#endif - eor r0,r0,r11,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r6,r6,r10 @ d+=h - eor r12,r12,r4 @ Maj(a,b,c) - add r10,r10,r0,ror#2 @ h+=Sigma0(a) - @ add r10,r10,r12 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#11*4] @ 26 - @ ldr r1,[sp,#8*4] - mov r0,r2,ror#7 - add r10,r10,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#10*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#3*4] - - add r12,r12,r0 - eor r0,r6,r6,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r6,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r9,r9,r2 @ h+=X[i] - str r2,[sp,#10*4] - eor r2,r7,r8 - add r9,r9,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r6 - add r9,r9,r12 @ h+=K256[i] - eor r2,r2,r8 @ Ch(e,f,g) - eor r0,r10,r10,ror#11 - add r9,r9,r2 @ h+=Ch(e,f,g) -#if 26==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 26<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r10,r11 @ a^b, b^c in next round -#else - ldr r2,[sp,#12*4] @ from future BODY_16_xx - eor r12,r10,r11 @ a^b, b^c in next round - ldr r1,[sp,#9*4] @ from future BODY_16_xx -#endif - eor r0,r0,r10,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r5,r5,r9 @ d+=h - eor r3,r3,r11 @ Maj(a,b,c) - add r9,r9,r0,ror#2 @ h+=Sigma0(a) - @ add r9,r9,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#12*4] @ 27 - @ ldr r1,[sp,#9*4] - mov r0,r2,ror#7 - add r9,r9,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#11*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#4*4] - - add r3,r3,r0 - eor r0,r5,r5,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r5,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r8,r8,r2 @ h+=X[i] - str r2,[sp,#11*4] - eor r2,r6,r7 - add r8,r8,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r5 - add r8,r8,r3 @ h+=K256[i] - eor r2,r2,r7 @ Ch(e,f,g) - eor r0,r9,r9,ror#11 - add r8,r8,r2 @ h+=Ch(e,f,g) -#if 27==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 27<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r9,r10 @ a^b, b^c in next round -#else - ldr r2,[sp,#13*4] @ from future BODY_16_xx - eor r3,r9,r10 @ a^b, b^c in next round - ldr r1,[sp,#10*4] @ from future BODY_16_xx -#endif - eor r0,r0,r9,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r4,r4,r8 @ d+=h - eor r12,r12,r10 @ Maj(a,b,c) - add r8,r8,r0,ror#2 @ h+=Sigma0(a) - @ add r8,r8,r12 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#13*4] @ 28 - @ ldr r1,[sp,#10*4] - mov r0,r2,ror#7 - add r8,r8,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#12*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#5*4] - - add r12,r12,r0 - eor r0,r4,r4,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r4,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r7,r7,r2 @ h+=X[i] - str r2,[sp,#12*4] - eor r2,r5,r6 - add r7,r7,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r4 - add r7,r7,r12 @ h+=K256[i] - eor r2,r2,r6 @ Ch(e,f,g) - eor r0,r8,r8,ror#11 - add r7,r7,r2 @ h+=Ch(e,f,g) -#if 28==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 28<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r8,r9 @ a^b, b^c in next round -#else - ldr r2,[sp,#14*4] @ from future BODY_16_xx - eor r12,r8,r9 @ a^b, b^c in next round - ldr r1,[sp,#11*4] @ from future BODY_16_xx -#endif - eor r0,r0,r8,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r11,r11,r7 @ d+=h - eor r3,r3,r9 @ Maj(a,b,c) - add r7,r7,r0,ror#2 @ h+=Sigma0(a) - @ add r7,r7,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#14*4] @ 29 - @ ldr r1,[sp,#11*4] - mov r0,r2,ror#7 - add r7,r7,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#13*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#6*4] - - add r3,r3,r0 - eor r0,r11,r11,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r11,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r6,r6,r2 @ h+=X[i] - str r2,[sp,#13*4] - eor r2,r4,r5 - add r6,r6,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r11 - add r6,r6,r3 @ h+=K256[i] - eor r2,r2,r5 @ Ch(e,f,g) - eor r0,r7,r7,ror#11 - add r6,r6,r2 @ h+=Ch(e,f,g) -#if 29==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 29<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r7,r8 @ a^b, b^c in next round -#else - ldr r2,[sp,#15*4] @ from future BODY_16_xx - eor r3,r7,r8 @ a^b, b^c in next round - ldr r1,[sp,#12*4] @ from future BODY_16_xx -#endif - eor r0,r0,r7,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r10,r10,r6 @ d+=h - eor r12,r12,r8 @ Maj(a,b,c) - add r6,r6,r0,ror#2 @ h+=Sigma0(a) - @ add r6,r6,r12 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#15*4] @ 30 - @ ldr r1,[sp,#12*4] - mov r0,r2,ror#7 - add r6,r6,r12 @ h+=Maj(a,b,c) from the past - mov r12,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r12,r12,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#14*4] - eor r12,r12,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#7*4] - - add r12,r12,r0 - eor r0,r10,r10,ror#5 @ from BODY_00_15 - add r2,r2,r12 - eor r0,r0,r10,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r12,[r14],#4 @ *K256++ - add r5,r5,r2 @ h+=X[i] - str r2,[sp,#14*4] - eor r2,r11,r4 - add r5,r5,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r10 - add r5,r5,r12 @ h+=K256[i] - eor r2,r2,r4 @ Ch(e,f,g) - eor r0,r6,r6,ror#11 - add r5,r5,r2 @ h+=Ch(e,f,g) -#if 30==31 - and r12,r12,#0xff - cmp r12,#0xf2 @ done? -#endif -#if 30<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r12,r6,r7 @ a^b, b^c in next round -#else - ldr r2,[sp,#0*4] @ from future BODY_16_xx - eor r12,r6,r7 @ a^b, b^c in next round - ldr r1,[sp,#13*4] @ from future BODY_16_xx -#endif - eor r0,r0,r6,ror#20 @ Sigma0(a) - and r3,r3,r12 @ (b^c)&=(a^b) - add r9,r9,r5 @ d+=h - eor r3,r3,r7 @ Maj(a,b,c) - add r5,r5,r0,ror#2 @ h+=Sigma0(a) - @ add r5,r5,r3 @ h+=Maj(a,b,c) - @ ldr r2,[sp,#0*4] @ 31 - @ ldr r1,[sp,#13*4] - mov r0,r2,ror#7 - add r5,r5,r3 @ h+=Maj(a,b,c) from the past - mov r3,r1,ror#17 - eor r0,r0,r2,ror#18 - eor r3,r3,r1,ror#19 - eor r0,r0,r2,lsr#3 @ sigma0(X[i+1]) - ldr r2,[sp,#15*4] - eor r3,r3,r1,lsr#10 @ sigma1(X[i+14]) - ldr r1,[sp,#8*4] - - add r3,r3,r0 - eor r0,r9,r9,ror#5 @ from BODY_00_15 - add r2,r2,r3 - eor r0,r0,r9,ror#19 @ Sigma1(e) - add r2,r2,r1 @ X[i] - ldr r3,[r14],#4 @ *K256++ - add r4,r4,r2 @ h+=X[i] - str r2,[sp,#15*4] - eor r2,r10,r11 - add r4,r4,r0,ror#6 @ h+=Sigma1(e) - and r2,r2,r9 - add r4,r4,r3 @ h+=K256[i] - eor r2,r2,r11 @ Ch(e,f,g) - eor r0,r5,r5,ror#11 - add r4,r4,r2 @ h+=Ch(e,f,g) -#if 31==31 - and r3,r3,#0xff - cmp r3,#0xf2 @ done? -#endif -#if 31<15 -# if __ARM_ARCH__>=7 - ldr r2,[r1],#4 @ prefetch -# else - ldrb r2,[r1,#3] -# endif - eor r3,r5,r6 @ a^b, b^c in next round -#else - ldr r2,[sp,#1*4] @ from future BODY_16_xx - eor r3,r5,r6 @ a^b, b^c in next round - ldr r1,[sp,#14*4] @ from future BODY_16_xx -#endif - eor r0,r0,r5,ror#20 @ Sigma0(a) - and r12,r12,r3 @ (b^c)&=(a^b) - add r8,r8,r4 @ d+=h - eor r12,r12,r6 @ Maj(a,b,c) - add r4,r4,r0,ror#2 @ h+=Sigma0(a) - @ add r4,r4,r12 @ h+=Maj(a,b,c) -#if __ARM_ARCH__>=7 - ite eq @ Thumb2 thing, sanity check in ARM -#endif - ldreq r3,[sp,#16*4] @ pull ctx - bne .Lrounds_16_xx - - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - ldr r0,[r3,#0] - ldr r2,[r3,#4] - ldr r12,[r3,#8] - add r4,r4,r0 - ldr r0,[r3,#12] - add r5,r5,r2 - ldr r2,[r3,#16] - add r6,r6,r12 - ldr r12,[r3,#20] - add r7,r7,r0 - ldr r0,[r3,#24] - add r8,r8,r2 - ldr r2,[r3,#28] - add r9,r9,r12 - ldr r1,[sp,#17*4] @ pull inp - ldr r12,[sp,#18*4] @ pull inp+len - add r10,r10,r0 - add r11,r11,r2 - stmia r3,{r4,r5,r6,r7,r8,r9,r10,r11} - cmp r1,r12 - sub r14,r14,#256 @ rewind Ktbl - bne .Loop - - add sp,sp,#19*4 @ destroy frame -#if __ARM_ARCH__>=5 - ldmia sp!,{r4-r11,pc} -#else - ldmia sp!,{r4-r11,lr} - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size sha256_block_data_order,.-sha256_block_data_order -#if __ARM_MAX_ARCH__>=7 -.arch armv7-a -.fpu neon - -.global sha256_block_data_order_neon -.type sha256_block_data_order_neon,%function -.align 4 -sha256_block_data_order_neon: -.LNEON: - stmdb sp!,{r4-r12,lr} - - sub r11,sp,#16*4+16 - adr r14,.Lsha256_block_data_order - sub r14,r14,#.Lsha256_block_data_order-K256 - bic r11,r11,#15 @ align for 128-bit stores - mov r12,sp - mov sp,r11 @ alloca - add r2,r1,r2,lsl#6 @ len to point at the end of inp - - vld1.8 {q0},[r1]! - vld1.8 {q1},[r1]! - vld1.8 {q2},[r1]! - vld1.8 {q3},[r1]! - vld1.32 {q8},[r14,:128]! - vld1.32 {q9},[r14,:128]! - vld1.32 {q10},[r14,:128]! - vld1.32 {q11},[r14,:128]! - vrev32.8 q0,q0 @ yes, even on - str r0,[sp,#64] - vrev32.8 q1,q1 @ big-endian - str r1,[sp,#68] - mov r1,sp - vrev32.8 q2,q2 - str r2,[sp,#72] - vrev32.8 q3,q3 - str r12,[sp,#76] @ save original sp - vadd.i32 q8,q8,q0 - vadd.i32 q9,q9,q1 - vst1.32 {q8},[r1,:128]! - vadd.i32 q10,q10,q2 - vst1.32 {q9},[r1,:128]! - vadd.i32 q11,q11,q3 - vst1.32 {q10},[r1,:128]! - vst1.32 {q11},[r1,:128]! - - ldmia r0,{r4-r11} - sub r1,r1,#64 - ldr r2,[sp,#0] - eor r12,r12,r12 - eor r3,r5,r6 - b .L_00_48 - -.align 4 -.L_00_48: - vext.8 q8,q0,q1,#4 - add r11,r11,r2 - eor r2,r9,r10 - eor r0,r8,r8,ror#5 - vext.8 q9,q2,q3,#4 - add r4,r4,r12 - and r2,r2,r8 - eor r12,r0,r8,ror#19 - vshr.u32 q10,q8,#7 - eor r0,r4,r4,ror#11 - eor r2,r2,r10 - vadd.i32 q0,q0,q9 - add r11,r11,r12,ror#6 - eor r12,r4,r5 - vshr.u32 q9,q8,#3 - eor r0,r0,r4,ror#20 - add r11,r11,r2 - vsli.32 q10,q8,#25 - ldr r2,[sp,#4] - and r3,r3,r12 - vshr.u32 q11,q8,#18 - add r7,r7,r11 - add r11,r11,r0,ror#2 - eor r3,r3,r5 - veor q9,q9,q10 - add r10,r10,r2 - vsli.32 q11,q8,#14 - eor r2,r8,r9 - eor r0,r7,r7,ror#5 - vshr.u32 d24,d7,#17 - add r11,r11,r3 - and r2,r2,r7 - veor q9,q9,q11 - eor r3,r0,r7,ror#19 - eor r0,r11,r11,ror#11 - vsli.32 d24,d7,#15 - eor r2,r2,r9 - add r10,r10,r3,ror#6 - vshr.u32 d25,d7,#10 - eor r3,r11,r4 - eor r0,r0,r11,ror#20 - vadd.i32 q0,q0,q9 - add r10,r10,r2 - ldr r2,[sp,#8] - veor d25,d25,d24 - and r12,r12,r3 - add r6,r6,r10 - vshr.u32 d24,d7,#19 - add r10,r10,r0,ror#2 - eor r12,r12,r4 - vsli.32 d24,d7,#13 - add r9,r9,r2 - eor r2,r7,r8 - veor d25,d25,d24 - eor r0,r6,r6,ror#5 - add r10,r10,r12 - vadd.i32 d0,d0,d25 - and r2,r2,r6 - eor r12,r0,r6,ror#19 - vshr.u32 d24,d0,#17 - eor r0,r10,r10,ror#11 - eor r2,r2,r8 - vsli.32 d24,d0,#15 - add r9,r9,r12,ror#6 - eor r12,r10,r11 - vshr.u32 d25,d0,#10 - eor r0,r0,r10,ror#20 - add r9,r9,r2 - veor d25,d25,d24 - ldr r2,[sp,#12] - and r3,r3,r12 - vshr.u32 d24,d0,#19 - add r5,r5,r9 - add r9,r9,r0,ror#2 - eor r3,r3,r11 - vld1.32 {q8},[r14,:128]! - add r8,r8,r2 - vsli.32 d24,d0,#13 - eor r2,r6,r7 - eor r0,r5,r5,ror#5 - veor d25,d25,d24 - add r9,r9,r3 - and r2,r2,r5 - vadd.i32 d1,d1,d25 - eor r3,r0,r5,ror#19 - eor r0,r9,r9,ror#11 - vadd.i32 q8,q8,q0 - eor r2,r2,r7 - add r8,r8,r3,ror#6 - eor r3,r9,r10 - eor r0,r0,r9,ror#20 - add r8,r8,r2 - ldr r2,[sp,#16] - and r12,r12,r3 - add r4,r4,r8 - vst1.32 {q8},[r1,:128]! - add r8,r8,r0,ror#2 - eor r12,r12,r10 - vext.8 q8,q1,q2,#4 - add r7,r7,r2 - eor r2,r5,r6 - eor r0,r4,r4,ror#5 - vext.8 q9,q3,q0,#4 - add r8,r8,r12 - and r2,r2,r4 - eor r12,r0,r4,ror#19 - vshr.u32 q10,q8,#7 - eor r0,r8,r8,ror#11 - eor r2,r2,r6 - vadd.i32 q1,q1,q9 - add r7,r7,r12,ror#6 - eor r12,r8,r9 - vshr.u32 q9,q8,#3 - eor r0,r0,r8,ror#20 - add r7,r7,r2 - vsli.32 q10,q8,#25 - ldr r2,[sp,#20] - and r3,r3,r12 - vshr.u32 q11,q8,#18 - add r11,r11,r7 - add r7,r7,r0,ror#2 - eor r3,r3,r9 - veor q9,q9,q10 - add r6,r6,r2 - vsli.32 q11,q8,#14 - eor r2,r4,r5 - eor r0,r11,r11,ror#5 - vshr.u32 d24,d1,#17 - add r7,r7,r3 - and r2,r2,r11 - veor q9,q9,q11 - eor r3,r0,r11,ror#19 - eor r0,r7,r7,ror#11 - vsli.32 d24,d1,#15 - eor r2,r2,r5 - add r6,r6,r3,ror#6 - vshr.u32 d25,d1,#10 - eor r3,r7,r8 - eor r0,r0,r7,ror#20 - vadd.i32 q1,q1,q9 - add r6,r6,r2 - ldr r2,[sp,#24] - veor d25,d25,d24 - and r12,r12,r3 - add r10,r10,r6 - vshr.u32 d24,d1,#19 - add r6,r6,r0,ror#2 - eor r12,r12,r8 - vsli.32 d24,d1,#13 - add r5,r5,r2 - eor r2,r11,r4 - veor d25,d25,d24 - eor r0,r10,r10,ror#5 - add r6,r6,r12 - vadd.i32 d2,d2,d25 - and r2,r2,r10 - eor r12,r0,r10,ror#19 - vshr.u32 d24,d2,#17 - eor r0,r6,r6,ror#11 - eor r2,r2,r4 - vsli.32 d24,d2,#15 - add r5,r5,r12,ror#6 - eor r12,r6,r7 - vshr.u32 d25,d2,#10 - eor r0,r0,r6,ror#20 - add r5,r5,r2 - veor d25,d25,d24 - ldr r2,[sp,#28] - and r3,r3,r12 - vshr.u32 d24,d2,#19 - add r9,r9,r5 - add r5,r5,r0,ror#2 - eor r3,r3,r7 - vld1.32 {q8},[r14,:128]! - add r4,r4,r2 - vsli.32 d24,d2,#13 - eor r2,r10,r11 - eor r0,r9,r9,ror#5 - veor d25,d25,d24 - add r5,r5,r3 - and r2,r2,r9 - vadd.i32 d3,d3,d25 - eor r3,r0,r9,ror#19 - eor r0,r5,r5,ror#11 - vadd.i32 q8,q8,q1 - eor r2,r2,r11 - add r4,r4,r3,ror#6 - eor r3,r5,r6 - eor r0,r0,r5,ror#20 - add r4,r4,r2 - ldr r2,[sp,#32] - and r12,r12,r3 - add r8,r8,r4 - vst1.32 {q8},[r1,:128]! - add r4,r4,r0,ror#2 - eor r12,r12,r6 - vext.8 q8,q2,q3,#4 - add r11,r11,r2 - eor r2,r9,r10 - eor r0,r8,r8,ror#5 - vext.8 q9,q0,q1,#4 - add r4,r4,r12 - and r2,r2,r8 - eor r12,r0,r8,ror#19 - vshr.u32 q10,q8,#7 - eor r0,r4,r4,ror#11 - eor r2,r2,r10 - vadd.i32 q2,q2,q9 - add r11,r11,r12,ror#6 - eor r12,r4,r5 - vshr.u32 q9,q8,#3 - eor r0,r0,r4,ror#20 - add r11,r11,r2 - vsli.32 q10,q8,#25 - ldr r2,[sp,#36] - and r3,r3,r12 - vshr.u32 q11,q8,#18 - add r7,r7,r11 - add r11,r11,r0,ror#2 - eor r3,r3,r5 - veor q9,q9,q10 - add r10,r10,r2 - vsli.32 q11,q8,#14 - eor r2,r8,r9 - eor r0,r7,r7,ror#5 - vshr.u32 d24,d3,#17 - add r11,r11,r3 - and r2,r2,r7 - veor q9,q9,q11 - eor r3,r0,r7,ror#19 - eor r0,r11,r11,ror#11 - vsli.32 d24,d3,#15 - eor r2,r2,r9 - add r10,r10,r3,ror#6 - vshr.u32 d25,d3,#10 - eor r3,r11,r4 - eor r0,r0,r11,ror#20 - vadd.i32 q2,q2,q9 - add r10,r10,r2 - ldr r2,[sp,#40] - veor d25,d25,d24 - and r12,r12,r3 - add r6,r6,r10 - vshr.u32 d24,d3,#19 - add r10,r10,r0,ror#2 - eor r12,r12,r4 - vsli.32 d24,d3,#13 - add r9,r9,r2 - eor r2,r7,r8 - veor d25,d25,d24 - eor r0,r6,r6,ror#5 - add r10,r10,r12 - vadd.i32 d4,d4,d25 - and r2,r2,r6 - eor r12,r0,r6,ror#19 - vshr.u32 d24,d4,#17 - eor r0,r10,r10,ror#11 - eor r2,r2,r8 - vsli.32 d24,d4,#15 - add r9,r9,r12,ror#6 - eor r12,r10,r11 - vshr.u32 d25,d4,#10 - eor r0,r0,r10,ror#20 - add r9,r9,r2 - veor d25,d25,d24 - ldr r2,[sp,#44] - and r3,r3,r12 - vshr.u32 d24,d4,#19 - add r5,r5,r9 - add r9,r9,r0,ror#2 - eor r3,r3,r11 - vld1.32 {q8},[r14,:128]! - add r8,r8,r2 - vsli.32 d24,d4,#13 - eor r2,r6,r7 - eor r0,r5,r5,ror#5 - veor d25,d25,d24 - add r9,r9,r3 - and r2,r2,r5 - vadd.i32 d5,d5,d25 - eor r3,r0,r5,ror#19 - eor r0,r9,r9,ror#11 - vadd.i32 q8,q8,q2 - eor r2,r2,r7 - add r8,r8,r3,ror#6 - eor r3,r9,r10 - eor r0,r0,r9,ror#20 - add r8,r8,r2 - ldr r2,[sp,#48] - and r12,r12,r3 - add r4,r4,r8 - vst1.32 {q8},[r1,:128]! - add r8,r8,r0,ror#2 - eor r12,r12,r10 - vext.8 q8,q3,q0,#4 - add r7,r7,r2 - eor r2,r5,r6 - eor r0,r4,r4,ror#5 - vext.8 q9,q1,q2,#4 - add r8,r8,r12 - and r2,r2,r4 - eor r12,r0,r4,ror#19 - vshr.u32 q10,q8,#7 - eor r0,r8,r8,ror#11 - eor r2,r2,r6 - vadd.i32 q3,q3,q9 - add r7,r7,r12,ror#6 - eor r12,r8,r9 - vshr.u32 q9,q8,#3 - eor r0,r0,r8,ror#20 - add r7,r7,r2 - vsli.32 q10,q8,#25 - ldr r2,[sp,#52] - and r3,r3,r12 - vshr.u32 q11,q8,#18 - add r11,r11,r7 - add r7,r7,r0,ror#2 - eor r3,r3,r9 - veor q9,q9,q10 - add r6,r6,r2 - vsli.32 q11,q8,#14 - eor r2,r4,r5 - eor r0,r11,r11,ror#5 - vshr.u32 d24,d5,#17 - add r7,r7,r3 - and r2,r2,r11 - veor q9,q9,q11 - eor r3,r0,r11,ror#19 - eor r0,r7,r7,ror#11 - vsli.32 d24,d5,#15 - eor r2,r2,r5 - add r6,r6,r3,ror#6 - vshr.u32 d25,d5,#10 - eor r3,r7,r8 - eor r0,r0,r7,ror#20 - vadd.i32 q3,q3,q9 - add r6,r6,r2 - ldr r2,[sp,#56] - veor d25,d25,d24 - and r12,r12,r3 - add r10,r10,r6 - vshr.u32 d24,d5,#19 - add r6,r6,r0,ror#2 - eor r12,r12,r8 - vsli.32 d24,d5,#13 - add r5,r5,r2 - eor r2,r11,r4 - veor d25,d25,d24 - eor r0,r10,r10,ror#5 - add r6,r6,r12 - vadd.i32 d6,d6,d25 - and r2,r2,r10 - eor r12,r0,r10,ror#19 - vshr.u32 d24,d6,#17 - eor r0,r6,r6,ror#11 - eor r2,r2,r4 - vsli.32 d24,d6,#15 - add r5,r5,r12,ror#6 - eor r12,r6,r7 - vshr.u32 d25,d6,#10 - eor r0,r0,r6,ror#20 - add r5,r5,r2 - veor d25,d25,d24 - ldr r2,[sp,#60] - and r3,r3,r12 - vshr.u32 d24,d6,#19 - add r9,r9,r5 - add r5,r5,r0,ror#2 - eor r3,r3,r7 - vld1.32 {q8},[r14,:128]! - add r4,r4,r2 - vsli.32 d24,d6,#13 - eor r2,r10,r11 - eor r0,r9,r9,ror#5 - veor d25,d25,d24 - add r5,r5,r3 - and r2,r2,r9 - vadd.i32 d7,d7,d25 - eor r3,r0,r9,ror#19 - eor r0,r5,r5,ror#11 - vadd.i32 q8,q8,q3 - eor r2,r2,r11 - add r4,r4,r3,ror#6 - eor r3,r5,r6 - eor r0,r0,r5,ror#20 - add r4,r4,r2 - ldr r2,[r14] - and r12,r12,r3 - add r8,r8,r4 - vst1.32 {q8},[r1,:128]! - add r4,r4,r0,ror#2 - eor r12,r12,r6 - teq r2,#0 @ check for K256 terminator - ldr r2,[sp,#0] - sub r1,r1,#64 - bne .L_00_48 - - ldr r1,[sp,#68] - ldr r0,[sp,#72] - sub r14,r14,#256 @ rewind r14 - teq r1,r0 - it eq - subeq r1,r1,#64 @ avoid SEGV - vld1.8 {q0},[r1]! @ load next input block - vld1.8 {q1},[r1]! - vld1.8 {q2},[r1]! - vld1.8 {q3},[r1]! - it ne - strne r1,[sp,#68] - mov r1,sp - add r11,r11,r2 - eor r2,r9,r10 - eor r0,r8,r8,ror#5 - add r4,r4,r12 - vld1.32 {q8},[r14,:128]! - and r2,r2,r8 - eor r12,r0,r8,ror#19 - eor r0,r4,r4,ror#11 - eor r2,r2,r10 - vrev32.8 q0,q0 - add r11,r11,r12,ror#6 - eor r12,r4,r5 - eor r0,r0,r4,ror#20 - add r11,r11,r2 - vadd.i32 q8,q8,q0 - ldr r2,[sp,#4] - and r3,r3,r12 - add r7,r7,r11 - add r11,r11,r0,ror#2 - eor r3,r3,r5 - add r10,r10,r2 - eor r2,r8,r9 - eor r0,r7,r7,ror#5 - add r11,r11,r3 - and r2,r2,r7 - eor r3,r0,r7,ror#19 - eor r0,r11,r11,ror#11 - eor r2,r2,r9 - add r10,r10,r3,ror#6 - eor r3,r11,r4 - eor r0,r0,r11,ror#20 - add r10,r10,r2 - ldr r2,[sp,#8] - and r12,r12,r3 - add r6,r6,r10 - add r10,r10,r0,ror#2 - eor r12,r12,r4 - add r9,r9,r2 - eor r2,r7,r8 - eor r0,r6,r6,ror#5 - add r10,r10,r12 - and r2,r2,r6 - eor r12,r0,r6,ror#19 - eor r0,r10,r10,ror#11 - eor r2,r2,r8 - add r9,r9,r12,ror#6 - eor r12,r10,r11 - eor r0,r0,r10,ror#20 - add r9,r9,r2 - ldr r2,[sp,#12] - and r3,r3,r12 - add r5,r5,r9 - add r9,r9,r0,ror#2 - eor r3,r3,r11 - add r8,r8,r2 - eor r2,r6,r7 - eor r0,r5,r5,ror#5 - add r9,r9,r3 - and r2,r2,r5 - eor r3,r0,r5,ror#19 - eor r0,r9,r9,ror#11 - eor r2,r2,r7 - add r8,r8,r3,ror#6 - eor r3,r9,r10 - eor r0,r0,r9,ror#20 - add r8,r8,r2 - ldr r2,[sp,#16] - and r12,r12,r3 - add r4,r4,r8 - add r8,r8,r0,ror#2 - eor r12,r12,r10 - vst1.32 {q8},[r1,:128]! - add r7,r7,r2 - eor r2,r5,r6 - eor r0,r4,r4,ror#5 - add r8,r8,r12 - vld1.32 {q8},[r14,:128]! - and r2,r2,r4 - eor r12,r0,r4,ror#19 - eor r0,r8,r8,ror#11 - eor r2,r2,r6 - vrev32.8 q1,q1 - add r7,r7,r12,ror#6 - eor r12,r8,r9 - eor r0,r0,r8,ror#20 - add r7,r7,r2 - vadd.i32 q8,q8,q1 - ldr r2,[sp,#20] - and r3,r3,r12 - add r11,r11,r7 - add r7,r7,r0,ror#2 - eor r3,r3,r9 - add r6,r6,r2 - eor r2,r4,r5 - eor r0,r11,r11,ror#5 - add r7,r7,r3 - and r2,r2,r11 - eor r3,r0,r11,ror#19 - eor r0,r7,r7,ror#11 - eor r2,r2,r5 - add r6,r6,r3,ror#6 - eor r3,r7,r8 - eor r0,r0,r7,ror#20 - add r6,r6,r2 - ldr r2,[sp,#24] - and r12,r12,r3 - add r10,r10,r6 - add r6,r6,r0,ror#2 - eor r12,r12,r8 - add r5,r5,r2 - eor r2,r11,r4 - eor r0,r10,r10,ror#5 - add r6,r6,r12 - and r2,r2,r10 - eor r12,r0,r10,ror#19 - eor r0,r6,r6,ror#11 - eor r2,r2,r4 - add r5,r5,r12,ror#6 - eor r12,r6,r7 - eor r0,r0,r6,ror#20 - add r5,r5,r2 - ldr r2,[sp,#28] - and r3,r3,r12 - add r9,r9,r5 - add r5,r5,r0,ror#2 - eor r3,r3,r7 - add r4,r4,r2 - eor r2,r10,r11 - eor r0,r9,r9,ror#5 - add r5,r5,r3 - and r2,r2,r9 - eor r3,r0,r9,ror#19 - eor r0,r5,r5,ror#11 - eor r2,r2,r11 - add r4,r4,r3,ror#6 - eor r3,r5,r6 - eor r0,r0,r5,ror#20 - add r4,r4,r2 - ldr r2,[sp,#32] - and r12,r12,r3 - add r8,r8,r4 - add r4,r4,r0,ror#2 - eor r12,r12,r6 - vst1.32 {q8},[r1,:128]! - add r11,r11,r2 - eor r2,r9,r10 - eor r0,r8,r8,ror#5 - add r4,r4,r12 - vld1.32 {q8},[r14,:128]! - and r2,r2,r8 - eor r12,r0,r8,ror#19 - eor r0,r4,r4,ror#11 - eor r2,r2,r10 - vrev32.8 q2,q2 - add r11,r11,r12,ror#6 - eor r12,r4,r5 - eor r0,r0,r4,ror#20 - add r11,r11,r2 - vadd.i32 q8,q8,q2 - ldr r2,[sp,#36] - and r3,r3,r12 - add r7,r7,r11 - add r11,r11,r0,ror#2 - eor r3,r3,r5 - add r10,r10,r2 - eor r2,r8,r9 - eor r0,r7,r7,ror#5 - add r11,r11,r3 - and r2,r2,r7 - eor r3,r0,r7,ror#19 - eor r0,r11,r11,ror#11 - eor r2,r2,r9 - add r10,r10,r3,ror#6 - eor r3,r11,r4 - eor r0,r0,r11,ror#20 - add r10,r10,r2 - ldr r2,[sp,#40] - and r12,r12,r3 - add r6,r6,r10 - add r10,r10,r0,ror#2 - eor r12,r12,r4 - add r9,r9,r2 - eor r2,r7,r8 - eor r0,r6,r6,ror#5 - add r10,r10,r12 - and r2,r2,r6 - eor r12,r0,r6,ror#19 - eor r0,r10,r10,ror#11 - eor r2,r2,r8 - add r9,r9,r12,ror#6 - eor r12,r10,r11 - eor r0,r0,r10,ror#20 - add r9,r9,r2 - ldr r2,[sp,#44] - and r3,r3,r12 - add r5,r5,r9 - add r9,r9,r0,ror#2 - eor r3,r3,r11 - add r8,r8,r2 - eor r2,r6,r7 - eor r0,r5,r5,ror#5 - add r9,r9,r3 - and r2,r2,r5 - eor r3,r0,r5,ror#19 - eor r0,r9,r9,ror#11 - eor r2,r2,r7 - add r8,r8,r3,ror#6 - eor r3,r9,r10 - eor r0,r0,r9,ror#20 - add r8,r8,r2 - ldr r2,[sp,#48] - and r12,r12,r3 - add r4,r4,r8 - add r8,r8,r0,ror#2 - eor r12,r12,r10 - vst1.32 {q8},[r1,:128]! - add r7,r7,r2 - eor r2,r5,r6 - eor r0,r4,r4,ror#5 - add r8,r8,r12 - vld1.32 {q8},[r14,:128]! - and r2,r2,r4 - eor r12,r0,r4,ror#19 - eor r0,r8,r8,ror#11 - eor r2,r2,r6 - vrev32.8 q3,q3 - add r7,r7,r12,ror#6 - eor r12,r8,r9 - eor r0,r0,r8,ror#20 - add r7,r7,r2 - vadd.i32 q8,q8,q3 - ldr r2,[sp,#52] - and r3,r3,r12 - add r11,r11,r7 - add r7,r7,r0,ror#2 - eor r3,r3,r9 - add r6,r6,r2 - eor r2,r4,r5 - eor r0,r11,r11,ror#5 - add r7,r7,r3 - and r2,r2,r11 - eor r3,r0,r11,ror#19 - eor r0,r7,r7,ror#11 - eor r2,r2,r5 - add r6,r6,r3,ror#6 - eor r3,r7,r8 - eor r0,r0,r7,ror#20 - add r6,r6,r2 - ldr r2,[sp,#56] - and r12,r12,r3 - add r10,r10,r6 - add r6,r6,r0,ror#2 - eor r12,r12,r8 - add r5,r5,r2 - eor r2,r11,r4 - eor r0,r10,r10,ror#5 - add r6,r6,r12 - and r2,r2,r10 - eor r12,r0,r10,ror#19 - eor r0,r6,r6,ror#11 - eor r2,r2,r4 - add r5,r5,r12,ror#6 - eor r12,r6,r7 - eor r0,r0,r6,ror#20 - add r5,r5,r2 - ldr r2,[sp,#60] - and r3,r3,r12 - add r9,r9,r5 - add r5,r5,r0,ror#2 - eor r3,r3,r7 - add r4,r4,r2 - eor r2,r10,r11 - eor r0,r9,r9,ror#5 - add r5,r5,r3 - and r2,r2,r9 - eor r3,r0,r9,ror#19 - eor r0,r5,r5,ror#11 - eor r2,r2,r11 - add r4,r4,r3,ror#6 - eor r3,r5,r6 - eor r0,r0,r5,ror#20 - add r4,r4,r2 - ldr r2,[sp,#64] - and r12,r12,r3 - add r8,r8,r4 - add r4,r4,r0,ror#2 - eor r12,r12,r6 - vst1.32 {q8},[r1,:128]! - ldr r0,[r2,#0] - add r4,r4,r12 @ h+=Maj(a,b,c) from the past - ldr r12,[r2,#4] - ldr r3,[r2,#8] - ldr r1,[r2,#12] - add r4,r4,r0 @ accumulate - ldr r0,[r2,#16] - add r5,r5,r12 - ldr r12,[r2,#20] - add r6,r6,r3 - ldr r3,[r2,#24] - add r7,r7,r1 - ldr r1,[r2,#28] - add r8,r8,r0 - str r4,[r2],#4 - add r9,r9,r12 - str r5,[r2],#4 - add r10,r10,r3 - str r6,[r2],#4 - add r11,r11,r1 - str r7,[r2],#4 - stmia r2,{r8-r11} - - ittte ne - movne r1,sp - ldrne r2,[sp,#0] - eorne r12,r12,r12 - ldreq sp,[sp,#76] @ restore original sp - itt ne - eorne r3,r5,r6 - bne .L_00_48 - - ldmia sp!,{r4-r12,pc} -.size sha256_block_data_order_neon,.-sha256_block_data_order_neon -#endif -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) - -# ifdef __thumb2__ -# define INST(a,b,c,d) .byte c,d|0xc,a,b -# else -# define INST(a,b,c,d) .byte a,b,c,d -# endif - -.type sha256_block_data_order_armv8,%function -.align 5 -sha256_block_data_order_armv8: -.LARMv8: - vld1.32 {q0,q1},[r0] -# ifdef __thumb2__ - adr r3,.LARMv8 - sub r3,r3,#.LARMv8-K256 -# else - adrl r3,K256 -# endif - add r2,r1,r2,lsl#6 @ len to point at the end of inp - -.Loop_v8: - vld1.8 {q8-q9},[r1]! - vld1.8 {q10-q11},[r1]! - vld1.32 {q12},[r3]! - vrev32.8 q8,q8 - vrev32.8 q9,q9 - vrev32.8 q10,q10 - vrev32.8 q11,q11 - vmov q14,q0 @ offload - vmov q15,q1 - teq r1,r2 - vld1.32 {q13},[r3]! - vadd.i32 q12,q12,q8 - INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9 - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11 - vld1.32 {q12},[r3]! - vadd.i32 q13,q13,q9 - INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8 - vld1.32 {q13},[r3]! - vadd.i32 q12,q12,q10 - INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11 - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9 - vld1.32 {q12},[r3]! - vadd.i32 q13,q13,q11 - INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10 - vld1.32 {q13},[r3]! - vadd.i32 q12,q12,q8 - INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9 - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11 - vld1.32 {q12},[r3]! - vadd.i32 q13,q13,q9 - INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8 - vld1.32 {q13},[r3]! - vadd.i32 q12,q12,q10 - INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11 - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9 - vld1.32 {q12},[r3]! - vadd.i32 q13,q13,q11 - INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10 - vld1.32 {q13},[r3]! - vadd.i32 q12,q12,q8 - INST(0xe2,0x03,0xfa,0xf3) @ sha256su0 q8,q9 - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - INST(0xe6,0x0c,0x64,0xf3) @ sha256su1 q8,q10,q11 - vld1.32 {q12},[r3]! - vadd.i32 q13,q13,q9 - INST(0xe4,0x23,0xfa,0xf3) @ sha256su0 q9,q10 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - INST(0xe0,0x2c,0x66,0xf3) @ sha256su1 q9,q11,q8 - vld1.32 {q13},[r3]! - vadd.i32 q12,q12,q10 - INST(0xe6,0x43,0xfa,0xf3) @ sha256su0 q10,q11 - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - INST(0xe2,0x4c,0x60,0xf3) @ sha256su1 q10,q8,q9 - vld1.32 {q12},[r3]! - vadd.i32 q13,q13,q11 - INST(0xe0,0x63,0xfa,0xf3) @ sha256su0 q11,q8 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - INST(0xe4,0x6c,0x62,0xf3) @ sha256su1 q11,q9,q10 - vld1.32 {q13},[r3]! - vadd.i32 q12,q12,q8 - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - - vld1.32 {q12},[r3]! - vadd.i32 q13,q13,q9 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - - vld1.32 {q13},[r3] - vadd.i32 q12,q12,q10 - sub r3,r3,#256-16 @ rewind - vmov q2,q0 - INST(0x68,0x0c,0x02,0xf3) @ sha256h q0,q1,q12 - INST(0x68,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q12 - - vadd.i32 q13,q13,q11 - vmov q2,q0 - INST(0x6a,0x0c,0x02,0xf3) @ sha256h q0,q1,q13 - INST(0x6a,0x2c,0x14,0xf3) @ sha256h2 q1,q2,q13 - - vadd.i32 q0,q0,q14 - vadd.i32 q1,q1,q15 - it ne - bne .Loop_v8 - - vst1.32 {q0,q1},[r0] - - bx lr @ bx lr -.size sha256_block_data_order_armv8,.-sha256_block_data_order_armv8 -#endif -.asciz "SHA256 block transform for ARMv4/NEON/ARMv8, CRYPTOGAMS by " -.align 2 -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.comm OPENSSL_armcap_P,4,4 -#endif diff --git a/arch/arm/crypto/sha512-core.S_shipped b/arch/arm/crypto/sha512-core.S_shipped deleted file mode 100644 index 03014624f2ab..000000000000 --- a/arch/arm/crypto/sha512-core.S_shipped +++ /dev/null @@ -1,1869 +0,0 @@ -@ SPDX-License-Identifier: GPL-2.0 - -@ This code is taken from the OpenSSL project but the author (Andy Polyakov) -@ has relicensed it under the GPLv2. Therefore this program is free software; -@ you can redistribute it and/or modify it under the terms of the GNU General -@ Public License version 2 as published by the Free Software Foundation. -@ -@ The original headers, including the original license headers, are -@ included below for completeness. - -@ ==================================================================== -@ Written by Andy Polyakov for the OpenSSL -@ project. The module is, however, dual licensed under OpenSSL and -@ CRYPTOGAMS licenses depending on where you obtain it. For further -@ details see https://www.openssl.org/~appro/cryptogams/. -@ ==================================================================== - -@ SHA512 block procedure for ARMv4. September 2007. - -@ This code is ~4.5 (four and a half) times faster than code generated -@ by gcc 3.4 and it spends ~72 clock cycles per byte [on single-issue -@ Xscale PXA250 core]. -@ -@ July 2010. -@ -@ Rescheduling for dual-issue pipeline resulted in 6% improvement on -@ Cortex A8 core and ~40 cycles per processed byte. - -@ February 2011. -@ -@ Profiler-assisted and platform-specific optimization resulted in 7% -@ improvement on Coxtex A8 core and ~38 cycles per byte. - -@ March 2011. -@ -@ Add NEON implementation. On Cortex A8 it was measured to process -@ one byte in 23.3 cycles or ~60% faster than integer-only code. - -@ August 2012. -@ -@ Improve NEON performance by 12% on Snapdragon S4. In absolute -@ terms it's 22.6 cycles per byte, which is disappointing result. -@ Technical writers asserted that 3-way S4 pipeline can sustain -@ multiple NEON instructions per cycle, but dual NEON issue could -@ not be observed, see https://www.openssl.org/~appro/Snapdragon-S4.html -@ for further details. On side note Cortex-A15 processes one byte in -@ 16 cycles. - -@ Byte order [in]dependence. ========================================= -@ -@ Originally caller was expected to maintain specific *dword* order in -@ h[0-7], namely with most significant dword at *lower* address, which -@ was reflected in below two parameters as 0 and 4. Now caller is -@ expected to maintain native byte order for whole 64-bit values. -#ifndef __KERNEL__ -# include "arm_arch.h" -# define VFP_ABI_PUSH vstmdb sp!,{d8-d15} -# define VFP_ABI_POP vldmia sp!,{d8-d15} -#else -# define __ARM_ARCH__ __LINUX_ARM_ARCH__ -# define __ARM_MAX_ARCH__ 7 -# define VFP_ABI_PUSH -# define VFP_ABI_POP -#endif - -#ifdef __ARMEL__ -# define LO 0 -# define HI 4 -# define WORD64(hi0,lo0,hi1,lo1) .word lo0,hi0, lo1,hi1 -#else -# define HI 0 -# define LO 4 -# define WORD64(hi0,lo0,hi1,lo1) .word hi0,lo0, hi1,lo1 -#endif - -.text -#if __ARM_ARCH__<7 -.code 32 -#else -.syntax unified -# ifdef __thumb2__ -.thumb -# else -.code 32 -# endif -#endif - -.type K512,%object -.align 5 -K512: -WORD64(0x428a2f98,0xd728ae22, 0x71374491,0x23ef65cd) -WORD64(0xb5c0fbcf,0xec4d3b2f, 0xe9b5dba5,0x8189dbbc) -WORD64(0x3956c25b,0xf348b538, 0x59f111f1,0xb605d019) -WORD64(0x923f82a4,0xaf194f9b, 0xab1c5ed5,0xda6d8118) -WORD64(0xd807aa98,0xa3030242, 0x12835b01,0x45706fbe) -WORD64(0x243185be,0x4ee4b28c, 0x550c7dc3,0xd5ffb4e2) -WORD64(0x72be5d74,0xf27b896f, 0x80deb1fe,0x3b1696b1) -WORD64(0x9bdc06a7,0x25c71235, 0xc19bf174,0xcf692694) -WORD64(0xe49b69c1,0x9ef14ad2, 0xefbe4786,0x384f25e3) -WORD64(0x0fc19dc6,0x8b8cd5b5, 0x240ca1cc,0x77ac9c65) -WORD64(0x2de92c6f,0x592b0275, 0x4a7484aa,0x6ea6e483) -WORD64(0x5cb0a9dc,0xbd41fbd4, 0x76f988da,0x831153b5) -WORD64(0x983e5152,0xee66dfab, 0xa831c66d,0x2db43210) -WORD64(0xb00327c8,0x98fb213f, 0xbf597fc7,0xbeef0ee4) -WORD64(0xc6e00bf3,0x3da88fc2, 0xd5a79147,0x930aa725) -WORD64(0x06ca6351,0xe003826f, 0x14292967,0x0a0e6e70) -WORD64(0x27b70a85,0x46d22ffc, 0x2e1b2138,0x5c26c926) -WORD64(0x4d2c6dfc,0x5ac42aed, 0x53380d13,0x9d95b3df) -WORD64(0x650a7354,0x8baf63de, 0x766a0abb,0x3c77b2a8) -WORD64(0x81c2c92e,0x47edaee6, 0x92722c85,0x1482353b) -WORD64(0xa2bfe8a1,0x4cf10364, 0xa81a664b,0xbc423001) -WORD64(0xc24b8b70,0xd0f89791, 0xc76c51a3,0x0654be30) -WORD64(0xd192e819,0xd6ef5218, 0xd6990624,0x5565a910) -WORD64(0xf40e3585,0x5771202a, 0x106aa070,0x32bbd1b8) -WORD64(0x19a4c116,0xb8d2d0c8, 0x1e376c08,0x5141ab53) -WORD64(0x2748774c,0xdf8eeb99, 0x34b0bcb5,0xe19b48a8) -WORD64(0x391c0cb3,0xc5c95a63, 0x4ed8aa4a,0xe3418acb) -WORD64(0x5b9cca4f,0x7763e373, 0x682e6ff3,0xd6b2b8a3) -WORD64(0x748f82ee,0x5defb2fc, 0x78a5636f,0x43172f60) -WORD64(0x84c87814,0xa1f0ab72, 0x8cc70208,0x1a6439ec) -WORD64(0x90befffa,0x23631e28, 0xa4506ceb,0xde82bde9) -WORD64(0xbef9a3f7,0xb2c67915, 0xc67178f2,0xe372532b) -WORD64(0xca273ece,0xea26619c, 0xd186b8c7,0x21c0c207) -WORD64(0xeada7dd6,0xcde0eb1e, 0xf57d4f7f,0xee6ed178) -WORD64(0x06f067aa,0x72176fba, 0x0a637dc5,0xa2c898a6) -WORD64(0x113f9804,0xbef90dae, 0x1b710b35,0x131c471b) -WORD64(0x28db77f5,0x23047d84, 0x32caab7b,0x40c72493) -WORD64(0x3c9ebe0a,0x15c9bebc, 0x431d67c4,0x9c100d4c) -WORD64(0x4cc5d4be,0xcb3e42b6, 0x597f299c,0xfc657e2a) -WORD64(0x5fcb6fab,0x3ad6faec, 0x6c44198c,0x4a475817) -.size K512,.-K512 -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.LOPENSSL_armcap: -.word OPENSSL_armcap_P-sha512_block_data_order -.skip 32-4 -#else -.skip 32 -#endif - -.global sha512_block_data_order -.type sha512_block_data_order,%function -sha512_block_data_order: -.Lsha512_block_data_order: -#if __ARM_ARCH__<7 - sub r3,pc,#8 @ sha512_block_data_order -#else - adr r3,.Lsha512_block_data_order -#endif -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) - ldr r12,.LOPENSSL_armcap - ldr r12,[r3,r12] @ OPENSSL_armcap_P - tst r12,#1 - bne .LNEON -#endif - add r2,r1,r2,lsl#7 @ len to point at the end of inp - stmdb sp!,{r4-r12,lr} - sub r14,r3,#672 @ K512 - sub sp,sp,#9*8 - - ldr r7,[r0,#32+LO] - ldr r8,[r0,#32+HI] - ldr r9, [r0,#48+LO] - ldr r10, [r0,#48+HI] - ldr r11, [r0,#56+LO] - ldr r12, [r0,#56+HI] -.Loop: - str r9, [sp,#48+0] - str r10, [sp,#48+4] - str r11, [sp,#56+0] - str r12, [sp,#56+4] - ldr r5,[r0,#0+LO] - ldr r6,[r0,#0+HI] - ldr r3,[r0,#8+LO] - ldr r4,[r0,#8+HI] - ldr r9, [r0,#16+LO] - ldr r10, [r0,#16+HI] - ldr r11, [r0,#24+LO] - ldr r12, [r0,#24+HI] - str r3,[sp,#8+0] - str r4,[sp,#8+4] - str r9, [sp,#16+0] - str r10, [sp,#16+4] - str r11, [sp,#24+0] - str r12, [sp,#24+4] - ldr r3,[r0,#40+LO] - ldr r4,[r0,#40+HI] - str r3,[sp,#40+0] - str r4,[sp,#40+4] - -.L00_15: -#if __ARM_ARCH__<7 - ldrb r3,[r1,#7] - ldrb r9, [r1,#6] - ldrb r10, [r1,#5] - ldrb r11, [r1,#4] - ldrb r4,[r1,#3] - ldrb r12, [r1,#2] - orr r3,r3,r9,lsl#8 - ldrb r9, [r1,#1] - orr r3,r3,r10,lsl#16 - ldrb r10, [r1],#8 - orr r3,r3,r11,lsl#24 - orr r4,r4,r12,lsl#8 - orr r4,r4,r9,lsl#16 - orr r4,r4,r10,lsl#24 -#else - ldr r3,[r1,#4] - ldr r4,[r1],#8 -#ifdef __ARMEL__ - rev r3,r3 - rev r4,r4 -#endif -#endif - @ Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41)) - @ LO lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23 - @ HI hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23 - mov r9,r7,lsr#14 - str r3,[sp,#64+0] - mov r10,r8,lsr#14 - str r4,[sp,#64+4] - eor r9,r9,r8,lsl#18 - ldr r11,[sp,#56+0] @ h.lo - eor r10,r10,r7,lsl#18 - ldr r12,[sp,#56+4] @ h.hi - eor r9,r9,r7,lsr#18 - eor r10,r10,r8,lsr#18 - eor r9,r9,r8,lsl#14 - eor r10,r10,r7,lsl#14 - eor r9,r9,r8,lsr#9 - eor r10,r10,r7,lsr#9 - eor r9,r9,r7,lsl#23 - eor r10,r10,r8,lsl#23 @ Sigma1(e) - adds r3,r3,r9 - ldr r9,[sp,#40+0] @ f.lo - adc r4,r4,r10 @ T += Sigma1(e) - ldr r10,[sp,#40+4] @ f.hi - adds r3,r3,r11 - ldr r11,[sp,#48+0] @ g.lo - adc r4,r4,r12 @ T += h - ldr r12,[sp,#48+4] @ g.hi - - eor r9,r9,r11 - str r7,[sp,#32+0] - eor r10,r10,r12 - str r8,[sp,#32+4] - and r9,r9,r7 - str r5,[sp,#0+0] - and r10,r10,r8 - str r6,[sp,#0+4] - eor r9,r9,r11 - ldr r11,[r14,#LO] @ K[i].lo - eor r10,r10,r12 @ Ch(e,f,g) - ldr r12,[r14,#HI] @ K[i].hi - - adds r3,r3,r9 - ldr r7,[sp,#24+0] @ d.lo - adc r4,r4,r10 @ T += Ch(e,f,g) - ldr r8,[sp,#24+4] @ d.hi - adds r3,r3,r11 - and r9,r11,#0xff - adc r4,r4,r12 @ T += K[i] - adds r7,r7,r3 - ldr r11,[sp,#8+0] @ b.lo - adc r8,r8,r4 @ d += T - teq r9,#148 - - ldr r12,[sp,#16+0] @ c.lo -#if __ARM_ARCH__>=7 - it eq @ Thumb2 thing, sanity check in ARM -#endif - orreq r14,r14,#1 - @ Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39)) - @ LO lo>>28^hi<<4 ^ hi>>2^lo<<30 ^ hi>>7^lo<<25 - @ HI hi>>28^lo<<4 ^ lo>>2^hi<<30 ^ lo>>7^hi<<25 - mov r9,r5,lsr#28 - mov r10,r6,lsr#28 - eor r9,r9,r6,lsl#4 - eor r10,r10,r5,lsl#4 - eor r9,r9,r6,lsr#2 - eor r10,r10,r5,lsr#2 - eor r9,r9,r5,lsl#30 - eor r10,r10,r6,lsl#30 - eor r9,r9,r6,lsr#7 - eor r10,r10,r5,lsr#7 - eor r9,r9,r5,lsl#25 - eor r10,r10,r6,lsl#25 @ Sigma0(a) - adds r3,r3,r9 - and r9,r5,r11 - adc r4,r4,r10 @ T += Sigma0(a) - - ldr r10,[sp,#8+4] @ b.hi - orr r5,r5,r11 - ldr r11,[sp,#16+4] @ c.hi - and r5,r5,r12 - and r12,r6,r10 - orr r6,r6,r10 - orr r5,r5,r9 @ Maj(a,b,c).lo - and r6,r6,r11 - adds r5,r5,r3 - orr r6,r6,r12 @ Maj(a,b,c).hi - sub sp,sp,#8 - adc r6,r6,r4 @ h += T - tst r14,#1 - add r14,r14,#8 - tst r14,#1 - beq .L00_15 - ldr r9,[sp,#184+0] - ldr r10,[sp,#184+4] - bic r14,r14,#1 -.L16_79: - @ sigma0(x) (ROTR((x),1) ^ ROTR((x),8) ^ ((x)>>7)) - @ LO lo>>1^hi<<31 ^ lo>>8^hi<<24 ^ lo>>7^hi<<25 - @ HI hi>>1^lo<<31 ^ hi>>8^lo<<24 ^ hi>>7 - mov r3,r9,lsr#1 - ldr r11,[sp,#80+0] - mov r4,r10,lsr#1 - ldr r12,[sp,#80+4] - eor r3,r3,r10,lsl#31 - eor r4,r4,r9,lsl#31 - eor r3,r3,r9,lsr#8 - eor r4,r4,r10,lsr#8 - eor r3,r3,r10,lsl#24 - eor r4,r4,r9,lsl#24 - eor r3,r3,r9,lsr#7 - eor r4,r4,r10,lsr#7 - eor r3,r3,r10,lsl#25 - - @ sigma1(x) (ROTR((x),19) ^ ROTR((x),61) ^ ((x)>>6)) - @ LO lo>>19^hi<<13 ^ hi>>29^lo<<3 ^ lo>>6^hi<<26 - @ HI hi>>19^lo<<13 ^ lo>>29^hi<<3 ^ hi>>6 - mov r9,r11,lsr#19 - mov r10,r12,lsr#19 - eor r9,r9,r12,lsl#13 - eor r10,r10,r11,lsl#13 - eor r9,r9,r12,lsr#29 - eor r10,r10,r11,lsr#29 - eor r9,r9,r11,lsl#3 - eor r10,r10,r12,lsl#3 - eor r9,r9,r11,lsr#6 - eor r10,r10,r12,lsr#6 - ldr r11,[sp,#120+0] - eor r9,r9,r12,lsl#26 - - ldr r12,[sp,#120+4] - adds r3,r3,r9 - ldr r9,[sp,#192+0] - adc r4,r4,r10 - - ldr r10,[sp,#192+4] - adds r3,r3,r11 - adc r4,r4,r12 - adds r3,r3,r9 - adc r4,r4,r10 - @ Sigma1(x) (ROTR((x),14) ^ ROTR((x),18) ^ ROTR((x),41)) - @ LO lo>>14^hi<<18 ^ lo>>18^hi<<14 ^ hi>>9^lo<<23 - @ HI hi>>14^lo<<18 ^ hi>>18^lo<<14 ^ lo>>9^hi<<23 - mov r9,r7,lsr#14 - str r3,[sp,#64+0] - mov r10,r8,lsr#14 - str r4,[sp,#64+4] - eor r9,r9,r8,lsl#18 - ldr r11,[sp,#56+0] @ h.lo - eor r10,r10,r7,lsl#18 - ldr r12,[sp,#56+4] @ h.hi - eor r9,r9,r7,lsr#18 - eor r10,r10,r8,lsr#18 - eor r9,r9,r8,lsl#14 - eor r10,r10,r7,lsl#14 - eor r9,r9,r8,lsr#9 - eor r10,r10,r7,lsr#9 - eor r9,r9,r7,lsl#23 - eor r10,r10,r8,lsl#23 @ Sigma1(e) - adds r3,r3,r9 - ldr r9,[sp,#40+0] @ f.lo - adc r4,r4,r10 @ T += Sigma1(e) - ldr r10,[sp,#40+4] @ f.hi - adds r3,r3,r11 - ldr r11,[sp,#48+0] @ g.lo - adc r4,r4,r12 @ T += h - ldr r12,[sp,#48+4] @ g.hi - - eor r9,r9,r11 - str r7,[sp,#32+0] - eor r10,r10,r12 - str r8,[sp,#32+4] - and r9,r9,r7 - str r5,[sp,#0+0] - and r10,r10,r8 - str r6,[sp,#0+4] - eor r9,r9,r11 - ldr r11,[r14,#LO] @ K[i].lo - eor r10,r10,r12 @ Ch(e,f,g) - ldr r12,[r14,#HI] @ K[i].hi - - adds r3,r3,r9 - ldr r7,[sp,#24+0] @ d.lo - adc r4,r4,r10 @ T += Ch(e,f,g) - ldr r8,[sp,#24+4] @ d.hi - adds r3,r3,r11 - and r9,r11,#0xff - adc r4,r4,r12 @ T += K[i] - adds r7,r7,r3 - ldr r11,[sp,#8+0] @ b.lo - adc r8,r8,r4 @ d += T - teq r9,#23 - - ldr r12,[sp,#16+0] @ c.lo -#if __ARM_ARCH__>=7 - it eq @ Thumb2 thing, sanity check in ARM -#endif - orreq r14,r14,#1 - @ Sigma0(x) (ROTR((x),28) ^ ROTR((x),34) ^ ROTR((x),39)) - @ LO lo>>28^hi<<4 ^ hi>>2^lo<<30 ^ hi>>7^lo<<25 - @ HI hi>>28^lo<<4 ^ lo>>2^hi<<30 ^ lo>>7^hi<<25 - mov r9,r5,lsr#28 - mov r10,r6,lsr#28 - eor r9,r9,r6,lsl#4 - eor r10,r10,r5,lsl#4 - eor r9,r9,r6,lsr#2 - eor r10,r10,r5,lsr#2 - eor r9,r9,r5,lsl#30 - eor r10,r10,r6,lsl#30 - eor r9,r9,r6,lsr#7 - eor r10,r10,r5,lsr#7 - eor r9,r9,r5,lsl#25 - eor r10,r10,r6,lsl#25 @ Sigma0(a) - adds r3,r3,r9 - and r9,r5,r11 - adc r4,r4,r10 @ T += Sigma0(a) - - ldr r10,[sp,#8+4] @ b.hi - orr r5,r5,r11 - ldr r11,[sp,#16+4] @ c.hi - and r5,r5,r12 - and r12,r6,r10 - orr r6,r6,r10 - orr r5,r5,r9 @ Maj(a,b,c).lo - and r6,r6,r11 - adds r5,r5,r3 - orr r6,r6,r12 @ Maj(a,b,c).hi - sub sp,sp,#8 - adc r6,r6,r4 @ h += T - tst r14,#1 - add r14,r14,#8 -#if __ARM_ARCH__>=7 - ittt eq @ Thumb2 thing, sanity check in ARM -#endif - ldreq r9,[sp,#184+0] - ldreq r10,[sp,#184+4] - beq .L16_79 - bic r14,r14,#1 - - ldr r3,[sp,#8+0] - ldr r4,[sp,#8+4] - ldr r9, [r0,#0+LO] - ldr r10, [r0,#0+HI] - ldr r11, [r0,#8+LO] - ldr r12, [r0,#8+HI] - adds r9,r5,r9 - str r9, [r0,#0+LO] - adc r10,r6,r10 - str r10, [r0,#0+HI] - adds r11,r3,r11 - str r11, [r0,#8+LO] - adc r12,r4,r12 - str r12, [r0,#8+HI] - - ldr r5,[sp,#16+0] - ldr r6,[sp,#16+4] - ldr r3,[sp,#24+0] - ldr r4,[sp,#24+4] - ldr r9, [r0,#16+LO] - ldr r10, [r0,#16+HI] - ldr r11, [r0,#24+LO] - ldr r12, [r0,#24+HI] - adds r9,r5,r9 - str r9, [r0,#16+LO] - adc r10,r6,r10 - str r10, [r0,#16+HI] - adds r11,r3,r11 - str r11, [r0,#24+LO] - adc r12,r4,r12 - str r12, [r0,#24+HI] - - ldr r3,[sp,#40+0] - ldr r4,[sp,#40+4] - ldr r9, [r0,#32+LO] - ldr r10, [r0,#32+HI] - ldr r11, [r0,#40+LO] - ldr r12, [r0,#40+HI] - adds r7,r7,r9 - str r7,[r0,#32+LO] - adc r8,r8,r10 - str r8,[r0,#32+HI] - adds r11,r3,r11 - str r11, [r0,#40+LO] - adc r12,r4,r12 - str r12, [r0,#40+HI] - - ldr r5,[sp,#48+0] - ldr r6,[sp,#48+4] - ldr r3,[sp,#56+0] - ldr r4,[sp,#56+4] - ldr r9, [r0,#48+LO] - ldr r10, [r0,#48+HI] - ldr r11, [r0,#56+LO] - ldr r12, [r0,#56+HI] - adds r9,r5,r9 - str r9, [r0,#48+LO] - adc r10,r6,r10 - str r10, [r0,#48+HI] - adds r11,r3,r11 - str r11, [r0,#56+LO] - adc r12,r4,r12 - str r12, [r0,#56+HI] - - add sp,sp,#640 - sub r14,r14,#640 - - teq r1,r2 - bne .Loop - - add sp,sp,#8*9 @ destroy frame -#if __ARM_ARCH__>=5 - ldmia sp!,{r4-r12,pc} -#else - ldmia sp!,{r4-r12,lr} - tst lr,#1 - moveq pc,lr @ be binary compatible with V4, yet - .word 0xe12fff1e @ interoperable with Thumb ISA:-) -#endif -.size sha512_block_data_order,.-sha512_block_data_order -#if __ARM_MAX_ARCH__>=7 -.arch armv7-a -.fpu neon - -.global sha512_block_data_order_neon -.type sha512_block_data_order_neon,%function -.align 4 -sha512_block_data_order_neon: -.LNEON: - dmb @ errata #451034 on early Cortex A8 - add r2,r1,r2,lsl#7 @ len to point at the end of inp - VFP_ABI_PUSH - adr r3,.Lsha512_block_data_order - sub r3,r3,.Lsha512_block_data_order-K512 - vldmia r0,{d16-d23} @ load context -.Loop_neon: - vshr.u64 d24,d20,#14 @ 0 -#if 0<16 - vld1.64 {d0},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d20,#18 -#if 0>0 - vadd.i64 d16,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d20,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d20,#50 - vsli.64 d25,d20,#46 - vmov d29,d20 - vsli.64 d26,d20,#23 -#if 0<16 && defined(__ARMEL__) - vrev64.8 d0,d0 -#endif - veor d25,d24 - vbsl d29,d21,d22 @ Ch(e,f,g) - vshr.u64 d24,d16,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d23 - vshr.u64 d25,d16,#34 - vsli.64 d24,d16,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d16,#39 - vadd.i64 d28,d0 - vsli.64 d25,d16,#30 - veor d30,d16,d17 - vsli.64 d26,d16,#25 - veor d23,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d18,d17 @ Maj(a,b,c) - veor d23,d26 @ Sigma0(a) - vadd.i64 d19,d27 - vadd.i64 d30,d27 - @ vadd.i64 d23,d30 - vshr.u64 d24,d19,#14 @ 1 -#if 1<16 - vld1.64 {d1},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d19,#18 -#if 1>0 - vadd.i64 d23,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d19,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d19,#50 - vsli.64 d25,d19,#46 - vmov d29,d19 - vsli.64 d26,d19,#23 -#if 1<16 && defined(__ARMEL__) - vrev64.8 d1,d1 -#endif - veor d25,d24 - vbsl d29,d20,d21 @ Ch(e,f,g) - vshr.u64 d24,d23,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d22 - vshr.u64 d25,d23,#34 - vsli.64 d24,d23,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d23,#39 - vadd.i64 d28,d1 - vsli.64 d25,d23,#30 - veor d30,d23,d16 - vsli.64 d26,d23,#25 - veor d22,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d17,d16 @ Maj(a,b,c) - veor d22,d26 @ Sigma0(a) - vadd.i64 d18,d27 - vadd.i64 d30,d27 - @ vadd.i64 d22,d30 - vshr.u64 d24,d18,#14 @ 2 -#if 2<16 - vld1.64 {d2},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d18,#18 -#if 2>0 - vadd.i64 d22,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d18,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d18,#50 - vsli.64 d25,d18,#46 - vmov d29,d18 - vsli.64 d26,d18,#23 -#if 2<16 && defined(__ARMEL__) - vrev64.8 d2,d2 -#endif - veor d25,d24 - vbsl d29,d19,d20 @ Ch(e,f,g) - vshr.u64 d24,d22,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d21 - vshr.u64 d25,d22,#34 - vsli.64 d24,d22,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d22,#39 - vadd.i64 d28,d2 - vsli.64 d25,d22,#30 - veor d30,d22,d23 - vsli.64 d26,d22,#25 - veor d21,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d16,d23 @ Maj(a,b,c) - veor d21,d26 @ Sigma0(a) - vadd.i64 d17,d27 - vadd.i64 d30,d27 - @ vadd.i64 d21,d30 - vshr.u64 d24,d17,#14 @ 3 -#if 3<16 - vld1.64 {d3},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d17,#18 -#if 3>0 - vadd.i64 d21,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d17,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d17,#50 - vsli.64 d25,d17,#46 - vmov d29,d17 - vsli.64 d26,d17,#23 -#if 3<16 && defined(__ARMEL__) - vrev64.8 d3,d3 -#endif - veor d25,d24 - vbsl d29,d18,d19 @ Ch(e,f,g) - vshr.u64 d24,d21,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d20 - vshr.u64 d25,d21,#34 - vsli.64 d24,d21,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d21,#39 - vadd.i64 d28,d3 - vsli.64 d25,d21,#30 - veor d30,d21,d22 - vsli.64 d26,d21,#25 - veor d20,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d23,d22 @ Maj(a,b,c) - veor d20,d26 @ Sigma0(a) - vadd.i64 d16,d27 - vadd.i64 d30,d27 - @ vadd.i64 d20,d30 - vshr.u64 d24,d16,#14 @ 4 -#if 4<16 - vld1.64 {d4},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d16,#18 -#if 4>0 - vadd.i64 d20,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d16,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d16,#50 - vsli.64 d25,d16,#46 - vmov d29,d16 - vsli.64 d26,d16,#23 -#if 4<16 && defined(__ARMEL__) - vrev64.8 d4,d4 -#endif - veor d25,d24 - vbsl d29,d17,d18 @ Ch(e,f,g) - vshr.u64 d24,d20,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d19 - vshr.u64 d25,d20,#34 - vsli.64 d24,d20,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d20,#39 - vadd.i64 d28,d4 - vsli.64 d25,d20,#30 - veor d30,d20,d21 - vsli.64 d26,d20,#25 - veor d19,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d22,d21 @ Maj(a,b,c) - veor d19,d26 @ Sigma0(a) - vadd.i64 d23,d27 - vadd.i64 d30,d27 - @ vadd.i64 d19,d30 - vshr.u64 d24,d23,#14 @ 5 -#if 5<16 - vld1.64 {d5},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d23,#18 -#if 5>0 - vadd.i64 d19,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d23,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d23,#50 - vsli.64 d25,d23,#46 - vmov d29,d23 - vsli.64 d26,d23,#23 -#if 5<16 && defined(__ARMEL__) - vrev64.8 d5,d5 -#endif - veor d25,d24 - vbsl d29,d16,d17 @ Ch(e,f,g) - vshr.u64 d24,d19,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d18 - vshr.u64 d25,d19,#34 - vsli.64 d24,d19,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d19,#39 - vadd.i64 d28,d5 - vsli.64 d25,d19,#30 - veor d30,d19,d20 - vsli.64 d26,d19,#25 - veor d18,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d21,d20 @ Maj(a,b,c) - veor d18,d26 @ Sigma0(a) - vadd.i64 d22,d27 - vadd.i64 d30,d27 - @ vadd.i64 d18,d30 - vshr.u64 d24,d22,#14 @ 6 -#if 6<16 - vld1.64 {d6},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d22,#18 -#if 6>0 - vadd.i64 d18,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d22,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d22,#50 - vsli.64 d25,d22,#46 - vmov d29,d22 - vsli.64 d26,d22,#23 -#if 6<16 && defined(__ARMEL__) - vrev64.8 d6,d6 -#endif - veor d25,d24 - vbsl d29,d23,d16 @ Ch(e,f,g) - vshr.u64 d24,d18,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d17 - vshr.u64 d25,d18,#34 - vsli.64 d24,d18,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d18,#39 - vadd.i64 d28,d6 - vsli.64 d25,d18,#30 - veor d30,d18,d19 - vsli.64 d26,d18,#25 - veor d17,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d20,d19 @ Maj(a,b,c) - veor d17,d26 @ Sigma0(a) - vadd.i64 d21,d27 - vadd.i64 d30,d27 - @ vadd.i64 d17,d30 - vshr.u64 d24,d21,#14 @ 7 -#if 7<16 - vld1.64 {d7},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d21,#18 -#if 7>0 - vadd.i64 d17,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d21,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d21,#50 - vsli.64 d25,d21,#46 - vmov d29,d21 - vsli.64 d26,d21,#23 -#if 7<16 && defined(__ARMEL__) - vrev64.8 d7,d7 -#endif - veor d25,d24 - vbsl d29,d22,d23 @ Ch(e,f,g) - vshr.u64 d24,d17,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d16 - vshr.u64 d25,d17,#34 - vsli.64 d24,d17,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d17,#39 - vadd.i64 d28,d7 - vsli.64 d25,d17,#30 - veor d30,d17,d18 - vsli.64 d26,d17,#25 - veor d16,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d19,d18 @ Maj(a,b,c) - veor d16,d26 @ Sigma0(a) - vadd.i64 d20,d27 - vadd.i64 d30,d27 - @ vadd.i64 d16,d30 - vshr.u64 d24,d20,#14 @ 8 -#if 8<16 - vld1.64 {d8},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d20,#18 -#if 8>0 - vadd.i64 d16,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d20,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d20,#50 - vsli.64 d25,d20,#46 - vmov d29,d20 - vsli.64 d26,d20,#23 -#if 8<16 && defined(__ARMEL__) - vrev64.8 d8,d8 -#endif - veor d25,d24 - vbsl d29,d21,d22 @ Ch(e,f,g) - vshr.u64 d24,d16,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d23 - vshr.u64 d25,d16,#34 - vsli.64 d24,d16,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d16,#39 - vadd.i64 d28,d8 - vsli.64 d25,d16,#30 - veor d30,d16,d17 - vsli.64 d26,d16,#25 - veor d23,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d18,d17 @ Maj(a,b,c) - veor d23,d26 @ Sigma0(a) - vadd.i64 d19,d27 - vadd.i64 d30,d27 - @ vadd.i64 d23,d30 - vshr.u64 d24,d19,#14 @ 9 -#if 9<16 - vld1.64 {d9},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d19,#18 -#if 9>0 - vadd.i64 d23,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d19,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d19,#50 - vsli.64 d25,d19,#46 - vmov d29,d19 - vsli.64 d26,d19,#23 -#if 9<16 && defined(__ARMEL__) - vrev64.8 d9,d9 -#endif - veor d25,d24 - vbsl d29,d20,d21 @ Ch(e,f,g) - vshr.u64 d24,d23,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d22 - vshr.u64 d25,d23,#34 - vsli.64 d24,d23,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d23,#39 - vadd.i64 d28,d9 - vsli.64 d25,d23,#30 - veor d30,d23,d16 - vsli.64 d26,d23,#25 - veor d22,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d17,d16 @ Maj(a,b,c) - veor d22,d26 @ Sigma0(a) - vadd.i64 d18,d27 - vadd.i64 d30,d27 - @ vadd.i64 d22,d30 - vshr.u64 d24,d18,#14 @ 10 -#if 10<16 - vld1.64 {d10},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d18,#18 -#if 10>0 - vadd.i64 d22,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d18,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d18,#50 - vsli.64 d25,d18,#46 - vmov d29,d18 - vsli.64 d26,d18,#23 -#if 10<16 && defined(__ARMEL__) - vrev64.8 d10,d10 -#endif - veor d25,d24 - vbsl d29,d19,d20 @ Ch(e,f,g) - vshr.u64 d24,d22,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d21 - vshr.u64 d25,d22,#34 - vsli.64 d24,d22,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d22,#39 - vadd.i64 d28,d10 - vsli.64 d25,d22,#30 - veor d30,d22,d23 - vsli.64 d26,d22,#25 - veor d21,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d16,d23 @ Maj(a,b,c) - veor d21,d26 @ Sigma0(a) - vadd.i64 d17,d27 - vadd.i64 d30,d27 - @ vadd.i64 d21,d30 - vshr.u64 d24,d17,#14 @ 11 -#if 11<16 - vld1.64 {d11},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d17,#18 -#if 11>0 - vadd.i64 d21,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d17,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d17,#50 - vsli.64 d25,d17,#46 - vmov d29,d17 - vsli.64 d26,d17,#23 -#if 11<16 && defined(__ARMEL__) - vrev64.8 d11,d11 -#endif - veor d25,d24 - vbsl d29,d18,d19 @ Ch(e,f,g) - vshr.u64 d24,d21,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d20 - vshr.u64 d25,d21,#34 - vsli.64 d24,d21,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d21,#39 - vadd.i64 d28,d11 - vsli.64 d25,d21,#30 - veor d30,d21,d22 - vsli.64 d26,d21,#25 - veor d20,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d23,d22 @ Maj(a,b,c) - veor d20,d26 @ Sigma0(a) - vadd.i64 d16,d27 - vadd.i64 d30,d27 - @ vadd.i64 d20,d30 - vshr.u64 d24,d16,#14 @ 12 -#if 12<16 - vld1.64 {d12},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d16,#18 -#if 12>0 - vadd.i64 d20,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d16,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d16,#50 - vsli.64 d25,d16,#46 - vmov d29,d16 - vsli.64 d26,d16,#23 -#if 12<16 && defined(__ARMEL__) - vrev64.8 d12,d12 -#endif - veor d25,d24 - vbsl d29,d17,d18 @ Ch(e,f,g) - vshr.u64 d24,d20,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d19 - vshr.u64 d25,d20,#34 - vsli.64 d24,d20,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d20,#39 - vadd.i64 d28,d12 - vsli.64 d25,d20,#30 - veor d30,d20,d21 - vsli.64 d26,d20,#25 - veor d19,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d22,d21 @ Maj(a,b,c) - veor d19,d26 @ Sigma0(a) - vadd.i64 d23,d27 - vadd.i64 d30,d27 - @ vadd.i64 d19,d30 - vshr.u64 d24,d23,#14 @ 13 -#if 13<16 - vld1.64 {d13},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d23,#18 -#if 13>0 - vadd.i64 d19,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d23,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d23,#50 - vsli.64 d25,d23,#46 - vmov d29,d23 - vsli.64 d26,d23,#23 -#if 13<16 && defined(__ARMEL__) - vrev64.8 d13,d13 -#endif - veor d25,d24 - vbsl d29,d16,d17 @ Ch(e,f,g) - vshr.u64 d24,d19,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d18 - vshr.u64 d25,d19,#34 - vsli.64 d24,d19,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d19,#39 - vadd.i64 d28,d13 - vsli.64 d25,d19,#30 - veor d30,d19,d20 - vsli.64 d26,d19,#25 - veor d18,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d21,d20 @ Maj(a,b,c) - veor d18,d26 @ Sigma0(a) - vadd.i64 d22,d27 - vadd.i64 d30,d27 - @ vadd.i64 d18,d30 - vshr.u64 d24,d22,#14 @ 14 -#if 14<16 - vld1.64 {d14},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d22,#18 -#if 14>0 - vadd.i64 d18,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d22,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d22,#50 - vsli.64 d25,d22,#46 - vmov d29,d22 - vsli.64 d26,d22,#23 -#if 14<16 && defined(__ARMEL__) - vrev64.8 d14,d14 -#endif - veor d25,d24 - vbsl d29,d23,d16 @ Ch(e,f,g) - vshr.u64 d24,d18,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d17 - vshr.u64 d25,d18,#34 - vsli.64 d24,d18,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d18,#39 - vadd.i64 d28,d14 - vsli.64 d25,d18,#30 - veor d30,d18,d19 - vsli.64 d26,d18,#25 - veor d17,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d20,d19 @ Maj(a,b,c) - veor d17,d26 @ Sigma0(a) - vadd.i64 d21,d27 - vadd.i64 d30,d27 - @ vadd.i64 d17,d30 - vshr.u64 d24,d21,#14 @ 15 -#if 15<16 - vld1.64 {d15},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d21,#18 -#if 15>0 - vadd.i64 d17,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d21,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d21,#50 - vsli.64 d25,d21,#46 - vmov d29,d21 - vsli.64 d26,d21,#23 -#if 15<16 && defined(__ARMEL__) - vrev64.8 d15,d15 -#endif - veor d25,d24 - vbsl d29,d22,d23 @ Ch(e,f,g) - vshr.u64 d24,d17,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d16 - vshr.u64 d25,d17,#34 - vsli.64 d24,d17,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d17,#39 - vadd.i64 d28,d15 - vsli.64 d25,d17,#30 - veor d30,d17,d18 - vsli.64 d26,d17,#25 - veor d16,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d19,d18 @ Maj(a,b,c) - veor d16,d26 @ Sigma0(a) - vadd.i64 d20,d27 - vadd.i64 d30,d27 - @ vadd.i64 d16,d30 - mov r12,#4 -.L16_79_neon: - subs r12,#1 - vshr.u64 q12,q7,#19 - vshr.u64 q13,q7,#61 - vadd.i64 d16,d30 @ h+=Maj from the past - vshr.u64 q15,q7,#6 - vsli.64 q12,q7,#45 - vext.8 q14,q0,q1,#8 @ X[i+1] - vsli.64 q13,q7,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q0,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q4,q5,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d20,#14 @ from NEON_00_15 - vadd.i64 q0,q14 - vshr.u64 d25,d20,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d20,#41 @ from NEON_00_15 - vadd.i64 q0,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d20,#50 - vsli.64 d25,d20,#46 - vmov d29,d20 - vsli.64 d26,d20,#23 -#if 16<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d21,d22 @ Ch(e,f,g) - vshr.u64 d24,d16,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d23 - vshr.u64 d25,d16,#34 - vsli.64 d24,d16,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d16,#39 - vadd.i64 d28,d0 - vsli.64 d25,d16,#30 - veor d30,d16,d17 - vsli.64 d26,d16,#25 - veor d23,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d18,d17 @ Maj(a,b,c) - veor d23,d26 @ Sigma0(a) - vadd.i64 d19,d27 - vadd.i64 d30,d27 - @ vadd.i64 d23,d30 - vshr.u64 d24,d19,#14 @ 17 -#if 17<16 - vld1.64 {d1},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d19,#18 -#if 17>0 - vadd.i64 d23,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d19,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d19,#50 - vsli.64 d25,d19,#46 - vmov d29,d19 - vsli.64 d26,d19,#23 -#if 17<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d20,d21 @ Ch(e,f,g) - vshr.u64 d24,d23,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d22 - vshr.u64 d25,d23,#34 - vsli.64 d24,d23,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d23,#39 - vadd.i64 d28,d1 - vsli.64 d25,d23,#30 - veor d30,d23,d16 - vsli.64 d26,d23,#25 - veor d22,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d17,d16 @ Maj(a,b,c) - veor d22,d26 @ Sigma0(a) - vadd.i64 d18,d27 - vadd.i64 d30,d27 - @ vadd.i64 d22,d30 - vshr.u64 q12,q0,#19 - vshr.u64 q13,q0,#61 - vadd.i64 d22,d30 @ h+=Maj from the past - vshr.u64 q15,q0,#6 - vsli.64 q12,q0,#45 - vext.8 q14,q1,q2,#8 @ X[i+1] - vsli.64 q13,q0,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q1,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q5,q6,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d18,#14 @ from NEON_00_15 - vadd.i64 q1,q14 - vshr.u64 d25,d18,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d18,#41 @ from NEON_00_15 - vadd.i64 q1,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d18,#50 - vsli.64 d25,d18,#46 - vmov d29,d18 - vsli.64 d26,d18,#23 -#if 18<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d19,d20 @ Ch(e,f,g) - vshr.u64 d24,d22,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d21 - vshr.u64 d25,d22,#34 - vsli.64 d24,d22,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d22,#39 - vadd.i64 d28,d2 - vsli.64 d25,d22,#30 - veor d30,d22,d23 - vsli.64 d26,d22,#25 - veor d21,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d16,d23 @ Maj(a,b,c) - veor d21,d26 @ Sigma0(a) - vadd.i64 d17,d27 - vadd.i64 d30,d27 - @ vadd.i64 d21,d30 - vshr.u64 d24,d17,#14 @ 19 -#if 19<16 - vld1.64 {d3},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d17,#18 -#if 19>0 - vadd.i64 d21,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d17,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d17,#50 - vsli.64 d25,d17,#46 - vmov d29,d17 - vsli.64 d26,d17,#23 -#if 19<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d18,d19 @ Ch(e,f,g) - vshr.u64 d24,d21,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d20 - vshr.u64 d25,d21,#34 - vsli.64 d24,d21,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d21,#39 - vadd.i64 d28,d3 - vsli.64 d25,d21,#30 - veor d30,d21,d22 - vsli.64 d26,d21,#25 - veor d20,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d23,d22 @ Maj(a,b,c) - veor d20,d26 @ Sigma0(a) - vadd.i64 d16,d27 - vadd.i64 d30,d27 - @ vadd.i64 d20,d30 - vshr.u64 q12,q1,#19 - vshr.u64 q13,q1,#61 - vadd.i64 d20,d30 @ h+=Maj from the past - vshr.u64 q15,q1,#6 - vsli.64 q12,q1,#45 - vext.8 q14,q2,q3,#8 @ X[i+1] - vsli.64 q13,q1,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q2,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q6,q7,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d16,#14 @ from NEON_00_15 - vadd.i64 q2,q14 - vshr.u64 d25,d16,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d16,#41 @ from NEON_00_15 - vadd.i64 q2,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d16,#50 - vsli.64 d25,d16,#46 - vmov d29,d16 - vsli.64 d26,d16,#23 -#if 20<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d17,d18 @ Ch(e,f,g) - vshr.u64 d24,d20,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d19 - vshr.u64 d25,d20,#34 - vsli.64 d24,d20,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d20,#39 - vadd.i64 d28,d4 - vsli.64 d25,d20,#30 - veor d30,d20,d21 - vsli.64 d26,d20,#25 - veor d19,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d22,d21 @ Maj(a,b,c) - veor d19,d26 @ Sigma0(a) - vadd.i64 d23,d27 - vadd.i64 d30,d27 - @ vadd.i64 d19,d30 - vshr.u64 d24,d23,#14 @ 21 -#if 21<16 - vld1.64 {d5},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d23,#18 -#if 21>0 - vadd.i64 d19,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d23,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d23,#50 - vsli.64 d25,d23,#46 - vmov d29,d23 - vsli.64 d26,d23,#23 -#if 21<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d16,d17 @ Ch(e,f,g) - vshr.u64 d24,d19,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d18 - vshr.u64 d25,d19,#34 - vsli.64 d24,d19,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d19,#39 - vadd.i64 d28,d5 - vsli.64 d25,d19,#30 - veor d30,d19,d20 - vsli.64 d26,d19,#25 - veor d18,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d21,d20 @ Maj(a,b,c) - veor d18,d26 @ Sigma0(a) - vadd.i64 d22,d27 - vadd.i64 d30,d27 - @ vadd.i64 d18,d30 - vshr.u64 q12,q2,#19 - vshr.u64 q13,q2,#61 - vadd.i64 d18,d30 @ h+=Maj from the past - vshr.u64 q15,q2,#6 - vsli.64 q12,q2,#45 - vext.8 q14,q3,q4,#8 @ X[i+1] - vsli.64 q13,q2,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q3,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q7,q0,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d22,#14 @ from NEON_00_15 - vadd.i64 q3,q14 - vshr.u64 d25,d22,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d22,#41 @ from NEON_00_15 - vadd.i64 q3,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d22,#50 - vsli.64 d25,d22,#46 - vmov d29,d22 - vsli.64 d26,d22,#23 -#if 22<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d23,d16 @ Ch(e,f,g) - vshr.u64 d24,d18,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d17 - vshr.u64 d25,d18,#34 - vsli.64 d24,d18,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d18,#39 - vadd.i64 d28,d6 - vsli.64 d25,d18,#30 - veor d30,d18,d19 - vsli.64 d26,d18,#25 - veor d17,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d20,d19 @ Maj(a,b,c) - veor d17,d26 @ Sigma0(a) - vadd.i64 d21,d27 - vadd.i64 d30,d27 - @ vadd.i64 d17,d30 - vshr.u64 d24,d21,#14 @ 23 -#if 23<16 - vld1.64 {d7},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d21,#18 -#if 23>0 - vadd.i64 d17,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d21,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d21,#50 - vsli.64 d25,d21,#46 - vmov d29,d21 - vsli.64 d26,d21,#23 -#if 23<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d22,d23 @ Ch(e,f,g) - vshr.u64 d24,d17,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d16 - vshr.u64 d25,d17,#34 - vsli.64 d24,d17,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d17,#39 - vadd.i64 d28,d7 - vsli.64 d25,d17,#30 - veor d30,d17,d18 - vsli.64 d26,d17,#25 - veor d16,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d19,d18 @ Maj(a,b,c) - veor d16,d26 @ Sigma0(a) - vadd.i64 d20,d27 - vadd.i64 d30,d27 - @ vadd.i64 d16,d30 - vshr.u64 q12,q3,#19 - vshr.u64 q13,q3,#61 - vadd.i64 d16,d30 @ h+=Maj from the past - vshr.u64 q15,q3,#6 - vsli.64 q12,q3,#45 - vext.8 q14,q4,q5,#8 @ X[i+1] - vsli.64 q13,q3,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q4,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q0,q1,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d20,#14 @ from NEON_00_15 - vadd.i64 q4,q14 - vshr.u64 d25,d20,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d20,#41 @ from NEON_00_15 - vadd.i64 q4,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d20,#50 - vsli.64 d25,d20,#46 - vmov d29,d20 - vsli.64 d26,d20,#23 -#if 24<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d21,d22 @ Ch(e,f,g) - vshr.u64 d24,d16,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d23 - vshr.u64 d25,d16,#34 - vsli.64 d24,d16,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d16,#39 - vadd.i64 d28,d8 - vsli.64 d25,d16,#30 - veor d30,d16,d17 - vsli.64 d26,d16,#25 - veor d23,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d18,d17 @ Maj(a,b,c) - veor d23,d26 @ Sigma0(a) - vadd.i64 d19,d27 - vadd.i64 d30,d27 - @ vadd.i64 d23,d30 - vshr.u64 d24,d19,#14 @ 25 -#if 25<16 - vld1.64 {d9},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d19,#18 -#if 25>0 - vadd.i64 d23,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d19,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d19,#50 - vsli.64 d25,d19,#46 - vmov d29,d19 - vsli.64 d26,d19,#23 -#if 25<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d20,d21 @ Ch(e,f,g) - vshr.u64 d24,d23,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d22 - vshr.u64 d25,d23,#34 - vsli.64 d24,d23,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d23,#39 - vadd.i64 d28,d9 - vsli.64 d25,d23,#30 - veor d30,d23,d16 - vsli.64 d26,d23,#25 - veor d22,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d17,d16 @ Maj(a,b,c) - veor d22,d26 @ Sigma0(a) - vadd.i64 d18,d27 - vadd.i64 d30,d27 - @ vadd.i64 d22,d30 - vshr.u64 q12,q4,#19 - vshr.u64 q13,q4,#61 - vadd.i64 d22,d30 @ h+=Maj from the past - vshr.u64 q15,q4,#6 - vsli.64 q12,q4,#45 - vext.8 q14,q5,q6,#8 @ X[i+1] - vsli.64 q13,q4,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q5,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q1,q2,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d18,#14 @ from NEON_00_15 - vadd.i64 q5,q14 - vshr.u64 d25,d18,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d18,#41 @ from NEON_00_15 - vadd.i64 q5,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d18,#50 - vsli.64 d25,d18,#46 - vmov d29,d18 - vsli.64 d26,d18,#23 -#if 26<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d19,d20 @ Ch(e,f,g) - vshr.u64 d24,d22,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d21 - vshr.u64 d25,d22,#34 - vsli.64 d24,d22,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d22,#39 - vadd.i64 d28,d10 - vsli.64 d25,d22,#30 - veor d30,d22,d23 - vsli.64 d26,d22,#25 - veor d21,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d16,d23 @ Maj(a,b,c) - veor d21,d26 @ Sigma0(a) - vadd.i64 d17,d27 - vadd.i64 d30,d27 - @ vadd.i64 d21,d30 - vshr.u64 d24,d17,#14 @ 27 -#if 27<16 - vld1.64 {d11},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d17,#18 -#if 27>0 - vadd.i64 d21,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d17,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d17,#50 - vsli.64 d25,d17,#46 - vmov d29,d17 - vsli.64 d26,d17,#23 -#if 27<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d18,d19 @ Ch(e,f,g) - vshr.u64 d24,d21,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d20 - vshr.u64 d25,d21,#34 - vsli.64 d24,d21,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d21,#39 - vadd.i64 d28,d11 - vsli.64 d25,d21,#30 - veor d30,d21,d22 - vsli.64 d26,d21,#25 - veor d20,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d23,d22 @ Maj(a,b,c) - veor d20,d26 @ Sigma0(a) - vadd.i64 d16,d27 - vadd.i64 d30,d27 - @ vadd.i64 d20,d30 - vshr.u64 q12,q5,#19 - vshr.u64 q13,q5,#61 - vadd.i64 d20,d30 @ h+=Maj from the past - vshr.u64 q15,q5,#6 - vsli.64 q12,q5,#45 - vext.8 q14,q6,q7,#8 @ X[i+1] - vsli.64 q13,q5,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q6,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q2,q3,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d16,#14 @ from NEON_00_15 - vadd.i64 q6,q14 - vshr.u64 d25,d16,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d16,#41 @ from NEON_00_15 - vadd.i64 q6,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d16,#50 - vsli.64 d25,d16,#46 - vmov d29,d16 - vsli.64 d26,d16,#23 -#if 28<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d17,d18 @ Ch(e,f,g) - vshr.u64 d24,d20,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d19 - vshr.u64 d25,d20,#34 - vsli.64 d24,d20,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d20,#39 - vadd.i64 d28,d12 - vsli.64 d25,d20,#30 - veor d30,d20,d21 - vsli.64 d26,d20,#25 - veor d19,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d22,d21 @ Maj(a,b,c) - veor d19,d26 @ Sigma0(a) - vadd.i64 d23,d27 - vadd.i64 d30,d27 - @ vadd.i64 d19,d30 - vshr.u64 d24,d23,#14 @ 29 -#if 29<16 - vld1.64 {d13},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d23,#18 -#if 29>0 - vadd.i64 d19,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d23,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d23,#50 - vsli.64 d25,d23,#46 - vmov d29,d23 - vsli.64 d26,d23,#23 -#if 29<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d16,d17 @ Ch(e,f,g) - vshr.u64 d24,d19,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d18 - vshr.u64 d25,d19,#34 - vsli.64 d24,d19,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d19,#39 - vadd.i64 d28,d13 - vsli.64 d25,d19,#30 - veor d30,d19,d20 - vsli.64 d26,d19,#25 - veor d18,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d21,d20 @ Maj(a,b,c) - veor d18,d26 @ Sigma0(a) - vadd.i64 d22,d27 - vadd.i64 d30,d27 - @ vadd.i64 d18,d30 - vshr.u64 q12,q6,#19 - vshr.u64 q13,q6,#61 - vadd.i64 d18,d30 @ h+=Maj from the past - vshr.u64 q15,q6,#6 - vsli.64 q12,q6,#45 - vext.8 q14,q7,q0,#8 @ X[i+1] - vsli.64 q13,q6,#3 - veor q15,q12 - vshr.u64 q12,q14,#1 - veor q15,q13 @ sigma1(X[i+14]) - vshr.u64 q13,q14,#8 - vadd.i64 q7,q15 - vshr.u64 q15,q14,#7 - vsli.64 q12,q14,#63 - vsli.64 q13,q14,#56 - vext.8 q14,q3,q4,#8 @ X[i+9] - veor q15,q12 - vshr.u64 d24,d22,#14 @ from NEON_00_15 - vadd.i64 q7,q14 - vshr.u64 d25,d22,#18 @ from NEON_00_15 - veor q15,q13 @ sigma0(X[i+1]) - vshr.u64 d26,d22,#41 @ from NEON_00_15 - vadd.i64 q7,q15 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d22,#50 - vsli.64 d25,d22,#46 - vmov d29,d22 - vsli.64 d26,d22,#23 -#if 30<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d23,d16 @ Ch(e,f,g) - vshr.u64 d24,d18,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d17 - vshr.u64 d25,d18,#34 - vsli.64 d24,d18,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d18,#39 - vadd.i64 d28,d14 - vsli.64 d25,d18,#30 - veor d30,d18,d19 - vsli.64 d26,d18,#25 - veor d17,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d20,d19 @ Maj(a,b,c) - veor d17,d26 @ Sigma0(a) - vadd.i64 d21,d27 - vadd.i64 d30,d27 - @ vadd.i64 d17,d30 - vshr.u64 d24,d21,#14 @ 31 -#if 31<16 - vld1.64 {d15},[r1]! @ handles unaligned -#endif - vshr.u64 d25,d21,#18 -#if 31>0 - vadd.i64 d17,d30 @ h+=Maj from the past -#endif - vshr.u64 d26,d21,#41 - vld1.64 {d28},[r3,:64]! @ K[i++] - vsli.64 d24,d21,#50 - vsli.64 d25,d21,#46 - vmov d29,d21 - vsli.64 d26,d21,#23 -#if 31<16 && defined(__ARMEL__) - vrev64.8 , -#endif - veor d25,d24 - vbsl d29,d22,d23 @ Ch(e,f,g) - vshr.u64 d24,d17,#28 - veor d26,d25 @ Sigma1(e) - vadd.i64 d27,d29,d16 - vshr.u64 d25,d17,#34 - vsli.64 d24,d17,#36 - vadd.i64 d27,d26 - vshr.u64 d26,d17,#39 - vadd.i64 d28,d15 - vsli.64 d25,d17,#30 - veor d30,d17,d18 - vsli.64 d26,d17,#25 - veor d16,d24,d25 - vadd.i64 d27,d28 - vbsl d30,d19,d18 @ Maj(a,b,c) - veor d16,d26 @ Sigma0(a) - vadd.i64 d20,d27 - vadd.i64 d30,d27 - @ vadd.i64 d16,d30 - bne .L16_79_neon - - vadd.i64 d16,d30 @ h+=Maj from the past - vldmia r0,{d24-d31} @ load context to temp - vadd.i64 q8,q12 @ vectorized accumulate - vadd.i64 q9,q13 - vadd.i64 q10,q14 - vadd.i64 q11,q15 - vstmia r0,{d16-d23} @ save context - teq r1,r2 - sub r3,#640 @ rewind K512 - bne .Loop_neon - - VFP_ABI_POP - bx lr @ .word 0xe12fff1e -.size sha512_block_data_order_neon,.-sha512_block_data_order_neon -#endif -.asciz "SHA512 block transform for ARMv4/NEON, CRYPTOGAMS by " -.align 2 -#if __ARM_MAX_ARCH__>=7 && !defined(__KERNEL__) -.comm OPENSSL_armcap_P,4,4 -#endif diff --git a/arch/arm/include/asm/cpuidle.h b/arch/arm/include/asm/cpuidle.h index bc4ffa7ca04c..397be5ed30e7 100644 --- a/arch/arm/include/asm/cpuidle.h +++ b/arch/arm/include/asm/cpuidle.h @@ -50,4 +50,9 @@ extern int arm_cpuidle_suspend(int index); extern int arm_cpuidle_init(int cpu); +struct arm_cpuidle_irq_context { }; + +#define arm_cpuidle_save_irq_context(c) (void)c +#define arm_cpuidle_restore_irq_context(c) (void)c + #endif diff --git a/arch/arm/kernel/perf_event_v7.c b/arch/arm/kernel/perf_event_v7.c index 2924d7910b10..eb2190477da1 100644 --- a/arch/arm/kernel/perf_event_v7.c +++ b/arch/arm/kernel/perf_event_v7.c @@ -773,10 +773,10 @@ static inline void armv7pmu_write_counter(struct perf_event *event, u64 value) pr_err("CPU%u writing wrong counter %d\n", smp_processor_id(), idx); } else if (idx == ARMV7_IDX_CYCLE_COUNTER) { - asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" (value)); + asm volatile("mcr p15, 0, %0, c9, c13, 0" : : "r" ((u32)value)); } else { armv7_pmnc_select_counter(idx); - asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" (value)); + asm volatile("mcr p15, 0, %0, c9, c13, 2" : : "r" ((u32)value)); } } diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index 000f672a94c9..007a44412e24 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -233,12 +233,38 @@ static struct platform_device *ixp46x_devices[] __initdata = { unsigned long ixp4xx_exp_bus_size; EXPORT_SYMBOL(ixp4xx_exp_bus_size); +static struct platform_device_info ixp_dev_info __initdata = { + .name = "ixp4xx_crypto", + .id = 0, + .dma_mask = DMA_BIT_MASK(32), +}; + +static int __init ixp_crypto_register(void) +{ + struct platform_device *pdev; + + if (!(~(*IXP4XX_EXP_CFG2) & (IXP4XX_FEATURE_HASH | + IXP4XX_FEATURE_AES | IXP4XX_FEATURE_DES))) { + printk(KERN_ERR "ixp_crypto: No HW crypto available\n"); + return -ENODEV; + } + + pdev = platform_device_register_full(&ixp_dev_info); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + + return 0; +} + void __init ixp4xx_sys_init(void) { ixp4xx_exp_bus_size = SZ_16M; platform_add_devices(ixp4xx_devices, ARRAY_SIZE(ixp4xx_devices)); + if (IS_ENABLED(CONFIG_CRYPTO_DEV_IXP4XX)) + ixp_crypto_register(); + if (cpu_is_ixp46x()) { int region; diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index 9f1d8566bbf9..dabe9b81012f 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1481,12 +1481,6 @@ menu "ARMv8.3 architectural features" config ARM64_PTR_AUTH bool "Enable support for pointer authentication" default y - depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC - # Modern compilers insert a .note.gnu.property section note for PAC - # which is only understood by binutils starting with version 2.33.1. - depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100) - depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE - depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) help Pointer authentication (part of the ARMv8.3 Extensions) provides instructions for signing and authenticating pointers against secret @@ -1498,13 +1492,6 @@ config ARM64_PTR_AUTH for each process at exec() time, with these keys being context-switched along with the process. - If the compiler supports the -mbranch-protection or - -msign-return-address flag (e.g. GCC 7 or later), then this option - will also cause the kernel itself to be compiled with return address - protection. In this case, and if the target hardware is known to - support pointer authentication, then CONFIG_STACKPROTECTOR can be - disabled with minimal loss of protection. - The feature is detected at runtime. If the feature is not present in hardware it will not be advertised to userspace/KVM guest nor will it be enabled. @@ -1515,6 +1502,24 @@ config ARM64_PTR_AUTH but with the feature disabled. On such a system, this option should not be selected. +config ARM64_PTR_AUTH_KERNEL + bool "Use pointer authentication for kernel" + default y + depends on ARM64_PTR_AUTH + depends on (CC_HAS_SIGN_RETURN_ADDRESS || CC_HAS_BRANCH_PROT_PAC_RET) && AS_HAS_PAC + # Modern compilers insert a .note.gnu.property section note for PAC + # which is only understood by binutils starting with version 2.33.1. + depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100) + depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE + depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS) + help + If the compiler supports the -mbranch-protection or + -msign-return-address flag (e.g. GCC 7 or later), then this option + will cause the kernel itself to be compiled with return address + protection. In this case, and if the target hardware is known to + support pointer authentication, then CONFIG_STACKPROTECTOR can be + disabled with minimal loss of protection. + This feature works with FUNCTION_GRAPH_TRACER option only if DYNAMIC_FTRACE_WITH_REGS is enabled. @@ -1606,7 +1611,7 @@ config ARM64_BTI_KERNEL bool "Use Branch Target Identification for kernel" default y depends on ARM64_BTI - depends on ARM64_PTR_AUTH + depends on ARM64_PTR_AUTH_KERNEL depends on CC_HAS_BRANCH_PROT_PAC_RET_BTI # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=94697 depends on !CC_IS_GCC || GCC_VERSION >= 100100 diff --git a/arch/arm64/Makefile b/arch/arm64/Makefile index 6bd79de8a78d..a63ffe84dbcb 100644 --- a/arch/arm64/Makefile +++ b/arch/arm64/Makefile @@ -70,7 +70,7 @@ endif # off, this will be overridden if we are using branch protection. branch-prot-flags-y += $(call cc-option,-mbranch-protection=none) -ifeq ($(CONFIG_ARM64_PTR_AUTH),y) +ifeq ($(CONFIG_ARM64_PTR_AUTH_KERNEL),y) branch-prot-flags-$(CONFIG_CC_HAS_SIGN_RETURN_ADDRESS) := -msign-return-address=all # We enable additional protection for leaf functions as there is some # narrow potential for ROP protection benefits and no substantial diff --git a/arch/arm64/crypto/Makefile b/arch/arm64/crypto/Makefile index d0901e610df3..09a805cc32d7 100644 --- a/arch/arm64/crypto/Makefile +++ b/arch/arm64/crypto/Makefile @@ -68,19 +68,13 @@ CFLAGS_aes-glue-ce.o := -DUSE_V8_CRYPTO_EXTENSIONS $(obj)/aes-glue-%.o: $(src)/aes-glue.c FORCE $(call if_changed_rule,cc_o_c) -ifdef REGENERATE_ARM64_CRYPTO quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $(<) void $(@) -$(src)/poly1305-core.S_shipped: $(src)/poly1305-armv8.pl +$(obj)/%-core.S: $(src)/%-armv8.pl $(call cmd,perlasm) -$(src)/sha256-core.S_shipped: $(src)/sha512-armv8.pl +$(obj)/sha256-core.S: $(src)/sha512-armv8.pl $(call cmd,perlasm) -$(src)/sha512-core.S_shipped: $(src)/sha512-armv8.pl - $(call cmd,perlasm) - -endif - clean-files += poly1305-core.S sha256-core.S sha512-core.S diff --git a/arch/arm64/crypto/poly1305-core.S_shipped b/arch/arm64/crypto/poly1305-core.S_shipped deleted file mode 100644 index fb2822abf63a..000000000000 --- a/arch/arm64/crypto/poly1305-core.S_shipped +++ /dev/null @@ -1,835 +0,0 @@ -#ifndef __KERNEL__ -# include "arm_arch.h" -.extern OPENSSL_armcap_P -#endif - -.text - -// forward "declarations" are required for Apple -.globl poly1305_blocks -.globl poly1305_emit - -.globl poly1305_init -.type poly1305_init,%function -.align 5 -poly1305_init: - cmp x1,xzr - stp xzr,xzr,[x0] // zero hash value - stp xzr,xzr,[x0,#16] // [along with is_base2_26] - - csel x0,xzr,x0,eq - b.eq .Lno_key - -#ifndef __KERNEL__ - adrp x17,OPENSSL_armcap_P - ldr w17,[x17,#:lo12:OPENSSL_armcap_P] -#endif - - ldp x7,x8,[x1] // load key - mov x9,#0xfffffffc0fffffff - movk x9,#0x0fff,lsl#48 -#ifdef __AARCH64EB__ - rev x7,x7 // flip bytes - rev x8,x8 -#endif - and x7,x7,x9 // &=0ffffffc0fffffff - and x9,x9,#-4 - and x8,x8,x9 // &=0ffffffc0ffffffc - mov w9,#-1 - stp x7,x8,[x0,#32] // save key value - str w9,[x0,#48] // impossible key power value - -#ifndef __KERNEL__ - tst w17,#ARMV7_NEON - - adr x12,.Lpoly1305_blocks - adr x7,.Lpoly1305_blocks_neon - adr x13,.Lpoly1305_emit - - csel x12,x12,x7,eq - -# ifdef __ILP32__ - stp w12,w13,[x2] -# else - stp x12,x13,[x2] -# endif -#endif - mov x0,#1 -.Lno_key: - ret -.size poly1305_init,.-poly1305_init - -.type poly1305_blocks,%function -.align 5 -poly1305_blocks: -.Lpoly1305_blocks: - ands x2,x2,#-16 - b.eq .Lno_data - - ldp x4,x5,[x0] // load hash value - ldp x6,x17,[x0,#16] // [along with is_base2_26] - ldp x7,x8,[x0,#32] // load key value - -#ifdef __AARCH64EB__ - lsr x12,x4,#32 - mov w13,w4 - lsr x14,x5,#32 - mov w15,w5 - lsr x16,x6,#32 -#else - mov w12,w4 - lsr x13,x4,#32 - mov w14,w5 - lsr x15,x5,#32 - mov w16,w6 -#endif - - add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64 - lsr x13,x14,#12 - adds x12,x12,x14,lsl#52 - add x13,x13,x15,lsl#14 - adc x13,x13,xzr - lsr x14,x16,#24 - adds x13,x13,x16,lsl#40 - adc x14,x14,xzr - - cmp x17,#0 // is_base2_26? - add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) - csel x4,x4,x12,eq // choose between radixes - csel x5,x5,x13,eq - csel x6,x6,x14,eq - -.Loop: - ldp x10,x11,[x1],#16 // load input - sub x2,x2,#16 -#ifdef __AARCH64EB__ - rev x10,x10 - rev x11,x11 -#endif - adds x4,x4,x10 // accumulate input - adcs x5,x5,x11 - - mul x12,x4,x7 // h0*r0 - adc x6,x6,x3 - umulh x13,x4,x7 - - mul x10,x5,x9 // h1*5*r1 - umulh x11,x5,x9 - - adds x12,x12,x10 - mul x10,x4,x8 // h0*r1 - adc x13,x13,x11 - umulh x14,x4,x8 - - adds x13,x13,x10 - mul x10,x5,x7 // h1*r0 - adc x14,x14,xzr - umulh x11,x5,x7 - - adds x13,x13,x10 - mul x10,x6,x9 // h2*5*r1 - adc x14,x14,x11 - mul x11,x6,x7 // h2*r0 - - adds x13,x13,x10 - adc x14,x14,x11 - - and x10,x14,#-4 // final reduction - and x6,x14,#3 - add x10,x10,x14,lsr#2 - adds x4,x12,x10 - adcs x5,x13,xzr - adc x6,x6,xzr - - cbnz x2,.Loop - - stp x4,x5,[x0] // store hash value - stp x6,xzr,[x0,#16] // [and clear is_base2_26] - -.Lno_data: - ret -.size poly1305_blocks,.-poly1305_blocks - -.type poly1305_emit,%function -.align 5 -poly1305_emit: -.Lpoly1305_emit: - ldp x4,x5,[x0] // load hash base 2^64 - ldp x6,x7,[x0,#16] // [along with is_base2_26] - ldp x10,x11,[x2] // load nonce - -#ifdef __AARCH64EB__ - lsr x12,x4,#32 - mov w13,w4 - lsr x14,x5,#32 - mov w15,w5 - lsr x16,x6,#32 -#else - mov w12,w4 - lsr x13,x4,#32 - mov w14,w5 - lsr x15,x5,#32 - mov w16,w6 -#endif - - add x12,x12,x13,lsl#26 // base 2^26 -> base 2^64 - lsr x13,x14,#12 - adds x12,x12,x14,lsl#52 - add x13,x13,x15,lsl#14 - adc x13,x13,xzr - lsr x14,x16,#24 - adds x13,x13,x16,lsl#40 - adc x14,x14,xzr - - cmp x7,#0 // is_base2_26? - csel x4,x4,x12,eq // choose between radixes - csel x5,x5,x13,eq - csel x6,x6,x14,eq - - adds x12,x4,#5 // compare to modulus - adcs x13,x5,xzr - adc x14,x6,xzr - - tst x14,#-4 // see if it's carried/borrowed - - csel x4,x4,x12,eq - csel x5,x5,x13,eq - -#ifdef __AARCH64EB__ - ror x10,x10,#32 // flip nonce words - ror x11,x11,#32 -#endif - adds x4,x4,x10 // accumulate nonce - adc x5,x5,x11 -#ifdef __AARCH64EB__ - rev x4,x4 // flip output bytes - rev x5,x5 -#endif - stp x4,x5,[x1] // write result - - ret -.size poly1305_emit,.-poly1305_emit -.type poly1305_mult,%function -.align 5 -poly1305_mult: - mul x12,x4,x7 // h0*r0 - umulh x13,x4,x7 - - mul x10,x5,x9 // h1*5*r1 - umulh x11,x5,x9 - - adds x12,x12,x10 - mul x10,x4,x8 // h0*r1 - adc x13,x13,x11 - umulh x14,x4,x8 - - adds x13,x13,x10 - mul x10,x5,x7 // h1*r0 - adc x14,x14,xzr - umulh x11,x5,x7 - - adds x13,x13,x10 - mul x10,x6,x9 // h2*5*r1 - adc x14,x14,x11 - mul x11,x6,x7 // h2*r0 - - adds x13,x13,x10 - adc x14,x14,x11 - - and x10,x14,#-4 // final reduction - and x6,x14,#3 - add x10,x10,x14,lsr#2 - adds x4,x12,x10 - adcs x5,x13,xzr - adc x6,x6,xzr - - ret -.size poly1305_mult,.-poly1305_mult - -.type poly1305_splat,%function -.align 4 -poly1305_splat: - and x12,x4,#0x03ffffff // base 2^64 -> base 2^26 - ubfx x13,x4,#26,#26 - extr x14,x5,x4,#52 - and x14,x14,#0x03ffffff - ubfx x15,x5,#14,#26 - extr x16,x6,x5,#40 - - str w12,[x0,#16*0] // r0 - add w12,w13,w13,lsl#2 // r1*5 - str w13,[x0,#16*1] // r1 - add w13,w14,w14,lsl#2 // r2*5 - str w12,[x0,#16*2] // s1 - str w14,[x0,#16*3] // r2 - add w14,w15,w15,lsl#2 // r3*5 - str w13,[x0,#16*4] // s2 - str w15,[x0,#16*5] // r3 - add w15,w16,w16,lsl#2 // r4*5 - str w14,[x0,#16*6] // s3 - str w16,[x0,#16*7] // r4 - str w15,[x0,#16*8] // s4 - - ret -.size poly1305_splat,.-poly1305_splat - -#ifdef __KERNEL__ -.globl poly1305_blocks_neon -#endif -.type poly1305_blocks_neon,%function -.align 5 -poly1305_blocks_neon: -.Lpoly1305_blocks_neon: - ldr x17,[x0,#24] - cmp x2,#128 - b.lo .Lpoly1305_blocks - - .inst 0xd503233f // paciasp - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - - stp d8,d9,[sp,#16] // meet ABI requirements - stp d10,d11,[sp,#32] - stp d12,d13,[sp,#48] - stp d14,d15,[sp,#64] - - cbz x17,.Lbase2_64_neon - - ldp w10,w11,[x0] // load hash value base 2^26 - ldp w12,w13,[x0,#8] - ldr w14,[x0,#16] - - tst x2,#31 - b.eq .Leven_neon - - ldp x7,x8,[x0,#32] // load key value - - add x4,x10,x11,lsl#26 // base 2^26 -> base 2^64 - lsr x5,x12,#12 - adds x4,x4,x12,lsl#52 - add x5,x5,x13,lsl#14 - adc x5,x5,xzr - lsr x6,x14,#24 - adds x5,x5,x14,lsl#40 - adc x14,x6,xzr // can be partially reduced... - - ldp x12,x13,[x1],#16 // load input - sub x2,x2,#16 - add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) - -#ifdef __AARCH64EB__ - rev x12,x12 - rev x13,x13 -#endif - adds x4,x4,x12 // accumulate input - adcs x5,x5,x13 - adc x6,x6,x3 - - bl poly1305_mult - - and x10,x4,#0x03ffffff // base 2^64 -> base 2^26 - ubfx x11,x4,#26,#26 - extr x12,x5,x4,#52 - and x12,x12,#0x03ffffff - ubfx x13,x5,#14,#26 - extr x14,x6,x5,#40 - - b .Leven_neon - -.align 4 -.Lbase2_64_neon: - ldp x7,x8,[x0,#32] // load key value - - ldp x4,x5,[x0] // load hash value base 2^64 - ldr x6,[x0,#16] - - tst x2,#31 - b.eq .Linit_neon - - ldp x12,x13,[x1],#16 // load input - sub x2,x2,#16 - add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) -#ifdef __AARCH64EB__ - rev x12,x12 - rev x13,x13 -#endif - adds x4,x4,x12 // accumulate input - adcs x5,x5,x13 - adc x6,x6,x3 - - bl poly1305_mult - -.Linit_neon: - ldr w17,[x0,#48] // first table element - and x10,x4,#0x03ffffff // base 2^64 -> base 2^26 - ubfx x11,x4,#26,#26 - extr x12,x5,x4,#52 - and x12,x12,#0x03ffffff - ubfx x13,x5,#14,#26 - extr x14,x6,x5,#40 - - cmp w17,#-1 // is value impossible? - b.ne .Leven_neon - - fmov d24,x10 - fmov d25,x11 - fmov d26,x12 - fmov d27,x13 - fmov d28,x14 - - ////////////////////////////////// initialize r^n table - mov x4,x7 // r^1 - add x9,x8,x8,lsr#2 // s1 = r1 + (r1 >> 2) - mov x5,x8 - mov x6,xzr - add x0,x0,#48+12 - bl poly1305_splat - - bl poly1305_mult // r^2 - sub x0,x0,#4 - bl poly1305_splat - - bl poly1305_mult // r^3 - sub x0,x0,#4 - bl poly1305_splat - - bl poly1305_mult // r^4 - sub x0,x0,#4 - bl poly1305_splat - sub x0,x0,#48 // restore original x0 - b .Ldo_neon - -.align 4 -.Leven_neon: - fmov d24,x10 - fmov d25,x11 - fmov d26,x12 - fmov d27,x13 - fmov d28,x14 - -.Ldo_neon: - ldp x8,x12,[x1,#32] // inp[2:3] - subs x2,x2,#64 - ldp x9,x13,[x1,#48] - add x16,x1,#96 - adr x17,.Lzeros - - lsl x3,x3,#24 - add x15,x0,#48 - -#ifdef __AARCH64EB__ - rev x8,x8 - rev x12,x12 - rev x9,x9 - rev x13,x13 -#endif - and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 - and x5,x9,#0x03ffffff - ubfx x6,x8,#26,#26 - ubfx x7,x9,#26,#26 - add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 - extr x8,x12,x8,#52 - extr x9,x13,x9,#52 - add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 - fmov d14,x4 - and x8,x8,#0x03ffffff - and x9,x9,#0x03ffffff - ubfx x10,x12,#14,#26 - ubfx x11,x13,#14,#26 - add x12,x3,x12,lsr#40 - add x13,x3,x13,lsr#40 - add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 - fmov d15,x6 - add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 - add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 - fmov d16,x8 - fmov d17,x10 - fmov d18,x12 - - ldp x8,x12,[x1],#16 // inp[0:1] - ldp x9,x13,[x1],#48 - - ld1 {v0.4s,v1.4s,v2.4s,v3.4s},[x15],#64 - ld1 {v4.4s,v5.4s,v6.4s,v7.4s},[x15],#64 - ld1 {v8.4s},[x15] - -#ifdef __AARCH64EB__ - rev x8,x8 - rev x12,x12 - rev x9,x9 - rev x13,x13 -#endif - and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 - and x5,x9,#0x03ffffff - ubfx x6,x8,#26,#26 - ubfx x7,x9,#26,#26 - add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 - extr x8,x12,x8,#52 - extr x9,x13,x9,#52 - add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 - fmov d9,x4 - and x8,x8,#0x03ffffff - and x9,x9,#0x03ffffff - ubfx x10,x12,#14,#26 - ubfx x11,x13,#14,#26 - add x12,x3,x12,lsr#40 - add x13,x3,x13,lsr#40 - add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 - fmov d10,x6 - add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 - add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 - movi v31.2d,#-1 - fmov d11,x8 - fmov d12,x10 - fmov d13,x12 - ushr v31.2d,v31.2d,#38 - - b.ls .Lskip_loop - -.align 4 -.Loop_neon: - //////////////////////////////////////////////////////////////// - // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2 - // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r - // ___________________/ - // ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2 - // ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r - // ___________________/ ____________________/ - // - // Note that we start with inp[2:3]*r^2. This is because it - // doesn't depend on reduction in previous iteration. - //////////////////////////////////////////////////////////////// - // d4 = h0*r4 + h1*r3 + h2*r2 + h3*r1 + h4*r0 - // d3 = h0*r3 + h1*r2 + h2*r1 + h3*r0 + h4*5*r4 - // d2 = h0*r2 + h1*r1 + h2*r0 + h3*5*r4 + h4*5*r3 - // d1 = h0*r1 + h1*r0 + h2*5*r4 + h3*5*r3 + h4*5*r2 - // d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1 - - subs x2,x2,#64 - umull v23.2d,v14.2s,v7.s[2] - csel x16,x17,x16,lo - umull v22.2d,v14.2s,v5.s[2] - umull v21.2d,v14.2s,v3.s[2] - ldp x8,x12,[x16],#16 // inp[2:3] (or zero) - umull v20.2d,v14.2s,v1.s[2] - ldp x9,x13,[x16],#48 - umull v19.2d,v14.2s,v0.s[2] -#ifdef __AARCH64EB__ - rev x8,x8 - rev x12,x12 - rev x9,x9 - rev x13,x13 -#endif - - umlal v23.2d,v15.2s,v5.s[2] - and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 - umlal v22.2d,v15.2s,v3.s[2] - and x5,x9,#0x03ffffff - umlal v21.2d,v15.2s,v1.s[2] - ubfx x6,x8,#26,#26 - umlal v20.2d,v15.2s,v0.s[2] - ubfx x7,x9,#26,#26 - umlal v19.2d,v15.2s,v8.s[2] - add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 - - umlal v23.2d,v16.2s,v3.s[2] - extr x8,x12,x8,#52 - umlal v22.2d,v16.2s,v1.s[2] - extr x9,x13,x9,#52 - umlal v21.2d,v16.2s,v0.s[2] - add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 - umlal v20.2d,v16.2s,v8.s[2] - fmov d14,x4 - umlal v19.2d,v16.2s,v6.s[2] - and x8,x8,#0x03ffffff - - umlal v23.2d,v17.2s,v1.s[2] - and x9,x9,#0x03ffffff - umlal v22.2d,v17.2s,v0.s[2] - ubfx x10,x12,#14,#26 - umlal v21.2d,v17.2s,v8.s[2] - ubfx x11,x13,#14,#26 - umlal v20.2d,v17.2s,v6.s[2] - add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 - umlal v19.2d,v17.2s,v4.s[2] - fmov d15,x6 - - add v11.2s,v11.2s,v26.2s - add x12,x3,x12,lsr#40 - umlal v23.2d,v18.2s,v0.s[2] - add x13,x3,x13,lsr#40 - umlal v22.2d,v18.2s,v8.s[2] - add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 - umlal v21.2d,v18.2s,v6.s[2] - add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 - umlal v20.2d,v18.2s,v4.s[2] - fmov d16,x8 - umlal v19.2d,v18.2s,v2.s[2] - fmov d17,x10 - - //////////////////////////////////////////////////////////////// - // (hash+inp[0:1])*r^4 and accumulate - - add v9.2s,v9.2s,v24.2s - fmov d18,x12 - umlal v22.2d,v11.2s,v1.s[0] - ldp x8,x12,[x1],#16 // inp[0:1] - umlal v19.2d,v11.2s,v6.s[0] - ldp x9,x13,[x1],#48 - umlal v23.2d,v11.2s,v3.s[0] - umlal v20.2d,v11.2s,v8.s[0] - umlal v21.2d,v11.2s,v0.s[0] -#ifdef __AARCH64EB__ - rev x8,x8 - rev x12,x12 - rev x9,x9 - rev x13,x13 -#endif - - add v10.2s,v10.2s,v25.2s - umlal v22.2d,v9.2s,v5.s[0] - umlal v23.2d,v9.2s,v7.s[0] - and x4,x8,#0x03ffffff // base 2^64 -> base 2^26 - umlal v21.2d,v9.2s,v3.s[0] - and x5,x9,#0x03ffffff - umlal v19.2d,v9.2s,v0.s[0] - ubfx x6,x8,#26,#26 - umlal v20.2d,v9.2s,v1.s[0] - ubfx x7,x9,#26,#26 - - add v12.2s,v12.2s,v27.2s - add x4,x4,x5,lsl#32 // bfi x4,x5,#32,#32 - umlal v22.2d,v10.2s,v3.s[0] - extr x8,x12,x8,#52 - umlal v23.2d,v10.2s,v5.s[0] - extr x9,x13,x9,#52 - umlal v19.2d,v10.2s,v8.s[0] - add x6,x6,x7,lsl#32 // bfi x6,x7,#32,#32 - umlal v21.2d,v10.2s,v1.s[0] - fmov d9,x4 - umlal v20.2d,v10.2s,v0.s[0] - and x8,x8,#0x03ffffff - - add v13.2s,v13.2s,v28.2s - and x9,x9,#0x03ffffff - umlal v22.2d,v12.2s,v0.s[0] - ubfx x10,x12,#14,#26 - umlal v19.2d,v12.2s,v4.s[0] - ubfx x11,x13,#14,#26 - umlal v23.2d,v12.2s,v1.s[0] - add x8,x8,x9,lsl#32 // bfi x8,x9,#32,#32 - umlal v20.2d,v12.2s,v6.s[0] - fmov d10,x6 - umlal v21.2d,v12.2s,v8.s[0] - add x12,x3,x12,lsr#40 - - umlal v22.2d,v13.2s,v8.s[0] - add x13,x3,x13,lsr#40 - umlal v19.2d,v13.2s,v2.s[0] - add x10,x10,x11,lsl#32 // bfi x10,x11,#32,#32 - umlal v23.2d,v13.2s,v0.s[0] - add x12,x12,x13,lsl#32 // bfi x12,x13,#32,#32 - umlal v20.2d,v13.2s,v4.s[0] - fmov d11,x8 - umlal v21.2d,v13.2s,v6.s[0] - fmov d12,x10 - fmov d13,x12 - - ///////////////////////////////////////////////////////////////// - // lazy reduction as discussed in "NEON crypto" by D.J. Bernstein - // and P. Schwabe - // - // [see discussion in poly1305-armv4 module] - - ushr v29.2d,v22.2d,#26 - xtn v27.2s,v22.2d - ushr v30.2d,v19.2d,#26 - and v19.16b,v19.16b,v31.16b - add v23.2d,v23.2d,v29.2d // h3 -> h4 - bic v27.2s,#0xfc,lsl#24 // &=0x03ffffff - add v20.2d,v20.2d,v30.2d // h0 -> h1 - - ushr v29.2d,v23.2d,#26 - xtn v28.2s,v23.2d - ushr v30.2d,v20.2d,#26 - xtn v25.2s,v20.2d - bic v28.2s,#0xfc,lsl#24 - add v21.2d,v21.2d,v30.2d // h1 -> h2 - - add v19.2d,v19.2d,v29.2d - shl v29.2d,v29.2d,#2 - shrn v30.2s,v21.2d,#26 - xtn v26.2s,v21.2d - add v19.2d,v19.2d,v29.2d // h4 -> h0 - bic v25.2s,#0xfc,lsl#24 - add v27.2s,v27.2s,v30.2s // h2 -> h3 - bic v26.2s,#0xfc,lsl#24 - - shrn v29.2s,v19.2d,#26 - xtn v24.2s,v19.2d - ushr v30.2s,v27.2s,#26 - bic v27.2s,#0xfc,lsl#24 - bic v24.2s,#0xfc,lsl#24 - add v25.2s,v25.2s,v29.2s // h0 -> h1 - add v28.2s,v28.2s,v30.2s // h3 -> h4 - - b.hi .Loop_neon - -.Lskip_loop: - dup v16.2d,v16.d[0] - add v11.2s,v11.2s,v26.2s - - //////////////////////////////////////////////////////////////// - // multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1 - - adds x2,x2,#32 - b.ne .Long_tail - - dup v16.2d,v11.d[0] - add v14.2s,v9.2s,v24.2s - add v17.2s,v12.2s,v27.2s - add v15.2s,v10.2s,v25.2s - add v18.2s,v13.2s,v28.2s - -.Long_tail: - dup v14.2d,v14.d[0] - umull2 v19.2d,v16.4s,v6.4s - umull2 v22.2d,v16.4s,v1.4s - umull2 v23.2d,v16.4s,v3.4s - umull2 v21.2d,v16.4s,v0.4s - umull2 v20.2d,v16.4s,v8.4s - - dup v15.2d,v15.d[0] - umlal2 v19.2d,v14.4s,v0.4s - umlal2 v21.2d,v14.4s,v3.4s - umlal2 v22.2d,v14.4s,v5.4s - umlal2 v23.2d,v14.4s,v7.4s - umlal2 v20.2d,v14.4s,v1.4s - - dup v17.2d,v17.d[0] - umlal2 v19.2d,v15.4s,v8.4s - umlal2 v22.2d,v15.4s,v3.4s - umlal2 v21.2d,v15.4s,v1.4s - umlal2 v23.2d,v15.4s,v5.4s - umlal2 v20.2d,v15.4s,v0.4s - - dup v18.2d,v18.d[0] - umlal2 v22.2d,v17.4s,v0.4s - umlal2 v23.2d,v17.4s,v1.4s - umlal2 v19.2d,v17.4s,v4.4s - umlal2 v20.2d,v17.4s,v6.4s - umlal2 v21.2d,v17.4s,v8.4s - - umlal2 v22.2d,v18.4s,v8.4s - umlal2 v19.2d,v18.4s,v2.4s - umlal2 v23.2d,v18.4s,v0.4s - umlal2 v20.2d,v18.4s,v4.4s - umlal2 v21.2d,v18.4s,v6.4s - - b.eq .Lshort_tail - - //////////////////////////////////////////////////////////////// - // (hash+inp[0:1])*r^4:r^3 and accumulate - - add v9.2s,v9.2s,v24.2s - umlal v22.2d,v11.2s,v1.2s - umlal v19.2d,v11.2s,v6.2s - umlal v23.2d,v11.2s,v3.2s - umlal v20.2d,v11.2s,v8.2s - umlal v21.2d,v11.2s,v0.2s - - add v10.2s,v10.2s,v25.2s - umlal v22.2d,v9.2s,v5.2s - umlal v19.2d,v9.2s,v0.2s - umlal v23.2d,v9.2s,v7.2s - umlal v20.2d,v9.2s,v1.2s - umlal v21.2d,v9.2s,v3.2s - - add v12.2s,v12.2s,v27.2s - umlal v22.2d,v10.2s,v3.2s - umlal v19.2d,v10.2s,v8.2s - umlal v23.2d,v10.2s,v5.2s - umlal v20.2d,v10.2s,v0.2s - umlal v21.2d,v10.2s,v1.2s - - add v13.2s,v13.2s,v28.2s - umlal v22.2d,v12.2s,v0.2s - umlal v19.2d,v12.2s,v4.2s - umlal v23.2d,v12.2s,v1.2s - umlal v20.2d,v12.2s,v6.2s - umlal v21.2d,v12.2s,v8.2s - - umlal v22.2d,v13.2s,v8.2s - umlal v19.2d,v13.2s,v2.2s - umlal v23.2d,v13.2s,v0.2s - umlal v20.2d,v13.2s,v4.2s - umlal v21.2d,v13.2s,v6.2s - -.Lshort_tail: - //////////////////////////////////////////////////////////////// - // horizontal add - - addp v22.2d,v22.2d,v22.2d - ldp d8,d9,[sp,#16] // meet ABI requirements - addp v19.2d,v19.2d,v19.2d - ldp d10,d11,[sp,#32] - addp v23.2d,v23.2d,v23.2d - ldp d12,d13,[sp,#48] - addp v20.2d,v20.2d,v20.2d - ldp d14,d15,[sp,#64] - addp v21.2d,v21.2d,v21.2d - ldr x30,[sp,#8] - - //////////////////////////////////////////////////////////////// - // lazy reduction, but without narrowing - - ushr v29.2d,v22.2d,#26 - and v22.16b,v22.16b,v31.16b - ushr v30.2d,v19.2d,#26 - and v19.16b,v19.16b,v31.16b - - add v23.2d,v23.2d,v29.2d // h3 -> h4 - add v20.2d,v20.2d,v30.2d // h0 -> h1 - - ushr v29.2d,v23.2d,#26 - and v23.16b,v23.16b,v31.16b - ushr v30.2d,v20.2d,#26 - and v20.16b,v20.16b,v31.16b - add v21.2d,v21.2d,v30.2d // h1 -> h2 - - add v19.2d,v19.2d,v29.2d - shl v29.2d,v29.2d,#2 - ushr v30.2d,v21.2d,#26 - and v21.16b,v21.16b,v31.16b - add v19.2d,v19.2d,v29.2d // h4 -> h0 - add v22.2d,v22.2d,v30.2d // h2 -> h3 - - ushr v29.2d,v19.2d,#26 - and v19.16b,v19.16b,v31.16b - ushr v30.2d,v22.2d,#26 - and v22.16b,v22.16b,v31.16b - add v20.2d,v20.2d,v29.2d // h0 -> h1 - add v23.2d,v23.2d,v30.2d // h3 -> h4 - - //////////////////////////////////////////////////////////////// - // write the result, can be partially reduced - - st4 {v19.s,v20.s,v21.s,v22.s}[0],[x0],#16 - mov x4,#1 - st1 {v23.s}[0],[x0] - str x4,[x0,#8] // set is_base2_26 - - ldr x29,[sp],#80 - .inst 0xd50323bf // autiasp - ret -.size poly1305_blocks_neon,.-poly1305_blocks_neon - -.align 5 -.Lzeros: -.long 0,0,0,0,0,0,0,0 -.asciz "Poly1305 for ARMv8, CRYPTOGAMS by @dot-asm" -.align 2 -#if !defined(__KERNEL__) && !defined(_WIN64) -.comm OPENSSL_armcap_P,4,4 -.hidden OPENSSL_armcap_P -#endif diff --git a/arch/arm64/crypto/sha256-core.S_shipped b/arch/arm64/crypto/sha256-core.S_shipped deleted file mode 100644 index 7c7ce2e3bad6..000000000000 --- a/arch/arm64/crypto/sha256-core.S_shipped +++ /dev/null @@ -1,2069 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -// This code is taken from the OpenSSL project but the author (Andy Polyakov) -// has relicensed it under the GPLv2. Therefore this program is free software; -// you can redistribute it and/or modify it under the terms of the GNU General -// Public License version 2 as published by the Free Software Foundation. -// -// The original headers, including the original license headers, are -// included below for completeness. - -// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the OpenSSL license (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html - -// ==================================================================== -// Written by Andy Polyakov for the OpenSSL -// project. The module is, however, dual licensed under OpenSSL and -// CRYPTOGAMS licenses depending on where you obtain it. For further -// details see http://www.openssl.org/~appro/cryptogams/. -// ==================================================================== -// -// SHA256/512 for ARMv8. -// -// Performance in cycles per processed byte and improvement coefficient -// over code generated with "default" compiler: -// -// SHA256-hw SHA256(*) SHA512 -// Apple A7 1.97 10.5 (+33%) 6.73 (-1%(**)) -// Cortex-A53 2.38 15.5 (+115%) 10.0 (+150%(***)) -// Cortex-A57 2.31 11.6 (+86%) 7.51 (+260%(***)) -// Denver 2.01 10.5 (+26%) 6.70 (+8%) -// X-Gene 20.0 (+100%) 12.8 (+300%(***)) -// Mongoose 2.36 13.0 (+50%) 8.36 (+33%) -// -// (*) Software SHA256 results are of lesser relevance, presented -// mostly for informational purposes. -// (**) The result is a trade-off: it's possible to improve it by -// 10% (or by 1 cycle per round), but at the cost of 20% loss -// on Cortex-A53 (or by 4 cycles per round). -// (***) Super-impressive coefficients over gcc-generated code are -// indication of some compiler "pathology", most notably code -// generated with -mgeneral-regs-only is significanty faster -// and the gap is only 40-90%. -// -// October 2016. -// -// Originally it was reckoned that it makes no sense to implement NEON -// version of SHA256 for 64-bit processors. This is because performance -// improvement on most wide-spread Cortex-A5x processors was observed -// to be marginal, same on Cortex-A53 and ~10% on A57. But then it was -// observed that 32-bit NEON SHA256 performs significantly better than -// 64-bit scalar version on *some* of the more recent processors. As -// result 64-bit NEON version of SHA256 was added to provide best -// all-round performance. For example it executes ~30% faster on X-Gene -// and Mongoose. [For reference, NEON version of SHA512 is bound to -// deliver much less improvement, likely *negative* on Cortex-A5x. -// Which is why NEON support is limited to SHA256.] - -#ifndef __KERNEL__ -# include "arm_arch.h" -#endif - -.text - -.extern OPENSSL_armcap_P -.globl sha256_block_data_order -.type sha256_block_data_order,%function -.align 6 -sha256_block_data_order: -#ifndef __KERNEL__ -# ifdef __ILP32__ - ldrsw x16,.LOPENSSL_armcap_P -# else - ldr x16,.LOPENSSL_armcap_P -# endif - adr x17,.LOPENSSL_armcap_P - add x16,x16,x17 - ldr w16,[x16] - tst w16,#ARMV8_SHA256 - b.ne .Lv8_entry - tst w16,#ARMV7_NEON - b.ne .Lneon_entry -#endif - stp x29,x30,[sp,#-128]! - add x29,sp,#0 - - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - sub sp,sp,#4*4 - - ldp w20,w21,[x0] // load context - ldp w22,w23,[x0,#2*4] - ldp w24,w25,[x0,#4*4] - add x2,x1,x2,lsl#6 // end of input - ldp w26,w27,[x0,#6*4] - adr x30,.LK256 - stp x0,x2,[x29,#96] - -.Loop: - ldp w3,w4,[x1],#2*4 - ldr w19,[x30],#4 // *K++ - eor w28,w21,w22 // magic seed - str x1,[x29,#112] -#ifndef __AARCH64EB__ - rev w3,w3 // 0 -#endif - ror w16,w24,#6 - add w27,w27,w19 // h+=K[i] - eor w6,w24,w24,ror#14 - and w17,w25,w24 - bic w19,w26,w24 - add w27,w27,w3 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w20,w21 // a^b, b^c in next round - eor w16,w16,w6,ror#11 // Sigma1(e) - ror w6,w20,#2 - add w27,w27,w17 // h+=Ch(e,f,g) - eor w17,w20,w20,ror#9 - add w27,w27,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w23,w23,w27 // d+=h - eor w28,w28,w21 // Maj(a,b,c) - eor w17,w6,w17,ror#13 // Sigma0(a) - add w27,w27,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w27,w27,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w4,w4 // 1 -#endif - ldp w5,w6,[x1],#2*4 - add w27,w27,w17 // h+=Sigma0(a) - ror w16,w23,#6 - add w26,w26,w28 // h+=K[i] - eor w7,w23,w23,ror#14 - and w17,w24,w23 - bic w28,w25,w23 - add w26,w26,w4 // h+=X[i] - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w27,w20 // a^b, b^c in next round - eor w16,w16,w7,ror#11 // Sigma1(e) - ror w7,w27,#2 - add w26,w26,w17 // h+=Ch(e,f,g) - eor w17,w27,w27,ror#9 - add w26,w26,w16 // h+=Sigma1(e) - and w19,w19,w28 // (b^c)&=(a^b) - add w22,w22,w26 // d+=h - eor w19,w19,w20 // Maj(a,b,c) - eor w17,w7,w17,ror#13 // Sigma0(a) - add w26,w26,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - //add w26,w26,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w5,w5 // 2 -#endif - add w26,w26,w17 // h+=Sigma0(a) - ror w16,w22,#6 - add w25,w25,w19 // h+=K[i] - eor w8,w22,w22,ror#14 - and w17,w23,w22 - bic w19,w24,w22 - add w25,w25,w5 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w26,w27 // a^b, b^c in next round - eor w16,w16,w8,ror#11 // Sigma1(e) - ror w8,w26,#2 - add w25,w25,w17 // h+=Ch(e,f,g) - eor w17,w26,w26,ror#9 - add w25,w25,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w21,w21,w25 // d+=h - eor w28,w28,w27 // Maj(a,b,c) - eor w17,w8,w17,ror#13 // Sigma0(a) - add w25,w25,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w25,w25,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w6,w6 // 3 -#endif - ldp w7,w8,[x1],#2*4 - add w25,w25,w17 // h+=Sigma0(a) - ror w16,w21,#6 - add w24,w24,w28 // h+=K[i] - eor w9,w21,w21,ror#14 - and w17,w22,w21 - bic w28,w23,w21 - add w24,w24,w6 // h+=X[i] - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w25,w26 // a^b, b^c in next round - eor w16,w16,w9,ror#11 // Sigma1(e) - ror w9,w25,#2 - add w24,w24,w17 // h+=Ch(e,f,g) - eor w17,w25,w25,ror#9 - add w24,w24,w16 // h+=Sigma1(e) - and w19,w19,w28 // (b^c)&=(a^b) - add w20,w20,w24 // d+=h - eor w19,w19,w26 // Maj(a,b,c) - eor w17,w9,w17,ror#13 // Sigma0(a) - add w24,w24,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - //add w24,w24,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w7,w7 // 4 -#endif - add w24,w24,w17 // h+=Sigma0(a) - ror w16,w20,#6 - add w23,w23,w19 // h+=K[i] - eor w10,w20,w20,ror#14 - and w17,w21,w20 - bic w19,w22,w20 - add w23,w23,w7 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w24,w25 // a^b, b^c in next round - eor w16,w16,w10,ror#11 // Sigma1(e) - ror w10,w24,#2 - add w23,w23,w17 // h+=Ch(e,f,g) - eor w17,w24,w24,ror#9 - add w23,w23,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w27,w27,w23 // d+=h - eor w28,w28,w25 // Maj(a,b,c) - eor w17,w10,w17,ror#13 // Sigma0(a) - add w23,w23,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w23,w23,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w8,w8 // 5 -#endif - ldp w9,w10,[x1],#2*4 - add w23,w23,w17 // h+=Sigma0(a) - ror w16,w27,#6 - add w22,w22,w28 // h+=K[i] - eor w11,w27,w27,ror#14 - and w17,w20,w27 - bic w28,w21,w27 - add w22,w22,w8 // h+=X[i] - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w23,w24 // a^b, b^c in next round - eor w16,w16,w11,ror#11 // Sigma1(e) - ror w11,w23,#2 - add w22,w22,w17 // h+=Ch(e,f,g) - eor w17,w23,w23,ror#9 - add w22,w22,w16 // h+=Sigma1(e) - and w19,w19,w28 // (b^c)&=(a^b) - add w26,w26,w22 // d+=h - eor w19,w19,w24 // Maj(a,b,c) - eor w17,w11,w17,ror#13 // Sigma0(a) - add w22,w22,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - //add w22,w22,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w9,w9 // 6 -#endif - add w22,w22,w17 // h+=Sigma0(a) - ror w16,w26,#6 - add w21,w21,w19 // h+=K[i] - eor w12,w26,w26,ror#14 - and w17,w27,w26 - bic w19,w20,w26 - add w21,w21,w9 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w22,w23 // a^b, b^c in next round - eor w16,w16,w12,ror#11 // Sigma1(e) - ror w12,w22,#2 - add w21,w21,w17 // h+=Ch(e,f,g) - eor w17,w22,w22,ror#9 - add w21,w21,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w25,w25,w21 // d+=h - eor w28,w28,w23 // Maj(a,b,c) - eor w17,w12,w17,ror#13 // Sigma0(a) - add w21,w21,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w21,w21,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w10,w10 // 7 -#endif - ldp w11,w12,[x1],#2*4 - add w21,w21,w17 // h+=Sigma0(a) - ror w16,w25,#6 - add w20,w20,w28 // h+=K[i] - eor w13,w25,w25,ror#14 - and w17,w26,w25 - bic w28,w27,w25 - add w20,w20,w10 // h+=X[i] - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w21,w22 // a^b, b^c in next round - eor w16,w16,w13,ror#11 // Sigma1(e) - ror w13,w21,#2 - add w20,w20,w17 // h+=Ch(e,f,g) - eor w17,w21,w21,ror#9 - add w20,w20,w16 // h+=Sigma1(e) - and w19,w19,w28 // (b^c)&=(a^b) - add w24,w24,w20 // d+=h - eor w19,w19,w22 // Maj(a,b,c) - eor w17,w13,w17,ror#13 // Sigma0(a) - add w20,w20,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - //add w20,w20,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w11,w11 // 8 -#endif - add w20,w20,w17 // h+=Sigma0(a) - ror w16,w24,#6 - add w27,w27,w19 // h+=K[i] - eor w14,w24,w24,ror#14 - and w17,w25,w24 - bic w19,w26,w24 - add w27,w27,w11 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w20,w21 // a^b, b^c in next round - eor w16,w16,w14,ror#11 // Sigma1(e) - ror w14,w20,#2 - add w27,w27,w17 // h+=Ch(e,f,g) - eor w17,w20,w20,ror#9 - add w27,w27,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w23,w23,w27 // d+=h - eor w28,w28,w21 // Maj(a,b,c) - eor w17,w14,w17,ror#13 // Sigma0(a) - add w27,w27,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w27,w27,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w12,w12 // 9 -#endif - ldp w13,w14,[x1],#2*4 - add w27,w27,w17 // h+=Sigma0(a) - ror w16,w23,#6 - add w26,w26,w28 // h+=K[i] - eor w15,w23,w23,ror#14 - and w17,w24,w23 - bic w28,w25,w23 - add w26,w26,w12 // h+=X[i] - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w27,w20 // a^b, b^c in next round - eor w16,w16,w15,ror#11 // Sigma1(e) - ror w15,w27,#2 - add w26,w26,w17 // h+=Ch(e,f,g) - eor w17,w27,w27,ror#9 - add w26,w26,w16 // h+=Sigma1(e) - and w19,w19,w28 // (b^c)&=(a^b) - add w22,w22,w26 // d+=h - eor w19,w19,w20 // Maj(a,b,c) - eor w17,w15,w17,ror#13 // Sigma0(a) - add w26,w26,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - //add w26,w26,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w13,w13 // 10 -#endif - add w26,w26,w17 // h+=Sigma0(a) - ror w16,w22,#6 - add w25,w25,w19 // h+=K[i] - eor w0,w22,w22,ror#14 - and w17,w23,w22 - bic w19,w24,w22 - add w25,w25,w13 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w26,w27 // a^b, b^c in next round - eor w16,w16,w0,ror#11 // Sigma1(e) - ror w0,w26,#2 - add w25,w25,w17 // h+=Ch(e,f,g) - eor w17,w26,w26,ror#9 - add w25,w25,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w21,w21,w25 // d+=h - eor w28,w28,w27 // Maj(a,b,c) - eor w17,w0,w17,ror#13 // Sigma0(a) - add w25,w25,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w25,w25,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w14,w14 // 11 -#endif - ldp w15,w0,[x1],#2*4 - add w25,w25,w17 // h+=Sigma0(a) - str w6,[sp,#12] - ror w16,w21,#6 - add w24,w24,w28 // h+=K[i] - eor w6,w21,w21,ror#14 - and w17,w22,w21 - bic w28,w23,w21 - add w24,w24,w14 // h+=X[i] - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w25,w26 // a^b, b^c in next round - eor w16,w16,w6,ror#11 // Sigma1(e) - ror w6,w25,#2 - add w24,w24,w17 // h+=Ch(e,f,g) - eor w17,w25,w25,ror#9 - add w24,w24,w16 // h+=Sigma1(e) - and w19,w19,w28 // (b^c)&=(a^b) - add w20,w20,w24 // d+=h - eor w19,w19,w26 // Maj(a,b,c) - eor w17,w6,w17,ror#13 // Sigma0(a) - add w24,w24,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - //add w24,w24,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w15,w15 // 12 -#endif - add w24,w24,w17 // h+=Sigma0(a) - str w7,[sp,#0] - ror w16,w20,#6 - add w23,w23,w19 // h+=K[i] - eor w7,w20,w20,ror#14 - and w17,w21,w20 - bic w19,w22,w20 - add w23,w23,w15 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w24,w25 // a^b, b^c in next round - eor w16,w16,w7,ror#11 // Sigma1(e) - ror w7,w24,#2 - add w23,w23,w17 // h+=Ch(e,f,g) - eor w17,w24,w24,ror#9 - add w23,w23,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w27,w27,w23 // d+=h - eor w28,w28,w25 // Maj(a,b,c) - eor w17,w7,w17,ror#13 // Sigma0(a) - add w23,w23,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w23,w23,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w0,w0 // 13 -#endif - ldp w1,w2,[x1] - add w23,w23,w17 // h+=Sigma0(a) - str w8,[sp,#4] - ror w16,w27,#6 - add w22,w22,w28 // h+=K[i] - eor w8,w27,w27,ror#14 - and w17,w20,w27 - bic w28,w21,w27 - add w22,w22,w0 // h+=X[i] - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w23,w24 // a^b, b^c in next round - eor w16,w16,w8,ror#11 // Sigma1(e) - ror w8,w23,#2 - add w22,w22,w17 // h+=Ch(e,f,g) - eor w17,w23,w23,ror#9 - add w22,w22,w16 // h+=Sigma1(e) - and w19,w19,w28 // (b^c)&=(a^b) - add w26,w26,w22 // d+=h - eor w19,w19,w24 // Maj(a,b,c) - eor w17,w8,w17,ror#13 // Sigma0(a) - add w22,w22,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - //add w22,w22,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w1,w1 // 14 -#endif - ldr w6,[sp,#12] - add w22,w22,w17 // h+=Sigma0(a) - str w9,[sp,#8] - ror w16,w26,#6 - add w21,w21,w19 // h+=K[i] - eor w9,w26,w26,ror#14 - and w17,w27,w26 - bic w19,w20,w26 - add w21,w21,w1 // h+=X[i] - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w22,w23 // a^b, b^c in next round - eor w16,w16,w9,ror#11 // Sigma1(e) - ror w9,w22,#2 - add w21,w21,w17 // h+=Ch(e,f,g) - eor w17,w22,w22,ror#9 - add w21,w21,w16 // h+=Sigma1(e) - and w28,w28,w19 // (b^c)&=(a^b) - add w25,w25,w21 // d+=h - eor w28,w28,w23 // Maj(a,b,c) - eor w17,w9,w17,ror#13 // Sigma0(a) - add w21,w21,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - //add w21,w21,w17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev w2,w2 // 15 -#endif - ldr w7,[sp,#0] - add w21,w21,w17 // h+=Sigma0(a) - str w10,[sp,#12] - ror w16,w25,#6 - add w20,w20,w28 // h+=K[i] - ror w9,w4,#7 - and w17,w26,w25 - ror w8,w1,#17 - bic w28,w27,w25 - ror w10,w21,#2 - add w20,w20,w2 // h+=X[i] - eor w16,w16,w25,ror#11 - eor w9,w9,w4,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w21,w22 // a^b, b^c in next round - eor w16,w16,w25,ror#25 // Sigma1(e) - eor w10,w10,w21,ror#13 - add w20,w20,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w8,w8,w1,ror#19 - eor w9,w9,w4,lsr#3 // sigma0(X[i+1]) - add w20,w20,w16 // h+=Sigma1(e) - eor w19,w19,w22 // Maj(a,b,c) - eor w17,w10,w21,ror#22 // Sigma0(a) - eor w8,w8,w1,lsr#10 // sigma1(X[i+14]) - add w3,w3,w12 - add w24,w24,w20 // d+=h - add w20,w20,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w3,w3,w9 - add w20,w20,w17 // h+=Sigma0(a) - add w3,w3,w8 -.Loop_16_xx: - ldr w8,[sp,#4] - str w11,[sp,#0] - ror w16,w24,#6 - add w27,w27,w19 // h+=K[i] - ror w10,w5,#7 - and w17,w25,w24 - ror w9,w2,#17 - bic w19,w26,w24 - ror w11,w20,#2 - add w27,w27,w3 // h+=X[i] - eor w16,w16,w24,ror#11 - eor w10,w10,w5,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w20,w21 // a^b, b^c in next round - eor w16,w16,w24,ror#25 // Sigma1(e) - eor w11,w11,w20,ror#13 - add w27,w27,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w9,w9,w2,ror#19 - eor w10,w10,w5,lsr#3 // sigma0(X[i+1]) - add w27,w27,w16 // h+=Sigma1(e) - eor w28,w28,w21 // Maj(a,b,c) - eor w17,w11,w20,ror#22 // Sigma0(a) - eor w9,w9,w2,lsr#10 // sigma1(X[i+14]) - add w4,w4,w13 - add w23,w23,w27 // d+=h - add w27,w27,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w4,w4,w10 - add w27,w27,w17 // h+=Sigma0(a) - add w4,w4,w9 - ldr w9,[sp,#8] - str w12,[sp,#4] - ror w16,w23,#6 - add w26,w26,w28 // h+=K[i] - ror w11,w6,#7 - and w17,w24,w23 - ror w10,w3,#17 - bic w28,w25,w23 - ror w12,w27,#2 - add w26,w26,w4 // h+=X[i] - eor w16,w16,w23,ror#11 - eor w11,w11,w6,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w27,w20 // a^b, b^c in next round - eor w16,w16,w23,ror#25 // Sigma1(e) - eor w12,w12,w27,ror#13 - add w26,w26,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w10,w10,w3,ror#19 - eor w11,w11,w6,lsr#3 // sigma0(X[i+1]) - add w26,w26,w16 // h+=Sigma1(e) - eor w19,w19,w20 // Maj(a,b,c) - eor w17,w12,w27,ror#22 // Sigma0(a) - eor w10,w10,w3,lsr#10 // sigma1(X[i+14]) - add w5,w5,w14 - add w22,w22,w26 // d+=h - add w26,w26,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w5,w5,w11 - add w26,w26,w17 // h+=Sigma0(a) - add w5,w5,w10 - ldr w10,[sp,#12] - str w13,[sp,#8] - ror w16,w22,#6 - add w25,w25,w19 // h+=K[i] - ror w12,w7,#7 - and w17,w23,w22 - ror w11,w4,#17 - bic w19,w24,w22 - ror w13,w26,#2 - add w25,w25,w5 // h+=X[i] - eor w16,w16,w22,ror#11 - eor w12,w12,w7,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w26,w27 // a^b, b^c in next round - eor w16,w16,w22,ror#25 // Sigma1(e) - eor w13,w13,w26,ror#13 - add w25,w25,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w11,w11,w4,ror#19 - eor w12,w12,w7,lsr#3 // sigma0(X[i+1]) - add w25,w25,w16 // h+=Sigma1(e) - eor w28,w28,w27 // Maj(a,b,c) - eor w17,w13,w26,ror#22 // Sigma0(a) - eor w11,w11,w4,lsr#10 // sigma1(X[i+14]) - add w6,w6,w15 - add w21,w21,w25 // d+=h - add w25,w25,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w6,w6,w12 - add w25,w25,w17 // h+=Sigma0(a) - add w6,w6,w11 - ldr w11,[sp,#0] - str w14,[sp,#12] - ror w16,w21,#6 - add w24,w24,w28 // h+=K[i] - ror w13,w8,#7 - and w17,w22,w21 - ror w12,w5,#17 - bic w28,w23,w21 - ror w14,w25,#2 - add w24,w24,w6 // h+=X[i] - eor w16,w16,w21,ror#11 - eor w13,w13,w8,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w25,w26 // a^b, b^c in next round - eor w16,w16,w21,ror#25 // Sigma1(e) - eor w14,w14,w25,ror#13 - add w24,w24,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w12,w12,w5,ror#19 - eor w13,w13,w8,lsr#3 // sigma0(X[i+1]) - add w24,w24,w16 // h+=Sigma1(e) - eor w19,w19,w26 // Maj(a,b,c) - eor w17,w14,w25,ror#22 // Sigma0(a) - eor w12,w12,w5,lsr#10 // sigma1(X[i+14]) - add w7,w7,w0 - add w20,w20,w24 // d+=h - add w24,w24,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w7,w7,w13 - add w24,w24,w17 // h+=Sigma0(a) - add w7,w7,w12 - ldr w12,[sp,#4] - str w15,[sp,#0] - ror w16,w20,#6 - add w23,w23,w19 // h+=K[i] - ror w14,w9,#7 - and w17,w21,w20 - ror w13,w6,#17 - bic w19,w22,w20 - ror w15,w24,#2 - add w23,w23,w7 // h+=X[i] - eor w16,w16,w20,ror#11 - eor w14,w14,w9,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w24,w25 // a^b, b^c in next round - eor w16,w16,w20,ror#25 // Sigma1(e) - eor w15,w15,w24,ror#13 - add w23,w23,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w13,w13,w6,ror#19 - eor w14,w14,w9,lsr#3 // sigma0(X[i+1]) - add w23,w23,w16 // h+=Sigma1(e) - eor w28,w28,w25 // Maj(a,b,c) - eor w17,w15,w24,ror#22 // Sigma0(a) - eor w13,w13,w6,lsr#10 // sigma1(X[i+14]) - add w8,w8,w1 - add w27,w27,w23 // d+=h - add w23,w23,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w8,w8,w14 - add w23,w23,w17 // h+=Sigma0(a) - add w8,w8,w13 - ldr w13,[sp,#8] - str w0,[sp,#4] - ror w16,w27,#6 - add w22,w22,w28 // h+=K[i] - ror w15,w10,#7 - and w17,w20,w27 - ror w14,w7,#17 - bic w28,w21,w27 - ror w0,w23,#2 - add w22,w22,w8 // h+=X[i] - eor w16,w16,w27,ror#11 - eor w15,w15,w10,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w23,w24 // a^b, b^c in next round - eor w16,w16,w27,ror#25 // Sigma1(e) - eor w0,w0,w23,ror#13 - add w22,w22,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w14,w14,w7,ror#19 - eor w15,w15,w10,lsr#3 // sigma0(X[i+1]) - add w22,w22,w16 // h+=Sigma1(e) - eor w19,w19,w24 // Maj(a,b,c) - eor w17,w0,w23,ror#22 // Sigma0(a) - eor w14,w14,w7,lsr#10 // sigma1(X[i+14]) - add w9,w9,w2 - add w26,w26,w22 // d+=h - add w22,w22,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w9,w9,w15 - add w22,w22,w17 // h+=Sigma0(a) - add w9,w9,w14 - ldr w14,[sp,#12] - str w1,[sp,#8] - ror w16,w26,#6 - add w21,w21,w19 // h+=K[i] - ror w0,w11,#7 - and w17,w27,w26 - ror w15,w8,#17 - bic w19,w20,w26 - ror w1,w22,#2 - add w21,w21,w9 // h+=X[i] - eor w16,w16,w26,ror#11 - eor w0,w0,w11,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w22,w23 // a^b, b^c in next round - eor w16,w16,w26,ror#25 // Sigma1(e) - eor w1,w1,w22,ror#13 - add w21,w21,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w15,w15,w8,ror#19 - eor w0,w0,w11,lsr#3 // sigma0(X[i+1]) - add w21,w21,w16 // h+=Sigma1(e) - eor w28,w28,w23 // Maj(a,b,c) - eor w17,w1,w22,ror#22 // Sigma0(a) - eor w15,w15,w8,lsr#10 // sigma1(X[i+14]) - add w10,w10,w3 - add w25,w25,w21 // d+=h - add w21,w21,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w10,w10,w0 - add w21,w21,w17 // h+=Sigma0(a) - add w10,w10,w15 - ldr w15,[sp,#0] - str w2,[sp,#12] - ror w16,w25,#6 - add w20,w20,w28 // h+=K[i] - ror w1,w12,#7 - and w17,w26,w25 - ror w0,w9,#17 - bic w28,w27,w25 - ror w2,w21,#2 - add w20,w20,w10 // h+=X[i] - eor w16,w16,w25,ror#11 - eor w1,w1,w12,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w21,w22 // a^b, b^c in next round - eor w16,w16,w25,ror#25 // Sigma1(e) - eor w2,w2,w21,ror#13 - add w20,w20,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w0,w0,w9,ror#19 - eor w1,w1,w12,lsr#3 // sigma0(X[i+1]) - add w20,w20,w16 // h+=Sigma1(e) - eor w19,w19,w22 // Maj(a,b,c) - eor w17,w2,w21,ror#22 // Sigma0(a) - eor w0,w0,w9,lsr#10 // sigma1(X[i+14]) - add w11,w11,w4 - add w24,w24,w20 // d+=h - add w20,w20,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w11,w11,w1 - add w20,w20,w17 // h+=Sigma0(a) - add w11,w11,w0 - ldr w0,[sp,#4] - str w3,[sp,#0] - ror w16,w24,#6 - add w27,w27,w19 // h+=K[i] - ror w2,w13,#7 - and w17,w25,w24 - ror w1,w10,#17 - bic w19,w26,w24 - ror w3,w20,#2 - add w27,w27,w11 // h+=X[i] - eor w16,w16,w24,ror#11 - eor w2,w2,w13,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w20,w21 // a^b, b^c in next round - eor w16,w16,w24,ror#25 // Sigma1(e) - eor w3,w3,w20,ror#13 - add w27,w27,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w1,w1,w10,ror#19 - eor w2,w2,w13,lsr#3 // sigma0(X[i+1]) - add w27,w27,w16 // h+=Sigma1(e) - eor w28,w28,w21 // Maj(a,b,c) - eor w17,w3,w20,ror#22 // Sigma0(a) - eor w1,w1,w10,lsr#10 // sigma1(X[i+14]) - add w12,w12,w5 - add w23,w23,w27 // d+=h - add w27,w27,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w12,w12,w2 - add w27,w27,w17 // h+=Sigma0(a) - add w12,w12,w1 - ldr w1,[sp,#8] - str w4,[sp,#4] - ror w16,w23,#6 - add w26,w26,w28 // h+=K[i] - ror w3,w14,#7 - and w17,w24,w23 - ror w2,w11,#17 - bic w28,w25,w23 - ror w4,w27,#2 - add w26,w26,w12 // h+=X[i] - eor w16,w16,w23,ror#11 - eor w3,w3,w14,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w27,w20 // a^b, b^c in next round - eor w16,w16,w23,ror#25 // Sigma1(e) - eor w4,w4,w27,ror#13 - add w26,w26,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w2,w2,w11,ror#19 - eor w3,w3,w14,lsr#3 // sigma0(X[i+1]) - add w26,w26,w16 // h+=Sigma1(e) - eor w19,w19,w20 // Maj(a,b,c) - eor w17,w4,w27,ror#22 // Sigma0(a) - eor w2,w2,w11,lsr#10 // sigma1(X[i+14]) - add w13,w13,w6 - add w22,w22,w26 // d+=h - add w26,w26,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w13,w13,w3 - add w26,w26,w17 // h+=Sigma0(a) - add w13,w13,w2 - ldr w2,[sp,#12] - str w5,[sp,#8] - ror w16,w22,#6 - add w25,w25,w19 // h+=K[i] - ror w4,w15,#7 - and w17,w23,w22 - ror w3,w12,#17 - bic w19,w24,w22 - ror w5,w26,#2 - add w25,w25,w13 // h+=X[i] - eor w16,w16,w22,ror#11 - eor w4,w4,w15,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w26,w27 // a^b, b^c in next round - eor w16,w16,w22,ror#25 // Sigma1(e) - eor w5,w5,w26,ror#13 - add w25,w25,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w3,w3,w12,ror#19 - eor w4,w4,w15,lsr#3 // sigma0(X[i+1]) - add w25,w25,w16 // h+=Sigma1(e) - eor w28,w28,w27 // Maj(a,b,c) - eor w17,w5,w26,ror#22 // Sigma0(a) - eor w3,w3,w12,lsr#10 // sigma1(X[i+14]) - add w14,w14,w7 - add w21,w21,w25 // d+=h - add w25,w25,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w14,w14,w4 - add w25,w25,w17 // h+=Sigma0(a) - add w14,w14,w3 - ldr w3,[sp,#0] - str w6,[sp,#12] - ror w16,w21,#6 - add w24,w24,w28 // h+=K[i] - ror w5,w0,#7 - and w17,w22,w21 - ror w4,w13,#17 - bic w28,w23,w21 - ror w6,w25,#2 - add w24,w24,w14 // h+=X[i] - eor w16,w16,w21,ror#11 - eor w5,w5,w0,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w25,w26 // a^b, b^c in next round - eor w16,w16,w21,ror#25 // Sigma1(e) - eor w6,w6,w25,ror#13 - add w24,w24,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w4,w4,w13,ror#19 - eor w5,w5,w0,lsr#3 // sigma0(X[i+1]) - add w24,w24,w16 // h+=Sigma1(e) - eor w19,w19,w26 // Maj(a,b,c) - eor w17,w6,w25,ror#22 // Sigma0(a) - eor w4,w4,w13,lsr#10 // sigma1(X[i+14]) - add w15,w15,w8 - add w20,w20,w24 // d+=h - add w24,w24,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w15,w15,w5 - add w24,w24,w17 // h+=Sigma0(a) - add w15,w15,w4 - ldr w4,[sp,#4] - str w7,[sp,#0] - ror w16,w20,#6 - add w23,w23,w19 // h+=K[i] - ror w6,w1,#7 - and w17,w21,w20 - ror w5,w14,#17 - bic w19,w22,w20 - ror w7,w24,#2 - add w23,w23,w15 // h+=X[i] - eor w16,w16,w20,ror#11 - eor w6,w6,w1,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w24,w25 // a^b, b^c in next round - eor w16,w16,w20,ror#25 // Sigma1(e) - eor w7,w7,w24,ror#13 - add w23,w23,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w5,w5,w14,ror#19 - eor w6,w6,w1,lsr#3 // sigma0(X[i+1]) - add w23,w23,w16 // h+=Sigma1(e) - eor w28,w28,w25 // Maj(a,b,c) - eor w17,w7,w24,ror#22 // Sigma0(a) - eor w5,w5,w14,lsr#10 // sigma1(X[i+14]) - add w0,w0,w9 - add w27,w27,w23 // d+=h - add w23,w23,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w0,w0,w6 - add w23,w23,w17 // h+=Sigma0(a) - add w0,w0,w5 - ldr w5,[sp,#8] - str w8,[sp,#4] - ror w16,w27,#6 - add w22,w22,w28 // h+=K[i] - ror w7,w2,#7 - and w17,w20,w27 - ror w6,w15,#17 - bic w28,w21,w27 - ror w8,w23,#2 - add w22,w22,w0 // h+=X[i] - eor w16,w16,w27,ror#11 - eor w7,w7,w2,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w23,w24 // a^b, b^c in next round - eor w16,w16,w27,ror#25 // Sigma1(e) - eor w8,w8,w23,ror#13 - add w22,w22,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w6,w6,w15,ror#19 - eor w7,w7,w2,lsr#3 // sigma0(X[i+1]) - add w22,w22,w16 // h+=Sigma1(e) - eor w19,w19,w24 // Maj(a,b,c) - eor w17,w8,w23,ror#22 // Sigma0(a) - eor w6,w6,w15,lsr#10 // sigma1(X[i+14]) - add w1,w1,w10 - add w26,w26,w22 // d+=h - add w22,w22,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w1,w1,w7 - add w22,w22,w17 // h+=Sigma0(a) - add w1,w1,w6 - ldr w6,[sp,#12] - str w9,[sp,#8] - ror w16,w26,#6 - add w21,w21,w19 // h+=K[i] - ror w8,w3,#7 - and w17,w27,w26 - ror w7,w0,#17 - bic w19,w20,w26 - ror w9,w22,#2 - add w21,w21,w1 // h+=X[i] - eor w16,w16,w26,ror#11 - eor w8,w8,w3,ror#18 - orr w17,w17,w19 // Ch(e,f,g) - eor w19,w22,w23 // a^b, b^c in next round - eor w16,w16,w26,ror#25 // Sigma1(e) - eor w9,w9,w22,ror#13 - add w21,w21,w17 // h+=Ch(e,f,g) - and w28,w28,w19 // (b^c)&=(a^b) - eor w7,w7,w0,ror#19 - eor w8,w8,w3,lsr#3 // sigma0(X[i+1]) - add w21,w21,w16 // h+=Sigma1(e) - eor w28,w28,w23 // Maj(a,b,c) - eor w17,w9,w22,ror#22 // Sigma0(a) - eor w7,w7,w0,lsr#10 // sigma1(X[i+14]) - add w2,w2,w11 - add w25,w25,w21 // d+=h - add w21,w21,w28 // h+=Maj(a,b,c) - ldr w28,[x30],#4 // *K++, w19 in next round - add w2,w2,w8 - add w21,w21,w17 // h+=Sigma0(a) - add w2,w2,w7 - ldr w7,[sp,#0] - str w10,[sp,#12] - ror w16,w25,#6 - add w20,w20,w28 // h+=K[i] - ror w9,w4,#7 - and w17,w26,w25 - ror w8,w1,#17 - bic w28,w27,w25 - ror w10,w21,#2 - add w20,w20,w2 // h+=X[i] - eor w16,w16,w25,ror#11 - eor w9,w9,w4,ror#18 - orr w17,w17,w28 // Ch(e,f,g) - eor w28,w21,w22 // a^b, b^c in next round - eor w16,w16,w25,ror#25 // Sigma1(e) - eor w10,w10,w21,ror#13 - add w20,w20,w17 // h+=Ch(e,f,g) - and w19,w19,w28 // (b^c)&=(a^b) - eor w8,w8,w1,ror#19 - eor w9,w9,w4,lsr#3 // sigma0(X[i+1]) - add w20,w20,w16 // h+=Sigma1(e) - eor w19,w19,w22 // Maj(a,b,c) - eor w17,w10,w21,ror#22 // Sigma0(a) - eor w8,w8,w1,lsr#10 // sigma1(X[i+14]) - add w3,w3,w12 - add w24,w24,w20 // d+=h - add w20,w20,w19 // h+=Maj(a,b,c) - ldr w19,[x30],#4 // *K++, w28 in next round - add w3,w3,w9 - add w20,w20,w17 // h+=Sigma0(a) - add w3,w3,w8 - cbnz w19,.Loop_16_xx - - ldp x0,x2,[x29,#96] - ldr x1,[x29,#112] - sub x30,x30,#260 // rewind - - ldp w3,w4,[x0] - ldp w5,w6,[x0,#2*4] - add x1,x1,#14*4 // advance input pointer - ldp w7,w8,[x0,#4*4] - add w20,w20,w3 - ldp w9,w10,[x0,#6*4] - add w21,w21,w4 - add w22,w22,w5 - add w23,w23,w6 - stp w20,w21,[x0] - add w24,w24,w7 - add w25,w25,w8 - stp w22,w23,[x0,#2*4] - add w26,w26,w9 - add w27,w27,w10 - cmp x1,x2 - stp w24,w25,[x0,#4*4] - stp w26,w27,[x0,#6*4] - b.ne .Loop - - ldp x19,x20,[x29,#16] - add sp,sp,#4*4 - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldp x29,x30,[sp],#128 - ret -.size sha256_block_data_order,.-sha256_block_data_order - -.align 6 -.type .LK256,%object -.LK256: - .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 - .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 - .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 - .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 - .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc - .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da - .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 - .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 - .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 - .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 - .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 - .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 - .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 - .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 - .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 - .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 - .long 0 //terminator -.size .LK256,.-.LK256 -#ifndef __KERNEL__ -.align 3 -.LOPENSSL_armcap_P: -# ifdef __ILP32__ - .long OPENSSL_armcap_P-. -# else - .quad OPENSSL_armcap_P-. -# endif -#endif -.asciz "SHA256 block transform for ARMv8, CRYPTOGAMS by " -.align 2 -#ifndef __KERNEL__ -.type sha256_block_armv8,%function -.align 6 -sha256_block_armv8: -.Lv8_entry: - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ld1 {v0.4s,v1.4s},[x0] - adr x3,.LK256 - -.Loop_hw: - ld1 {v4.16b-v7.16b},[x1],#64 - sub x2,x2,#1 - ld1 {v16.4s},[x3],#16 - rev32 v4.16b,v4.16b - rev32 v5.16b,v5.16b - rev32 v6.16b,v6.16b - rev32 v7.16b,v7.16b - orr v18.16b,v0.16b,v0.16b // offload - orr v19.16b,v1.16b,v1.16b - ld1 {v17.4s},[x3],#16 - add v16.4s,v16.4s,v4.4s - .inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - .inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b - ld1 {v16.4s},[x3],#16 - add v17.4s,v17.4s,v5.4s - .inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - .inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b - ld1 {v17.4s},[x3],#16 - add v16.4s,v16.4s,v6.4s - .inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - .inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b - ld1 {v16.4s},[x3],#16 - add v17.4s,v17.4s,v7.4s - .inst 0x5e282887 //sha256su0 v7.16b,v4.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - .inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b - ld1 {v17.4s},[x3],#16 - add v16.4s,v16.4s,v4.4s - .inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - .inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b - ld1 {v16.4s},[x3],#16 - add v17.4s,v17.4s,v5.4s - .inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - .inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b - ld1 {v17.4s},[x3],#16 - add v16.4s,v16.4s,v6.4s - .inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - .inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b - ld1 {v16.4s},[x3],#16 - add v17.4s,v17.4s,v7.4s - .inst 0x5e282887 //sha256su0 v7.16b,v4.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - .inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b - ld1 {v17.4s},[x3],#16 - add v16.4s,v16.4s,v4.4s - .inst 0x5e2828a4 //sha256su0 v4.16b,v5.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - .inst 0x5e0760c4 //sha256su1 v4.16b,v6.16b,v7.16b - ld1 {v16.4s},[x3],#16 - add v17.4s,v17.4s,v5.4s - .inst 0x5e2828c5 //sha256su0 v5.16b,v6.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - .inst 0x5e0460e5 //sha256su1 v5.16b,v7.16b,v4.16b - ld1 {v17.4s},[x3],#16 - add v16.4s,v16.4s,v6.4s - .inst 0x5e2828e6 //sha256su0 v6.16b,v7.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - .inst 0x5e056086 //sha256su1 v6.16b,v4.16b,v5.16b - ld1 {v16.4s},[x3],#16 - add v17.4s,v17.4s,v7.4s - .inst 0x5e282887 //sha256su0 v7.16b,v4.16b - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - .inst 0x5e0660a7 //sha256su1 v7.16b,v5.16b,v6.16b - ld1 {v17.4s},[x3],#16 - add v16.4s,v16.4s,v4.4s - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - - ld1 {v16.4s},[x3],#16 - add v17.4s,v17.4s,v5.4s - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - - ld1 {v17.4s},[x3] - add v16.4s,v16.4s,v6.4s - sub x3,x3,#64*4-16 // rewind - orr v2.16b,v0.16b,v0.16b - .inst 0x5e104020 //sha256h v0.16b,v1.16b,v16.4s - .inst 0x5e105041 //sha256h2 v1.16b,v2.16b,v16.4s - - add v17.4s,v17.4s,v7.4s - orr v2.16b,v0.16b,v0.16b - .inst 0x5e114020 //sha256h v0.16b,v1.16b,v17.4s - .inst 0x5e115041 //sha256h2 v1.16b,v2.16b,v17.4s - - add v0.4s,v0.4s,v18.4s - add v1.4s,v1.4s,v19.4s - - cbnz x2,.Loop_hw - - st1 {v0.4s,v1.4s},[x0] - - ldr x29,[sp],#16 - ret -.size sha256_block_armv8,.-sha256_block_armv8 -#endif -#ifdef __KERNEL__ -.globl sha256_block_neon -#endif -.type sha256_block_neon,%function -.align 4 -sha256_block_neon: -.Lneon_entry: - stp x29, x30, [sp, #-16]! - mov x29, sp - sub sp,sp,#16*4 - - adr x16,.LK256 - add x2,x1,x2,lsl#6 // len to point at the end of inp - - ld1 {v0.16b},[x1], #16 - ld1 {v1.16b},[x1], #16 - ld1 {v2.16b},[x1], #16 - ld1 {v3.16b},[x1], #16 - ld1 {v4.4s},[x16], #16 - ld1 {v5.4s},[x16], #16 - ld1 {v6.4s},[x16], #16 - ld1 {v7.4s},[x16], #16 - rev32 v0.16b,v0.16b // yes, even on - rev32 v1.16b,v1.16b // big-endian - rev32 v2.16b,v2.16b - rev32 v3.16b,v3.16b - mov x17,sp - add v4.4s,v4.4s,v0.4s - add v5.4s,v5.4s,v1.4s - add v6.4s,v6.4s,v2.4s - st1 {v4.4s-v5.4s},[x17], #32 - add v7.4s,v7.4s,v3.4s - st1 {v6.4s-v7.4s},[x17] - sub x17,x17,#32 - - ldp w3,w4,[x0] - ldp w5,w6,[x0,#8] - ldp w7,w8,[x0,#16] - ldp w9,w10,[x0,#24] - ldr w12,[sp,#0] - mov w13,wzr - eor w14,w4,w5 - mov w15,wzr - b .L_00_48 - -.align 4 -.L_00_48: - ext v4.16b,v0.16b,v1.16b,#4 - add w10,w10,w12 - add w3,w3,w15 - and w12,w8,w7 - bic w15,w9,w7 - ext v7.16b,v2.16b,v3.16b,#4 - eor w11,w7,w7,ror#5 - add w3,w3,w13 - mov d19,v3.d[1] - orr w12,w12,w15 - eor w11,w11,w7,ror#19 - ushr v6.4s,v4.4s,#7 - eor w15,w3,w3,ror#11 - ushr v5.4s,v4.4s,#3 - add w10,w10,w12 - add v0.4s,v0.4s,v7.4s - ror w11,w11,#6 - sli v6.4s,v4.4s,#25 - eor w13,w3,w4 - eor w15,w15,w3,ror#20 - ushr v7.4s,v4.4s,#18 - add w10,w10,w11 - ldr w12,[sp,#4] - and w14,w14,w13 - eor v5.16b,v5.16b,v6.16b - ror w15,w15,#2 - add w6,w6,w10 - sli v7.4s,v4.4s,#14 - eor w14,w14,w4 - ushr v16.4s,v19.4s,#17 - add w9,w9,w12 - add w10,w10,w15 - and w12,w7,w6 - eor v5.16b,v5.16b,v7.16b - bic w15,w8,w6 - eor w11,w6,w6,ror#5 - sli v16.4s,v19.4s,#15 - add w10,w10,w14 - orr w12,w12,w15 - ushr v17.4s,v19.4s,#10 - eor w11,w11,w6,ror#19 - eor w15,w10,w10,ror#11 - ushr v7.4s,v19.4s,#19 - add w9,w9,w12 - ror w11,w11,#6 - add v0.4s,v0.4s,v5.4s - eor w14,w10,w3 - eor w15,w15,w10,ror#20 - sli v7.4s,v19.4s,#13 - add w9,w9,w11 - ldr w12,[sp,#8] - and w13,w13,w14 - eor v17.16b,v17.16b,v16.16b - ror w15,w15,#2 - add w5,w5,w9 - eor w13,w13,w3 - eor v17.16b,v17.16b,v7.16b - add w8,w8,w12 - add w9,w9,w15 - and w12,w6,w5 - add v0.4s,v0.4s,v17.4s - bic w15,w7,w5 - eor w11,w5,w5,ror#5 - add w9,w9,w13 - ushr v18.4s,v0.4s,#17 - orr w12,w12,w15 - ushr v19.4s,v0.4s,#10 - eor w11,w11,w5,ror#19 - eor w15,w9,w9,ror#11 - sli v18.4s,v0.4s,#15 - add w8,w8,w12 - ushr v17.4s,v0.4s,#19 - ror w11,w11,#6 - eor w13,w9,w10 - eor v19.16b,v19.16b,v18.16b - eor w15,w15,w9,ror#20 - add w8,w8,w11 - sli v17.4s,v0.4s,#13 - ldr w12,[sp,#12] - and w14,w14,w13 - ror w15,w15,#2 - ld1 {v4.4s},[x16], #16 - add w4,w4,w8 - eor v19.16b,v19.16b,v17.16b - eor w14,w14,w10 - eor v17.16b,v17.16b,v17.16b - add w7,w7,w12 - add w8,w8,w15 - and w12,w5,w4 - mov v17.d[1],v19.d[0] - bic w15,w6,w4 - eor w11,w4,w4,ror#5 - add w8,w8,w14 - add v0.4s,v0.4s,v17.4s - orr w12,w12,w15 - eor w11,w11,w4,ror#19 - eor w15,w8,w8,ror#11 - add v4.4s,v4.4s,v0.4s - add w7,w7,w12 - ror w11,w11,#6 - eor w14,w8,w9 - eor w15,w15,w8,ror#20 - add w7,w7,w11 - ldr w12,[sp,#16] - and w13,w13,w14 - ror w15,w15,#2 - add w3,w3,w7 - eor w13,w13,w9 - st1 {v4.4s},[x17], #16 - ext v4.16b,v1.16b,v2.16b,#4 - add w6,w6,w12 - add w7,w7,w15 - and w12,w4,w3 - bic w15,w5,w3 - ext v7.16b,v3.16b,v0.16b,#4 - eor w11,w3,w3,ror#5 - add w7,w7,w13 - mov d19,v0.d[1] - orr w12,w12,w15 - eor w11,w11,w3,ror#19 - ushr v6.4s,v4.4s,#7 - eor w15,w7,w7,ror#11 - ushr v5.4s,v4.4s,#3 - add w6,w6,w12 - add v1.4s,v1.4s,v7.4s - ror w11,w11,#6 - sli v6.4s,v4.4s,#25 - eor w13,w7,w8 - eor w15,w15,w7,ror#20 - ushr v7.4s,v4.4s,#18 - add w6,w6,w11 - ldr w12,[sp,#20] - and w14,w14,w13 - eor v5.16b,v5.16b,v6.16b - ror w15,w15,#2 - add w10,w10,w6 - sli v7.4s,v4.4s,#14 - eor w14,w14,w8 - ushr v16.4s,v19.4s,#17 - add w5,w5,w12 - add w6,w6,w15 - and w12,w3,w10 - eor v5.16b,v5.16b,v7.16b - bic w15,w4,w10 - eor w11,w10,w10,ror#5 - sli v16.4s,v19.4s,#15 - add w6,w6,w14 - orr w12,w12,w15 - ushr v17.4s,v19.4s,#10 - eor w11,w11,w10,ror#19 - eor w15,w6,w6,ror#11 - ushr v7.4s,v19.4s,#19 - add w5,w5,w12 - ror w11,w11,#6 - add v1.4s,v1.4s,v5.4s - eor w14,w6,w7 - eor w15,w15,w6,ror#20 - sli v7.4s,v19.4s,#13 - add w5,w5,w11 - ldr w12,[sp,#24] - and w13,w13,w14 - eor v17.16b,v17.16b,v16.16b - ror w15,w15,#2 - add w9,w9,w5 - eor w13,w13,w7 - eor v17.16b,v17.16b,v7.16b - add w4,w4,w12 - add w5,w5,w15 - and w12,w10,w9 - add v1.4s,v1.4s,v17.4s - bic w15,w3,w9 - eor w11,w9,w9,ror#5 - add w5,w5,w13 - ushr v18.4s,v1.4s,#17 - orr w12,w12,w15 - ushr v19.4s,v1.4s,#10 - eor w11,w11,w9,ror#19 - eor w15,w5,w5,ror#11 - sli v18.4s,v1.4s,#15 - add w4,w4,w12 - ushr v17.4s,v1.4s,#19 - ror w11,w11,#6 - eor w13,w5,w6 - eor v19.16b,v19.16b,v18.16b - eor w15,w15,w5,ror#20 - add w4,w4,w11 - sli v17.4s,v1.4s,#13 - ldr w12,[sp,#28] - and w14,w14,w13 - ror w15,w15,#2 - ld1 {v4.4s},[x16], #16 - add w8,w8,w4 - eor v19.16b,v19.16b,v17.16b - eor w14,w14,w6 - eor v17.16b,v17.16b,v17.16b - add w3,w3,w12 - add w4,w4,w15 - and w12,w9,w8 - mov v17.d[1],v19.d[0] - bic w15,w10,w8 - eor w11,w8,w8,ror#5 - add w4,w4,w14 - add v1.4s,v1.4s,v17.4s - orr w12,w12,w15 - eor w11,w11,w8,ror#19 - eor w15,w4,w4,ror#11 - add v4.4s,v4.4s,v1.4s - add w3,w3,w12 - ror w11,w11,#6 - eor w14,w4,w5 - eor w15,w15,w4,ror#20 - add w3,w3,w11 - ldr w12,[sp,#32] - and w13,w13,w14 - ror w15,w15,#2 - add w7,w7,w3 - eor w13,w13,w5 - st1 {v4.4s},[x17], #16 - ext v4.16b,v2.16b,v3.16b,#4 - add w10,w10,w12 - add w3,w3,w15 - and w12,w8,w7 - bic w15,w9,w7 - ext v7.16b,v0.16b,v1.16b,#4 - eor w11,w7,w7,ror#5 - add w3,w3,w13 - mov d19,v1.d[1] - orr w12,w12,w15 - eor w11,w11,w7,ror#19 - ushr v6.4s,v4.4s,#7 - eor w15,w3,w3,ror#11 - ushr v5.4s,v4.4s,#3 - add w10,w10,w12 - add v2.4s,v2.4s,v7.4s - ror w11,w11,#6 - sli v6.4s,v4.4s,#25 - eor w13,w3,w4 - eor w15,w15,w3,ror#20 - ushr v7.4s,v4.4s,#18 - add w10,w10,w11 - ldr w12,[sp,#36] - and w14,w14,w13 - eor v5.16b,v5.16b,v6.16b - ror w15,w15,#2 - add w6,w6,w10 - sli v7.4s,v4.4s,#14 - eor w14,w14,w4 - ushr v16.4s,v19.4s,#17 - add w9,w9,w12 - add w10,w10,w15 - and w12,w7,w6 - eor v5.16b,v5.16b,v7.16b - bic w15,w8,w6 - eor w11,w6,w6,ror#5 - sli v16.4s,v19.4s,#15 - add w10,w10,w14 - orr w12,w12,w15 - ushr v17.4s,v19.4s,#10 - eor w11,w11,w6,ror#19 - eor w15,w10,w10,ror#11 - ushr v7.4s,v19.4s,#19 - add w9,w9,w12 - ror w11,w11,#6 - add v2.4s,v2.4s,v5.4s - eor w14,w10,w3 - eor w15,w15,w10,ror#20 - sli v7.4s,v19.4s,#13 - add w9,w9,w11 - ldr w12,[sp,#40] - and w13,w13,w14 - eor v17.16b,v17.16b,v16.16b - ror w15,w15,#2 - add w5,w5,w9 - eor w13,w13,w3 - eor v17.16b,v17.16b,v7.16b - add w8,w8,w12 - add w9,w9,w15 - and w12,w6,w5 - add v2.4s,v2.4s,v17.4s - bic w15,w7,w5 - eor w11,w5,w5,ror#5 - add w9,w9,w13 - ushr v18.4s,v2.4s,#17 - orr w12,w12,w15 - ushr v19.4s,v2.4s,#10 - eor w11,w11,w5,ror#19 - eor w15,w9,w9,ror#11 - sli v18.4s,v2.4s,#15 - add w8,w8,w12 - ushr v17.4s,v2.4s,#19 - ror w11,w11,#6 - eor w13,w9,w10 - eor v19.16b,v19.16b,v18.16b - eor w15,w15,w9,ror#20 - add w8,w8,w11 - sli v17.4s,v2.4s,#13 - ldr w12,[sp,#44] - and w14,w14,w13 - ror w15,w15,#2 - ld1 {v4.4s},[x16], #16 - add w4,w4,w8 - eor v19.16b,v19.16b,v17.16b - eor w14,w14,w10 - eor v17.16b,v17.16b,v17.16b - add w7,w7,w12 - add w8,w8,w15 - and w12,w5,w4 - mov v17.d[1],v19.d[0] - bic w15,w6,w4 - eor w11,w4,w4,ror#5 - add w8,w8,w14 - add v2.4s,v2.4s,v17.4s - orr w12,w12,w15 - eor w11,w11,w4,ror#19 - eor w15,w8,w8,ror#11 - add v4.4s,v4.4s,v2.4s - add w7,w7,w12 - ror w11,w11,#6 - eor w14,w8,w9 - eor w15,w15,w8,ror#20 - add w7,w7,w11 - ldr w12,[sp,#48] - and w13,w13,w14 - ror w15,w15,#2 - add w3,w3,w7 - eor w13,w13,w9 - st1 {v4.4s},[x17], #16 - ext v4.16b,v3.16b,v0.16b,#4 - add w6,w6,w12 - add w7,w7,w15 - and w12,w4,w3 - bic w15,w5,w3 - ext v7.16b,v1.16b,v2.16b,#4 - eor w11,w3,w3,ror#5 - add w7,w7,w13 - mov d19,v2.d[1] - orr w12,w12,w15 - eor w11,w11,w3,ror#19 - ushr v6.4s,v4.4s,#7 - eor w15,w7,w7,ror#11 - ushr v5.4s,v4.4s,#3 - add w6,w6,w12 - add v3.4s,v3.4s,v7.4s - ror w11,w11,#6 - sli v6.4s,v4.4s,#25 - eor w13,w7,w8 - eor w15,w15,w7,ror#20 - ushr v7.4s,v4.4s,#18 - add w6,w6,w11 - ldr w12,[sp,#52] - and w14,w14,w13 - eor v5.16b,v5.16b,v6.16b - ror w15,w15,#2 - add w10,w10,w6 - sli v7.4s,v4.4s,#14 - eor w14,w14,w8 - ushr v16.4s,v19.4s,#17 - add w5,w5,w12 - add w6,w6,w15 - and w12,w3,w10 - eor v5.16b,v5.16b,v7.16b - bic w15,w4,w10 - eor w11,w10,w10,ror#5 - sli v16.4s,v19.4s,#15 - add w6,w6,w14 - orr w12,w12,w15 - ushr v17.4s,v19.4s,#10 - eor w11,w11,w10,ror#19 - eor w15,w6,w6,ror#11 - ushr v7.4s,v19.4s,#19 - add w5,w5,w12 - ror w11,w11,#6 - add v3.4s,v3.4s,v5.4s - eor w14,w6,w7 - eor w15,w15,w6,ror#20 - sli v7.4s,v19.4s,#13 - add w5,w5,w11 - ldr w12,[sp,#56] - and w13,w13,w14 - eor v17.16b,v17.16b,v16.16b - ror w15,w15,#2 - add w9,w9,w5 - eor w13,w13,w7 - eor v17.16b,v17.16b,v7.16b - add w4,w4,w12 - add w5,w5,w15 - and w12,w10,w9 - add v3.4s,v3.4s,v17.4s - bic w15,w3,w9 - eor w11,w9,w9,ror#5 - add w5,w5,w13 - ushr v18.4s,v3.4s,#17 - orr w12,w12,w15 - ushr v19.4s,v3.4s,#10 - eor w11,w11,w9,ror#19 - eor w15,w5,w5,ror#11 - sli v18.4s,v3.4s,#15 - add w4,w4,w12 - ushr v17.4s,v3.4s,#19 - ror w11,w11,#6 - eor w13,w5,w6 - eor v19.16b,v19.16b,v18.16b - eor w15,w15,w5,ror#20 - add w4,w4,w11 - sli v17.4s,v3.4s,#13 - ldr w12,[sp,#60] - and w14,w14,w13 - ror w15,w15,#2 - ld1 {v4.4s},[x16], #16 - add w8,w8,w4 - eor v19.16b,v19.16b,v17.16b - eor w14,w14,w6 - eor v17.16b,v17.16b,v17.16b - add w3,w3,w12 - add w4,w4,w15 - and w12,w9,w8 - mov v17.d[1],v19.d[0] - bic w15,w10,w8 - eor w11,w8,w8,ror#5 - add w4,w4,w14 - add v3.4s,v3.4s,v17.4s - orr w12,w12,w15 - eor w11,w11,w8,ror#19 - eor w15,w4,w4,ror#11 - add v4.4s,v4.4s,v3.4s - add w3,w3,w12 - ror w11,w11,#6 - eor w14,w4,w5 - eor w15,w15,w4,ror#20 - add w3,w3,w11 - ldr w12,[x16] - and w13,w13,w14 - ror w15,w15,#2 - add w7,w7,w3 - eor w13,w13,w5 - st1 {v4.4s},[x17], #16 - cmp w12,#0 // check for K256 terminator - ldr w12,[sp,#0] - sub x17,x17,#64 - bne .L_00_48 - - sub x16,x16,#256 // rewind x16 - cmp x1,x2 - mov x17, #64 - csel x17, x17, xzr, eq - sub x1,x1,x17 // avoid SEGV - mov x17,sp - add w10,w10,w12 - add w3,w3,w15 - and w12,w8,w7 - ld1 {v0.16b},[x1],#16 - bic w15,w9,w7 - eor w11,w7,w7,ror#5 - ld1 {v4.4s},[x16],#16 - add w3,w3,w13 - orr w12,w12,w15 - eor w11,w11,w7,ror#19 - eor w15,w3,w3,ror#11 - rev32 v0.16b,v0.16b - add w10,w10,w12 - ror w11,w11,#6 - eor w13,w3,w4 - eor w15,w15,w3,ror#20 - add v4.4s,v4.4s,v0.4s - add w10,w10,w11 - ldr w12,[sp,#4] - and w14,w14,w13 - ror w15,w15,#2 - add w6,w6,w10 - eor w14,w14,w4 - add w9,w9,w12 - add w10,w10,w15 - and w12,w7,w6 - bic w15,w8,w6 - eor w11,w6,w6,ror#5 - add w10,w10,w14 - orr w12,w12,w15 - eor w11,w11,w6,ror#19 - eor w15,w10,w10,ror#11 - add w9,w9,w12 - ror w11,w11,#6 - eor w14,w10,w3 - eor w15,w15,w10,ror#20 - add w9,w9,w11 - ldr w12,[sp,#8] - and w13,w13,w14 - ror w15,w15,#2 - add w5,w5,w9 - eor w13,w13,w3 - add w8,w8,w12 - add w9,w9,w15 - and w12,w6,w5 - bic w15,w7,w5 - eor w11,w5,w5,ror#5 - add w9,w9,w13 - orr w12,w12,w15 - eor w11,w11,w5,ror#19 - eor w15,w9,w9,ror#11 - add w8,w8,w12 - ror w11,w11,#6 - eor w13,w9,w10 - eor w15,w15,w9,ror#20 - add w8,w8,w11 - ldr w12,[sp,#12] - and w14,w14,w13 - ror w15,w15,#2 - add w4,w4,w8 - eor w14,w14,w10 - add w7,w7,w12 - add w8,w8,w15 - and w12,w5,w4 - bic w15,w6,w4 - eor w11,w4,w4,ror#5 - add w8,w8,w14 - orr w12,w12,w15 - eor w11,w11,w4,ror#19 - eor w15,w8,w8,ror#11 - add w7,w7,w12 - ror w11,w11,#6 - eor w14,w8,w9 - eor w15,w15,w8,ror#20 - add w7,w7,w11 - ldr w12,[sp,#16] - and w13,w13,w14 - ror w15,w15,#2 - add w3,w3,w7 - eor w13,w13,w9 - st1 {v4.4s},[x17], #16 - add w6,w6,w12 - add w7,w7,w15 - and w12,w4,w3 - ld1 {v1.16b},[x1],#16 - bic w15,w5,w3 - eor w11,w3,w3,ror#5 - ld1 {v4.4s},[x16],#16 - add w7,w7,w13 - orr w12,w12,w15 - eor w11,w11,w3,ror#19 - eor w15,w7,w7,ror#11 - rev32 v1.16b,v1.16b - add w6,w6,w12 - ror w11,w11,#6 - eor w13,w7,w8 - eor w15,w15,w7,ror#20 - add v4.4s,v4.4s,v1.4s - add w6,w6,w11 - ldr w12,[sp,#20] - and w14,w14,w13 - ror w15,w15,#2 - add w10,w10,w6 - eor w14,w14,w8 - add w5,w5,w12 - add w6,w6,w15 - and w12,w3,w10 - bic w15,w4,w10 - eor w11,w10,w10,ror#5 - add w6,w6,w14 - orr w12,w12,w15 - eor w11,w11,w10,ror#19 - eor w15,w6,w6,ror#11 - add w5,w5,w12 - ror w11,w11,#6 - eor w14,w6,w7 - eor w15,w15,w6,ror#20 - add w5,w5,w11 - ldr w12,[sp,#24] - and w13,w13,w14 - ror w15,w15,#2 - add w9,w9,w5 - eor w13,w13,w7 - add w4,w4,w12 - add w5,w5,w15 - and w12,w10,w9 - bic w15,w3,w9 - eor w11,w9,w9,ror#5 - add w5,w5,w13 - orr w12,w12,w15 - eor w11,w11,w9,ror#19 - eor w15,w5,w5,ror#11 - add w4,w4,w12 - ror w11,w11,#6 - eor w13,w5,w6 - eor w15,w15,w5,ror#20 - add w4,w4,w11 - ldr w12,[sp,#28] - and w14,w14,w13 - ror w15,w15,#2 - add w8,w8,w4 - eor w14,w14,w6 - add w3,w3,w12 - add w4,w4,w15 - and w12,w9,w8 - bic w15,w10,w8 - eor w11,w8,w8,ror#5 - add w4,w4,w14 - orr w12,w12,w15 - eor w11,w11,w8,ror#19 - eor w15,w4,w4,ror#11 - add w3,w3,w12 - ror w11,w11,#6 - eor w14,w4,w5 - eor w15,w15,w4,ror#20 - add w3,w3,w11 - ldr w12,[sp,#32] - and w13,w13,w14 - ror w15,w15,#2 - add w7,w7,w3 - eor w13,w13,w5 - st1 {v4.4s},[x17], #16 - add w10,w10,w12 - add w3,w3,w15 - and w12,w8,w7 - ld1 {v2.16b},[x1],#16 - bic w15,w9,w7 - eor w11,w7,w7,ror#5 - ld1 {v4.4s},[x16],#16 - add w3,w3,w13 - orr w12,w12,w15 - eor w11,w11,w7,ror#19 - eor w15,w3,w3,ror#11 - rev32 v2.16b,v2.16b - add w10,w10,w12 - ror w11,w11,#6 - eor w13,w3,w4 - eor w15,w15,w3,ror#20 - add v4.4s,v4.4s,v2.4s - add w10,w10,w11 - ldr w12,[sp,#36] - and w14,w14,w13 - ror w15,w15,#2 - add w6,w6,w10 - eor w14,w14,w4 - add w9,w9,w12 - add w10,w10,w15 - and w12,w7,w6 - bic w15,w8,w6 - eor w11,w6,w6,ror#5 - add w10,w10,w14 - orr w12,w12,w15 - eor w11,w11,w6,ror#19 - eor w15,w10,w10,ror#11 - add w9,w9,w12 - ror w11,w11,#6 - eor w14,w10,w3 - eor w15,w15,w10,ror#20 - add w9,w9,w11 - ldr w12,[sp,#40] - and w13,w13,w14 - ror w15,w15,#2 - add w5,w5,w9 - eor w13,w13,w3 - add w8,w8,w12 - add w9,w9,w15 - and w12,w6,w5 - bic w15,w7,w5 - eor w11,w5,w5,ror#5 - add w9,w9,w13 - orr w12,w12,w15 - eor w11,w11,w5,ror#19 - eor w15,w9,w9,ror#11 - add w8,w8,w12 - ror w11,w11,#6 - eor w13,w9,w10 - eor w15,w15,w9,ror#20 - add w8,w8,w11 - ldr w12,[sp,#44] - and w14,w14,w13 - ror w15,w15,#2 - add w4,w4,w8 - eor w14,w14,w10 - add w7,w7,w12 - add w8,w8,w15 - and w12,w5,w4 - bic w15,w6,w4 - eor w11,w4,w4,ror#5 - add w8,w8,w14 - orr w12,w12,w15 - eor w11,w11,w4,ror#19 - eor w15,w8,w8,ror#11 - add w7,w7,w12 - ror w11,w11,#6 - eor w14,w8,w9 - eor w15,w15,w8,ror#20 - add w7,w7,w11 - ldr w12,[sp,#48] - and w13,w13,w14 - ror w15,w15,#2 - add w3,w3,w7 - eor w13,w13,w9 - st1 {v4.4s},[x17], #16 - add w6,w6,w12 - add w7,w7,w15 - and w12,w4,w3 - ld1 {v3.16b},[x1],#16 - bic w15,w5,w3 - eor w11,w3,w3,ror#5 - ld1 {v4.4s},[x16],#16 - add w7,w7,w13 - orr w12,w12,w15 - eor w11,w11,w3,ror#19 - eor w15,w7,w7,ror#11 - rev32 v3.16b,v3.16b - add w6,w6,w12 - ror w11,w11,#6 - eor w13,w7,w8 - eor w15,w15,w7,ror#20 - add v4.4s,v4.4s,v3.4s - add w6,w6,w11 - ldr w12,[sp,#52] - and w14,w14,w13 - ror w15,w15,#2 - add w10,w10,w6 - eor w14,w14,w8 - add w5,w5,w12 - add w6,w6,w15 - and w12,w3,w10 - bic w15,w4,w10 - eor w11,w10,w10,ror#5 - add w6,w6,w14 - orr w12,w12,w15 - eor w11,w11,w10,ror#19 - eor w15,w6,w6,ror#11 - add w5,w5,w12 - ror w11,w11,#6 - eor w14,w6,w7 - eor w15,w15,w6,ror#20 - add w5,w5,w11 - ldr w12,[sp,#56] - and w13,w13,w14 - ror w15,w15,#2 - add w9,w9,w5 - eor w13,w13,w7 - add w4,w4,w12 - add w5,w5,w15 - and w12,w10,w9 - bic w15,w3,w9 - eor w11,w9,w9,ror#5 - add w5,w5,w13 - orr w12,w12,w15 - eor w11,w11,w9,ror#19 - eor w15,w5,w5,ror#11 - add w4,w4,w12 - ror w11,w11,#6 - eor w13,w5,w6 - eor w15,w15,w5,ror#20 - add w4,w4,w11 - ldr w12,[sp,#60] - and w14,w14,w13 - ror w15,w15,#2 - add w8,w8,w4 - eor w14,w14,w6 - add w3,w3,w12 - add w4,w4,w15 - and w12,w9,w8 - bic w15,w10,w8 - eor w11,w8,w8,ror#5 - add w4,w4,w14 - orr w12,w12,w15 - eor w11,w11,w8,ror#19 - eor w15,w4,w4,ror#11 - add w3,w3,w12 - ror w11,w11,#6 - eor w14,w4,w5 - eor w15,w15,w4,ror#20 - add w3,w3,w11 - and w13,w13,w14 - ror w15,w15,#2 - add w7,w7,w3 - eor w13,w13,w5 - st1 {v4.4s},[x17], #16 - add w3,w3,w15 // h+=Sigma0(a) from the past - ldp w11,w12,[x0,#0] - add w3,w3,w13 // h+=Maj(a,b,c) from the past - ldp w13,w14,[x0,#8] - add w3,w3,w11 // accumulate - add w4,w4,w12 - ldp w11,w12,[x0,#16] - add w5,w5,w13 - add w6,w6,w14 - ldp w13,w14,[x0,#24] - add w7,w7,w11 - add w8,w8,w12 - ldr w12,[sp,#0] - stp w3,w4,[x0,#0] - add w9,w9,w13 - mov w13,wzr - stp w5,w6,[x0,#8] - add w10,w10,w14 - stp w7,w8,[x0,#16] - eor w14,w4,w5 - stp w9,w10,[x0,#24] - mov w15,wzr - mov x17,sp - b.ne .L_00_48 - - ldr x29,[x29] - add sp,sp,#16*4+16 - ret -.size sha256_block_neon,.-sha256_block_neon -#ifndef __KERNEL__ -.comm OPENSSL_armcap_P,4,4 -#endif diff --git a/arch/arm64/crypto/sha512-core.S_shipped b/arch/arm64/crypto/sha512-core.S_shipped deleted file mode 100644 index e063a6106720..000000000000 --- a/arch/arm64/crypto/sha512-core.S_shipped +++ /dev/null @@ -1,1093 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 - -// This code is taken from the OpenSSL project but the author (Andy Polyakov) -// has relicensed it under the GPLv2. Therefore this program is free software; -// you can redistribute it and/or modify it under the terms of the GNU General -// Public License version 2 as published by the Free Software Foundation. -// -// The original headers, including the original license headers, are -// included below for completeness. - -// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. -// -// Licensed under the OpenSSL license (the "License"). You may not use -// this file except in compliance with the License. You can obtain a copy -// in the file LICENSE in the source distribution or at -// https://www.openssl.org/source/license.html - -// ==================================================================== -// Written by Andy Polyakov for the OpenSSL -// project. The module is, however, dual licensed under OpenSSL and -// CRYPTOGAMS licenses depending on where you obtain it. For further -// details see http://www.openssl.org/~appro/cryptogams/. -// ==================================================================== -// -// SHA256/512 for ARMv8. -// -// Performance in cycles per processed byte and improvement coefficient -// over code generated with "default" compiler: -// -// SHA256-hw SHA256(*) SHA512 -// Apple A7 1.97 10.5 (+33%) 6.73 (-1%(**)) -// Cortex-A53 2.38 15.5 (+115%) 10.0 (+150%(***)) -// Cortex-A57 2.31 11.6 (+86%) 7.51 (+260%(***)) -// Denver 2.01 10.5 (+26%) 6.70 (+8%) -// X-Gene 20.0 (+100%) 12.8 (+300%(***)) -// Mongoose 2.36 13.0 (+50%) 8.36 (+33%) -// -// (*) Software SHA256 results are of lesser relevance, presented -// mostly for informational purposes. -// (**) The result is a trade-off: it's possible to improve it by -// 10% (or by 1 cycle per round), but at the cost of 20% loss -// on Cortex-A53 (or by 4 cycles per round). -// (***) Super-impressive coefficients over gcc-generated code are -// indication of some compiler "pathology", most notably code -// generated with -mgeneral-regs-only is significanty faster -// and the gap is only 40-90%. -// -// October 2016. -// -// Originally it was reckoned that it makes no sense to implement NEON -// version of SHA256 for 64-bit processors. This is because performance -// improvement on most wide-spread Cortex-A5x processors was observed -// to be marginal, same on Cortex-A53 and ~10% on A57. But then it was -// observed that 32-bit NEON SHA256 performs significantly better than -// 64-bit scalar version on *some* of the more recent processors. As -// result 64-bit NEON version of SHA256 was added to provide best -// all-round performance. For example it executes ~30% faster on X-Gene -// and Mongoose. [For reference, NEON version of SHA512 is bound to -// deliver much less improvement, likely *negative* on Cortex-A5x. -// Which is why NEON support is limited to SHA256.] - -#ifndef __KERNEL__ -# include "arm_arch.h" -#endif - -.text - -.extern OPENSSL_armcap_P -.globl sha512_block_data_order -.type sha512_block_data_order,%function -.align 6 -sha512_block_data_order: - stp x29,x30,[sp,#-128]! - add x29,sp,#0 - - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - sub sp,sp,#4*8 - - ldp x20,x21,[x0] // load context - ldp x22,x23,[x0,#2*8] - ldp x24,x25,[x0,#4*8] - add x2,x1,x2,lsl#7 // end of input - ldp x26,x27,[x0,#6*8] - adr x30,.LK512 - stp x0,x2,[x29,#96] - -.Loop: - ldp x3,x4,[x1],#2*8 - ldr x19,[x30],#8 // *K++ - eor x28,x21,x22 // magic seed - str x1,[x29,#112] -#ifndef __AARCH64EB__ - rev x3,x3 // 0 -#endif - ror x16,x24,#14 - add x27,x27,x19 // h+=K[i] - eor x6,x24,x24,ror#23 - and x17,x25,x24 - bic x19,x26,x24 - add x27,x27,x3 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x20,x21 // a^b, b^c in next round - eor x16,x16,x6,ror#18 // Sigma1(e) - ror x6,x20,#28 - add x27,x27,x17 // h+=Ch(e,f,g) - eor x17,x20,x20,ror#5 - add x27,x27,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x23,x23,x27 // d+=h - eor x28,x28,x21 // Maj(a,b,c) - eor x17,x6,x17,ror#34 // Sigma0(a) - add x27,x27,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x27,x27,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x4,x4 // 1 -#endif - ldp x5,x6,[x1],#2*8 - add x27,x27,x17 // h+=Sigma0(a) - ror x16,x23,#14 - add x26,x26,x28 // h+=K[i] - eor x7,x23,x23,ror#23 - and x17,x24,x23 - bic x28,x25,x23 - add x26,x26,x4 // h+=X[i] - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x27,x20 // a^b, b^c in next round - eor x16,x16,x7,ror#18 // Sigma1(e) - ror x7,x27,#28 - add x26,x26,x17 // h+=Ch(e,f,g) - eor x17,x27,x27,ror#5 - add x26,x26,x16 // h+=Sigma1(e) - and x19,x19,x28 // (b^c)&=(a^b) - add x22,x22,x26 // d+=h - eor x19,x19,x20 // Maj(a,b,c) - eor x17,x7,x17,ror#34 // Sigma0(a) - add x26,x26,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - //add x26,x26,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x5,x5 // 2 -#endif - add x26,x26,x17 // h+=Sigma0(a) - ror x16,x22,#14 - add x25,x25,x19 // h+=K[i] - eor x8,x22,x22,ror#23 - and x17,x23,x22 - bic x19,x24,x22 - add x25,x25,x5 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x26,x27 // a^b, b^c in next round - eor x16,x16,x8,ror#18 // Sigma1(e) - ror x8,x26,#28 - add x25,x25,x17 // h+=Ch(e,f,g) - eor x17,x26,x26,ror#5 - add x25,x25,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x21,x21,x25 // d+=h - eor x28,x28,x27 // Maj(a,b,c) - eor x17,x8,x17,ror#34 // Sigma0(a) - add x25,x25,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x25,x25,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x6,x6 // 3 -#endif - ldp x7,x8,[x1],#2*8 - add x25,x25,x17 // h+=Sigma0(a) - ror x16,x21,#14 - add x24,x24,x28 // h+=K[i] - eor x9,x21,x21,ror#23 - and x17,x22,x21 - bic x28,x23,x21 - add x24,x24,x6 // h+=X[i] - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x25,x26 // a^b, b^c in next round - eor x16,x16,x9,ror#18 // Sigma1(e) - ror x9,x25,#28 - add x24,x24,x17 // h+=Ch(e,f,g) - eor x17,x25,x25,ror#5 - add x24,x24,x16 // h+=Sigma1(e) - and x19,x19,x28 // (b^c)&=(a^b) - add x20,x20,x24 // d+=h - eor x19,x19,x26 // Maj(a,b,c) - eor x17,x9,x17,ror#34 // Sigma0(a) - add x24,x24,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - //add x24,x24,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x7,x7 // 4 -#endif - add x24,x24,x17 // h+=Sigma0(a) - ror x16,x20,#14 - add x23,x23,x19 // h+=K[i] - eor x10,x20,x20,ror#23 - and x17,x21,x20 - bic x19,x22,x20 - add x23,x23,x7 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x24,x25 // a^b, b^c in next round - eor x16,x16,x10,ror#18 // Sigma1(e) - ror x10,x24,#28 - add x23,x23,x17 // h+=Ch(e,f,g) - eor x17,x24,x24,ror#5 - add x23,x23,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x27,x27,x23 // d+=h - eor x28,x28,x25 // Maj(a,b,c) - eor x17,x10,x17,ror#34 // Sigma0(a) - add x23,x23,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x23,x23,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x8,x8 // 5 -#endif - ldp x9,x10,[x1],#2*8 - add x23,x23,x17 // h+=Sigma0(a) - ror x16,x27,#14 - add x22,x22,x28 // h+=K[i] - eor x11,x27,x27,ror#23 - and x17,x20,x27 - bic x28,x21,x27 - add x22,x22,x8 // h+=X[i] - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x23,x24 // a^b, b^c in next round - eor x16,x16,x11,ror#18 // Sigma1(e) - ror x11,x23,#28 - add x22,x22,x17 // h+=Ch(e,f,g) - eor x17,x23,x23,ror#5 - add x22,x22,x16 // h+=Sigma1(e) - and x19,x19,x28 // (b^c)&=(a^b) - add x26,x26,x22 // d+=h - eor x19,x19,x24 // Maj(a,b,c) - eor x17,x11,x17,ror#34 // Sigma0(a) - add x22,x22,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - //add x22,x22,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x9,x9 // 6 -#endif - add x22,x22,x17 // h+=Sigma0(a) - ror x16,x26,#14 - add x21,x21,x19 // h+=K[i] - eor x12,x26,x26,ror#23 - and x17,x27,x26 - bic x19,x20,x26 - add x21,x21,x9 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x22,x23 // a^b, b^c in next round - eor x16,x16,x12,ror#18 // Sigma1(e) - ror x12,x22,#28 - add x21,x21,x17 // h+=Ch(e,f,g) - eor x17,x22,x22,ror#5 - add x21,x21,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x25,x25,x21 // d+=h - eor x28,x28,x23 // Maj(a,b,c) - eor x17,x12,x17,ror#34 // Sigma0(a) - add x21,x21,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x21,x21,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x10,x10 // 7 -#endif - ldp x11,x12,[x1],#2*8 - add x21,x21,x17 // h+=Sigma0(a) - ror x16,x25,#14 - add x20,x20,x28 // h+=K[i] - eor x13,x25,x25,ror#23 - and x17,x26,x25 - bic x28,x27,x25 - add x20,x20,x10 // h+=X[i] - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x21,x22 // a^b, b^c in next round - eor x16,x16,x13,ror#18 // Sigma1(e) - ror x13,x21,#28 - add x20,x20,x17 // h+=Ch(e,f,g) - eor x17,x21,x21,ror#5 - add x20,x20,x16 // h+=Sigma1(e) - and x19,x19,x28 // (b^c)&=(a^b) - add x24,x24,x20 // d+=h - eor x19,x19,x22 // Maj(a,b,c) - eor x17,x13,x17,ror#34 // Sigma0(a) - add x20,x20,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - //add x20,x20,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x11,x11 // 8 -#endif - add x20,x20,x17 // h+=Sigma0(a) - ror x16,x24,#14 - add x27,x27,x19 // h+=K[i] - eor x14,x24,x24,ror#23 - and x17,x25,x24 - bic x19,x26,x24 - add x27,x27,x11 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x20,x21 // a^b, b^c in next round - eor x16,x16,x14,ror#18 // Sigma1(e) - ror x14,x20,#28 - add x27,x27,x17 // h+=Ch(e,f,g) - eor x17,x20,x20,ror#5 - add x27,x27,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x23,x23,x27 // d+=h - eor x28,x28,x21 // Maj(a,b,c) - eor x17,x14,x17,ror#34 // Sigma0(a) - add x27,x27,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x27,x27,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x12,x12 // 9 -#endif - ldp x13,x14,[x1],#2*8 - add x27,x27,x17 // h+=Sigma0(a) - ror x16,x23,#14 - add x26,x26,x28 // h+=K[i] - eor x15,x23,x23,ror#23 - and x17,x24,x23 - bic x28,x25,x23 - add x26,x26,x12 // h+=X[i] - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x27,x20 // a^b, b^c in next round - eor x16,x16,x15,ror#18 // Sigma1(e) - ror x15,x27,#28 - add x26,x26,x17 // h+=Ch(e,f,g) - eor x17,x27,x27,ror#5 - add x26,x26,x16 // h+=Sigma1(e) - and x19,x19,x28 // (b^c)&=(a^b) - add x22,x22,x26 // d+=h - eor x19,x19,x20 // Maj(a,b,c) - eor x17,x15,x17,ror#34 // Sigma0(a) - add x26,x26,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - //add x26,x26,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x13,x13 // 10 -#endif - add x26,x26,x17 // h+=Sigma0(a) - ror x16,x22,#14 - add x25,x25,x19 // h+=K[i] - eor x0,x22,x22,ror#23 - and x17,x23,x22 - bic x19,x24,x22 - add x25,x25,x13 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x26,x27 // a^b, b^c in next round - eor x16,x16,x0,ror#18 // Sigma1(e) - ror x0,x26,#28 - add x25,x25,x17 // h+=Ch(e,f,g) - eor x17,x26,x26,ror#5 - add x25,x25,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x21,x21,x25 // d+=h - eor x28,x28,x27 // Maj(a,b,c) - eor x17,x0,x17,ror#34 // Sigma0(a) - add x25,x25,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x25,x25,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x14,x14 // 11 -#endif - ldp x15,x0,[x1],#2*8 - add x25,x25,x17 // h+=Sigma0(a) - str x6,[sp,#24] - ror x16,x21,#14 - add x24,x24,x28 // h+=K[i] - eor x6,x21,x21,ror#23 - and x17,x22,x21 - bic x28,x23,x21 - add x24,x24,x14 // h+=X[i] - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x25,x26 // a^b, b^c in next round - eor x16,x16,x6,ror#18 // Sigma1(e) - ror x6,x25,#28 - add x24,x24,x17 // h+=Ch(e,f,g) - eor x17,x25,x25,ror#5 - add x24,x24,x16 // h+=Sigma1(e) - and x19,x19,x28 // (b^c)&=(a^b) - add x20,x20,x24 // d+=h - eor x19,x19,x26 // Maj(a,b,c) - eor x17,x6,x17,ror#34 // Sigma0(a) - add x24,x24,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - //add x24,x24,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x15,x15 // 12 -#endif - add x24,x24,x17 // h+=Sigma0(a) - str x7,[sp,#0] - ror x16,x20,#14 - add x23,x23,x19 // h+=K[i] - eor x7,x20,x20,ror#23 - and x17,x21,x20 - bic x19,x22,x20 - add x23,x23,x15 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x24,x25 // a^b, b^c in next round - eor x16,x16,x7,ror#18 // Sigma1(e) - ror x7,x24,#28 - add x23,x23,x17 // h+=Ch(e,f,g) - eor x17,x24,x24,ror#5 - add x23,x23,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x27,x27,x23 // d+=h - eor x28,x28,x25 // Maj(a,b,c) - eor x17,x7,x17,ror#34 // Sigma0(a) - add x23,x23,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x23,x23,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x0,x0 // 13 -#endif - ldp x1,x2,[x1] - add x23,x23,x17 // h+=Sigma0(a) - str x8,[sp,#8] - ror x16,x27,#14 - add x22,x22,x28 // h+=K[i] - eor x8,x27,x27,ror#23 - and x17,x20,x27 - bic x28,x21,x27 - add x22,x22,x0 // h+=X[i] - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x23,x24 // a^b, b^c in next round - eor x16,x16,x8,ror#18 // Sigma1(e) - ror x8,x23,#28 - add x22,x22,x17 // h+=Ch(e,f,g) - eor x17,x23,x23,ror#5 - add x22,x22,x16 // h+=Sigma1(e) - and x19,x19,x28 // (b^c)&=(a^b) - add x26,x26,x22 // d+=h - eor x19,x19,x24 // Maj(a,b,c) - eor x17,x8,x17,ror#34 // Sigma0(a) - add x22,x22,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - //add x22,x22,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x1,x1 // 14 -#endif - ldr x6,[sp,#24] - add x22,x22,x17 // h+=Sigma0(a) - str x9,[sp,#16] - ror x16,x26,#14 - add x21,x21,x19 // h+=K[i] - eor x9,x26,x26,ror#23 - and x17,x27,x26 - bic x19,x20,x26 - add x21,x21,x1 // h+=X[i] - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x22,x23 // a^b, b^c in next round - eor x16,x16,x9,ror#18 // Sigma1(e) - ror x9,x22,#28 - add x21,x21,x17 // h+=Ch(e,f,g) - eor x17,x22,x22,ror#5 - add x21,x21,x16 // h+=Sigma1(e) - and x28,x28,x19 // (b^c)&=(a^b) - add x25,x25,x21 // d+=h - eor x28,x28,x23 // Maj(a,b,c) - eor x17,x9,x17,ror#34 // Sigma0(a) - add x21,x21,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - //add x21,x21,x17 // h+=Sigma0(a) -#ifndef __AARCH64EB__ - rev x2,x2 // 15 -#endif - ldr x7,[sp,#0] - add x21,x21,x17 // h+=Sigma0(a) - str x10,[sp,#24] - ror x16,x25,#14 - add x20,x20,x28 // h+=K[i] - ror x9,x4,#1 - and x17,x26,x25 - ror x8,x1,#19 - bic x28,x27,x25 - ror x10,x21,#28 - add x20,x20,x2 // h+=X[i] - eor x16,x16,x25,ror#18 - eor x9,x9,x4,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x21,x22 // a^b, b^c in next round - eor x16,x16,x25,ror#41 // Sigma1(e) - eor x10,x10,x21,ror#34 - add x20,x20,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x8,x8,x1,ror#61 - eor x9,x9,x4,lsr#7 // sigma0(X[i+1]) - add x20,x20,x16 // h+=Sigma1(e) - eor x19,x19,x22 // Maj(a,b,c) - eor x17,x10,x21,ror#39 // Sigma0(a) - eor x8,x8,x1,lsr#6 // sigma1(X[i+14]) - add x3,x3,x12 - add x24,x24,x20 // d+=h - add x20,x20,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x3,x3,x9 - add x20,x20,x17 // h+=Sigma0(a) - add x3,x3,x8 -.Loop_16_xx: - ldr x8,[sp,#8] - str x11,[sp,#0] - ror x16,x24,#14 - add x27,x27,x19 // h+=K[i] - ror x10,x5,#1 - and x17,x25,x24 - ror x9,x2,#19 - bic x19,x26,x24 - ror x11,x20,#28 - add x27,x27,x3 // h+=X[i] - eor x16,x16,x24,ror#18 - eor x10,x10,x5,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x20,x21 // a^b, b^c in next round - eor x16,x16,x24,ror#41 // Sigma1(e) - eor x11,x11,x20,ror#34 - add x27,x27,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x9,x9,x2,ror#61 - eor x10,x10,x5,lsr#7 // sigma0(X[i+1]) - add x27,x27,x16 // h+=Sigma1(e) - eor x28,x28,x21 // Maj(a,b,c) - eor x17,x11,x20,ror#39 // Sigma0(a) - eor x9,x9,x2,lsr#6 // sigma1(X[i+14]) - add x4,x4,x13 - add x23,x23,x27 // d+=h - add x27,x27,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x4,x4,x10 - add x27,x27,x17 // h+=Sigma0(a) - add x4,x4,x9 - ldr x9,[sp,#16] - str x12,[sp,#8] - ror x16,x23,#14 - add x26,x26,x28 // h+=K[i] - ror x11,x6,#1 - and x17,x24,x23 - ror x10,x3,#19 - bic x28,x25,x23 - ror x12,x27,#28 - add x26,x26,x4 // h+=X[i] - eor x16,x16,x23,ror#18 - eor x11,x11,x6,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x27,x20 // a^b, b^c in next round - eor x16,x16,x23,ror#41 // Sigma1(e) - eor x12,x12,x27,ror#34 - add x26,x26,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x10,x10,x3,ror#61 - eor x11,x11,x6,lsr#7 // sigma0(X[i+1]) - add x26,x26,x16 // h+=Sigma1(e) - eor x19,x19,x20 // Maj(a,b,c) - eor x17,x12,x27,ror#39 // Sigma0(a) - eor x10,x10,x3,lsr#6 // sigma1(X[i+14]) - add x5,x5,x14 - add x22,x22,x26 // d+=h - add x26,x26,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x5,x5,x11 - add x26,x26,x17 // h+=Sigma0(a) - add x5,x5,x10 - ldr x10,[sp,#24] - str x13,[sp,#16] - ror x16,x22,#14 - add x25,x25,x19 // h+=K[i] - ror x12,x7,#1 - and x17,x23,x22 - ror x11,x4,#19 - bic x19,x24,x22 - ror x13,x26,#28 - add x25,x25,x5 // h+=X[i] - eor x16,x16,x22,ror#18 - eor x12,x12,x7,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x26,x27 // a^b, b^c in next round - eor x16,x16,x22,ror#41 // Sigma1(e) - eor x13,x13,x26,ror#34 - add x25,x25,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x11,x11,x4,ror#61 - eor x12,x12,x7,lsr#7 // sigma0(X[i+1]) - add x25,x25,x16 // h+=Sigma1(e) - eor x28,x28,x27 // Maj(a,b,c) - eor x17,x13,x26,ror#39 // Sigma0(a) - eor x11,x11,x4,lsr#6 // sigma1(X[i+14]) - add x6,x6,x15 - add x21,x21,x25 // d+=h - add x25,x25,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x6,x6,x12 - add x25,x25,x17 // h+=Sigma0(a) - add x6,x6,x11 - ldr x11,[sp,#0] - str x14,[sp,#24] - ror x16,x21,#14 - add x24,x24,x28 // h+=K[i] - ror x13,x8,#1 - and x17,x22,x21 - ror x12,x5,#19 - bic x28,x23,x21 - ror x14,x25,#28 - add x24,x24,x6 // h+=X[i] - eor x16,x16,x21,ror#18 - eor x13,x13,x8,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x25,x26 // a^b, b^c in next round - eor x16,x16,x21,ror#41 // Sigma1(e) - eor x14,x14,x25,ror#34 - add x24,x24,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x12,x12,x5,ror#61 - eor x13,x13,x8,lsr#7 // sigma0(X[i+1]) - add x24,x24,x16 // h+=Sigma1(e) - eor x19,x19,x26 // Maj(a,b,c) - eor x17,x14,x25,ror#39 // Sigma0(a) - eor x12,x12,x5,lsr#6 // sigma1(X[i+14]) - add x7,x7,x0 - add x20,x20,x24 // d+=h - add x24,x24,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x7,x7,x13 - add x24,x24,x17 // h+=Sigma0(a) - add x7,x7,x12 - ldr x12,[sp,#8] - str x15,[sp,#0] - ror x16,x20,#14 - add x23,x23,x19 // h+=K[i] - ror x14,x9,#1 - and x17,x21,x20 - ror x13,x6,#19 - bic x19,x22,x20 - ror x15,x24,#28 - add x23,x23,x7 // h+=X[i] - eor x16,x16,x20,ror#18 - eor x14,x14,x9,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x24,x25 // a^b, b^c in next round - eor x16,x16,x20,ror#41 // Sigma1(e) - eor x15,x15,x24,ror#34 - add x23,x23,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x13,x13,x6,ror#61 - eor x14,x14,x9,lsr#7 // sigma0(X[i+1]) - add x23,x23,x16 // h+=Sigma1(e) - eor x28,x28,x25 // Maj(a,b,c) - eor x17,x15,x24,ror#39 // Sigma0(a) - eor x13,x13,x6,lsr#6 // sigma1(X[i+14]) - add x8,x8,x1 - add x27,x27,x23 // d+=h - add x23,x23,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x8,x8,x14 - add x23,x23,x17 // h+=Sigma0(a) - add x8,x8,x13 - ldr x13,[sp,#16] - str x0,[sp,#8] - ror x16,x27,#14 - add x22,x22,x28 // h+=K[i] - ror x15,x10,#1 - and x17,x20,x27 - ror x14,x7,#19 - bic x28,x21,x27 - ror x0,x23,#28 - add x22,x22,x8 // h+=X[i] - eor x16,x16,x27,ror#18 - eor x15,x15,x10,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x23,x24 // a^b, b^c in next round - eor x16,x16,x27,ror#41 // Sigma1(e) - eor x0,x0,x23,ror#34 - add x22,x22,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x14,x14,x7,ror#61 - eor x15,x15,x10,lsr#7 // sigma0(X[i+1]) - add x22,x22,x16 // h+=Sigma1(e) - eor x19,x19,x24 // Maj(a,b,c) - eor x17,x0,x23,ror#39 // Sigma0(a) - eor x14,x14,x7,lsr#6 // sigma1(X[i+14]) - add x9,x9,x2 - add x26,x26,x22 // d+=h - add x22,x22,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x9,x9,x15 - add x22,x22,x17 // h+=Sigma0(a) - add x9,x9,x14 - ldr x14,[sp,#24] - str x1,[sp,#16] - ror x16,x26,#14 - add x21,x21,x19 // h+=K[i] - ror x0,x11,#1 - and x17,x27,x26 - ror x15,x8,#19 - bic x19,x20,x26 - ror x1,x22,#28 - add x21,x21,x9 // h+=X[i] - eor x16,x16,x26,ror#18 - eor x0,x0,x11,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x22,x23 // a^b, b^c in next round - eor x16,x16,x26,ror#41 // Sigma1(e) - eor x1,x1,x22,ror#34 - add x21,x21,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x15,x15,x8,ror#61 - eor x0,x0,x11,lsr#7 // sigma0(X[i+1]) - add x21,x21,x16 // h+=Sigma1(e) - eor x28,x28,x23 // Maj(a,b,c) - eor x17,x1,x22,ror#39 // Sigma0(a) - eor x15,x15,x8,lsr#6 // sigma1(X[i+14]) - add x10,x10,x3 - add x25,x25,x21 // d+=h - add x21,x21,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x10,x10,x0 - add x21,x21,x17 // h+=Sigma0(a) - add x10,x10,x15 - ldr x15,[sp,#0] - str x2,[sp,#24] - ror x16,x25,#14 - add x20,x20,x28 // h+=K[i] - ror x1,x12,#1 - and x17,x26,x25 - ror x0,x9,#19 - bic x28,x27,x25 - ror x2,x21,#28 - add x20,x20,x10 // h+=X[i] - eor x16,x16,x25,ror#18 - eor x1,x1,x12,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x21,x22 // a^b, b^c in next round - eor x16,x16,x25,ror#41 // Sigma1(e) - eor x2,x2,x21,ror#34 - add x20,x20,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x0,x0,x9,ror#61 - eor x1,x1,x12,lsr#7 // sigma0(X[i+1]) - add x20,x20,x16 // h+=Sigma1(e) - eor x19,x19,x22 // Maj(a,b,c) - eor x17,x2,x21,ror#39 // Sigma0(a) - eor x0,x0,x9,lsr#6 // sigma1(X[i+14]) - add x11,x11,x4 - add x24,x24,x20 // d+=h - add x20,x20,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x11,x11,x1 - add x20,x20,x17 // h+=Sigma0(a) - add x11,x11,x0 - ldr x0,[sp,#8] - str x3,[sp,#0] - ror x16,x24,#14 - add x27,x27,x19 // h+=K[i] - ror x2,x13,#1 - and x17,x25,x24 - ror x1,x10,#19 - bic x19,x26,x24 - ror x3,x20,#28 - add x27,x27,x11 // h+=X[i] - eor x16,x16,x24,ror#18 - eor x2,x2,x13,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x20,x21 // a^b, b^c in next round - eor x16,x16,x24,ror#41 // Sigma1(e) - eor x3,x3,x20,ror#34 - add x27,x27,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x1,x1,x10,ror#61 - eor x2,x2,x13,lsr#7 // sigma0(X[i+1]) - add x27,x27,x16 // h+=Sigma1(e) - eor x28,x28,x21 // Maj(a,b,c) - eor x17,x3,x20,ror#39 // Sigma0(a) - eor x1,x1,x10,lsr#6 // sigma1(X[i+14]) - add x12,x12,x5 - add x23,x23,x27 // d+=h - add x27,x27,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x12,x12,x2 - add x27,x27,x17 // h+=Sigma0(a) - add x12,x12,x1 - ldr x1,[sp,#16] - str x4,[sp,#8] - ror x16,x23,#14 - add x26,x26,x28 // h+=K[i] - ror x3,x14,#1 - and x17,x24,x23 - ror x2,x11,#19 - bic x28,x25,x23 - ror x4,x27,#28 - add x26,x26,x12 // h+=X[i] - eor x16,x16,x23,ror#18 - eor x3,x3,x14,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x27,x20 // a^b, b^c in next round - eor x16,x16,x23,ror#41 // Sigma1(e) - eor x4,x4,x27,ror#34 - add x26,x26,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x2,x2,x11,ror#61 - eor x3,x3,x14,lsr#7 // sigma0(X[i+1]) - add x26,x26,x16 // h+=Sigma1(e) - eor x19,x19,x20 // Maj(a,b,c) - eor x17,x4,x27,ror#39 // Sigma0(a) - eor x2,x2,x11,lsr#6 // sigma1(X[i+14]) - add x13,x13,x6 - add x22,x22,x26 // d+=h - add x26,x26,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x13,x13,x3 - add x26,x26,x17 // h+=Sigma0(a) - add x13,x13,x2 - ldr x2,[sp,#24] - str x5,[sp,#16] - ror x16,x22,#14 - add x25,x25,x19 // h+=K[i] - ror x4,x15,#1 - and x17,x23,x22 - ror x3,x12,#19 - bic x19,x24,x22 - ror x5,x26,#28 - add x25,x25,x13 // h+=X[i] - eor x16,x16,x22,ror#18 - eor x4,x4,x15,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x26,x27 // a^b, b^c in next round - eor x16,x16,x22,ror#41 // Sigma1(e) - eor x5,x5,x26,ror#34 - add x25,x25,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x3,x3,x12,ror#61 - eor x4,x4,x15,lsr#7 // sigma0(X[i+1]) - add x25,x25,x16 // h+=Sigma1(e) - eor x28,x28,x27 // Maj(a,b,c) - eor x17,x5,x26,ror#39 // Sigma0(a) - eor x3,x3,x12,lsr#6 // sigma1(X[i+14]) - add x14,x14,x7 - add x21,x21,x25 // d+=h - add x25,x25,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x14,x14,x4 - add x25,x25,x17 // h+=Sigma0(a) - add x14,x14,x3 - ldr x3,[sp,#0] - str x6,[sp,#24] - ror x16,x21,#14 - add x24,x24,x28 // h+=K[i] - ror x5,x0,#1 - and x17,x22,x21 - ror x4,x13,#19 - bic x28,x23,x21 - ror x6,x25,#28 - add x24,x24,x14 // h+=X[i] - eor x16,x16,x21,ror#18 - eor x5,x5,x0,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x25,x26 // a^b, b^c in next round - eor x16,x16,x21,ror#41 // Sigma1(e) - eor x6,x6,x25,ror#34 - add x24,x24,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x4,x4,x13,ror#61 - eor x5,x5,x0,lsr#7 // sigma0(X[i+1]) - add x24,x24,x16 // h+=Sigma1(e) - eor x19,x19,x26 // Maj(a,b,c) - eor x17,x6,x25,ror#39 // Sigma0(a) - eor x4,x4,x13,lsr#6 // sigma1(X[i+14]) - add x15,x15,x8 - add x20,x20,x24 // d+=h - add x24,x24,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x15,x15,x5 - add x24,x24,x17 // h+=Sigma0(a) - add x15,x15,x4 - ldr x4,[sp,#8] - str x7,[sp,#0] - ror x16,x20,#14 - add x23,x23,x19 // h+=K[i] - ror x6,x1,#1 - and x17,x21,x20 - ror x5,x14,#19 - bic x19,x22,x20 - ror x7,x24,#28 - add x23,x23,x15 // h+=X[i] - eor x16,x16,x20,ror#18 - eor x6,x6,x1,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x24,x25 // a^b, b^c in next round - eor x16,x16,x20,ror#41 // Sigma1(e) - eor x7,x7,x24,ror#34 - add x23,x23,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x5,x5,x14,ror#61 - eor x6,x6,x1,lsr#7 // sigma0(X[i+1]) - add x23,x23,x16 // h+=Sigma1(e) - eor x28,x28,x25 // Maj(a,b,c) - eor x17,x7,x24,ror#39 // Sigma0(a) - eor x5,x5,x14,lsr#6 // sigma1(X[i+14]) - add x0,x0,x9 - add x27,x27,x23 // d+=h - add x23,x23,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x0,x0,x6 - add x23,x23,x17 // h+=Sigma0(a) - add x0,x0,x5 - ldr x5,[sp,#16] - str x8,[sp,#8] - ror x16,x27,#14 - add x22,x22,x28 // h+=K[i] - ror x7,x2,#1 - and x17,x20,x27 - ror x6,x15,#19 - bic x28,x21,x27 - ror x8,x23,#28 - add x22,x22,x0 // h+=X[i] - eor x16,x16,x27,ror#18 - eor x7,x7,x2,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x23,x24 // a^b, b^c in next round - eor x16,x16,x27,ror#41 // Sigma1(e) - eor x8,x8,x23,ror#34 - add x22,x22,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x6,x6,x15,ror#61 - eor x7,x7,x2,lsr#7 // sigma0(X[i+1]) - add x22,x22,x16 // h+=Sigma1(e) - eor x19,x19,x24 // Maj(a,b,c) - eor x17,x8,x23,ror#39 // Sigma0(a) - eor x6,x6,x15,lsr#6 // sigma1(X[i+14]) - add x1,x1,x10 - add x26,x26,x22 // d+=h - add x22,x22,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x1,x1,x7 - add x22,x22,x17 // h+=Sigma0(a) - add x1,x1,x6 - ldr x6,[sp,#24] - str x9,[sp,#16] - ror x16,x26,#14 - add x21,x21,x19 // h+=K[i] - ror x8,x3,#1 - and x17,x27,x26 - ror x7,x0,#19 - bic x19,x20,x26 - ror x9,x22,#28 - add x21,x21,x1 // h+=X[i] - eor x16,x16,x26,ror#18 - eor x8,x8,x3,ror#8 - orr x17,x17,x19 // Ch(e,f,g) - eor x19,x22,x23 // a^b, b^c in next round - eor x16,x16,x26,ror#41 // Sigma1(e) - eor x9,x9,x22,ror#34 - add x21,x21,x17 // h+=Ch(e,f,g) - and x28,x28,x19 // (b^c)&=(a^b) - eor x7,x7,x0,ror#61 - eor x8,x8,x3,lsr#7 // sigma0(X[i+1]) - add x21,x21,x16 // h+=Sigma1(e) - eor x28,x28,x23 // Maj(a,b,c) - eor x17,x9,x22,ror#39 // Sigma0(a) - eor x7,x7,x0,lsr#6 // sigma1(X[i+14]) - add x2,x2,x11 - add x25,x25,x21 // d+=h - add x21,x21,x28 // h+=Maj(a,b,c) - ldr x28,[x30],#8 // *K++, x19 in next round - add x2,x2,x8 - add x21,x21,x17 // h+=Sigma0(a) - add x2,x2,x7 - ldr x7,[sp,#0] - str x10,[sp,#24] - ror x16,x25,#14 - add x20,x20,x28 // h+=K[i] - ror x9,x4,#1 - and x17,x26,x25 - ror x8,x1,#19 - bic x28,x27,x25 - ror x10,x21,#28 - add x20,x20,x2 // h+=X[i] - eor x16,x16,x25,ror#18 - eor x9,x9,x4,ror#8 - orr x17,x17,x28 // Ch(e,f,g) - eor x28,x21,x22 // a^b, b^c in next round - eor x16,x16,x25,ror#41 // Sigma1(e) - eor x10,x10,x21,ror#34 - add x20,x20,x17 // h+=Ch(e,f,g) - and x19,x19,x28 // (b^c)&=(a^b) - eor x8,x8,x1,ror#61 - eor x9,x9,x4,lsr#7 // sigma0(X[i+1]) - add x20,x20,x16 // h+=Sigma1(e) - eor x19,x19,x22 // Maj(a,b,c) - eor x17,x10,x21,ror#39 // Sigma0(a) - eor x8,x8,x1,lsr#6 // sigma1(X[i+14]) - add x3,x3,x12 - add x24,x24,x20 // d+=h - add x20,x20,x19 // h+=Maj(a,b,c) - ldr x19,[x30],#8 // *K++, x28 in next round - add x3,x3,x9 - add x20,x20,x17 // h+=Sigma0(a) - add x3,x3,x8 - cbnz x19,.Loop_16_xx - - ldp x0,x2,[x29,#96] - ldr x1,[x29,#112] - sub x30,x30,#648 // rewind - - ldp x3,x4,[x0] - ldp x5,x6,[x0,#2*8] - add x1,x1,#14*8 // advance input pointer - ldp x7,x8,[x0,#4*8] - add x20,x20,x3 - ldp x9,x10,[x0,#6*8] - add x21,x21,x4 - add x22,x22,x5 - add x23,x23,x6 - stp x20,x21,[x0] - add x24,x24,x7 - add x25,x25,x8 - stp x22,x23,[x0,#2*8] - add x26,x26,x9 - add x27,x27,x10 - cmp x1,x2 - stp x24,x25,[x0,#4*8] - stp x26,x27,[x0,#6*8] - b.ne .Loop - - ldp x19,x20,[x29,#16] - add sp,sp,#4*8 - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldp x29,x30,[sp],#128 - ret -.size sha512_block_data_order,.-sha512_block_data_order - -.align 6 -.type .LK512,%object -.LK512: - .quad 0x428a2f98d728ae22,0x7137449123ef65cd - .quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc - .quad 0x3956c25bf348b538,0x59f111f1b605d019 - .quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118 - .quad 0xd807aa98a3030242,0x12835b0145706fbe - .quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2 - .quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1 - .quad 0x9bdc06a725c71235,0xc19bf174cf692694 - .quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3 - .quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65 - .quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483 - .quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5 - .quad 0x983e5152ee66dfab,0xa831c66d2db43210 - .quad 0xb00327c898fb213f,0xbf597fc7beef0ee4 - .quad 0xc6e00bf33da88fc2,0xd5a79147930aa725 - .quad 0x06ca6351e003826f,0x142929670a0e6e70 - .quad 0x27b70a8546d22ffc,0x2e1b21385c26c926 - .quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df - .quad 0x650a73548baf63de,0x766a0abb3c77b2a8 - .quad 0x81c2c92e47edaee6,0x92722c851482353b - .quad 0xa2bfe8a14cf10364,0xa81a664bbc423001 - .quad 0xc24b8b70d0f89791,0xc76c51a30654be30 - .quad 0xd192e819d6ef5218,0xd69906245565a910 - .quad 0xf40e35855771202a,0x106aa07032bbd1b8 - .quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53 - .quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8 - .quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb - .quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3 - .quad 0x748f82ee5defb2fc,0x78a5636f43172f60 - .quad 0x84c87814a1f0ab72,0x8cc702081a6439ec - .quad 0x90befffa23631e28,0xa4506cebde82bde9 - .quad 0xbef9a3f7b2c67915,0xc67178f2e372532b - .quad 0xca273eceea26619c,0xd186b8c721c0c207 - .quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178 - .quad 0x06f067aa72176fba,0x0a637dc5a2c898a6 - .quad 0x113f9804bef90dae,0x1b710b35131c471b - .quad 0x28db77f523047d84,0x32caab7b40c72493 - .quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c - .quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a - .quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817 - .quad 0 // terminator -.size .LK512,.-.LK512 -#ifndef __KERNEL__ -.align 3 -.LOPENSSL_armcap_P: -# ifdef __ILP32__ - .long OPENSSL_armcap_P-. -# else - .quad OPENSSL_armcap_P-. -# endif -#endif -.asciz "SHA512 block transform for ARMv8, CRYPTOGAMS by " -.align 2 -#ifndef __KERNEL__ -.comm OPENSSL_armcap_P,4,4 -#endif diff --git a/arch/arm64/include/asm/alternative-macros.h b/arch/arm64/include/asm/alternative-macros.h index 8a078fc662ac..7e157ab6cd50 100644 --- a/arch/arm64/include/asm/alternative-macros.h +++ b/arch/arm64/include/asm/alternative-macros.h @@ -3,12 +3,10 @@ #define __ASM_ALTERNATIVE_MACROS_H #include +#include #define ARM64_CB_PATCH ARM64_NCAPS -/* A64 instructions are always 32 bits. */ -#define AARCH64_INSN_SIZE 4 - #ifndef __ASSEMBLY__ #include @@ -197,11 +195,6 @@ alternative_endif #define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...) \ alternative_insn insn1, insn2, cap, IS_ENABLED(cfg) -.macro user_alt, label, oldinstr, newinstr, cond -9999: alternative_insn "\oldinstr", "\newinstr", \cond - _asm_extable 9999b, \label -.endm - #endif /* __ASSEMBLY__ */ /* diff --git a/arch/arm64/include/asm/arch_gicv3.h b/arch/arm64/include/asm/arch_gicv3.h index 934b9be582d2..4ad22c3135db 100644 --- a/arch/arm64/include/asm/arch_gicv3.h +++ b/arch/arm64/include/asm/arch_gicv3.h @@ -124,7 +124,8 @@ static inline u32 gic_read_rpr(void) #define gic_read_lpir(c) readq_relaxed(c) #define gic_write_lpir(v, c) writeq_relaxed(v, c) -#define gic_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l)) +#define gic_flush_dcache_to_poc(a,l) \ + dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l)) #define gits_read_baser(c) readq_relaxed(c) #define gits_write_baser(v, c) writeq_relaxed(v, c) diff --git a/arch/arm64/include/asm/asm-prototypes.h b/arch/arm64/include/asm/asm-prototypes.h index 1c9a3a0c5fa5..ec1d9655f885 100644 --- a/arch/arm64/include/asm/asm-prototypes.h +++ b/arch/arm64/include/asm/asm-prototypes.h @@ -23,4 +23,10 @@ long long __ashlti3(long long a, int b); long long __ashrti3(long long a, int b); long long __lshrti3(long long a, int b); +/* + * This function uses a custom calling convention and cannot be called from C so + * this prototype is not entirely accurate. + */ +void __hwasan_tag_mismatch(unsigned long addr, unsigned long access_info); + #endif /* __ASM_PROTOTYPES_H */ diff --git a/arch/arm64/include/asm/asm_pointer_auth.h b/arch/arm64/include/asm/asm_pointer_auth.h index 8ca2dc0661ee..f1bba5fc61c4 100644 --- a/arch/arm64/include/asm/asm_pointer_auth.h +++ b/arch/arm64/include/asm/asm_pointer_auth.h @@ -7,19 +7,7 @@ #include #include -#ifdef CONFIG_ARM64_PTR_AUTH -/* - * thread.keys_user.ap* as offset exceeds the #imm offset range - * so use the base value of ldp as thread.keys_user and offset as - * thread.keys_user.ap*. - */ - .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 - mov \tmp1, #THREAD_KEYS_USER - add \tmp1, \tsk, \tmp1 - ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA] - msr_s SYS_APIAKEYLO_EL1, \tmp2 - msr_s SYS_APIAKEYHI_EL1, \tmp3 - .endm +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 mov \tmp1, #THREAD_KEYS_KERNEL @@ -42,6 +30,33 @@ alternative_if ARM64_HAS_ADDRESS_AUTH alternative_else_nop_endif .endm +#else /* CONFIG_ARM64_PTR_AUTH_KERNEL */ + + .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 + .endm + + .macro ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 + .endm + + .macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3 + .endm + +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ + +#ifdef CONFIG_ARM64_PTR_AUTH +/* + * thread.keys_user.ap* as offset exceeds the #imm offset range + * so use the base value of ldp as thread.keys_user and offset as + * thread.keys_user.ap*. + */ + .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 + mov \tmp1, #THREAD_KEYS_USER + add \tmp1, \tsk, \tmp1 + ldp \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA] + msr_s SYS_APIAKEYLO_EL1, \tmp2 + msr_s SYS_APIAKEYHI_EL1, \tmp3 + .endm + .macro __ptrauth_keys_init_cpu tsk, tmp1, tmp2, tmp3 mrs \tmp1, id_aa64isar1_el1 ubfx \tmp1, \tmp1, #ID_AA64ISAR1_APA_SHIFT, #8 @@ -64,17 +79,11 @@ alternative_else_nop_endif .Lno_addr_auth\@: .endm -#else /* CONFIG_ARM64_PTR_AUTH */ +#else /* !CONFIG_ARM64_PTR_AUTH */ .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3 .endm - .macro ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3 - .endm - - .macro ptrauth_keys_install_kernel tsk, tmp1, tmp2, tmp3 - .endm - #endif /* CONFIG_ARM64_PTR_AUTH */ #endif /* __ASM_ASM_POINTER_AUTH_H */ diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h index 8418c1bd8f04..89faca0e740d 100644 --- a/arch/arm64/include/asm/assembler.h +++ b/arch/arm64/include/asm/assembler.h @@ -130,15 +130,27 @@ alternative_endif .endm /* - * Emit an entry into the exception table + * Create an exception table entry for `insn`, which will branch to `fixup` + * when an unhandled fault is taken. */ - .macro _asm_extable, from, to + .macro _asm_extable, insn, fixup .pushsection __ex_table, "a" .align 3 - .long (\from - .), (\to - .) + .long (\insn - .), (\fixup - .) .popsection .endm +/* + * Create an exception table entry for `insn` if `fixup` is provided. Otherwise + * do nothing. + */ + .macro _cond_extable, insn, fixup + .ifnc \fixup, + _asm_extable \insn, \fixup + .endif + .endm + + #define USER(l, x...) \ 9999: x; \ _asm_extable 9999b, l @@ -232,15 +244,23 @@ lr .req x30 // link register * @dst: destination register */ #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) - .macro this_cpu_offset, dst + .macro get_this_cpu_offset, dst mrs \dst, tpidr_el2 .endm #else - .macro this_cpu_offset, dst + .macro get_this_cpu_offset, dst alternative_if_not ARM64_HAS_VIRT_HOST_EXTN mrs \dst, tpidr_el1 alternative_else mrs \dst, tpidr_el2 +alternative_endif + .endm + + .macro set_this_cpu_offset, src +alternative_if_not ARM64_HAS_VIRT_HOST_EXTN + msr tpidr_el1, \src +alternative_else + msr tpidr_el2, \src alternative_endif .endm #endif @@ -253,7 +273,7 @@ alternative_endif .macro adr_this_cpu, dst, sym, tmp adrp \tmp, \sym add \dst, \tmp, #:lo12:\sym - this_cpu_offset \tmp + get_this_cpu_offset \tmp add \dst, \dst, \tmp .endm @@ -264,7 +284,7 @@ alternative_endif */ .macro ldr_this_cpu dst, sym, tmp adr_l \dst, \sym - this_cpu_offset \tmp + get_this_cpu_offset \tmp ldr \dst, [\dst, \tmp] .endm @@ -375,51 +395,53 @@ alternative_cb_end bfi \tcr, \tmp0, \pos, #3 .endm -/* - * Macro to perform a data cache maintenance for the interval - * [kaddr, kaddr + size) - * - * op: operation passed to dc instruction - * domain: domain used in dsb instruciton - * kaddr: starting virtual address of the region - * size: size of the region - * Corrupts: kaddr, size, tmp1, tmp2 - */ - .macro __dcache_op_workaround_clean_cache, op, kaddr + .macro __dcache_op_workaround_clean_cache, op, addr alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE - dc \op, \kaddr + dc \op, \addr alternative_else - dc civac, \kaddr + dc civac, \addr alternative_endif .endm - .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 +/* + * Macro to perform a data cache maintenance for the interval + * [start, end) + * + * op: operation passed to dc instruction + * domain: domain used in dsb instruciton + * start: starting virtual address of the region + * end: end virtual address of the region + * fixup: optional label to branch to on user fault + * Corrupts: start, end, tmp1, tmp2 + */ + .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup dcache_line_size \tmp1, \tmp2 - add \size, \kaddr, \size sub \tmp2, \tmp1, #1 - bic \kaddr, \kaddr, \tmp2 -9998: + bic \start, \start, \tmp2 +.Ldcache_op\@: .ifc \op, cvau - __dcache_op_workaround_clean_cache \op, \kaddr + __dcache_op_workaround_clean_cache \op, \start .else .ifc \op, cvac - __dcache_op_workaround_clean_cache \op, \kaddr + __dcache_op_workaround_clean_cache \op, \start .else .ifc \op, cvap - sys 3, c7, c12, 1, \kaddr // dc cvap + sys 3, c7, c12, 1, \start // dc cvap .else .ifc \op, cvadp - sys 3, c7, c13, 1, \kaddr // dc cvadp + sys 3, c7, c13, 1, \start // dc cvadp .else - dc \op, \kaddr + dc \op, \start .endif .endif .endif .endif - add \kaddr, \kaddr, \tmp1 - cmp \kaddr, \size - b.lo 9998b + add \start, \start, \tmp1 + cmp \start, \end + b.lo .Ldcache_op\@ dsb \domain + + _cond_extable .Ldcache_op\@, \fixup .endm /* @@ -427,20 +449,22 @@ alternative_endif * [start, end) * * start, end: virtual addresses describing the region - * label: A label to branch to on user fault. + * fixup: optional label to branch to on user fault * Corrupts: tmp1, tmp2 */ - .macro invalidate_icache_by_line start, end, tmp1, tmp2, label + .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup icache_line_size \tmp1, \tmp2 sub \tmp2, \tmp1, #1 bic \tmp2, \start, \tmp2 -9997: -USER(\label, ic ivau, \tmp2) // invalidate I line PoU +.Licache_op\@: + ic ivau, \tmp2 // invalidate I line PoU add \tmp2, \tmp2, \tmp1 cmp \tmp2, \end - b.lo 9997b + b.lo .Licache_op\@ dsb ish isb + + _cond_extable .Licache_op\@, \fixup .endm /* @@ -745,7 +769,7 @@ USER(\label, ic ivau, \tmp2) // invalidate I line PoU cbz \tmp, \lbl #endif adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING - this_cpu_offset \tmp2 + get_this_cpu_offset \tmp2 ldr w\tmp, [\tmp, \tmp2] cbnz w\tmp, \lbl // yield on pending softirq in task context .Lnoyield_\@: diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index a074459f8f2f..a9c0716e7440 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -47,7 +47,7 @@ * cache before the transfer is done, causing old data to be seen by * the CPU. */ -#define ARCH_DMA_MINALIGN (128) +#define ARCH_DMA_MINALIGN L1_CACHE_BYTES #ifdef CONFIG_KASAN_SW_TAGS #define ARCH_SLAB_MINALIGN (1ULL << KASAN_SHADOW_SCALE_SHIFT) diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 52e5c1623224..543c997eb3b7 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -30,45 +30,58 @@ * the implementation assumes non-aliasing VIPT D-cache and (aliasing) * VIPT I-cache. * - * flush_icache_range(start, end) + * All functions below apply to the interval [start, end) + * - start - virtual start address (inclusive) + * - end - virtual end address (exclusive) * - * Ensure coherency between the I-cache and the D-cache in the - * region described by start, end. - * - start - virtual start address - * - end - virtual end address + * caches_clean_inval_pou(start, end) * - * invalidate_icache_range(start, end) + * Ensure coherency between the I-cache and the D-cache region to + * the Point of Unification. * - * Invalidate the I-cache in the region described by start, end. - * - start - virtual start address - * - end - virtual end address + * caches_clean_inval_user_pou(start, end) * - * __flush_cache_user_range(start, end) + * Ensure coherency between the I-cache and the D-cache region to + * the Point of Unification. + * Use only if the region might access user memory. * - * Ensure coherency between the I-cache and the D-cache in the - * region described by start, end. - * - start - virtual start address - * - end - virtual end address + * icache_inval_pou(start, end) * - * __flush_dcache_area(kaddr, size) + * Invalidate I-cache region to the Point of Unification. * - * Ensure that the data held in page is written back. - * - kaddr - page address - * - size - region size + * dcache_clean_inval_poc(start, end) + * + * Clean and invalidate D-cache region to the Point of Coherency. + * + * dcache_inval_poc(start, end) + * + * Invalidate D-cache region to the Point of Coherency. + * + * dcache_clean_poc(start, end) + * + * Clean D-cache region to the Point of Coherency. + * + * dcache_clean_pop(start, end) + * + * Clean D-cache region to the Point of Persistence. + * + * dcache_clean_pou(start, end) + * + * Clean D-cache region to the Point of Unification. */ -extern void __flush_icache_range(unsigned long start, unsigned long end); -extern int invalidate_icache_range(unsigned long start, unsigned long end); -extern void __flush_dcache_area(void *addr, size_t len); -extern void __inval_dcache_area(void *addr, size_t len); -extern void __clean_dcache_area_poc(void *addr, size_t len); -extern void __clean_dcache_area_pop(void *addr, size_t len); -extern void __clean_dcache_area_pou(void *addr, size_t len); -extern long __flush_cache_user_range(unsigned long start, unsigned long end); -extern void sync_icache_aliases(void *kaddr, unsigned long len); +extern void caches_clean_inval_pou(unsigned long start, unsigned long end); +extern void icache_inval_pou(unsigned long start, unsigned long end); +extern void dcache_clean_inval_poc(unsigned long start, unsigned long end); +extern void dcache_inval_poc(unsigned long start, unsigned long end); +extern void dcache_clean_poc(unsigned long start, unsigned long end); +extern void dcache_clean_pop(unsigned long start, unsigned long end); +extern void dcache_clean_pou(unsigned long start, unsigned long end); +extern long caches_clean_inval_user_pou(unsigned long start, unsigned long end); +extern void sync_icache_aliases(unsigned long start, unsigned long end); static inline void flush_icache_range(unsigned long start, unsigned long end) { - __flush_icache_range(start, end); + caches_clean_inval_pou(start, end); /* * IPI all online CPUs so that they undergo a context synchronization @@ -122,7 +135,7 @@ extern void copy_to_user_page(struct vm_area_struct *, struct page *, #define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE 1 extern void flush_dcache_page(struct page *); -static __always_inline void __flush_icache_all(void) +static __always_inline void icache_inval_all_pou(void) { if (cpus_have_const_cap(ARM64_HAS_CACHE_DIC)) return; diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index 7faae6ff3ab4..0f6d16faa540 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -12,26 +12,7 @@ /* * Records attributes of an individual CPU. */ -struct cpuinfo_arm64 { - struct cpu cpu; - struct kobject kobj; - u32 reg_ctr; - u32 reg_cntfrq; - u32 reg_dczid; - u32 reg_midr; - u32 reg_revidr; - - u64 reg_id_aa64dfr0; - u64 reg_id_aa64dfr1; - u64 reg_id_aa64isar0; - u64 reg_id_aa64isar1; - u64 reg_id_aa64mmfr0; - u64 reg_id_aa64mmfr1; - u64 reg_id_aa64mmfr2; - u64 reg_id_aa64pfr0; - u64 reg_id_aa64pfr1; - u64 reg_id_aa64zfr0; - +struct cpuinfo_32bit { u32 reg_id_dfr0; u32 reg_id_dfr1; u32 reg_id_isar0; @@ -54,6 +35,30 @@ struct cpuinfo_arm64 { u32 reg_mvfr0; u32 reg_mvfr1; u32 reg_mvfr2; +}; + +struct cpuinfo_arm64 { + struct cpu cpu; + struct kobject kobj; + u64 reg_ctr; + u64 reg_cntfrq; + u64 reg_dczid; + u64 reg_midr; + u64 reg_revidr; + u64 reg_gmid; + + u64 reg_id_aa64dfr0; + u64 reg_id_aa64dfr1; + u64 reg_id_aa64isar0; + u64 reg_id_aa64isar1; + u64 reg_id_aa64mmfr0; + u64 reg_id_aa64mmfr1; + u64 reg_id_aa64mmfr2; + u64 reg_id_aa64pfr0; + u64 reg_id_aa64pfr1; + u64 reg_id_aa64zfr0; + + struct cpuinfo_32bit aarch32; /* pseudo-ZCR for recording maximum ZCR_EL1 LEN value: */ u64 reg_zcr; diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 338840c00e8e..9bb9d11750d7 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -619,6 +619,13 @@ static inline bool id_aa64pfr0_sve(u64 pfr0) return val > 0; } +static inline bool id_aa64pfr1_mte(u64 pfr1) +{ + u32 val = cpuid_feature_extract_unsigned_field(pfr1, ID_AA64PFR1_MTE_SHIFT); + + return val >= ID_AA64PFR1_MTE; +} + void __init setup_cpu_features(void); void check_local_cpu_capabilities(void); @@ -630,9 +637,15 @@ static inline bool cpu_supports_mixed_endian_el0(void) return id_aa64mmfr0_mixed_endian_el0(read_cpuid(ID_AA64MMFR0_EL1)); } +const struct cpumask *system_32bit_el0_cpumask(void); +DECLARE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); + static inline bool system_supports_32bit_el0(void) { - return cpus_have_const_cap(ARM64_HAS_32BIT_EL0); + u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); + + return static_branch_unlikely(&arm64_mismatched_32bit_el0) || + id_aa64pfr0_32bit_el0(pfr0); } static inline bool system_supports_4kb_granule(void) diff --git a/arch/arm64/include/asm/cpuidle.h b/arch/arm64/include/asm/cpuidle.h index 3c5ddb429ea2..14a19d1141bd 100644 --- a/arch/arm64/include/asm/cpuidle.h +++ b/arch/arm64/include/asm/cpuidle.h @@ -18,4 +18,39 @@ static inline int arm_cpuidle_suspend(int index) return -EOPNOTSUPP; } #endif + +#ifdef CONFIG_ARM64_PSEUDO_NMI +#include + +struct arm_cpuidle_irq_context { + unsigned long pmr; + unsigned long daif_bits; +}; + +#define arm_cpuidle_save_irq_context(__c) \ + do { \ + struct arm_cpuidle_irq_context *c = __c; \ + if (system_uses_irq_prio_masking()) { \ + c->daif_bits = read_sysreg(daif); \ + write_sysreg(c->daif_bits | PSR_I_BIT | PSR_F_BIT, \ + daif); \ + c->pmr = gic_read_pmr(); \ + gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); \ + } \ + } while (0) + +#define arm_cpuidle_restore_irq_context(__c) \ + do { \ + struct arm_cpuidle_irq_context *c = __c; \ + if (system_uses_irq_prio_masking()) { \ + gic_write_pmr(c->pmr); \ + write_sysreg(c->daif_bits, daif); \ + } \ + } while (0) +#else +struct arm_cpuidle_irq_context { }; + +#define arm_cpuidle_save_irq_context(c) (void)c +#define arm_cpuidle_restore_irq_context(c) (void)c +#endif #endif diff --git a/arch/arm64/include/asm/efi.h b/arch/arm64/include/asm/efi.h index 3578aba9c608..1bed37eb013a 100644 --- a/arch/arm64/include/asm/efi.h +++ b/arch/arm64/include/asm/efi.h @@ -137,7 +137,7 @@ void efi_virtmap_unload(void); static inline void efi_capsule_flush_cache_range(void *addr, int size) { - __flush_dcache_area(addr, size); + dcache_clean_inval_poc((unsigned long)addr, (unsigned long)addr + size); } #endif /* _ASM_EFI_H */ diff --git a/arch/arm64/include/asm/exception.h b/arch/arm64/include/asm/exception.h index 6546158d2f2d..4afbc45b8bb0 100644 --- a/arch/arm64/include/asm/exception.h +++ b/arch/arm64/include/asm/exception.h @@ -31,20 +31,35 @@ static inline u32 disr_to_esr(u64 disr) return esr; } -asmlinkage void el1_sync_handler(struct pt_regs *regs); -asmlinkage void el0_sync_handler(struct pt_regs *regs); -asmlinkage void el0_sync_compat_handler(struct pt_regs *regs); +asmlinkage void handle_bad_stack(struct pt_regs *regs); -asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs); -asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs); +asmlinkage void el1t_64_sync_handler(struct pt_regs *regs); +asmlinkage void el1t_64_irq_handler(struct pt_regs *regs); +asmlinkage void el1t_64_fiq_handler(struct pt_regs *regs); +asmlinkage void el1t_64_error_handler(struct pt_regs *regs); + +asmlinkage void el1h_64_sync_handler(struct pt_regs *regs); +asmlinkage void el1h_64_irq_handler(struct pt_regs *regs); +asmlinkage void el1h_64_fiq_handler(struct pt_regs *regs); +asmlinkage void el1h_64_error_handler(struct pt_regs *regs); + +asmlinkage void el0t_64_sync_handler(struct pt_regs *regs); +asmlinkage void el0t_64_irq_handler(struct pt_regs *regs); +asmlinkage void el0t_64_fiq_handler(struct pt_regs *regs); +asmlinkage void el0t_64_error_handler(struct pt_regs *regs); + +asmlinkage void el0t_32_sync_handler(struct pt_regs *regs); +asmlinkage void el0t_32_irq_handler(struct pt_regs *regs); +asmlinkage void el0t_32_fiq_handler(struct pt_regs *regs); +asmlinkage void el0t_32_error_handler(struct pt_regs *regs); + +asmlinkage void call_on_irq_stack(struct pt_regs *regs, + void (*func)(struct pt_regs *)); asmlinkage void enter_from_user_mode(void); asmlinkage void exit_to_user_mode(void); -void arm64_enter_nmi(struct pt_regs *regs); -void arm64_exit_nmi(struct pt_regs *regs); void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs); void do_undefinstr(struct pt_regs *regs); void do_bti(struct pt_regs *regs); -asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr); void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, struct pt_regs *regs); void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs); @@ -57,4 +72,7 @@ void do_cp15instr(unsigned int esr, struct pt_regs *regs); void do_el0_svc(struct pt_regs *regs); void do_el0_svc_compat(struct pt_regs *regs); void do_ptrauth_fault(struct pt_regs *regs, unsigned int esr); +void do_serror(struct pt_regs *regs, unsigned int esr); + +void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far); #endif /* __ASM_EXCEPTION_H */ diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h index 2599504674b5..c072161d5c65 100644 --- a/arch/arm64/include/asm/fpsimd.h +++ b/arch/arm64/include/asm/fpsimd.h @@ -69,7 +69,7 @@ static inline void *sve_pffr(struct thread_struct *thread) extern void sve_save_state(void *state, u32 *pfpsr); extern void sve_load_state(void const *state, u32 const *pfpsr, unsigned long vq_minus_1); -extern void sve_flush_live(void); +extern void sve_flush_live(unsigned long vq_minus_1); extern void sve_load_from_fpsimd_state(struct user_fpsimd_state const *state, unsigned long vq_minus_1); extern unsigned int sve_get_vl(void); diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h index a2563992d2dc..059204477ce6 100644 --- a/arch/arm64/include/asm/fpsimdmacros.h +++ b/arch/arm64/include/asm/fpsimdmacros.h @@ -213,8 +213,10 @@ mov v\nz\().16b, v\nz\().16b .endm -.macro sve_flush +.macro sve_flush_z _for n, 0, 31, _sve_flush_z \n +.endm +.macro sve_flush_p_ffr _for n, 0, 15, _sve_pfalse \n _sve_wrffr 0 .endm diff --git a/arch/arm64/include/asm/insn-def.h b/arch/arm64/include/asm/insn-def.h new file mode 100644 index 000000000000..2c075f615c6a --- /dev/null +++ b/arch/arm64/include/asm/insn-def.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +#ifndef __ASM_INSN_DEF_H +#define __ASM_INSN_DEF_H + +/* A64 instructions are always 32 bits. */ +#define AARCH64_INSN_SIZE 4 + +#endif /* __ASM_INSN_DEF_H */ diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 4ebb9c054ccc..6b776c8667b2 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -10,7 +10,7 @@ #include #include -#include +#include #ifndef __ASSEMBLY__ /* @@ -30,6 +30,7 @@ */ enum aarch64_insn_encoding_class { AARCH64_INSN_CLS_UNKNOWN, /* UNALLOCATED */ + AARCH64_INSN_CLS_SVE, /* SVE instructions */ AARCH64_INSN_CLS_DP_IMM, /* Data processing - immediate */ AARCH64_INSN_CLS_DP_REG, /* Data processing - register */ AARCH64_INSN_CLS_DP_FPSIMD, /* Data processing - SIMD and FP */ @@ -294,6 +295,12 @@ __AARCH64_INSN_FUNCS(adr, 0x9F000000, 0x10000000) __AARCH64_INSN_FUNCS(adrp, 0x9F000000, 0x90000000) __AARCH64_INSN_FUNCS(prfm, 0x3FC00000, 0x39800000) __AARCH64_INSN_FUNCS(prfm_lit, 0xFF000000, 0xD8000000) +__AARCH64_INSN_FUNCS(store_imm, 0x3FC00000, 0x39000000) +__AARCH64_INSN_FUNCS(load_imm, 0x3FC00000, 0x39400000) +__AARCH64_INSN_FUNCS(store_pre, 0x3FE00C00, 0x38000C00) +__AARCH64_INSN_FUNCS(load_pre, 0x3FE00C00, 0x38400C00) +__AARCH64_INSN_FUNCS(store_post, 0x3FE00C00, 0x38000400) +__AARCH64_INSN_FUNCS(load_post, 0x3FE00C00, 0x38400400) __AARCH64_INSN_FUNCS(str_reg, 0x3FE0EC00, 0x38206800) __AARCH64_INSN_FUNCS(ldadd, 0x3F20FC00, 0x38200000) __AARCH64_INSN_FUNCS(ldr_reg, 0x3FE0EC00, 0x38606800) @@ -302,6 +309,8 @@ __AARCH64_INSN_FUNCS(ldrsw_lit, 0xFF000000, 0x98000000) __AARCH64_INSN_FUNCS(exclusive, 0x3F800000, 0x08000000) __AARCH64_INSN_FUNCS(load_ex, 0x3F400000, 0x08400000) __AARCH64_INSN_FUNCS(store_ex, 0x3F400000, 0x08000000) +__AARCH64_INSN_FUNCS(stp, 0x7FC00000, 0x29000000) +__AARCH64_INSN_FUNCS(ldp, 0x7FC00000, 0x29400000) __AARCH64_INSN_FUNCS(stp_post, 0x7FC00000, 0x28800000) __AARCH64_INSN_FUNCS(ldp_post, 0x7FC00000, 0x28C00000) __AARCH64_INSN_FUNCS(stp_pre, 0x7FC00000, 0x29800000) @@ -334,6 +343,7 @@ __AARCH64_INSN_FUNCS(rev64, 0x7FFFFC00, 0x5AC00C00) __AARCH64_INSN_FUNCS(and, 0x7F200000, 0x0A000000) __AARCH64_INSN_FUNCS(bic, 0x7F200000, 0x0A200000) __AARCH64_INSN_FUNCS(orr, 0x7F200000, 0x2A000000) +__AARCH64_INSN_FUNCS(mov_reg, 0x7FE0FFE0, 0x2A0003E0) __AARCH64_INSN_FUNCS(orn, 0x7F200000, 0x2A200000) __AARCH64_INSN_FUNCS(eor, 0x7F200000, 0x4A000000) __AARCH64_INSN_FUNCS(eon, 0x7F200000, 0x4A200000) @@ -368,6 +378,14 @@ __AARCH64_INSN_FUNCS(eret_auth, 0xFFFFFBFF, 0xD69F0BFF) __AARCH64_INSN_FUNCS(mrs, 0xFFF00000, 0xD5300000) __AARCH64_INSN_FUNCS(msr_imm, 0xFFF8F01F, 0xD500401F) __AARCH64_INSN_FUNCS(msr_reg, 0xFFF00000, 0xD5100000) +__AARCH64_INSN_FUNCS(dmb, 0xFFFFF0FF, 0xD50330BF) +__AARCH64_INSN_FUNCS(dsb_base, 0xFFFFF0FF, 0xD503309F) +__AARCH64_INSN_FUNCS(dsb_nxs, 0xFFFFF3FF, 0xD503323F) +__AARCH64_INSN_FUNCS(isb, 0xFFFFF0FF, 0xD50330DF) +__AARCH64_INSN_FUNCS(sb, 0xFFFFFFFF, 0xD50330FF) +__AARCH64_INSN_FUNCS(clrex, 0xFFFFF0FF, 0xD503305F) +__AARCH64_INSN_FUNCS(ssbb, 0xFFFFFFFF, 0xD503309F) +__AARCH64_INSN_FUNCS(pssbb, 0xFFFFFFFF, 0xD503349F) #undef __AARCH64_INSN_FUNCS @@ -379,8 +397,47 @@ static inline bool aarch64_insn_is_adr_adrp(u32 insn) return aarch64_insn_is_adr(insn) || aarch64_insn_is_adrp(insn); } -int aarch64_insn_read(void *addr, u32 *insnp); -int aarch64_insn_write(void *addr, u32 insn); +static inline bool aarch64_insn_is_dsb(u32 insn) +{ + return aarch64_insn_is_dsb_base(insn) || aarch64_insn_is_dsb_nxs(insn); +} + +static inline bool aarch64_insn_is_barrier(u32 insn) +{ + return aarch64_insn_is_dmb(insn) || aarch64_insn_is_dsb(insn) || + aarch64_insn_is_isb(insn) || aarch64_insn_is_sb(insn) || + aarch64_insn_is_clrex(insn) || aarch64_insn_is_ssbb(insn) || + aarch64_insn_is_pssbb(insn); +} + +static inline bool aarch64_insn_is_store_single(u32 insn) +{ + return aarch64_insn_is_store_imm(insn) || + aarch64_insn_is_store_pre(insn) || + aarch64_insn_is_store_post(insn); +} + +static inline bool aarch64_insn_is_store_pair(u32 insn) +{ + return aarch64_insn_is_stp(insn) || + aarch64_insn_is_stp_pre(insn) || + aarch64_insn_is_stp_post(insn); +} + +static inline bool aarch64_insn_is_load_single(u32 insn) +{ + return aarch64_insn_is_load_imm(insn) || + aarch64_insn_is_load_pre(insn) || + aarch64_insn_is_load_post(insn); +} + +static inline bool aarch64_insn_is_load_pair(u32 insn) +{ + return aarch64_insn_is_ldp(insn) || + aarch64_insn_is_ldp_pre(insn) || + aarch64_insn_is_ldp_post(insn); +} + enum aarch64_insn_encoding_class aarch64_get_insn_class(u32 insn); bool aarch64_insn_uses_literal(u32 insn); bool aarch64_insn_is_branch(u32 insn); @@ -487,9 +544,6 @@ u32 aarch64_insn_gen_prefetch(enum aarch64_insn_register base, s32 aarch64_get_branch_offset(u32 insn); u32 aarch64_set_branch_offset(u32 insn, s32 offset); -int aarch64_insn_patch_text_nosync(void *addr, u32 insn); -int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); - s32 aarch64_insn_adrp_get_offset(u32 insn); u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset); @@ -506,6 +560,7 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn); typedef bool (pstate_check_t)(unsigned long); extern pstate_check_t * const aarch32_opcode_cond_checks[16]; + #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ diff --git a/arch/arm64/include/asm/kernel-pgtable.h b/arch/arm64/include/asm/kernel-pgtable.h index d44df9d62fc9..3512184cfec1 100644 --- a/arch/arm64/include/asm/kernel-pgtable.h +++ b/arch/arm64/include/asm/kernel-pgtable.h @@ -18,9 +18,9 @@ * 64K (section size = 512M). */ #ifdef CONFIG_ARM64_4K_PAGES -#define ARM64_SWAPPER_USES_SECTION_MAPS 1 +#define ARM64_KERNEL_USES_PMD_MAPS 1 #else -#define ARM64_SWAPPER_USES_SECTION_MAPS 0 +#define ARM64_KERNEL_USES_PMD_MAPS 0 #endif /* @@ -33,7 +33,7 @@ * VA range, so pages required to map highest possible PA are reserved in all * cases. */ -#if ARM64_SWAPPER_USES_SECTION_MAPS +#if ARM64_KERNEL_USES_PMD_MAPS #define SWAPPER_PGTABLE_LEVELS (CONFIG_PGTABLE_LEVELS - 1) #define IDMAP_PGTABLE_LEVELS (ARM64_HW_PGTABLE_LEVELS(PHYS_MASK_SHIFT) - 1) #else @@ -90,9 +90,9 @@ #define IDMAP_DIR_SIZE (IDMAP_PGTABLE_LEVELS * PAGE_SIZE) /* Initial memory map size */ -#if ARM64_SWAPPER_USES_SECTION_MAPS -#define SWAPPER_BLOCK_SHIFT SECTION_SHIFT -#define SWAPPER_BLOCK_SIZE SECTION_SIZE +#if ARM64_KERNEL_USES_PMD_MAPS +#define SWAPPER_BLOCK_SHIFT PMD_SHIFT +#define SWAPPER_BLOCK_SIZE PMD_SIZE #define SWAPPER_TABLE_SHIFT PUD_SHIFT #else #define SWAPPER_BLOCK_SHIFT PAGE_SHIFT @@ -100,16 +100,13 @@ #define SWAPPER_TABLE_SHIFT PMD_SHIFT #endif -/* The size of the initial kernel direct mapping */ -#define SWAPPER_INIT_MAP_SIZE (_AC(1, UL) << SWAPPER_TABLE_SHIFT) - /* * Initial memory map attributes. */ #define SWAPPER_PTE_FLAGS (PTE_TYPE_PAGE | PTE_AF | PTE_SHARED) #define SWAPPER_PMD_FLAGS (PMD_TYPE_SECT | PMD_SECT_AF | PMD_SECT_S) -#if ARM64_SWAPPER_USES_SECTION_MAPS +#if ARM64_KERNEL_USES_PMD_MAPS #define SWAPPER_MM_MMUFLAGS (PMD_ATTRINDX(MT_NORMAL) | SWAPPER_PMD_FLAGS) #else #define SWAPPER_MM_MMUFLAGS (PTE_ATTRINDX(MT_NORMAL) | SWAPPER_PTE_FLAGS) @@ -125,7 +122,7 @@ #if defined(CONFIG_ARM64_4K_PAGES) #define ARM64_MEMSTART_SHIFT PUD_SHIFT #elif defined(CONFIG_ARM64_16K_PAGES) -#define ARM64_MEMSTART_SHIFT (PMD_SHIFT + 5) +#define ARM64_MEMSTART_SHIFT CONT_PMD_SHIFT #else #define ARM64_MEMSTART_SHIFT PMD_SHIFT #endif diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 692c9049befa..d436831dd706 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -12,7 +12,8 @@ #include /* Hyp Configuration Register (HCR) bits */ -#define HCR_ATA (UL(1) << 56) +#define HCR_ATA_SHIFT 56 +#define HCR_ATA (UL(1) << HCR_ATA_SHIFT) #define HCR_FWB (UL(1) << 46) #define HCR_API (UL(1) << 41) #define HCR_APK (UL(1) << 40) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 5e9b33cbac51..9f0bf2109be7 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -8,6 +8,7 @@ #define __ARM_KVM_ASM_H__ #include +#include #include #define ARM_EXIT_WITH_SERROR_BIT 31 diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 01b9857757f2..fd418955e31e 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -84,6 +84,9 @@ static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu) if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE) || vcpu_el1_is_32bit(vcpu)) vcpu->arch.hcr_el2 |= HCR_TID2; + + if (kvm_has_mte(vcpu->kvm)) + vcpu->arch.hcr_el2 |= HCR_ATA; } static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 7cd7d5c8c4bc..41911585ae0c 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -46,6 +46,7 @@ #define KVM_REQ_VCPU_RESET KVM_ARCH_REQ(2) #define KVM_REQ_RECORD_STEAL KVM_ARCH_REQ(3) #define KVM_REQ_RELOAD_GICv4 KVM_ARCH_REQ(4) +#define KVM_REQ_RELOAD_PMU KVM_ARCH_REQ(5) #define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \ KVM_DIRTY_LOG_INITIALLY_SET) @@ -132,6 +133,9 @@ struct kvm_arch { u8 pfr0_csv2; u8 pfr0_csv3; + + /* Memory Tagging Extension enabled for the guest */ + bool mte_enabled; }; struct kvm_vcpu_fault_info { @@ -206,6 +210,12 @@ enum vcpu_sysreg { CNTP_CVAL_EL0, CNTP_CTL_EL0, + /* Memory Tagging Extension registers */ + RGSR_EL1, /* Random Allocation Tag Seed Register */ + GCR_EL1, /* Tag Control Register */ + TFSR_EL1, /* Tag Fault Status Register (EL1) */ + TFSRE0_EL1, /* Tag Fault Status Register (EL0) */ + /* 32bit specific registers. Keep them at the end of the range */ DACR32_EL2, /* Domain Access Control Register */ IFSR32_EL2, /* Instruction Fault Status Register */ @@ -556,16 +566,11 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) } struct kvm_vm_stat { - ulong remote_tlb_flush; + struct kvm_vm_stat_generic generic; }; struct kvm_vcpu_stat { - u64 halt_successful_poll; - u64 halt_attempted_poll; - u64 halt_poll_success_ns; - u64 halt_poll_fail_ns; - u64 halt_poll_invalid; - u64 halt_wakeup; + struct kvm_vcpu_stat_generic generic; u64 hvc_exit_stat; u64 wfe_exit_stat; u64 wfi_exit_stat; @@ -721,6 +726,9 @@ int kvm_arm_vcpu_arch_get_attr(struct kvm_vcpu *vcpu, int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr); +long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, + struct kvm_arm_copy_mte_tags *copy_tags); + /* Guest/host FPSIMD coordination helpers */ int kvm_arch_vcpu_run_map_fp(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_load_fp(struct kvm_vcpu *vcpu); @@ -769,6 +777,7 @@ bool kvm_arm_vcpu_is_finalized(struct kvm_vcpu *vcpu); #define kvm_arm_vcpu_sve_finalized(vcpu) \ ((vcpu)->arch.flags & KVM_ARM64_VCPU_SVE_FINALIZED) +#define kvm_has_mte(kvm) (system_supports_mte() && (kvm)->arch.mte_enabled) #define kvm_vcpu_has_pmu(vcpu) \ (test_bit(KVM_ARM_VCPU_PMU_V3, (vcpu)->arch.features)) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 25ed956f9af1..b52c5c4b9a3d 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -180,17 +180,16 @@ static inline void *__kvm_vector_slot2addr(void *base, struct kvm; -#define kvm_flush_dcache_to_poc(a,l) __flush_dcache_area((a), (l)) +#define kvm_flush_dcache_to_poc(a,l) \ + dcache_clean_inval_poc((unsigned long)(a), (unsigned long)(a)+(l)) static inline bool vcpu_has_cache_enabled(struct kvm_vcpu *vcpu) { return (vcpu_read_sys_reg(vcpu, SCTLR_EL1) & 0b101) == 0b101; } -static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size) +static inline void __clean_dcache_guest_page(void *va, size_t size) { - void *va = page_address(pfn_to_page(pfn)); - /* * With FWB, we ensure that the guest always accesses memory using * cacheable attributes, and we don't have to clean to PoC when @@ -203,18 +202,14 @@ static inline void __clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size) kvm_flush_dcache_to_poc(va, size); } -static inline void __invalidate_icache_guest_page(kvm_pfn_t pfn, - unsigned long size) +static inline void __invalidate_icache_guest_page(void *va, size_t size) { if (icache_is_aliasing()) { /* any kind of VIPT cache */ - __flush_icache_all(); + icache_inval_all_pou(); } else if (is_kernel_in_hyp_mode() || !icache_is_vpipt()) { /* PIPT or VPIPT at EL2 (see comment in __kvm_tlb_flush_vmid_ipa) */ - void *va = page_address(pfn_to_page(pfn)); - - invalidate_icache_range((unsigned long)va, - (unsigned long)va + size); + icache_inval_pou((unsigned long)va, (unsigned long)va + size); } } diff --git a/arch/arm64/include/asm/kvm_mte.h b/arch/arm64/include/asm/kvm_mte.h new file mode 100644 index 000000000000..de002636eb1f --- /dev/null +++ b/arch/arm64/include/asm/kvm_mte.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2020-2021 ARM Ltd. + */ +#ifndef __ASM_KVM_MTE_H +#define __ASM_KVM_MTE_H + +#ifdef __ASSEMBLY__ + +#include + +#ifdef CONFIG_ARM64_MTE + +.macro mte_switch_to_guest g_ctxt, h_ctxt, reg1 +alternative_if_not ARM64_MTE + b .L__skip_switch\@ +alternative_else_nop_endif + mrs \reg1, hcr_el2 + tbz \reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@ + + mrs_s \reg1, SYS_RGSR_EL1 + str \reg1, [\h_ctxt, #CPU_RGSR_EL1] + mrs_s \reg1, SYS_GCR_EL1 + str \reg1, [\h_ctxt, #CPU_GCR_EL1] + + ldr \reg1, [\g_ctxt, #CPU_RGSR_EL1] + msr_s SYS_RGSR_EL1, \reg1 + ldr \reg1, [\g_ctxt, #CPU_GCR_EL1] + msr_s SYS_GCR_EL1, \reg1 + +.L__skip_switch\@: +.endm + +.macro mte_switch_to_hyp g_ctxt, h_ctxt, reg1 +alternative_if_not ARM64_MTE + b .L__skip_switch\@ +alternative_else_nop_endif + mrs \reg1, hcr_el2 + tbz \reg1, #(HCR_ATA_SHIFT), .L__skip_switch\@ + + mrs_s \reg1, SYS_RGSR_EL1 + str \reg1, [\g_ctxt, #CPU_RGSR_EL1] + mrs_s \reg1, SYS_GCR_EL1 + str \reg1, [\g_ctxt, #CPU_GCR_EL1] + + ldr \reg1, [\h_ctxt, #CPU_RGSR_EL1] + msr_s SYS_RGSR_EL1, \reg1 + ldr \reg1, [\h_ctxt, #CPU_GCR_EL1] + msr_s SYS_GCR_EL1, \reg1 + + isb + +.L__skip_switch\@: +.endm + +#else /* !CONFIG_ARM64_MTE */ + +.macro mte_switch_to_guest g_ctxt, h_ctxt, reg1 +.endm + +.macro mte_switch_to_hyp g_ctxt, h_ctxt, reg1 +.endm + +#endif /* CONFIG_ARM64_MTE */ +#endif /* __ASSEMBLY__ */ +#endif /* __ASM_KVM_MTE_H */ diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h index c3674c47d48c..f004c0115d89 100644 --- a/arch/arm64/include/asm/kvm_pgtable.h +++ b/arch/arm64/include/asm/kvm_pgtable.h @@ -27,23 +27,29 @@ typedef u64 kvm_pte_t; /** * struct kvm_pgtable_mm_ops - Memory management callbacks. - * @zalloc_page: Allocate a single zeroed memory page. The @arg parameter - * can be used by the walker to pass a memcache. The - * initial refcount of the page is 1. - * @zalloc_pages_exact: Allocate an exact number of zeroed memory pages. The - * @size parameter is in bytes, and is rounded-up to the - * next page boundary. The resulting allocation is - * physically contiguous. - * @free_pages_exact: Free an exact number of memory pages previously - * allocated by zalloc_pages_exact. - * @get_page: Increment the refcount on a page. - * @put_page: Decrement the refcount on a page. When the refcount - * reaches 0 the page is automatically freed. - * @page_count: Return the refcount of a page. - * @phys_to_virt: Convert a physical address into a virtual address mapped - * in the current context. - * @virt_to_phys: Convert a virtual address mapped in the current context - * into a physical address. + * @zalloc_page: Allocate a single zeroed memory page. + * The @arg parameter can be used by the walker + * to pass a memcache. The initial refcount of + * the page is 1. + * @zalloc_pages_exact: Allocate an exact number of zeroed memory pages. + * The @size parameter is in bytes, and is rounded + * up to the next page boundary. The resulting + * allocation is physically contiguous. + * @free_pages_exact: Free an exact number of memory pages previously + * allocated by zalloc_pages_exact. + * @get_page: Increment the refcount on a page. + * @put_page: Decrement the refcount on a page. When the + * refcount reaches 0 the page is automatically + * freed. + * @page_count: Return the refcount of a page. + * @phys_to_virt: Convert a physical address into a virtual + * address mapped in the current context. + * @virt_to_phys: Convert a virtual address mapped in the current + * context into a physical address. + * @dcache_clean_inval_poc: Clean and invalidate the data cache to the PoC + * for the specified memory address range. + * @icache_inval_pou: Invalidate the instruction cache to the PoU + * for the specified memory address range. */ struct kvm_pgtable_mm_ops { void* (*zalloc_page)(void *arg); @@ -54,6 +60,8 @@ struct kvm_pgtable_mm_ops { int (*page_count)(void *addr); void* (*phys_to_virt)(phys_addr_t phys); phys_addr_t (*virt_to_phys)(void *addr); + void (*dcache_clean_inval_poc)(void *addr, size_t size); + void (*icache_inval_pou)(void *addr, size_t size); }; /** diff --git a/arch/arm64/include/asm/linkage.h b/arch/arm64/include/asm/linkage.h index ba89a9af820a..9906541a6861 100644 --- a/arch/arm64/include/asm/linkage.h +++ b/arch/arm64/include/asm/linkage.h @@ -56,8 +56,16 @@ SYM_FUNC_START_ALIAS(__pi_##x); \ SYM_FUNC_START_WEAK(x) +#define SYM_FUNC_START_WEAK_ALIAS_PI(x) \ + SYM_FUNC_START_ALIAS(__pi_##x); \ + SYM_START(x, SYM_L_WEAK, SYM_A_ALIGN) + #define SYM_FUNC_END_PI(x) \ SYM_FUNC_END(x); \ SYM_FUNC_END_ALIAS(__pi_##x) +#define SYM_FUNC_END_ALIAS_PI(x) \ + SYM_FUNC_END_ALIAS(x); \ + SYM_FUNC_END_ALIAS(__pi_##x) + #endif diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 87b90dc27a43..7b360960cc35 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -135,10 +135,8 @@ #define MT_NORMAL 0 #define MT_NORMAL_TAGGED 1 #define MT_NORMAL_NC 2 -#define MT_NORMAL_WT 3 -#define MT_DEVICE_nGnRnE 4 -#define MT_DEVICE_nGnRE 5 -#define MT_DEVICE_GRE 6 +#define MT_DEVICE_nGnRnE 3 +#define MT_DEVICE_nGnRE 4 /* * Memory types for Stage-2 translation diff --git a/arch/arm64/include/asm/mmu_context.h b/arch/arm64/include/asm/mmu_context.h index d3cef9133539..eeb210997149 100644 --- a/arch/arm64/include/asm/mmu_context.h +++ b/arch/arm64/include/asm/mmu_context.h @@ -177,9 +177,9 @@ static inline void update_saved_ttbr0(struct task_struct *tsk, return; if (mm == &init_mm) - ttbr = __pa_symbol(reserved_pg_dir); + ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); else - ttbr = virt_to_phys(mm->pgd) | ASID(mm) << 48; + ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) | ASID(mm) << 48; WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr); } diff --git a/arch/arm64/include/asm/module.lds.h b/arch/arm64/include/asm/module.lds.h index 810045628c66..a11ccadd47d2 100644 --- a/arch/arm64/include/asm/module.lds.h +++ b/arch/arm64/include/asm/module.lds.h @@ -1,7 +1,20 @@ -#ifdef CONFIG_ARM64_MODULE_PLTS SECTIONS { +#ifdef CONFIG_ARM64_MODULE_PLTS .plt 0 (NOLOAD) : { BYTE(0) } .init.plt 0 (NOLOAD) : { BYTE(0) } .text.ftrace_trampoline 0 (NOLOAD) : { BYTE(0) } -} #endif + +#ifdef CONFIG_KASAN_SW_TAGS + /* + * Outlined checks go into comdat-deduplicated sections named .text.hot. + * Because they are in comdats they are not combined by the linker and + * we otherwise end up with multiple sections with the same .text.hot + * name in the .ko file. The kernel module loader warns if it sees + * multiple sections with the same name so we use this sections + * directive to force them into a single section and silence the + * warning. + */ + .text.hot : { *(.text.hot) } +#endif +} diff --git a/arch/arm64/include/asm/mte-def.h b/arch/arm64/include/asm/mte-def.h index cf241b0f0a42..626d359b396e 100644 --- a/arch/arm64/include/asm/mte-def.h +++ b/arch/arm64/include/asm/mte-def.h @@ -7,6 +7,7 @@ #define MTE_GRANULE_SIZE UL(16) #define MTE_GRANULE_MASK (~(MTE_GRANULE_SIZE - 1)) +#define MTE_GRANULES_PER_PAGE (PAGE_SIZE / MTE_GRANULE_SIZE) #define MTE_TAG_SHIFT 56 #define MTE_TAG_SIZE 4 #define MTE_TAG_MASK GENMASK((MTE_TAG_SHIFT + (MTE_TAG_SIZE - 1)), MTE_TAG_SHIFT) diff --git a/arch/arm64/include/asm/mte-kasan.h b/arch/arm64/include/asm/mte-kasan.h index ddd4d17cf9a0..d952352bd008 100644 --- a/arch/arm64/include/asm/mte-kasan.h +++ b/arch/arm64/include/asm/mte-kasan.h @@ -48,43 +48,84 @@ static inline u8 mte_get_random_tag(void) return mte_get_ptr_tag(addr); } +static inline u64 __stg_post(u64 p) +{ + asm volatile(__MTE_PREAMBLE "stg %0, [%0], #16" + : "+r"(p) + : + : "memory"); + return p; +} + +static inline u64 __stzg_post(u64 p) +{ + asm volatile(__MTE_PREAMBLE "stzg %0, [%0], #16" + : "+r"(p) + : + : "memory"); + return p; +} + +static inline void __dc_gva(u64 p) +{ + asm volatile(__MTE_PREAMBLE "dc gva, %0" : : "r"(p) : "memory"); +} + +static inline void __dc_gzva(u64 p) +{ + asm volatile(__MTE_PREAMBLE "dc gzva, %0" : : "r"(p) : "memory"); +} + /* * Assign allocation tags for a region of memory based on the pointer tag. * Note: The address must be non-NULL and MTE_GRANULE_SIZE aligned and - * size must be non-zero and MTE_GRANULE_SIZE aligned. + * size must be MTE_GRANULE_SIZE aligned. */ -static inline void mte_set_mem_tag_range(void *addr, size_t size, - u8 tag, bool init) +static inline void mte_set_mem_tag_range(void *addr, size_t size, u8 tag, + bool init) { - u64 curr, end; + u64 curr, mask, dczid_bs, end1, end2, end3; - if (!size) - return; + /* Read DC G(Z)VA block size from the system register. */ + dczid_bs = 4ul << (read_cpuid(DCZID_EL0) & 0xf); curr = (u64)__tag_set(addr, tag); - end = curr + size; + mask = dczid_bs - 1; + /* STG/STZG up to the end of the first block. */ + end1 = curr | mask; + end3 = curr + size; + /* DC GVA / GZVA in [end1, end2) */ + end2 = end3 & ~mask; /* - * 'asm volatile' is required to prevent the compiler to move - * the statement outside of the loop. + * The following code uses STG on the first DC GVA block even if the + * start address is aligned - it appears to be faster than an alignment + * check + conditional branch. Also, if the range size is at least 2 DC + * GVA blocks, the first two loops can use post-condition to save one + * branch each. */ - if (init) { - do { - asm volatile(__MTE_PREAMBLE "stzg %0, [%0]" - : - : "r" (curr) - : "memory"); - curr += MTE_GRANULE_SIZE; - } while (curr != end); - } else { - do { - asm volatile(__MTE_PREAMBLE "stg %0, [%0]" - : - : "r" (curr) - : "memory"); - curr += MTE_GRANULE_SIZE; - } while (curr != end); - } +#define SET_MEMTAG_RANGE(stg_post, dc_gva) \ + do { \ + if (size >= 2 * dczid_bs) { \ + do { \ + curr = stg_post(curr); \ + } while (curr < end1); \ + \ + do { \ + dc_gva(curr); \ + curr += dczid_bs; \ + } while (curr < end2); \ + } \ + \ + while (curr < end3) \ + curr = stg_post(curr); \ + } while (0) + + if (init) + SET_MEMTAG_RANGE(__stzg_post, __dc_gzva); + else + SET_MEMTAG_RANGE(__stg_post, __dc_gva); +#undef SET_MEMTAG_RANGE } void mte_enable_kernel_sync(void); diff --git a/arch/arm64/include/asm/mte.h b/arch/arm64/include/asm/mte.h index bc88a1ced0d7..58c7f80f5596 100644 --- a/arch/arm64/include/asm/mte.h +++ b/arch/arm64/include/asm/mte.h @@ -37,7 +37,8 @@ void mte_free_tag_storage(char *storage); /* track which pages have valid allocation tags */ #define PG_mte_tagged PG_arch_2 -void mte_sync_tags(pte_t *ptep, pte_t pte); +void mte_zero_clear_page_tags(void *addr); +void mte_sync_tags(pte_t old_pte, pte_t pte); void mte_copy_page_tags(void *kto, const void *kfrom); void mte_thread_init_user(void); void mte_thread_switch(struct task_struct *next); @@ -53,7 +54,10 @@ int mte_ptrace_copy_tags(struct task_struct *child, long request, /* unused if !CONFIG_ARM64_MTE, silence the compiler */ #define PG_mte_tagged 0 -static inline void mte_sync_tags(pte_t *ptep, pte_t pte) +static inline void mte_zero_clear_page_tags(void *addr) +{ +} +static inline void mte_sync_tags(pte_t old_pte, pte_t pte) { } static inline void mte_copy_page_tags(void *kto, const void *kfrom) diff --git a/arch/arm64/include/asm/page.h b/arch/arm64/include/asm/page.h index 012cffc574e8..ed1b9dcf12b2 100644 --- a/arch/arm64/include/asm/page.h +++ b/arch/arm64/include/asm/page.h @@ -13,6 +13,7 @@ #ifndef __ASSEMBLY__ #include /* for READ_IMPLIES_EXEC */ +#include /* for gfp_t */ #include struct page; @@ -28,9 +29,12 @@ void copy_user_highpage(struct page *to, struct page *from, void copy_highpage(struct page *to, struct page *from); #define __HAVE_ARCH_COPY_HIGHPAGE -#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ - alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) -#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, + unsigned long vaddr); +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE + +void tag_clear_highpage(struct page *to); +#define __HAVE_ARCH_TAG_CLEAR_HIGHPAGE #define clear_user_page(page, vaddr, pg) clear_page(page) #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) diff --git a/arch/arm64/include/asm/patching.h b/arch/arm64/include/asm/patching.h new file mode 100644 index 000000000000..6bf5adc56295 --- /dev/null +++ b/arch/arm64/include/asm/patching.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_PATCHING_H +#define __ASM_PATCHING_H + +#include + +int aarch64_insn_read(void *addr, u32 *insnp); +int aarch64_insn_write(void *addr, u32 insn); + +int aarch64_insn_patch_text_nosync(void *addr, u32 insn); +int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); + +#endif /* __ASM_PATCHING_H */ diff --git a/arch/arm64/include/asm/perf_event.h b/arch/arm64/include/asm/perf_event.h index 60731f602d3e..4ef6f19331f9 100644 --- a/arch/arm64/include/asm/perf_event.h +++ b/arch/arm64/include/asm/perf_event.h @@ -239,6 +239,11 @@ /* PMMIR_EL1.SLOTS mask */ #define ARMV8_PMU_SLOTS_MASK 0xff +#define ARMV8_PMU_BUS_SLOTS_SHIFT 8 +#define ARMV8_PMU_BUS_SLOTS_MASK 0xff +#define ARMV8_PMU_BUS_WIDTH_SHIFT 16 +#define ARMV8_PMU_BUS_WIDTH_MASK 0xf + #ifdef CONFIG_PERF_EVENTS struct pt_regs; extern unsigned long perf_instruction_pointer(struct pt_regs *regs); diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index b82575a33f8b..40085e53f573 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -71,13 +71,6 @@ #define PGDIR_MASK (~(PGDIR_SIZE-1)) #define PTRS_PER_PGD (1 << (VA_BITS - PGDIR_SHIFT)) -/* - * Section address mask and size definitions. - */ -#define SECTION_SHIFT PMD_SHIFT -#define SECTION_SIZE (_AC(1, UL) << SECTION_SHIFT) -#define SECTION_MASK (~(SECTION_SIZE-1)) - /* * Contiguous page definitions. */ diff --git a/arch/arm64/include/asm/pgtable-prot.h b/arch/arm64/include/asm/pgtable-prot.h index 938092df76cf..7032f04c8ac6 100644 --- a/arch/arm64/include/asm/pgtable-prot.h +++ b/arch/arm64/include/asm/pgtable-prot.h @@ -55,7 +55,6 @@ extern bool arm64_use_ng_mappings; #define PROT_DEVICE_nGnRnE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRnE)) #define PROT_DEVICE_nGnRE (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_DEVICE_nGnRE)) #define PROT_NORMAL_NC (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_NC)) -#define PROT_NORMAL_WT (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_WT)) #define PROT_NORMAL (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL)) #define PROT_NORMAL_TAGGED (PROT_DEFAULT | PTE_PXN | PTE_UXN | PTE_WRITE | PTE_ATTRINDX(MT_NORMAL_TAGGED)) diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h index 0b10204e72fc..c0ba8cdfa10a 100644 --- a/arch/arm64/include/asm/pgtable.h +++ b/arch/arm64/include/asm/pgtable.h @@ -314,9 +314,25 @@ static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, if (pte_present(pte) && pte_user_exec(pte) && !pte_special(pte)) __sync_icache_dcache(pte); - if (system_supports_mte() && - pte_present(pte) && pte_tagged(pte) && !pte_special(pte)) - mte_sync_tags(ptep, pte); + /* + * If the PTE would provide user space access to the tags associated + * with it then ensure that the MTE tags are synchronised. Although + * pte_access_permitted() returns false for exec only mappings, they + * don't expose tags (instruction fetches don't check tags). + */ + if (system_supports_mte() && pte_access_permitted(pte, false) && + !pte_special(pte)) { + pte_t old_pte = READ_ONCE(*ptep); + /* + * We only need to synchronise if the new PTE has tags enabled + * or if swapping in (in which case another mapping may have + * set tags in the past even if this PTE isn't tagged). + * (!pte_none() && !pte_present()) is an open coded version of + * is_swap_pte() + */ + if (pte_tagged(pte) || (!pte_none(old_pte) && !pte_present(old_pte))) + mte_sync_tags(old_pte, pte); + } __check_racy_pte_update(mm, ptep, pte); @@ -511,13 +527,12 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, #define pmd_none(pmd) (!pmd_val(pmd)) -#define pmd_bad(pmd) (!(pmd_val(pmd) & PMD_TABLE_BIT)) - #define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ PMD_TYPE_TABLE) #define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \ PMD_TYPE_SECT) #define pmd_leaf(pmd) pmd_sect(pmd) +#define pmd_bad(pmd) (!pmd_table(pmd)) #define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE) #define pte_leaf_size(pte) (pte_cont(pte) ? CONT_PTE_SIZE : PAGE_SIZE) @@ -604,7 +619,7 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd) pr_err("%s:%d: bad pmd %016llx.\n", __FILE__, __LINE__, pmd_val(e)) #define pud_none(pud) (!pud_val(pud)) -#define pud_bad(pud) (!(pud_val(pud) & PUD_TABLE_BIT)) +#define pud_bad(pud) (!pud_table(pud)) #define pud_present(pud) pte_present(pud_pte(pud)) #define pud_leaf(pud) pud_sect(pud) #define pud_valid(pud) pte_valid(pud_pte(pud)) diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h index d50416be99be..28a78b67d9b4 100644 --- a/arch/arm64/include/asm/pointer_auth.h +++ b/arch/arm64/include/asm/pointer_auth.h @@ -31,10 +31,6 @@ struct ptrauth_keys_user { struct ptrauth_key apga; }; -struct ptrauth_keys_kernel { - struct ptrauth_key apia; -}; - #define __ptrauth_key_install_nosync(k, v) \ do { \ struct ptrauth_key __pki_v = (v); \ @@ -42,6 +38,29 @@ do { \ write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1); \ } while (0) +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL + +struct ptrauth_keys_kernel { + struct ptrauth_key apia; +}; + +static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys) +{ + if (system_supports_address_auth()) + get_random_bytes(&keys->apia, sizeof(keys->apia)); +} + +static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys) +{ + if (!system_supports_address_auth()) + return; + + __ptrauth_key_install_nosync(APIA, keys->apia); + isb(); +} + +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ + static inline void ptrauth_keys_install_user(struct ptrauth_keys_user *keys) { if (system_supports_address_auth()) { @@ -69,21 +88,6 @@ static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys) ptrauth_keys_install_user(keys); } -static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys) -{ - if (system_supports_address_auth()) - get_random_bytes(&keys->apia, sizeof(keys->apia)); -} - -static __always_inline void ptrauth_keys_switch_kernel(struct ptrauth_keys_kernel *keys) -{ - if (!system_supports_address_auth()) - return; - - __ptrauth_key_install_nosync(APIA, keys->apia); - isb(); -} - extern int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg); extern int ptrauth_set_enabled_keys(struct task_struct *tsk, unsigned long keys, @@ -121,11 +125,6 @@ static __always_inline void ptrauth_enable(void) #define ptrauth_thread_switch_user(tsk) \ ptrauth_keys_install_user(&(tsk)->thread.keys_user) -#define ptrauth_thread_init_kernel(tsk) \ - ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel) -#define ptrauth_thread_switch_kernel(tsk) \ - ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel) - #else /* CONFIG_ARM64_PTR_AUTH */ #define ptrauth_enable() #define ptrauth_prctl_reset_keys(tsk, arg) (-EINVAL) @@ -134,11 +133,19 @@ static __always_inline void ptrauth_enable(void) #define ptrauth_strip_insn_pac(lr) (lr) #define ptrauth_suspend_exit() #define ptrauth_thread_init_user() -#define ptrauth_thread_init_kernel(tsk) #define ptrauth_thread_switch_user(tsk) -#define ptrauth_thread_switch_kernel(tsk) #endif /* CONFIG_ARM64_PTR_AUTH */ +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL +#define ptrauth_thread_init_kernel(tsk) \ + ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel) +#define ptrauth_thread_switch_kernel(tsk) \ + ptrauth_keys_switch_kernel(&(tsk)->thread.keys_kernel) +#else +#define ptrauth_thread_init_kernel(tsk) +#define ptrauth_thread_switch_kernel(tsk) +#endif /* CONFIG_ARM64_PTR_AUTH_KERNEL */ + #define PR_PAC_ENABLED_KEYS_MASK \ (PR_PAC_APIAKEY | PR_PAC_APIBKEY | PR_PAC_APDAKEY | PR_PAC_APDBKEY) diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h index 35cb45e3d9ff..ed99ba8bb269 100644 --- a/arch/arm64/include/asm/processor.h +++ b/arch/arm64/include/asm/processor.h @@ -151,8 +151,10 @@ struct thread_struct { struct debug_info debug; /* debugging */ #ifdef CONFIG_ARM64_PTR_AUTH struct ptrauth_keys_user keys_user; +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL struct ptrauth_keys_kernel keys_kernel; #endif +#endif #ifdef CONFIG_ARM64_MTE u64 gcr_user_excl; #endif @@ -260,8 +262,6 @@ void set_task_sctlr_el1(u64 sctlr); extern struct task_struct *cpu_switch_to(struct task_struct *prev, struct task_struct *next); -asmlinkage void arm64_preempt_schedule_irq(void); - #define task_pt_regs(p) \ ((struct pt_regs *)(THREAD_SIZE + task_stack_page(p)) - 1) @@ -332,13 +332,13 @@ long get_tagged_addr_ctrl(struct task_struct *task); * of header definitions for the use of task_stack_page. */ -#define current_top_of_stack() \ -({ \ - struct stack_info _info; \ - BUG_ON(!on_accessible_stack(current, current_stack_pointer, &_info)); \ - _info.high; \ +#define current_top_of_stack() \ +({ \ + struct stack_info _info; \ + BUG_ON(!on_accessible_stack(current, current_stack_pointer, 1, &_info)); \ + _info.high; \ }) -#define on_thread_stack() (on_task_stack(current, current_stack_pointer, NULL)) +#define on_thread_stack() (on_task_stack(current, current_stack_pointer, 1, NULL)) #endif /* __ASSEMBLY__ */ #endif /* __ASM_PROCESSOR_H */ diff --git a/arch/arm64/include/asm/scs.h b/arch/arm64/include/asm/scs.h index eaa2cd92e4c1..8297bccf0784 100644 --- a/arch/arm64/include/asm/scs.h +++ b/arch/arm64/include/asm/scs.h @@ -9,18 +9,18 @@ #ifdef CONFIG_SHADOW_CALL_STACK scs_sp .req x18 - .macro scs_load tsk, tmp + .macro scs_load tsk ldr scs_sp, [\tsk, #TSK_TI_SCS_SP] .endm - .macro scs_save tsk, tmp + .macro scs_save tsk str scs_sp, [\tsk, #TSK_TI_SCS_SP] .endm #else - .macro scs_load tsk, tmp + .macro scs_load tsk .endm - .macro scs_save tsk, tmp + .macro scs_save tsk .endm #endif /* CONFIG_SHADOW_CALL_STACK */ diff --git a/arch/arm64/include/asm/sdei.h b/arch/arm64/include/asm/sdei.h index 63e0b92a5fbb..7bea1d705dd6 100644 --- a/arch/arm64/include/asm/sdei.h +++ b/arch/arm64/include/asm/sdei.h @@ -37,13 +37,17 @@ struct sdei_registered_event; asmlinkage unsigned long __sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg); +unsigned long do_sdei_event(struct pt_regs *regs, + struct sdei_registered_event *arg); + unsigned long sdei_arch_get_entry_point(int conduit); #define sdei_arch_get_entry_point(x) sdei_arch_get_entry_point(x) struct stack_info; -bool _on_sdei_stack(unsigned long sp, struct stack_info *info); -static inline bool on_sdei_stack(unsigned long sp, +bool _on_sdei_stack(unsigned long sp, unsigned long size, + struct stack_info *info); +static inline bool on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info) { if (!IS_ENABLED(CONFIG_VMAP_STACK)) @@ -51,7 +55,7 @@ static inline bool on_sdei_stack(unsigned long sp, if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE)) return false; if (in_nmi()) - return _on_sdei_stack(sp, info); + return _on_sdei_stack(sp, size, info); return false; } diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h index 566e973e38c2..b93637756f91 100644 --- a/arch/arm64/include/asm/smp.h +++ b/arch/arm64/include/asm/smp.h @@ -73,12 +73,10 @@ asmlinkage void secondary_start_kernel(void); /* * Initial data for bringing up a secondary CPU. - * @stack - sp for the secondary CPU * @status - Result passed back from the secondary CPU to * indicate failure. */ struct secondary_data { - void *stack; struct task_struct *task; long status; }; diff --git a/arch/arm64/include/asm/stacktrace.h b/arch/arm64/include/asm/stacktrace.h index 4b33ca620679..1801399204d7 100644 --- a/arch/arm64/include/asm/stacktrace.h +++ b/arch/arm64/include/asm/stacktrace.h @@ -69,14 +69,14 @@ extern void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk, DECLARE_PER_CPU(unsigned long *, irq_stack_ptr); -static inline bool on_stack(unsigned long sp, unsigned long low, - unsigned long high, enum stack_type type, - struct stack_info *info) +static inline bool on_stack(unsigned long sp, unsigned long size, + unsigned long low, unsigned long high, + enum stack_type type, struct stack_info *info) { if (!low) return false; - if (sp < low || sp >= high) + if (sp < low || sp + size < sp || sp + size > high) return false; if (info) { @@ -87,38 +87,38 @@ static inline bool on_stack(unsigned long sp, unsigned long low, return true; } -static inline bool on_irq_stack(unsigned long sp, +static inline bool on_irq_stack(unsigned long sp, unsigned long size, struct stack_info *info) { unsigned long low = (unsigned long)raw_cpu_read(irq_stack_ptr); unsigned long high = low + IRQ_STACK_SIZE; - return on_stack(sp, low, high, STACK_TYPE_IRQ, info); + return on_stack(sp, size, low, high, STACK_TYPE_IRQ, info); } static inline bool on_task_stack(const struct task_struct *tsk, - unsigned long sp, + unsigned long sp, unsigned long size, struct stack_info *info) { unsigned long low = (unsigned long)task_stack_page(tsk); unsigned long high = low + THREAD_SIZE; - return on_stack(sp, low, high, STACK_TYPE_TASK, info); + return on_stack(sp, size, low, high, STACK_TYPE_TASK, info); } #ifdef CONFIG_VMAP_STACK DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); -static inline bool on_overflow_stack(unsigned long sp, +static inline bool on_overflow_stack(unsigned long sp, unsigned long size, struct stack_info *info) { unsigned long low = (unsigned long)raw_cpu_ptr(overflow_stack); unsigned long high = low + OVERFLOW_STACK_SIZE; - return on_stack(sp, low, high, STACK_TYPE_OVERFLOW, info); + return on_stack(sp, size, low, high, STACK_TYPE_OVERFLOW, info); } #else -static inline bool on_overflow_stack(unsigned long sp, +static inline bool on_overflow_stack(unsigned long sp, unsigned long size, struct stack_info *info) { return false; } #endif @@ -128,21 +128,21 @@ static inline bool on_overflow_stack(unsigned long sp, * context. */ static inline bool on_accessible_stack(const struct task_struct *tsk, - unsigned long sp, + unsigned long sp, unsigned long size, struct stack_info *info) { if (info) info->type = STACK_TYPE_UNKNOWN; - if (on_task_stack(tsk, sp, info)) + if (on_task_stack(tsk, sp, size, info)) return true; if (tsk != current || preemptible()) return false; - if (on_irq_stack(sp, info)) + if (on_irq_stack(sp, size, info)) return true; - if (on_overflow_stack(sp, info)) + if (on_overflow_stack(sp, size, info)) return true; - if (on_sdei_stack(sp, info)) + if (on_sdei_stack(sp, size, info)) return true; return false; diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 65d15700a168..7b9c3acba684 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -651,7 +651,8 @@ #define INIT_SCTLR_EL2_MMU_ON \ (SCTLR_ELx_M | SCTLR_ELx_C | SCTLR_ELx_SA | SCTLR_ELx_I | \ - SCTLR_ELx_IESB | SCTLR_ELx_WXN | ENDIAN_SET_EL2 | SCTLR_EL2_RES1) + SCTLR_ELx_IESB | SCTLR_ELx_WXN | ENDIAN_SET_EL2 | \ + SCTLR_ELx_ITFSB | SCTLR_EL2_RES1) #define INIT_SCTLR_EL2_MMU_OFF \ (SCTLR_EL2_RES1 | ENDIAN_SET_EL2) @@ -703,9 +704,7 @@ /* MAIR_ELx memory attributes (used by Linux) */ #define MAIR_ATTR_DEVICE_nGnRnE UL(0x00) #define MAIR_ATTR_DEVICE_nGnRE UL(0x04) -#define MAIR_ATTR_DEVICE_GRE UL(0x0c) #define MAIR_ATTR_NORMAL_NC UL(0x44) -#define MAIR_ATTR_NORMAL_WT UL(0xbb) #define MAIR_ATTR_NORMAL_TAGGED UL(0xf0) #define MAIR_ATTR_NORMAL UL(0xff) #define MAIR_ATTR_MASK UL(0xff) diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index 61c97d3b58c7..c995d1f4594f 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -28,6 +28,10 @@ static void tlb_flush(struct mmu_gather *tlb); */ static inline int tlb_get_level(struct mmu_gather *tlb) { + /* The TTL field is only valid for the leaf entry. */ + if (tlb->freed_tables) + return 0; + if (tlb->cleared_ptes && !(tlb->cleared_pmds || tlb->cleared_puds || tlb->cleared_p4ds)) diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 24223adae150..b3edde68bc3e 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -184,6 +184,17 @@ struct kvm_vcpu_events { __u32 reserved[12]; }; +struct kvm_arm_copy_mte_tags { + __u64 guest_ipa; + __u64 length; + void __user *addr; + __u64 flags; + __u64 reserved[2]; +}; + +#define KVM_ARM_TAGS_TO_GUEST 0 +#define KVM_ARM_TAGS_FROM_GUEST 1 + /* If you need to interpret the index values, here is the key: */ #define KVM_REG_ARM_COPROC_MASK 0x000000000FFF0000 #define KVM_REG_ARM_COPROC_SHIFT 16 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index 6cc97730790e..cce308586fcc 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -14,15 +14,22 @@ CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_syscall.o = -fstack-protector -fstack-protector-strong CFLAGS_syscall.o += -fno-stack-protector +# It's not safe to invoke KCOV when portions of the kernel environment aren't +# available or are out-of-sync with HW state. Since `noinstr` doesn't always +# inhibit KCOV instrumentation, disable it for the entire compilation unit. +KCOV_INSTRUMENT_entry.o := n +KCOV_INSTRUMENT_idle.o := n + # Object file lists. obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ entry-common.o entry-fpsimd.o process.o ptrace.o \ setup.o signal.o sys.o stacktrace.o time.o traps.o \ - io.o vdso.o hyp-stub.o psci.o cpu_ops.o insn.o \ + io.o vdso.o hyp-stub.o psci.o cpu_ops.o \ return_address.o cpuinfo.o cpu_errata.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ - syscall.o proton-pack.o idreg-override.o + syscall.o proton-pack.o idreg-override.o idle.o \ + patching.o targets += efi-entry.o diff --git a/arch/arm64/kernel/acpi.c b/arch/arm64/kernel/acpi.c index cada0b816c8a..f3851724fe35 100644 --- a/arch/arm64/kernel/acpi.c +++ b/arch/arm64/kernel/acpi.c @@ -239,6 +239,18 @@ void __init acpi_boot_table_init(void) } } +static pgprot_t __acpi_get_writethrough_mem_attribute(void) +{ + /* + * Although UEFI specifies the use of Normal Write-through for + * EFI_MEMORY_WT, it is seldom used in practice and not implemented + * by most (all?) CPUs. Rather than allocate a MAIR just for this + * purpose, emit a warning and use Normal Non-cacheable instead. + */ + pr_warn_once("No MAIR allocation for EFI_MEMORY_WT; treating as Normal Non-cacheable\n"); + return __pgprot(PROT_NORMAL_NC); +} + pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) { /* @@ -246,7 +258,7 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) * types" of UEFI 2.5 section 2.3.6.1, each EFI memory type is * mapped to a corresponding MAIR attribute encoding. * The EFI memory attribute advises all possible capabilities - * of a memory region. We use the most efficient capability. + * of a memory region. */ u64 attr; @@ -254,10 +266,10 @@ pgprot_t __acpi_get_mem_attribute(phys_addr_t addr) attr = efi_mem_attributes(addr); if (attr & EFI_MEMORY_WB) return PAGE_KERNEL; - if (attr & EFI_MEMORY_WT) - return __pgprot(PROT_NORMAL_WT); if (attr & EFI_MEMORY_WC) return __pgprot(PROT_NORMAL_NC); + if (attr & EFI_MEMORY_WT) + return __acpi_get_writethrough_mem_attribute(); return __pgprot(PROT_DEVICE_nGnRnE); } @@ -340,10 +352,10 @@ void __iomem *acpi_os_ioremap(acpi_physical_address phys, acpi_size size) default: if (region->attribute & EFI_MEMORY_WB) prot = PAGE_KERNEL; - else if (region->attribute & EFI_MEMORY_WT) - prot = __pgprot(PROT_NORMAL_WT); else if (region->attribute & EFI_MEMORY_WC) prot = __pgprot(PROT_NORMAL_NC); + else if (region->attribute & EFI_MEMORY_WT) + prot = __acpi_get_writethrough_mem_attribute(); } } return __ioremap(phys, size, prot); diff --git a/arch/arm64/kernel/alternative.c b/arch/arm64/kernel/alternative.c index c906d20c7b52..3fb79b76e9d9 100644 --- a/arch/arm64/kernel/alternative.c +++ b/arch/arm64/kernel/alternative.c @@ -181,7 +181,7 @@ static void __nocfi __apply_alternatives(struct alt_region *region, bool is_modu */ if (!is_module) { dsb(ish); - __flush_icache_all(); + icache_inval_all_pou(); isb(); /* Ignore ARM64_CB bit from feature mask */ diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c index 0cb34ccb6e73..c85670692afa 100644 --- a/arch/arm64/kernel/asm-offsets.c +++ b/arch/arm64/kernel/asm-offsets.c @@ -27,6 +27,7 @@ int main(void) { DEFINE(TSK_ACTIVE_MM, offsetof(struct task_struct, active_mm)); + DEFINE(TSK_CPU, offsetof(struct task_struct, cpu)); BLANK(); DEFINE(TSK_TI_FLAGS, offsetof(struct task_struct, thread_info.flags)); DEFINE(TSK_TI_PREEMPT, offsetof(struct task_struct, thread_info.preempt_count)); @@ -46,6 +47,8 @@ int main(void) DEFINE(THREAD_SCTLR_USER, offsetof(struct task_struct, thread.sctlr_user)); #ifdef CONFIG_ARM64_PTR_AUTH DEFINE(THREAD_KEYS_USER, offsetof(struct task_struct, thread.keys_user)); +#endif +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL DEFINE(THREAD_KEYS_KERNEL, offsetof(struct task_struct, thread.keys_kernel)); #endif #ifdef CONFIG_ARM64_MTE @@ -99,7 +102,6 @@ int main(void) DEFINE(SOFTIRQ_SHIFT, SOFTIRQ_SHIFT); DEFINE(IRQ_CPUSTAT_SOFTIRQ_PENDING, offsetof(irq_cpustat_t, __softirq_pending)); BLANK(); - DEFINE(CPU_BOOT_STACK, offsetof(struct secondary_data, stack)); DEFINE(CPU_BOOT_TASK, offsetof(struct secondary_data, task)); BLANK(); DEFINE(FTR_OVR_VAL_OFFSET, offsetof(struct arm64_ftr_override, val)); @@ -111,6 +113,8 @@ int main(void) DEFINE(VCPU_WORKAROUND_FLAGS, offsetof(struct kvm_vcpu, arch.workaround_flags)); DEFINE(VCPU_HCR_EL2, offsetof(struct kvm_vcpu, arch.hcr_el2)); DEFINE(CPU_USER_PT_REGS, offsetof(struct kvm_cpu_context, regs)); + DEFINE(CPU_RGSR_EL1, offsetof(struct kvm_cpu_context, sys_regs[RGSR_EL1])); + DEFINE(CPU_GCR_EL1, offsetof(struct kvm_cpu_context, sys_regs[GCR_EL1])); DEFINE(CPU_APIAKEYLO_EL1, offsetof(struct kvm_cpu_context, sys_regs[APIAKEYLO_EL1])); DEFINE(CPU_APIBKEYLO_EL1, offsetof(struct kvm_cpu_context, sys_regs[APIBKEYLO_EL1])); DEFINE(CPU_APDAKEYLO_EL1, offsetof(struct kvm_cpu_context, sys_regs[APDAKEYLO_EL1])); @@ -138,6 +142,15 @@ int main(void) DEFINE(ARM_SMCCC_RES_X2_OFFS, offsetof(struct arm_smccc_res, a2)); DEFINE(ARM_SMCCC_QUIRK_ID_OFFS, offsetof(struct arm_smccc_quirk, id)); DEFINE(ARM_SMCCC_QUIRK_STATE_OFFS, offsetof(struct arm_smccc_quirk, state)); + DEFINE(ARM_SMCCC_1_2_REGS_X0_OFFS, offsetof(struct arm_smccc_1_2_regs, a0)); + DEFINE(ARM_SMCCC_1_2_REGS_X2_OFFS, offsetof(struct arm_smccc_1_2_regs, a2)); + DEFINE(ARM_SMCCC_1_2_REGS_X4_OFFS, offsetof(struct arm_smccc_1_2_regs, a4)); + DEFINE(ARM_SMCCC_1_2_REGS_X6_OFFS, offsetof(struct arm_smccc_1_2_regs, a6)); + DEFINE(ARM_SMCCC_1_2_REGS_X8_OFFS, offsetof(struct arm_smccc_1_2_regs, a8)); + DEFINE(ARM_SMCCC_1_2_REGS_X10_OFFS, offsetof(struct arm_smccc_1_2_regs, a10)); + DEFINE(ARM_SMCCC_1_2_REGS_X12_OFFS, offsetof(struct arm_smccc_1_2_regs, a12)); + DEFINE(ARM_SMCCC_1_2_REGS_X14_OFFS, offsetof(struct arm_smccc_1_2_regs, a14)); + DEFINE(ARM_SMCCC_1_2_REGS_X16_OFFS, offsetof(struct arm_smccc_1_2_regs, a16)); BLANK(); DEFINE(HIBERN_PBE_ORIG, offsetof(struct pbe, orig_address)); DEFINE(HIBERN_PBE_ADDR, offsetof(struct pbe, address)); @@ -153,7 +166,9 @@ int main(void) #endif #ifdef CONFIG_ARM64_PTR_AUTH DEFINE(PTRAUTH_USER_KEY_APIA, offsetof(struct ptrauth_keys_user, apia)); +#ifdef CONFIG_ARM64_PTR_AUTH_KERNEL DEFINE(PTRAUTH_KERNEL_KEY_APIA, offsetof(struct ptrauth_keys_kernel, apia)); +#endif BLANK(); #endif return 0; diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c index efed2830d141..125d5c9471ac 100644 --- a/arch/arm64/kernel/cpufeature.c +++ b/arch/arm64/kernel/cpufeature.c @@ -76,6 +76,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,24 @@ DECLARE_BITMAP(boot_capabilities, ARM64_NPATCHABLE); bool arm64_use_ng_mappings = false; EXPORT_SYMBOL(arm64_use_ng_mappings); +/* + * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs + * support it? + */ +static bool __read_mostly allow_mismatched_32bit_el0; + +/* + * Static branch enabled only if allow_mismatched_32bit_el0 is set and we have + * seen at least one CPU capable of 32-bit EL0. + */ +DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0); + +/* + * Mask of CPUs supporting 32-bit EL0. + * Only valid if arm64_mismatched_32bit_el0 is enabled. + */ +static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly; + /* * Flag to indicate if we have computed the system wide * capabilities based on the boot time active CPUs. This @@ -400,6 +419,11 @@ static const struct arm64_ftr_bits ftr_dczid[] = { ARM64_FTR_END, }; +static const struct arm64_ftr_bits ftr_gmid[] = { + ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, SYS_GMID_EL1_BS_SHIFT, 4, 0), + ARM64_FTR_END, +}; + static const struct arm64_ftr_bits ftr_id_isar0[] = { ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DIVIDE_SHIFT, 4, 0), ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_DEBUG_SHIFT, 4, 0), @@ -617,6 +641,9 @@ static const struct __ftr_reg_entry { /* Op1 = 0, CRn = 1, CRm = 2 */ ARM64_FTR_REG(SYS_ZCR_EL1, ftr_zcr), + /* Op1 = 1, CRn = 0, CRm = 0 */ + ARM64_FTR_REG(SYS_GMID_EL1, ftr_gmid), + /* Op1 = 3, CRn = 0, CRm = 0 */ { SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 }, ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid), @@ -767,7 +794,7 @@ static void __init sort_ftr_regs(void) * Any bits that are not covered by an arm64_ftr_bits entry are considered * RES0 for the system-wide value, and must strictly match. */ -static void __init init_cpu_ftr_reg(u32 sys_reg, u64 new) +static void init_cpu_ftr_reg(u32 sys_reg, u64 new) { u64 val = 0; u64 strict_mask = ~0x0ULL; @@ -863,6 +890,31 @@ static void __init init_cpu_hwcaps_indirect_list(void) static void __init setup_boot_cpu_capabilities(void); +static void init_32bit_cpu_features(struct cpuinfo_32bit *info) +{ + init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); + init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1); + init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0); + init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1); + init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2); + init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3); + init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4); + init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5); + init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6); + init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0); + init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1); + init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2); + init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3); + init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4); + init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5); + init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0); + init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1); + init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2); + init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0); + init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1); + init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); +} + void __init init_cpu_features(struct cpuinfo_arm64 *info) { /* Before we start using the tables, make sure it is sorted */ @@ -882,35 +934,17 @@ void __init init_cpu_features(struct cpuinfo_arm64 *info) init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1); init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0); - if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { - init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); - init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1); - init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0); - init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1); - init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2); - init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3); - init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4); - init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5); - init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6); - init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0); - init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1); - init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2); - init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3); - init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4); - init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5); - init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0); - init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1); - init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2); - init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0); - init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1); - init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); - } + if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) + init_32bit_cpu_features(&info->aarch32); if (id_aa64pfr0_sve(info->reg_id_aa64pfr0)) { init_cpu_ftr_reg(SYS_ZCR_EL1, info->reg_zcr); sve_init_vq_map(); } + if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) + init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid); + /* * Initialize the indirect array of CPU hwcaps capabilities pointers * before we handle the boot CPU below. @@ -975,20 +1009,28 @@ static void relax_cpu_ftr_reg(u32 sys_id, int field) WARN_ON(!ftrp->width); } -static int update_32bit_cpu_features(int cpu, struct cpuinfo_arm64 *info, - struct cpuinfo_arm64 *boot) +static void lazy_init_32bit_cpu_features(struct cpuinfo_arm64 *info, + struct cpuinfo_arm64 *boot) +{ + static bool boot_cpu_32bit_regs_overridden = false; + + if (!allow_mismatched_32bit_el0 || boot_cpu_32bit_regs_overridden) + return; + + if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0)) + return; + + boot->aarch32 = info->aarch32; + init_32bit_cpu_features(&boot->aarch32); + boot_cpu_32bit_regs_overridden = true; +} + +static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info, + struct cpuinfo_32bit *boot) { int taint = 0; u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1); - /* - * If we don't have AArch32 at all then skip the checks entirely - * as the register values may be UNKNOWN and we're not going to be - * using them for anything. - */ - if (!id_aa64pfr0_32bit_el0(pfr0)) - return taint; - /* * If we don't have AArch32 at EL1, then relax the strictness of * EL1-dependent register fields to avoid spurious sanity check fails. @@ -1135,10 +1177,29 @@ void update_cpu_features(int cpu, } /* + * The kernel uses the LDGM/STGM instructions and the number of tags + * they read/write depends on the GMID_EL1.BS field. Check that the + * value is the same on all CPUs. + */ + if (IS_ENABLED(CONFIG_ARM64_MTE) && + id_aa64pfr1_mte(info->reg_id_aa64pfr1)) { + taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu, + info->reg_gmid, boot->reg_gmid); + } + + /* + * If we don't have AArch32 at all then skip the checks entirely + * as the register values may be UNKNOWN and we're not going to be + * using them for anything. + * * This relies on a sanitised view of the AArch64 ID registers * (e.g. SYS_ID_AA64PFR0_EL1), so we call it last. */ - taint |= update_32bit_cpu_features(cpu, info, boot); + if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { + lazy_init_32bit_cpu_features(info, boot); + taint |= update_32bit_cpu_features(cpu, &info->aarch32, + &boot->aarch32); + } /* * Mismatched CPU features are a recipe for disaster. Don't even @@ -1248,6 +1309,28 @@ has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope) return feature_matches(val, entry); } +const struct cpumask *system_32bit_el0_cpumask(void) +{ + if (!system_supports_32bit_el0()) + return cpu_none_mask; + + if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) + return cpu_32bit_el0_mask; + + return cpu_possible_mask; +} + +static bool has_32bit_el0(const struct arm64_cpu_capabilities *entry, int scope) +{ + if (!has_cpuid_feature(entry, scope)) + return allow_mismatched_32bit_el0; + + if (scope == SCOPE_SYSTEM) + pr_info("detected: 32-bit EL0 Support\n"); + + return true; +} + static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope) { bool has_sre; @@ -1866,10 +1949,9 @@ static const struct arm64_cpu_capabilities arm64_features[] = { .cpu_enable = cpu_copy_el2regs, }, { - .desc = "32-bit EL0 Support", - .capability = ARM64_HAS_32BIT_EL0, + .capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE, .type = ARM64_CPUCAP_SYSTEM_FEATURE, - .matches = has_cpuid_feature, + .matches = has_32bit_el0, .sys_reg = SYS_ID_AA64PFR0_EL1, .sign = FTR_UNSIGNED, .field_pos = ID_AA64PFR0_EL0_SHIFT, @@ -2378,7 +2460,7 @@ static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = { {}, }; -static void __init cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap) +static void cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap) { switch (cap->hwcap_type) { case CAP_HWCAP: @@ -2423,7 +2505,7 @@ static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap) return rc; } -static void __init setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps) +static void setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps) { /* We support emulation of accesses to CPU ID feature registers */ cpu_set_named_feature(CPUID); @@ -2598,7 +2680,7 @@ static void check_early_cpu_features(void) } static void -verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) +__verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) { for (; caps->matches; caps++) @@ -2609,6 +2691,14 @@ verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps) } } +static void verify_local_elf_hwcaps(void) +{ + __verify_local_elf_hwcaps(arm64_elf_hwcaps); + + if (id_aa64pfr0_32bit_el0(read_cpuid(ID_AA64PFR0_EL1))) + __verify_local_elf_hwcaps(compat_elf_hwcaps); +} + static void verify_sve_features(void) { u64 safe_zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1); @@ -2673,11 +2763,7 @@ static void verify_local_cpu_capabilities(void) * on all secondary CPUs. */ verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU); - - verify_local_elf_hwcaps(arm64_elf_hwcaps); - - if (system_supports_32bit_el0()) - verify_local_elf_hwcaps(compat_elf_hwcaps); + verify_local_elf_hwcaps(); if (system_supports_sve()) verify_sve_features(); @@ -2812,6 +2898,34 @@ void __init setup_cpu_features(void) ARCH_DMA_MINALIGN); } +static int enable_mismatched_32bit_el0(unsigned int cpu) +{ + struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu); + bool cpu_32bit = id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0); + + if (cpu_32bit) { + cpumask_set_cpu(cpu, cpu_32bit_el0_mask); + static_branch_enable_cpuslocked(&arm64_mismatched_32bit_el0); + setup_elf_hwcaps(compat_elf_hwcaps); + } + + return 0; +} + +static int __init init_32bit_el0_mask(void) +{ + if (!allow_mismatched_32bit_el0) + return 0; + + if (!zalloc_cpumask_var(&cpu_32bit_el0_mask, GFP_KERNEL)) + return -ENOMEM; + + return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "arm64/mismatched_32bit_el0:online", + enable_mismatched_32bit_el0, NULL); +} +subsys_initcall_sync(init_32bit_el0_mask); + static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap) { cpu_replace_ttbr1(lm_alias(swapper_pg_dir)); @@ -2905,8 +3019,8 @@ static int emulate_mrs(struct pt_regs *regs, u32 insn) } static struct undef_hook mrs_hook = { - .instr_mask = 0xfff00000, - .instr_val = 0xd5300000, + .instr_mask = 0xffff0000, + .instr_val = 0xd5380000, .pstate_mask = PSR_AA32_MODE_MASK, .pstate_val = PSR_MODE_EL0t, .fn = emulate_mrs, diff --git a/arch/arm64/kernel/cpuinfo.c b/arch/arm64/kernel/cpuinfo.c index 51fcf99d5351..87731fea5e41 100644 --- a/arch/arm64/kernel/cpuinfo.c +++ b/arch/arm64/kernel/cpuinfo.c @@ -246,7 +246,7 @@ static struct kobj_type cpuregs_kobj_type = { struct cpuinfo_arm64 *info = kobj_to_cpuinfo(kobj); \ \ if (info->reg_midr) \ - return sprintf(buf, "0x%016x\n", info->reg_##_field); \ + return sprintf(buf, "0x%016llx\n", info->reg_##_field); \ else \ return 0; \ } \ @@ -344,6 +344,32 @@ static void cpuinfo_detect_icache_policy(struct cpuinfo_arm64 *info) pr_info("Detected %s I-cache on CPU%d\n", icache_policy_str[l1ip], cpu); } +static void __cpuinfo_store_cpu_32bit(struct cpuinfo_32bit *info) +{ + info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1); + info->reg_id_dfr1 = read_cpuid(ID_DFR1_EL1); + info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1); + info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1); + info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1); + info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1); + info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1); + info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1); + info->reg_id_isar6 = read_cpuid(ID_ISAR6_EL1); + info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1); + info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1); + info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1); + info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1); + info->reg_id_mmfr4 = read_cpuid(ID_MMFR4_EL1); + info->reg_id_mmfr5 = read_cpuid(ID_MMFR5_EL1); + info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1); + info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1); + info->reg_id_pfr2 = read_cpuid(ID_PFR2_EL1); + + info->reg_mvfr0 = read_cpuid(MVFR0_EL1); + info->reg_mvfr1 = read_cpuid(MVFR1_EL1); + info->reg_mvfr2 = read_cpuid(MVFR2_EL1); +} + static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) { info->reg_cntfrq = arch_timer_get_cntfrq(); @@ -371,31 +397,11 @@ static void __cpuinfo_store_cpu(struct cpuinfo_arm64 *info) info->reg_id_aa64pfr1 = read_cpuid(ID_AA64PFR1_EL1); info->reg_id_aa64zfr0 = read_cpuid(ID_AA64ZFR0_EL1); - /* Update the 32bit ID registers only if AArch32 is implemented */ - if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { - info->reg_id_dfr0 = read_cpuid(ID_DFR0_EL1); - info->reg_id_dfr1 = read_cpuid(ID_DFR1_EL1); - info->reg_id_isar0 = read_cpuid(ID_ISAR0_EL1); - info->reg_id_isar1 = read_cpuid(ID_ISAR1_EL1); - info->reg_id_isar2 = read_cpuid(ID_ISAR2_EL1); - info->reg_id_isar3 = read_cpuid(ID_ISAR3_EL1); - info->reg_id_isar4 = read_cpuid(ID_ISAR4_EL1); - info->reg_id_isar5 = read_cpuid(ID_ISAR5_EL1); - info->reg_id_isar6 = read_cpuid(ID_ISAR6_EL1); - info->reg_id_mmfr0 = read_cpuid(ID_MMFR0_EL1); - info->reg_id_mmfr1 = read_cpuid(ID_MMFR1_EL1); - info->reg_id_mmfr2 = read_cpuid(ID_MMFR2_EL1); - info->reg_id_mmfr3 = read_cpuid(ID_MMFR3_EL1); - info->reg_id_mmfr4 = read_cpuid(ID_MMFR4_EL1); - info->reg_id_mmfr5 = read_cpuid(ID_MMFR5_EL1); - info->reg_id_pfr0 = read_cpuid(ID_PFR0_EL1); - info->reg_id_pfr1 = read_cpuid(ID_PFR1_EL1); - info->reg_id_pfr2 = read_cpuid(ID_PFR2_EL1); + if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) + info->reg_gmid = read_cpuid(GMID_EL1); - info->reg_mvfr0 = read_cpuid(MVFR0_EL1); - info->reg_mvfr1 = read_cpuid(MVFR1_EL1); - info->reg_mvfr2 = read_cpuid(MVFR2_EL1); - } + if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) + __cpuinfo_store_cpu_32bit(&info->aarch32); if (IS_ENABLED(CONFIG_ARM64_SVE) && id_aa64pfr0_sve(info->reg_id_aa64pfr0)) diff --git a/arch/arm64/kernel/efi-entry.S b/arch/arm64/kernel/efi-entry.S index 0073b24b5d25..61a87fa1c305 100644 --- a/arch/arm64/kernel/efi-entry.S +++ b/arch/arm64/kernel/efi-entry.S @@ -28,7 +28,8 @@ SYM_CODE_START(efi_enter_kernel) * stale icache entries from before relocation. */ ldr w1, =kernel_size - bl __clean_dcache_area_poc + add x1, x0, x1 + bl dcache_clean_poc ic ialluis /* @@ -36,8 +37,8 @@ SYM_CODE_START(efi_enter_kernel) * so that we can safely disable the MMU and caches. */ adr x0, 0f - ldr w1, 3f - bl __clean_dcache_area_poc + adr x1, 3f + bl dcache_clean_poc 0: /* Turn off Dcache and MMU */ mrs x0, CurrentEL @@ -64,5 +65,5 @@ SYM_CODE_START(efi_enter_kernel) mov x2, xzr mov x3, xzr br x19 +3: SYM_CODE_END(efi_enter_kernel) -3: .long . - 0b diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c index 340d04e13617..12ce14a98b7c 100644 --- a/arch/arm64/kernel/entry-common.c +++ b/arch/arm64/kernel/entry-common.c @@ -6,7 +6,11 @@ */ #include +#include +#include #include +#include +#include #include #include @@ -15,7 +19,11 @@ #include #include #include +#include +#include +#include #include +#include /* * This is intended to match the logic in irqentry_enter(), handling the kernel @@ -67,7 +75,7 @@ static void noinstr exit_to_kernel_mode(struct pt_regs *regs) } } -void noinstr arm64_enter_nmi(struct pt_regs *regs) +static void noinstr arm64_enter_nmi(struct pt_regs *regs) { regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); @@ -80,7 +88,7 @@ void noinstr arm64_enter_nmi(struct pt_regs *regs) ftrace_nmi_enter(); } -void noinstr arm64_exit_nmi(struct pt_regs *regs) +static void noinstr arm64_exit_nmi(struct pt_regs *regs) { bool restore = regs->lockdep_hardirqs; @@ -97,7 +105,7 @@ void noinstr arm64_exit_nmi(struct pt_regs *regs) __nmi_exit(); } -asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) +static void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) { if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) arm64_enter_nmi(regs); @@ -105,7 +113,7 @@ asmlinkage void noinstr enter_el1_irq_or_nmi(struct pt_regs *regs) enter_from_kernel_mode(regs); } -asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) +static void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) { if (IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && !interrupts_enabled(regs)) arm64_exit_nmi(regs); @@ -113,6 +121,65 @@ asmlinkage void noinstr exit_el1_irq_or_nmi(struct pt_regs *regs) exit_to_kernel_mode(regs); } +static void __sched arm64_preempt_schedule_irq(void) +{ + lockdep_assert_irqs_disabled(); + + /* + * DAIF.DA are cleared at the start of IRQ/FIQ handling, and when GIC + * priority masking is used the GIC irqchip driver will clear DAIF.IF + * using gic_arch_enable_irqs() for normal IRQs. If anything is set in + * DAIF we must have handled an NMI, so skip preemption. + */ + if (system_uses_irq_prio_masking() && read_sysreg(daif)) + return; + + /* + * Preempting a task from an IRQ means we leave copies of PSTATE + * on the stack. cpufeature's enable calls may modify PSTATE, but + * resuming one of these preempted tasks would undo those changes. + * + * Only allow a task to be preempted once cpufeatures have been + * enabled. + */ + if (system_capabilities_finalized()) + preempt_schedule_irq(); +} + +static void do_interrupt_handler(struct pt_regs *regs, + void (*handler)(struct pt_regs *)) +{ + if (on_thread_stack()) + call_on_irq_stack(regs, handler); + else + handler(regs); +} + +extern void (*handle_arch_irq)(struct pt_regs *); +extern void (*handle_arch_fiq)(struct pt_regs *); + +static void noinstr __panic_unhandled(struct pt_regs *regs, const char *vector, + unsigned int esr) +{ + arm64_enter_nmi(regs); + + console_verbose(); + + pr_crit("Unhandled %s exception on CPU%d, ESR 0x%08x -- %s\n", + vector, smp_processor_id(), esr, + esr_get_class_string(esr)); + + __show_regs(regs); + panic("Unhandled exception"); +} + +#define UNHANDLED(el, regsize, vector) \ +asmlinkage void noinstr el##_##regsize##_##vector##_handler(struct pt_regs *regs) \ +{ \ + const char *desc = #regsize "-bit " #el " " #vector; \ + __panic_unhandled(regs, desc, read_sysreg(esr_el1)); \ +} + #ifdef CONFIG_ARM64_ERRATUM_1463225 static DEFINE_PER_CPU(int, __in_cortex_a76_erratum_1463225_wa); @@ -162,6 +229,11 @@ static bool cortex_a76_erratum_1463225_debug_handler(struct pt_regs *regs) } #endif /* CONFIG_ARM64_ERRATUM_1463225 */ +UNHANDLED(el1t, 64, sync) +UNHANDLED(el1t, 64, irq) +UNHANDLED(el1t, 64, fiq) +UNHANDLED(el1t, 64, error) + static void noinstr el1_abort(struct pt_regs *regs, unsigned long esr) { unsigned long far = read_sysreg(far_el1); @@ -193,15 +265,6 @@ static void noinstr el1_undef(struct pt_regs *regs) exit_to_kernel_mode(regs); } -static void noinstr el1_inv(struct pt_regs *regs, unsigned long esr) -{ - enter_from_kernel_mode(regs); - local_daif_inherit(regs); - bad_mode(regs, 0, esr); - local_daif_mask(); - exit_to_kernel_mode(regs); -} - static void noinstr arm64_enter_el1_dbg(struct pt_regs *regs) { regs->lockdep_hardirqs = lockdep_hardirqs_enabled(); @@ -245,7 +308,7 @@ static void noinstr el1_fpac(struct pt_regs *regs, unsigned long esr) exit_to_kernel_mode(regs); } -asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs) +asmlinkage void noinstr el1h_64_sync_handler(struct pt_regs *regs) { unsigned long esr = read_sysreg(esr_el1); @@ -275,10 +338,50 @@ asmlinkage void noinstr el1_sync_handler(struct pt_regs *regs) el1_fpac(regs, esr); break; default: - el1_inv(regs, esr); + __panic_unhandled(regs, "64-bit el1h sync", esr); } } +static void noinstr el1_interrupt(struct pt_regs *regs, + void (*handler)(struct pt_regs *)) +{ + write_sysreg(DAIF_PROCCTX_NOIRQ, daif); + + enter_el1_irq_or_nmi(regs); + do_interrupt_handler(regs, handler); + + /* + * Note: thread_info::preempt_count includes both thread_info::count + * and thread_info::need_resched, and is not equivalent to + * preempt_count(). + */ + if (IS_ENABLED(CONFIG_PREEMPTION) && + READ_ONCE(current_thread_info()->preempt_count) == 0) + arm64_preempt_schedule_irq(); + + exit_el1_irq_or_nmi(regs); +} + +asmlinkage void noinstr el1h_64_irq_handler(struct pt_regs *regs) +{ + el1_interrupt(regs, handle_arch_irq); +} + +asmlinkage void noinstr el1h_64_fiq_handler(struct pt_regs *regs) +{ + el1_interrupt(regs, handle_arch_fiq); +} + +asmlinkage void noinstr el1h_64_error_handler(struct pt_regs *regs) +{ + unsigned long esr = read_sysreg(esr_el1); + + local_daif_restore(DAIF_ERRCTX); + arm64_enter_nmi(regs); + do_serror(regs, esr); + arm64_exit_nmi(regs); +} + asmlinkage void noinstr enter_from_user_mode(void) { lockdep_hardirqs_off(CALLER_ADDR0); @@ -398,7 +501,7 @@ static void noinstr el0_dbg(struct pt_regs *regs, unsigned long esr) enter_from_user_mode(); do_debug_exception(far, esr, regs); - local_daif_restore(DAIF_PROCCTX_NOIRQ); + local_daif_restore(DAIF_PROCCTX); } static void noinstr el0_svc(struct pt_regs *regs) @@ -415,7 +518,7 @@ static void noinstr el0_fpac(struct pt_regs *regs, unsigned long esr) do_ptrauth_fault(regs, esr); } -asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs) +asmlinkage void noinstr el0t_64_sync_handler(struct pt_regs *regs) { unsigned long esr = read_sysreg(esr_el1); @@ -468,6 +571,56 @@ asmlinkage void noinstr el0_sync_handler(struct pt_regs *regs) } } +static void noinstr el0_interrupt(struct pt_regs *regs, + void (*handler)(struct pt_regs *)) +{ + enter_from_user_mode(); + + write_sysreg(DAIF_PROCCTX_NOIRQ, daif); + + if (regs->pc & BIT(55)) + arm64_apply_bp_hardening(); + + do_interrupt_handler(regs, handler); +} + +static void noinstr __el0_irq_handler_common(struct pt_regs *regs) +{ + el0_interrupt(regs, handle_arch_irq); +} + +asmlinkage void noinstr el0t_64_irq_handler(struct pt_regs *regs) +{ + __el0_irq_handler_common(regs); +} + +static void noinstr __el0_fiq_handler_common(struct pt_regs *regs) +{ + el0_interrupt(regs, handle_arch_fiq); +} + +asmlinkage void noinstr el0t_64_fiq_handler(struct pt_regs *regs) +{ + __el0_fiq_handler_common(regs); +} + +static void __el0_error_handler_common(struct pt_regs *regs) +{ + unsigned long esr = read_sysreg(esr_el1); + + enter_from_user_mode(); + local_daif_restore(DAIF_ERRCTX); + arm64_enter_nmi(regs); + do_serror(regs, esr); + arm64_exit_nmi(regs); + local_daif_restore(DAIF_PROCCTX); +} + +asmlinkage void noinstr el0t_64_error_handler(struct pt_regs *regs) +{ + __el0_error_handler_common(regs); +} + #ifdef CONFIG_COMPAT static void noinstr el0_cp15(struct pt_regs *regs, unsigned long esr) { @@ -483,7 +636,7 @@ static void noinstr el0_svc_compat(struct pt_regs *regs) do_el0_svc_compat(regs); } -asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs) +asmlinkage void noinstr el0t_32_sync_handler(struct pt_regs *regs) { unsigned long esr = read_sysreg(esr_el1); @@ -526,4 +679,71 @@ asmlinkage void noinstr el0_sync_compat_handler(struct pt_regs *regs) el0_inv(regs, esr); } } + +asmlinkage void noinstr el0t_32_irq_handler(struct pt_regs *regs) +{ + __el0_irq_handler_common(regs); +} + +asmlinkage void noinstr el0t_32_fiq_handler(struct pt_regs *regs) +{ + __el0_fiq_handler_common(regs); +} + +asmlinkage void noinstr el0t_32_error_handler(struct pt_regs *regs) +{ + __el0_error_handler_common(regs); +} +#else /* CONFIG_COMPAT */ +UNHANDLED(el0t, 32, sync) +UNHANDLED(el0t, 32, irq) +UNHANDLED(el0t, 32, fiq) +UNHANDLED(el0t, 32, error) #endif /* CONFIG_COMPAT */ + +#ifdef CONFIG_VMAP_STACK +asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs) +{ + unsigned int esr = read_sysreg(esr_el1); + unsigned long far = read_sysreg(far_el1); + + arm64_enter_nmi(regs); + panic_bad_stack(regs, esr, far); +} +#endif /* CONFIG_VMAP_STACK */ + +#ifdef CONFIG_ARM_SDE_INTERFACE +asmlinkage noinstr unsigned long +__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) +{ + unsigned long ret; + + /* + * We didn't take an exception to get here, so the HW hasn't + * set/cleared bits in PSTATE that we may rely on. + * + * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to + * whether PSTATE bits are inherited unchanged or generated from + * scratch, and the TF-A implementation always clears PAN and always + * clears UAO. There are no other known implementations. + * + * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how + * PSTATE is modified upon architectural exceptions, and so PAN is + * either inherited or set per SCTLR_ELx.SPAN, and UAO is always + * cleared. + * + * We must explicitly reset PAN to the expected state, including + * clearing it when the host isn't using it, in case a VM had it set. + */ + if (system_uses_hw_pan()) + set_pstate_pan(1); + else if (cpu_has_pan()) + set_pstate_pan(0); + + arm64_enter_nmi(regs); + ret = do_sdei_event(regs, arg); + arm64_exit_nmi(regs); + + return ret; +} +#endif /* CONFIG_ARM_SDE_INTERFACE */ diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S index 3ecec60d3295..0a7a64753878 100644 --- a/arch/arm64/kernel/entry-fpsimd.S +++ b/arch/arm64/kernel/entry-fpsimd.S @@ -63,16 +63,24 @@ SYM_FUNC_END(sve_set_vq) * and the rest zeroed. All the other SVE registers will be zeroed. */ SYM_FUNC_START(sve_load_from_fpsimd_state) - sve_load_vq x1, x2, x3 - fpsimd_restore x0, 8 - _for n, 0, 15, _sve_pfalse \n - _sve_wrffr 0 - ret + sve_load_vq x1, x2, x3 + fpsimd_restore x0, 8 + sve_flush_p_ffr + ret SYM_FUNC_END(sve_load_from_fpsimd_state) -/* Zero all SVE registers but the first 128-bits of each vector */ +/* + * Zero all SVE registers but the first 128-bits of each vector + * + * VQ must already be configured by caller, any further updates of VQ + * will need to ensure that the register state remains valid. + * + * x0 = VQ - 1 + */ SYM_FUNC_START(sve_flush_live) - sve_flush + cbz x0, 1f // A VQ-1 of 0 is 128 bits so no extra Z state + sve_flush_z +1: sve_flush_p_ffr ret SYM_FUNC_END(sve_flush_live) diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S index 3513984a88bd..863d44f73028 100644 --- a/arch/arm64/kernel/entry.S +++ b/arch/arm64/kernel/entry.S @@ -33,12 +33,6 @@ * Context tracking and irqflag tracing need to instrument transitions between * user and kernel mode. */ - .macro user_exit_irqoff -#if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS) - bl enter_from_user_mode -#endif - .endm - .macro user_enter_irqoff #if defined(CONFIG_CONTEXT_TRACKING) || defined(CONFIG_TRACE_IRQFLAGS) bl exit_to_user_mode @@ -51,16 +45,7 @@ .endr .endm -/* - * Bad Abort numbers - *----------------- - */ -#define BAD_SYNC 0 -#define BAD_IRQ 1 -#define BAD_FIQ 2 -#define BAD_ERROR 3 - - .macro kernel_ventry, el, label, regsize = 64 + .macro kernel_ventry, el:req, ht:req, regsize:req, label:req .align 7 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0 .if \el == 0 @@ -87,7 +72,7 @@ alternative_else_nop_endif tbnz x0, #THREAD_SHIFT, 0f sub x0, sp, x0 // x0'' = sp' - x0' = (sp + x0) - sp = x0 sub sp, sp, x0 // sp'' = sp' - x0 = (sp + x0) - x0 = sp - b el\()\el\()_\label + b el\el\ht\()_\regsize\()_\label 0: /* @@ -119,7 +104,7 @@ alternative_else_nop_endif sub sp, sp, x0 mrs x0, tpidrro_el0 #endif - b el\()\el\()_\label + b el\el\ht\()_\regsize\()_\label .endm .macro tramp_alias, dst, sym @@ -275,7 +260,7 @@ alternative_else_nop_endif mte_set_kernel_gcr x22, x23 - scs_load tsk, x20 + scs_load tsk .else add x21, sp, #PT_REGS_SIZE get_current_task tsk @@ -285,7 +270,7 @@ alternative_else_nop_endif stp lr, x21, [sp, #S_LR] /* - * For exceptions from EL0, create a terminal frame record. + * For exceptions from EL0, create a final frame record. * For exceptions from EL1, create a synthetic frame record so the * interrupted code shows up in the backtrace. */ @@ -375,7 +360,7 @@ alternative_if ARM64_WORKAROUND_845719 alternative_else_nop_endif #endif 3: - scs_save tsk, x0 + scs_save tsk #ifdef CONFIG_ARM64_PTR_AUTH alternative_if ARM64_HAS_ADDRESS_AUTH @@ -486,63 +471,12 @@ SYM_CODE_START_LOCAL(__swpan_exit_el0) SYM_CODE_END(__swpan_exit_el0) #endif - .macro irq_stack_entry - mov x19, sp // preserve the original sp -#ifdef CONFIG_SHADOW_CALL_STACK - mov x24, scs_sp // preserve the original shadow stack -#endif - - /* - * Compare sp with the base of the task stack. - * If the top ~(THREAD_SIZE - 1) bits match, we are on a task stack, - * and should switch to the irq stack. - */ - ldr x25, [tsk, TSK_STACK] - eor x25, x25, x19 - and x25, x25, #~(THREAD_SIZE - 1) - cbnz x25, 9998f - - ldr_this_cpu x25, irq_stack_ptr, x26 - mov x26, #IRQ_STACK_SIZE - add x26, x25, x26 - - /* switch to the irq stack */ - mov sp, x26 - -#ifdef CONFIG_SHADOW_CALL_STACK - /* also switch to the irq shadow stack */ - ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x26 -#endif - -9998: - .endm - - /* - * The callee-saved regs (x19-x29) should be preserved between - * irq_stack_entry and irq_stack_exit, but note that kernel_entry - * uses x20-x23 to store data for later use. - */ - .macro irq_stack_exit - mov sp, x19 -#ifdef CONFIG_SHADOW_CALL_STACK - mov scs_sp, x24 -#endif - .endm - /* GPRs used by entry code */ tsk .req x28 // current thread_info /* * Interrupt handling. */ - .macro irq_handler, handler:req - ldr_l x1, \handler - mov x0, sp - irq_stack_entry - blr x1 - irq_stack_exit - .endm - .macro gic_prio_kentry_setup, tmp:req #ifdef CONFIG_ARM64_PSEUDO_NMI alternative_if ARM64_HAS_IRQ_PRIO_MASKING @@ -552,45 +486,6 @@ tsk .req x28 // current thread_info #endif .endm - .macro el1_interrupt_handler, handler:req - enable_da - - mov x0, sp - bl enter_el1_irq_or_nmi - - irq_handler \handler - -#ifdef CONFIG_PREEMPTION - ldr x24, [tsk, #TSK_TI_PREEMPT] // get preempt count -alternative_if ARM64_HAS_IRQ_PRIO_MASKING - /* - * DA were cleared at start of handling, and IF are cleared by - * the GIC irqchip driver using gic_arch_enable_irqs() for - * normal IRQs. If anything is set, it means we come back from - * an NMI instead of a normal IRQ, so skip preemption - */ - mrs x0, daif - orr x24, x24, x0 -alternative_else_nop_endif - cbnz x24, 1f // preempt count != 0 || NMI return path - bl arm64_preempt_schedule_irq // irq en/disable is done inside -1: -#endif - - mov x0, sp - bl exit_el1_irq_or_nmi - .endm - - .macro el0_interrupt_handler, handler:req - user_exit_irqoff - enable_da - - tbz x22, #55, 1f - bl do_el0_irq_bp_hardening -1: - irq_handler \handler - .endm - .text /* @@ -600,32 +495,25 @@ alternative_else_nop_endif .align 11 SYM_CODE_START(vectors) - kernel_ventry 1, sync_invalid // Synchronous EL1t - kernel_ventry 1, irq_invalid // IRQ EL1t - kernel_ventry 1, fiq_invalid // FIQ EL1t - kernel_ventry 1, error_invalid // Error EL1t + kernel_ventry 1, t, 64, sync // Synchronous EL1t + kernel_ventry 1, t, 64, irq // IRQ EL1t + kernel_ventry 1, t, 64, fiq // FIQ EL1h + kernel_ventry 1, t, 64, error // Error EL1t - kernel_ventry 1, sync // Synchronous EL1h - kernel_ventry 1, irq // IRQ EL1h - kernel_ventry 1, fiq // FIQ EL1h - kernel_ventry 1, error // Error EL1h + kernel_ventry 1, h, 64, sync // Synchronous EL1h + kernel_ventry 1, h, 64, irq // IRQ EL1h + kernel_ventry 1, h, 64, fiq // FIQ EL1h + kernel_ventry 1, h, 64, error // Error EL1h - kernel_ventry 0, sync // Synchronous 64-bit EL0 - kernel_ventry 0, irq // IRQ 64-bit EL0 - kernel_ventry 0, fiq // FIQ 64-bit EL0 - kernel_ventry 0, error // Error 64-bit EL0 + kernel_ventry 0, t, 64, sync // Synchronous 64-bit EL0 + kernel_ventry 0, t, 64, irq // IRQ 64-bit EL0 + kernel_ventry 0, t, 64, fiq // FIQ 64-bit EL0 + kernel_ventry 0, t, 64, error // Error 64-bit EL0 -#ifdef CONFIG_COMPAT - kernel_ventry 0, sync_compat, 32 // Synchronous 32-bit EL0 - kernel_ventry 0, irq_compat, 32 // IRQ 32-bit EL0 - kernel_ventry 0, fiq_compat, 32 // FIQ 32-bit EL0 - kernel_ventry 0, error_compat, 32 // Error 32-bit EL0 -#else - kernel_ventry 0, sync_invalid, 32 // Synchronous 32-bit EL0 - kernel_ventry 0, irq_invalid, 32 // IRQ 32-bit EL0 - kernel_ventry 0, fiq_invalid, 32 // FIQ 32-bit EL0 - kernel_ventry 0, error_invalid, 32 // Error 32-bit EL0 -#endif + kernel_ventry 0, t, 32, sync // Synchronous 32-bit EL0 + kernel_ventry 0, t, 32, irq // IRQ 32-bit EL0 + kernel_ventry 0, t, 32, fiq // FIQ 32-bit EL0 + kernel_ventry 0, t, 32, error // Error 32-bit EL0 SYM_CODE_END(vectors) #ifdef CONFIG_VMAP_STACK @@ -656,147 +544,46 @@ __bad_stack: ASM_BUG() #endif /* CONFIG_VMAP_STACK */ -/* - * Invalid mode handlers - */ - .macro inv_entry, el, reason, regsize = 64 + + .macro entry_handler el:req, ht:req, regsize:req, label:req +SYM_CODE_START_LOCAL(el\el\ht\()_\regsize\()_\label) kernel_entry \el, \regsize mov x0, sp - mov x1, #\reason - mrs x2, esr_el1 - bl bad_mode - ASM_BUG() + bl el\el\ht\()_\regsize\()_\label\()_handler + .if \el == 0 + b ret_to_user + .else + b ret_to_kernel + .endif +SYM_CODE_END(el\el\ht\()_\regsize\()_\label) .endm -SYM_CODE_START_LOCAL(el0_sync_invalid) - inv_entry 0, BAD_SYNC -SYM_CODE_END(el0_sync_invalid) - -SYM_CODE_START_LOCAL(el0_irq_invalid) - inv_entry 0, BAD_IRQ -SYM_CODE_END(el0_irq_invalid) - -SYM_CODE_START_LOCAL(el0_fiq_invalid) - inv_entry 0, BAD_FIQ -SYM_CODE_END(el0_fiq_invalid) - -SYM_CODE_START_LOCAL(el0_error_invalid) - inv_entry 0, BAD_ERROR -SYM_CODE_END(el0_error_invalid) - -SYM_CODE_START_LOCAL(el1_sync_invalid) - inv_entry 1, BAD_SYNC -SYM_CODE_END(el1_sync_invalid) - -SYM_CODE_START_LOCAL(el1_irq_invalid) - inv_entry 1, BAD_IRQ -SYM_CODE_END(el1_irq_invalid) - -SYM_CODE_START_LOCAL(el1_fiq_invalid) - inv_entry 1, BAD_FIQ -SYM_CODE_END(el1_fiq_invalid) - -SYM_CODE_START_LOCAL(el1_error_invalid) - inv_entry 1, BAD_ERROR -SYM_CODE_END(el1_error_invalid) - /* - * EL1 mode handlers. + * Early exception handlers */ - .align 6 -SYM_CODE_START_LOCAL_NOALIGN(el1_sync) - kernel_entry 1 - mov x0, sp - bl el1_sync_handler + entry_handler 1, t, 64, sync + entry_handler 1, t, 64, irq + entry_handler 1, t, 64, fiq + entry_handler 1, t, 64, error + + entry_handler 1, h, 64, sync + entry_handler 1, h, 64, irq + entry_handler 1, h, 64, fiq + entry_handler 1, h, 64, error + + entry_handler 0, t, 64, sync + entry_handler 0, t, 64, irq + entry_handler 0, t, 64, fiq + entry_handler 0, t, 64, error + + entry_handler 0, t, 32, sync + entry_handler 0, t, 32, irq + entry_handler 0, t, 32, fiq + entry_handler 0, t, 32, error + +SYM_CODE_START_LOCAL(ret_to_kernel) kernel_exit 1 -SYM_CODE_END(el1_sync) - - .align 6 -SYM_CODE_START_LOCAL_NOALIGN(el1_irq) - kernel_entry 1 - el1_interrupt_handler handle_arch_irq - kernel_exit 1 -SYM_CODE_END(el1_irq) - -SYM_CODE_START_LOCAL_NOALIGN(el1_fiq) - kernel_entry 1 - el1_interrupt_handler handle_arch_fiq - kernel_exit 1 -SYM_CODE_END(el1_fiq) - -/* - * EL0 mode handlers. - */ - .align 6 -SYM_CODE_START_LOCAL_NOALIGN(el0_sync) - kernel_entry 0 - mov x0, sp - bl el0_sync_handler - b ret_to_user -SYM_CODE_END(el0_sync) - -#ifdef CONFIG_COMPAT - .align 6 -SYM_CODE_START_LOCAL_NOALIGN(el0_sync_compat) - kernel_entry 0, 32 - mov x0, sp - bl el0_sync_compat_handler - b ret_to_user -SYM_CODE_END(el0_sync_compat) - - .align 6 -SYM_CODE_START_LOCAL_NOALIGN(el0_irq_compat) - kernel_entry 0, 32 - b el0_irq_naked -SYM_CODE_END(el0_irq_compat) - -SYM_CODE_START_LOCAL_NOALIGN(el0_fiq_compat) - kernel_entry 0, 32 - b el0_fiq_naked -SYM_CODE_END(el0_fiq_compat) - -SYM_CODE_START_LOCAL_NOALIGN(el0_error_compat) - kernel_entry 0, 32 - b el0_error_naked -SYM_CODE_END(el0_error_compat) -#endif - - .align 6 -SYM_CODE_START_LOCAL_NOALIGN(el0_irq) - kernel_entry 0 -el0_irq_naked: - el0_interrupt_handler handle_arch_irq - b ret_to_user -SYM_CODE_END(el0_irq) - -SYM_CODE_START_LOCAL_NOALIGN(el0_fiq) - kernel_entry 0 -el0_fiq_naked: - el0_interrupt_handler handle_arch_fiq - b ret_to_user -SYM_CODE_END(el0_fiq) - -SYM_CODE_START_LOCAL(el1_error) - kernel_entry 1 - mrs x1, esr_el1 - enable_dbg - mov x0, sp - bl do_serror - kernel_exit 1 -SYM_CODE_END(el1_error) - -SYM_CODE_START_LOCAL(el0_error) - kernel_entry 0 -el0_error_naked: - mrs x25, esr_el1 - user_exit_irqoff - enable_dbg - mov x0, sp - mov x1, x25 - bl do_serror - enable_da - b ret_to_user -SYM_CODE_END(el0_error) +SYM_CODE_END(ret_to_kernel) /* * "slow" syscall return path. @@ -979,8 +766,8 @@ SYM_FUNC_START(cpu_switch_to) mov sp, x9 msr sp_el0, x1 ptrauth_keys_install_kernel x1, x8, x9, x10 - scs_save x0, x8 - scs_load x1, x8 + scs_save x0 + scs_load x1 ret SYM_FUNC_END(cpu_switch_to) NOKPROBE(cpu_switch_to) @@ -998,6 +785,42 @@ SYM_CODE_START(ret_from_fork) SYM_CODE_END(ret_from_fork) NOKPROBE(ret_from_fork) +/* + * void call_on_irq_stack(struct pt_regs *regs, + * void (*func)(struct pt_regs *)); + * + * Calls func(regs) using this CPU's irq stack and shadow irq stack. + */ +SYM_FUNC_START(call_on_irq_stack) +#ifdef CONFIG_SHADOW_CALL_STACK + stp scs_sp, xzr, [sp, #-16]! + ldr_this_cpu scs_sp, irq_shadow_call_stack_ptr, x17 +#endif + /* Create a frame record to save our LR and SP (implicit in FP) */ + stp x29, x30, [sp, #-16]! + mov x29, sp + + ldr_this_cpu x16, irq_stack_ptr, x17 + mov x15, #IRQ_STACK_SIZE + add x16, x16, x15 + + /* Move to the new stack and call the function there */ + mov sp, x16 + blr x1 + + /* + * Restore the SP from the FP, and restore the FP and LR from the frame + * record. + */ + mov sp, x29 + ldp x29, x30, [sp], #16 +#ifdef CONFIG_SHADOW_CALL_STACK + ldp scs_sp, xzr, [sp], #16 +#endif + ret +SYM_FUNC_END(call_on_irq_stack) +NOKPROBE(call_on_irq_stack) + #ifdef CONFIG_ARM_SDE_INTERFACE #include diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c index ad3dd34a83cf..e57b23f95284 100644 --- a/arch/arm64/kernel/fpsimd.c +++ b/arch/arm64/kernel/fpsimd.c @@ -957,8 +957,10 @@ void do_sve_acc(unsigned int esr, struct pt_regs *regs) * disabling the trap, otherwise update our in-memory copy. */ if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) { - sve_set_vq(sve_vq_from_vl(current->thread.sve_vl) - 1); - sve_flush_live(); + unsigned long vq_minus_one = + sve_vq_from_vl(current->thread.sve_vl) - 1; + sve_set_vq(vq_minus_one); + sve_flush_live(vq_minus_one); fpsimd_bind_task_to_cpu(); } else { fpsimd_to_sve(current); diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index b5d3ddaf69d9..7f467bd9db7a 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -15,6 +15,7 @@ #include #include #include +#include #ifdef CONFIG_DYNAMIC_FTRACE /* diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 96873dfa67fd..c5c994a73a64 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -117,8 +118,8 @@ SYM_CODE_START_LOCAL(preserve_boot_args) dmb sy // needed before dc ivac with // MMU off - mov x1, #0x20 // 4 x 8 bytes - b __inval_dcache_area // tail call + add x1, x0, #0x20 // 4 x 8 bytes + b dcache_inval_poc // tail call SYM_CODE_END(preserve_boot_args) /* @@ -195,7 +196,7 @@ SYM_CODE_END(preserve_boot_args) and \iend, \iend, \istart // iend = (vend >> shift) & (ptrs - 1) mov \istart, \ptrs mul \istart, \istart, \count - add \iend, \iend, \istart // iend += (count - 1) * ptrs + add \iend, \iend, \istart // iend += count * ptrs // our entries span multiple tables lsr \istart, \vstart, \shift @@ -268,8 +269,7 @@ SYM_FUNC_START_LOCAL(__create_page_tables) */ adrp x0, init_pg_dir adrp x1, init_pg_end - sub x1, x1, x0 - bl __inval_dcache_area + bl dcache_inval_poc /* * Clear the init page tables. @@ -354,7 +354,6 @@ SYM_FUNC_START_LOCAL(__create_page_tables) #endif 1: ldr_l x4, idmap_ptrs_per_pgd - mov x5, x3 // __pa(__idmap_text_start) adr_l x6, __idmap_text_end // __pa(__idmap_text_end) map_memory x0, x1, x3, x6, x7, x3, x4, x10, x11, x12, x13, x14 @@ -382,39 +381,57 @@ SYM_FUNC_START_LOCAL(__create_page_tables) adrp x0, idmap_pg_dir adrp x1, idmap_pg_end - sub x1, x1, x0 - bl __inval_dcache_area + bl dcache_inval_poc adrp x0, init_pg_dir adrp x1, init_pg_end - sub x1, x1, x0 - bl __inval_dcache_area + bl dcache_inval_poc ret x28 SYM_FUNC_END(__create_page_tables) + /* + * Initialize CPU registers with task-specific and cpu-specific context. + * + * Create a final frame record at task_pt_regs(current)->stackframe, so + * that the unwinder can identify the final frame record of any task by + * its location in the task stack. We reserve the entire pt_regs space + * for consistency with user tasks and kthreads. + */ + .macro init_cpu_task tsk, tmp1, tmp2 + msr sp_el0, \tsk + + ldr \tmp1, [\tsk, #TSK_STACK] + add sp, \tmp1, #THREAD_SIZE + sub sp, sp, #PT_REGS_SIZE + + stp xzr, xzr, [sp, #S_STACKFRAME] + add x29, sp, #S_STACKFRAME + + scs_load \tsk + + adr_l \tmp1, __per_cpu_offset + ldr w\tmp2, [\tsk, #TSK_CPU] + ldr \tmp1, [\tmp1, \tmp2, lsl #3] + set_this_cpu_offset \tmp1 + .endm + /* * The following fragment of code is executed with the MMU enabled. * * x0 = __PHYS_OFFSET */ SYM_FUNC_START_LOCAL(__primary_switched) - adrp x4, init_thread_union - add sp, x4, #THREAD_SIZE - adr_l x5, init_task - msr sp_el0, x5 // Save thread_info + adr_l x4, init_task + init_cpu_task x4, x5, x6 adr_l x8, vectors // load VBAR_EL1 with virtual msr vbar_el1, x8 // vector table address isb - stp xzr, x30, [sp, #-16]! + stp x29, x30, [sp, #-16]! mov x29, sp -#ifdef CONFIG_SHADOW_CALL_STACK - adr_l scs_sp, init_shadow_call_stack // Set shadow call stack -#endif - str_l x21, __fdt_pointer, x5 // Save FDT pointer ldr_l x4, kimage_vaddr // Save the offset between @@ -446,10 +463,9 @@ SYM_FUNC_START_LOCAL(__primary_switched) 0: #endif bl switch_to_vhe // Prefer VHE if possible - add sp, sp, #16 - mov x29, #0 - mov x30, #0 - b start_kernel + ldp x29, x30, [sp], #16 + bl start_kernel + ASM_BUG() SYM_FUNC_END(__primary_switched) .pushsection ".rodata", "a" @@ -551,7 +567,7 @@ SYM_FUNC_START_LOCAL(set_cpu_boot_mode_flag) cmp w0, #BOOT_CPU_MODE_EL2 b.ne 1f add x1, x1, #4 -1: str w0, [x1] // This CPU has booted in EL1 +1: str w0, [x1] // Save CPU boot mode dmb sy dc ivac, x1 // Invalidate potentially stale cache line ret @@ -632,21 +648,17 @@ SYM_FUNC_START_LOCAL(__secondary_switched) isb adr_l x0, secondary_data - ldr x1, [x0, #CPU_BOOT_STACK] // get secondary_data.stack - cbz x1, __secondary_too_slow - mov sp, x1 ldr x2, [x0, #CPU_BOOT_TASK] cbz x2, __secondary_too_slow - msr sp_el0, x2 - scs_load x2, x3 - mov x29, #0 - mov x30, #0 + + init_cpu_task x2, x1, x3 #ifdef CONFIG_ARM64_PTR_AUTH ptrauth_keys_init_cpu x2, x3, x4, x5 #endif - b secondary_start_kernel + bl secondary_start_kernel + ASM_BUG() SYM_FUNC_END(__secondary_switched) SYM_FUNC_START_LOCAL(__secondary_too_slow) diff --git a/arch/arm64/kernel/hibernate-asm.S b/arch/arm64/kernel/hibernate-asm.S index 8ccca660034e..81c0186a5e32 100644 --- a/arch/arm64/kernel/hibernate-asm.S +++ b/arch/arm64/kernel/hibernate-asm.S @@ -45,7 +45,7 @@ * Because this code has to be copied to a 'safe' page, it can't call out to * other functions by PC-relative address. Also remember that it may be * mid-way through over-writing other functions. For this reason it contains - * code from flush_icache_range() and uses the copy_page() macro. + * code from caches_clean_inval_pou() and uses the copy_page() macro. * * This 'safe' page is mapped via ttbr0, and executed from there. This function * switches to a copy of the linear map in ttbr1, performs the restore, then @@ -87,11 +87,12 @@ SYM_CODE_START(swsusp_arch_suspend_exit) copy_page x0, x1, x2, x3, x4, x5, x6, x7, x8, x9 add x1, x10, #PAGE_SIZE - /* Clean the copied page to PoU - based on flush_icache_range() */ + /* Clean the copied page to PoU - based on caches_clean_inval_pou() */ raw_dcache_line_size x2, x3 sub x3, x2, #1 bic x4, x10, x3 -2: dc cvau, x4 /* clean D line / unified line */ +2: /* clean D line / unified line */ +alternative_insn "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE add x4, x4, x2 cmp x4, x1 b.lo 2b diff --git a/arch/arm64/kernel/hibernate.c b/arch/arm64/kernel/hibernate.c index b1cef371df2b..46a0b4d6e251 100644 --- a/arch/arm64/kernel/hibernate.c +++ b/arch/arm64/kernel/hibernate.c @@ -210,7 +210,7 @@ static int create_safe_exec_page(void *src_start, size_t length, return -ENOMEM; memcpy(page, src_start, length); - __flush_icache_range((unsigned long)page, (unsigned long)page + length); + caches_clean_inval_pou((unsigned long)page, (unsigned long)page + length); rc = trans_pgd_idmap_page(&trans_info, &trans_ttbr0, &t0sz, page); if (rc) return rc; @@ -240,8 +240,6 @@ static int create_safe_exec_page(void *src_start, size_t length, return 0; } -#define dcache_clean_range(start, end) __flush_dcache_area(start, (end - start)) - #ifdef CONFIG_ARM64_MTE static DEFINE_XARRAY(mte_pages); @@ -383,13 +381,18 @@ int swsusp_arch_suspend(void) ret = swsusp_save(); } else { /* Clean kernel core startup/idle code to PoC*/ - dcache_clean_range(__mmuoff_data_start, __mmuoff_data_end); - dcache_clean_range(__idmap_text_start, __idmap_text_end); + dcache_clean_inval_poc((unsigned long)__mmuoff_data_start, + (unsigned long)__mmuoff_data_end); + dcache_clean_inval_poc((unsigned long)__idmap_text_start, + (unsigned long)__idmap_text_end); /* Clean kvm setup code to PoC? */ if (el2_reset_needed()) { - dcache_clean_range(__hyp_idmap_text_start, __hyp_idmap_text_end); - dcache_clean_range(__hyp_text_start, __hyp_text_end); + dcache_clean_inval_poc( + (unsigned long)__hyp_idmap_text_start, + (unsigned long)__hyp_idmap_text_end); + dcache_clean_inval_poc((unsigned long)__hyp_text_start, + (unsigned long)__hyp_text_end); } swsusp_mte_restore_tags(); @@ -474,7 +477,8 @@ int swsusp_arch_resume(void) * The hibernate exit text contains a set of el2 vectors, that will * be executed at el2 with the mmu off in order to reload hyp-stub. */ - __flush_dcache_area(hibernate_exit, exit_size); + dcache_clean_inval_poc((unsigned long)hibernate_exit, + (unsigned long)hibernate_exit + exit_size); /* * KASLR will cause the el2 vectors to be in a different location in diff --git a/arch/arm64/kernel/idle.c b/arch/arm64/kernel/idle.c new file mode 100644 index 000000000000..a2cfbacec2bb --- /dev/null +++ b/arch/arm64/kernel/idle.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Low-level idle sequences + */ + +#include +#include + +#include +#include +#include +#include + +/* + * cpu_do_idle() + * + * Idle the processor (wait for interrupt). + * + * If the CPU supports priority masking we must do additional work to + * ensure that interrupts are not masked at the PMR (because the core will + * not wake up if we block the wake up signal in the interrupt controller). + */ +void noinstr cpu_do_idle(void) +{ + struct arm_cpuidle_irq_context context; + + arm_cpuidle_save_irq_context(&context); + + dsb(sy); + wfi(); + + arm_cpuidle_restore_irq_context(&context); +} + +/* + * This is our default idle handler. + */ +void noinstr arch_cpu_idle(void) +{ + /* + * This should do all the clock switching and wait for interrupt + * tricks + */ + cpu_do_idle(); + raw_local_irq_enable(); +} diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index e628c8ce1ffe..53a381a7f65d 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -237,7 +237,8 @@ asmlinkage void __init init_feature_override(void) for (i = 0; i < ARRAY_SIZE(regs); i++) { if (regs[i]->override) - __flush_dcache_area(regs[i]->override, + dcache_clean_inval_poc((unsigned long)regs[i]->override, + (unsigned long)regs[i]->override + sizeof(*regs[i]->override)); } } diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index bcf3c2755370..c96a9a0043bf 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -35,7 +35,7 @@ __efistub_strnlen = __pi_strnlen; __efistub_strcmp = __pi_strcmp; __efistub_strncmp = __pi_strncmp; __efistub_strrchr = __pi_strrchr; -__efistub___clean_dcache_area_poc = __pi___clean_dcache_area_poc; +__efistub_dcache_clean_poc = __pi_dcache_clean_poc; #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) __efistub___memcpy = __pi_memcpy; diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c index 9a8a0ae1e75f..fc98037e1220 100644 --- a/arch/arm64/kernel/jump_label.c +++ b/arch/arm64/kernel/jump_label.c @@ -8,6 +8,7 @@ #include #include #include +#include void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type) diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index 341342b207f6..cfa2cfde3019 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -72,7 +72,9 @@ u64 __init kaslr_early_init(void) * we end up running with module randomization disabled. */ module_alloc_base = (u64)_etext - MODULES_VSIZE; - __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base)); + dcache_clean_inval_poc((unsigned long)&module_alloc_base, + (unsigned long)&module_alloc_base + + sizeof(module_alloc_base)); /* * Try to map the FDT early. If this fails, we simply bail, @@ -170,8 +172,12 @@ u64 __init kaslr_early_init(void) module_alloc_base += (module_range * (seed & ((1 << 21) - 1))) >> 21; module_alloc_base &= PAGE_MASK; - __flush_dcache_area(&module_alloc_base, sizeof(module_alloc_base)); - __flush_dcache_area(&memstart_offset_seed, sizeof(memstart_offset_seed)); + dcache_clean_inval_poc((unsigned long)&module_alloc_base, + (unsigned long)&module_alloc_base + + sizeof(module_alloc_base)); + dcache_clean_inval_poc((unsigned long)&memstart_offset_seed, + (unsigned long)&memstart_offset_seed + + sizeof(memstart_offset_seed)); return offset; } diff --git a/arch/arm64/kernel/kgdb.c b/arch/arm64/kernel/kgdb.c index 1a157ca33262..2aede780fb80 100644 --- a/arch/arm64/kernel/kgdb.c +++ b/arch/arm64/kernel/kgdb.c @@ -17,6 +17,7 @@ #include #include +#include #include struct dbg_reg_def_t dbg_reg_def[DBG_MAX_REG_NUM] = { diff --git a/arch/arm64/kernel/machine_kexec.c b/arch/arm64/kernel/machine_kexec.c index 90a335c74442..03ceabe4d912 100644 --- a/arch/arm64/kernel/machine_kexec.c +++ b/arch/arm64/kernel/machine_kexec.c @@ -68,10 +68,16 @@ int machine_kexec_post_load(struct kimage *kimage) kimage->arch.kern_reloc = __pa(reloc_code); kexec_image_info(kimage); - /* Flush the reloc_code in preparation for its execution. */ - __flush_dcache_area(reloc_code, arm64_relocate_new_kernel_size); - flush_icache_range((uintptr_t)reloc_code, (uintptr_t)reloc_code + - arm64_relocate_new_kernel_size); + /* + * For execution with the MMU off, reloc_code needs to be cleaned to the + * PoC and invalidated from the I-cache. + */ + dcache_clean_inval_poc((unsigned long)reloc_code, + (unsigned long)reloc_code + + arm64_relocate_new_kernel_size); + icache_inval_pou((uintptr_t)reloc_code, + (uintptr_t)reloc_code + + arm64_relocate_new_kernel_size); return 0; } @@ -102,16 +108,18 @@ static void kexec_list_flush(struct kimage *kimage) for (entry = &kimage->head; ; entry++) { unsigned int flag; - void *addr; + unsigned long addr; /* flush the list entries. */ - __flush_dcache_area(entry, sizeof(kimage_entry_t)); + dcache_clean_inval_poc((unsigned long)entry, + (unsigned long)entry + + sizeof(kimage_entry_t)); flag = *entry & IND_FLAGS; if (flag == IND_DONE) break; - addr = phys_to_virt(*entry & PAGE_MASK); + addr = (unsigned long)phys_to_virt(*entry & PAGE_MASK); switch (flag) { case IND_INDIRECTION: @@ -120,7 +128,7 @@ static void kexec_list_flush(struct kimage *kimage) break; case IND_SOURCE: /* flush the source pages. */ - __flush_dcache_area(addr, PAGE_SIZE); + dcache_clean_inval_poc(addr, addr + PAGE_SIZE); break; case IND_DESTINATION: break; @@ -147,8 +155,10 @@ static void kexec_segment_flush(const struct kimage *kimage) kimage->segment[i].memsz, kimage->segment[i].memsz / PAGE_SIZE); - __flush_dcache_area(phys_to_virt(kimage->segment[i].mem), - kimage->segment[i].memsz); + dcache_clean_inval_poc( + (unsigned long)phys_to_virt(kimage->segment[i].mem), + (unsigned long)phys_to_virt(kimage->segment[i].mem) + + kimage->segment[i].memsz); } } diff --git a/arch/arm64/kernel/mte.c b/arch/arm64/kernel/mte.c index 125a10e413e9..69b3fde8759e 100644 --- a/arch/arm64/kernel/mte.c +++ b/arch/arm64/kernel/mte.c @@ -32,10 +32,9 @@ DEFINE_STATIC_KEY_FALSE(mte_async_mode); EXPORT_SYMBOL_GPL(mte_async_mode); #endif -static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) +static void mte_sync_page_tags(struct page *page, pte_t old_pte, + bool check_swap, bool pte_is_tagged) { - pte_t old_pte = READ_ONCE(*ptep); - if (check_swap && is_swap_pte(old_pte)) { swp_entry_t entry = pte_to_swp_entry(old_pte); @@ -43,6 +42,9 @@ static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) return; } + if (!pte_is_tagged) + return; + page_kasan_tag_reset(page); /* * We need smp_wmb() in between setting the flags and clearing the @@ -55,16 +57,22 @@ static void mte_sync_page_tags(struct page *page, pte_t *ptep, bool check_swap) mte_clear_page_tags(page_address(page)); } -void mte_sync_tags(pte_t *ptep, pte_t pte) +void mte_sync_tags(pte_t old_pte, pte_t pte) { struct page *page = pte_page(pte); long i, nr_pages = compound_nr(page); bool check_swap = nr_pages == 1; + bool pte_is_tagged = pte_tagged(pte); + + /* Early out if there's nothing to do */ + if (!check_swap && !pte_is_tagged) + return; /* if PG_mte_tagged is set, tags have already been initialised */ for (i = 0; i < nr_pages; i++, page++) { if (!test_and_set_bit(PG_mte_tagged, &page->flags)) - mte_sync_page_tags(page, ptep, check_swap); + mte_sync_page_tags(page, old_pte, check_swap, + pte_is_tagged); } } diff --git a/arch/arm64/kernel/patching.c b/arch/arm64/kernel/patching.c new file mode 100644 index 000000000000..771f543464e0 --- /dev/null +++ b/arch/arm64/kernel/patching.c @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +static DEFINE_RAW_SPINLOCK(patch_lock); + +static bool is_exit_text(unsigned long addr) +{ + /* discarded with init text/data */ + return system_state < SYSTEM_RUNNING && + addr >= (unsigned long)__exittext_begin && + addr < (unsigned long)__exittext_end; +} + +static bool is_image_text(unsigned long addr) +{ + return core_kernel_text(addr) || is_exit_text(addr); +} + +static void __kprobes *patch_map(void *addr, int fixmap) +{ + unsigned long uintaddr = (uintptr_t) addr; + bool image = is_image_text(uintaddr); + struct page *page; + + if (image) + page = phys_to_page(__pa_symbol(addr)); + else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) + page = vmalloc_to_page(addr); + else + return addr; + + BUG_ON(!page); + return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + + (uintaddr & ~PAGE_MASK)); +} + +static void __kprobes patch_unmap(int fixmap) +{ + clear_fixmap(fixmap); +} +/* + * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always + * little-endian. + */ +int __kprobes aarch64_insn_read(void *addr, u32 *insnp) +{ + int ret; + __le32 val; + + ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE); + if (!ret) + *insnp = le32_to_cpu(val); + + return ret; +} + +static int __kprobes __aarch64_insn_write(void *addr, __le32 insn) +{ + void *waddr = addr; + unsigned long flags = 0; + int ret; + + raw_spin_lock_irqsave(&patch_lock, flags); + waddr = patch_map(addr, FIX_TEXT_POKE0); + + ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE); + + patch_unmap(FIX_TEXT_POKE0); + raw_spin_unlock_irqrestore(&patch_lock, flags); + + return ret; +} + +int __kprobes aarch64_insn_write(void *addr, u32 insn) +{ + return __aarch64_insn_write(addr, cpu_to_le32(insn)); +} + +int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn) +{ + u32 *tp = addr; + int ret; + + /* A64 instructions must be word aligned */ + if ((uintptr_t)tp & 0x3) + return -EINVAL; + + ret = aarch64_insn_write(tp, insn); + if (ret == 0) + caches_clean_inval_pou((uintptr_t)tp, + (uintptr_t)tp + AARCH64_INSN_SIZE); + + return ret; +} + +struct aarch64_insn_patch { + void **text_addrs; + u32 *new_insns; + int insn_cnt; + atomic_t cpu_count; +}; + +static int __kprobes aarch64_insn_patch_text_cb(void *arg) +{ + int i, ret = 0; + struct aarch64_insn_patch *pp = arg; + + /* The first CPU becomes master */ + if (atomic_inc_return(&pp->cpu_count) == 1) { + for (i = 0; ret == 0 && i < pp->insn_cnt; i++) + ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i], + pp->new_insns[i]); + /* Notify other processors with an additional increment. */ + atomic_inc(&pp->cpu_count); + } else { + while (atomic_read(&pp->cpu_count) <= num_online_cpus()) + cpu_relax(); + isb(); + } + + return ret; +} + +int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) +{ + struct aarch64_insn_patch patch = { + .text_addrs = addrs, + .new_insns = insns, + .insn_cnt = cnt, + .cpu_count = ATOMIC_INIT(0), + }; + + if (cnt <= 0) + return -EINVAL; + + return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch, + cpu_online_mask); +} diff --git a/arch/arm64/kernel/perf_callchain.c b/arch/arm64/kernel/perf_callchain.c index 88ff471b0bce..4a72c2727309 100644 --- a/arch/arm64/kernel/perf_callchain.c +++ b/arch/arm64/kernel/perf_callchain.c @@ -116,7 +116,7 @@ void perf_callchain_user(struct perf_callchain_entry_ctx *entry, tail = (struct frame_tail __user *)regs->regs[29]; while (entry->nr < entry->max_stack && - tail && !((unsigned long)tail & 0xf)) + tail && !((unsigned long)tail & 0x7)) tail = user_backtrace(tail, entry); } else { #ifdef CONFIG_COMPAT diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index f594957e29bd..d07788dad388 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -165,10 +165,7 @@ armv8pmu_events_sysfs_show(struct device *dev, } #define ARMV8_EVENT_ATTR(name, config) \ - (&((struct perf_pmu_events_attr) { \ - .attr = __ATTR(name, 0444, armv8pmu_events_sysfs_show, NULL), \ - .id = config, \ - }).attr.attr) + PMU_EVENT_ATTR_ID(name, armv8pmu_events_sysfs_show, config) static struct attribute *armv8_pmuv3_event_attrs[] = { ARMV8_EVENT_ATTR(sw_incr, ARMV8_PMUV3_PERFCTR_SW_INCR), @@ -312,13 +309,46 @@ static ssize_t slots_show(struct device *dev, struct device_attribute *attr, struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); u32 slots = cpu_pmu->reg_pmmir & ARMV8_PMU_SLOTS_MASK; - return snprintf(page, PAGE_SIZE, "0x%08x\n", slots); + return sysfs_emit(page, "0x%08x\n", slots); } static DEVICE_ATTR_RO(slots); +static ssize_t bus_slots_show(struct device *dev, struct device_attribute *attr, + char *page) +{ + struct pmu *pmu = dev_get_drvdata(dev); + struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); + u32 bus_slots = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_SLOTS_SHIFT) + & ARMV8_PMU_BUS_SLOTS_MASK; + + return sysfs_emit(page, "0x%08x\n", bus_slots); +} + +static DEVICE_ATTR_RO(bus_slots); + +static ssize_t bus_width_show(struct device *dev, struct device_attribute *attr, + char *page) +{ + struct pmu *pmu = dev_get_drvdata(dev); + struct arm_pmu *cpu_pmu = container_of(pmu, struct arm_pmu, pmu); + u32 bus_width = (cpu_pmu->reg_pmmir >> ARMV8_PMU_BUS_WIDTH_SHIFT) + & ARMV8_PMU_BUS_WIDTH_MASK; + u32 val = 0; + + /* Encoded as Log2(number of bytes), plus one */ + if (bus_width > 2 && bus_width < 13) + val = 1 << (bus_width - 1); + + return sysfs_emit(page, "0x%08x\n", val); +} + +static DEVICE_ATTR_RO(bus_width); + static struct attribute *armv8_pmuv3_caps_attrs[] = { &dev_attr_slots.attr, + &dev_attr_bus_slots.attr, + &dev_attr_bus_width.attr, NULL, }; diff --git a/arch/arm64/kernel/probes/kprobes.c b/arch/arm64/kernel/probes/kprobes.c index 004b86eff9c2..6dbcc89f6662 100644 --- a/arch/arm64/kernel/probes/kprobes.c +++ b/arch/arm64/kernel/probes/kprobes.c @@ -7,26 +7,28 @@ * Copyright (C) 2013 Linaro Limited. * Author: Sandeepa Prabhu */ +#include #include #include #include -#include -#include -#include #include #include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include + +#include +#include +#include +#include #include +#include +#include #include +#include +#include #include "decode-insn.h" diff --git a/arch/arm64/kernel/probes/simulate-insn.c b/arch/arm64/kernel/probes/simulate-insn.c index 25f67ec59635..22d0b3252476 100644 --- a/arch/arm64/kernel/probes/simulate-insn.c +++ b/arch/arm64/kernel/probes/simulate-insn.c @@ -10,6 +10,7 @@ #include #include +#include #include "simulate-insn.h" diff --git a/arch/arm64/kernel/probes/uprobes.c b/arch/arm64/kernel/probes/uprobes.c index 2c247634552b..9be668f3f034 100644 --- a/arch/arm64/kernel/probes/uprobes.c +++ b/arch/arm64/kernel/probes/uprobes.c @@ -21,7 +21,7 @@ void arch_uprobe_copy_ixol(struct page *page, unsigned long vaddr, memcpy(dst, src, len); /* flush caches (dcache/icache) */ - sync_icache_aliases(dst, len); + sync_icache_aliases((unsigned long)dst, (unsigned long)dst + len); kunmap_atomic(xol_page_kaddr); } diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c index 05d42ad9e3c5..b8b960d43536 100644 --- a/arch/arm64/kernel/process.c +++ b/arch/arm64/kernel/process.c @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -48,7 +47,6 @@ #include #include -#include #include #include #include @@ -74,63 +72,6 @@ EXPORT_SYMBOL(__stack_chk_guard); void (*pm_power_off)(void); EXPORT_SYMBOL_GPL(pm_power_off); -static void noinstr __cpu_do_idle(void) -{ - dsb(sy); - wfi(); -} - -static void noinstr __cpu_do_idle_irqprio(void) -{ - unsigned long pmr; - unsigned long daif_bits; - - daif_bits = read_sysreg(daif); - write_sysreg(daif_bits | PSR_I_BIT | PSR_F_BIT, daif); - - /* - * Unmask PMR before going idle to make sure interrupts can - * be raised. - */ - pmr = gic_read_pmr(); - gic_write_pmr(GIC_PRIO_IRQON | GIC_PRIO_PSR_I_SET); - - __cpu_do_idle(); - - gic_write_pmr(pmr); - write_sysreg(daif_bits, daif); -} - -/* - * cpu_do_idle() - * - * Idle the processor (wait for interrupt). - * - * If the CPU supports priority masking we must do additional work to - * ensure that interrupts are not masked at the PMR (because the core will - * not wake up if we block the wake up signal in the interrupt controller). - */ -void noinstr cpu_do_idle(void) -{ - if (system_uses_irq_prio_masking()) - __cpu_do_idle_irqprio(); - else - __cpu_do_idle(); -} - -/* - * This is our default idle handler. - */ -void noinstr arch_cpu_idle(void) -{ - /* - * This should do all the clock switching and wait for interrupt - * tricks - */ - cpu_do_idle(); - raw_local_irq_enable(); -} - #ifdef CONFIG_HOTPLUG_CPU void arch_cpu_idle_dead(void) { @@ -433,6 +374,11 @@ int copy_thread(unsigned long clone_flags, unsigned long stack_start, } p->thread.cpu_context.pc = (unsigned long)ret_from_fork; p->thread.cpu_context.sp = (unsigned long)childregs; + /* + * For the benefit of the unwinder, set up childregs->stackframe + * as the final frame for the new task. + */ + p->thread.cpu_context.fp = (unsigned long)childregs->stackframe; ptrace_hw_copy_thread(p); @@ -525,6 +471,15 @@ static void erratum_1418040_thread_switch(struct task_struct *prev, write_sysreg(val, cntkctl_el1); } +static void compat_thread_switch(struct task_struct *next) +{ + if (!is_compat_thread(task_thread_info(next))) + return; + + if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) + set_tsk_thread_flag(next, TIF_NOTIFY_RESUME); +} + static void update_sctlr_el1(u64 sctlr) { /* @@ -566,6 +521,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev, ssbs_thread_switch(next); erratum_1418040_thread_switch(prev, next); ptrauth_thread_switch_user(next); + compat_thread_switch(next); /* * vendor hook is needed before the dsb(), * because MPAM is related to cache maintenance. @@ -638,8 +594,15 @@ unsigned long arch_align_stack(unsigned long sp) */ void arch_setup_new_exec(void) { - current->mm->context.flags = is_compat_task() ? MMCF_AARCH32 : 0; + unsigned long mmflags = 0; + if (is_compat_task()) { + mmflags = MMCF_AARCH32; + if (static_branch_unlikely(&arm64_mismatched_32bit_el0)) + set_tsk_thread_flag(current, TIF_NOTIFY_RESUME); + } + + current->mm->context.flags = mmflags; ptrauth_thread_init_user(); mte_thread_init_user(); @@ -729,22 +692,6 @@ static int __init tagged_addr_init(void) core_initcall(tagged_addr_init); #endif /* CONFIG_ARM64_TAGGED_ADDR_ABI */ -asmlinkage void __sched arm64_preempt_schedule_irq(void) -{ - lockdep_assert_irqs_disabled(); - - /* - * Preempting a task from an IRQ means we leave copies of PSTATE - * on the stack. cpufeature's enable calls may modify PSTATE, but - * resuming one of these preempted tasks would undo those changes. - * - * Only allow a task to be preempted once cpufeatures have been - * enabled. - */ - if (system_capabilities_finalized()) - preempt_schedule_irq(); -} - #ifdef CONFIG_BINFMT_ELF int arch_elf_adjust_prot(int prot, const struct arch_elf_state *state, bool has_interp, bool is_interp) diff --git a/arch/arm64/kernel/ptrace.c b/arch/arm64/kernel/ptrace.c index eb2f73939b7b..499b6b2f9757 100644 --- a/arch/arm64/kernel/ptrace.c +++ b/arch/arm64/kernel/ptrace.c @@ -122,7 +122,7 @@ static bool regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) { return ((addr & ~(THREAD_SIZE - 1)) == (kernel_stack_pointer(regs) & ~(THREAD_SIZE - 1))) || - on_irq_stack(addr, NULL); + on_irq_stack(addr, sizeof(unsigned long), NULL); } /** diff --git a/arch/arm64/kernel/sdei.c b/arch/arm64/kernel/sdei.c index 2c7ca449dd51..47f77d1234cb 100644 --- a/arch/arm64/kernel/sdei.c +++ b/arch/arm64/kernel/sdei.c @@ -162,31 +162,33 @@ static int init_sdei_scs(void) return err; } -static bool on_sdei_normal_stack(unsigned long sp, struct stack_info *info) +static bool on_sdei_normal_stack(unsigned long sp, unsigned long size, + struct stack_info *info) { unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_normal_ptr); unsigned long high = low + SDEI_STACK_SIZE; - return on_stack(sp, low, high, STACK_TYPE_SDEI_NORMAL, info); + return on_stack(sp, size, low, high, STACK_TYPE_SDEI_NORMAL, info); } -static bool on_sdei_critical_stack(unsigned long sp, struct stack_info *info) +static bool on_sdei_critical_stack(unsigned long sp, unsigned long size, + struct stack_info *info) { unsigned long low = (unsigned long)raw_cpu_read(sdei_stack_critical_ptr); unsigned long high = low + SDEI_STACK_SIZE; - return on_stack(sp, low, high, STACK_TYPE_SDEI_CRITICAL, info); + return on_stack(sp, size, low, high, STACK_TYPE_SDEI_CRITICAL, info); } -bool _on_sdei_stack(unsigned long sp, struct stack_info *info) +bool _on_sdei_stack(unsigned long sp, unsigned long size, struct stack_info *info) { if (!IS_ENABLED(CONFIG_VMAP_STACK)) return false; - if (on_sdei_critical_stack(sp, info)) + if (on_sdei_critical_stack(sp, size, info)) return true; - if (on_sdei_normal_stack(sp, info)) + if (on_sdei_normal_stack(sp, size, info)) return true; return false; @@ -231,13 +233,13 @@ unsigned long sdei_arch_get_entry_point(int conduit) } /* - * __sdei_handler() returns one of: + * do_sdei_event() returns one of: * SDEI_EV_HANDLED - success, return to the interrupted context. * SDEI_EV_FAILED - failure, return this error code to firmare. * virtual-address - success, return to this address. */ -static __kprobes unsigned long _sdei_handler(struct pt_regs *regs, - struct sdei_registered_event *arg) +unsigned long __kprobes do_sdei_event(struct pt_regs *regs, + struct sdei_registered_event *arg) { u32 mode; int i, err = 0; @@ -292,45 +294,3 @@ static __kprobes unsigned long _sdei_handler(struct pt_regs *regs, return vbar + 0x480; } - -static void __kprobes notrace __sdei_pstate_entry(void) -{ - /* - * The original SDEI spec (ARM DEN 0054A) can be read ambiguously as to - * whether PSTATE bits are inherited unchanged or generated from - * scratch, and the TF-A implementation always clears PAN and always - * clears UAO. There are no other known implementations. - * - * Subsequent revisions (ARM DEN 0054B) follow the usual rules for how - * PSTATE is modified upon architectural exceptions, and so PAN is - * either inherited or set per SCTLR_ELx.SPAN, and UAO is always - * cleared. - * - * We must explicitly reset PAN to the expected state, including - * clearing it when the host isn't using it, in case a VM had it set. - */ - if (system_uses_hw_pan()) - set_pstate_pan(1); - else if (cpu_has_pan()) - set_pstate_pan(0); -} - -asmlinkage noinstr unsigned long -__sdei_handler(struct pt_regs *regs, struct sdei_registered_event *arg) -{ - unsigned long ret; - - /* - * We didn't take an exception to get here, so the HW hasn't - * set/cleared bits in PSTATE that we may rely on. Initialize PAN. - */ - __sdei_pstate_entry(); - - arm64_enter_nmi(regs); - - ret = _sdei_handler(regs, arg); - - arm64_exit_nmi(regs); - - return ret; -} diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 61845c0821d9..8ed66142b088 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -87,12 +87,6 @@ void __init smp_setup_processor_id(void) u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; set_cpu_logical_map(0, mpidr); - /* - * clear __my_cpu_offset on boot CPU to avoid hang caused by - * using percpu variable early, for example, lockdep will - * access percpu variable inside lock_release - */ - set_my_cpu_offset(0); pr_info("Booting Linux on physical CPU 0x%010lx [0x%08x]\n", (unsigned long)mpidr, read_cpuid_id()); } @@ -381,7 +375,7 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p) * faults in case uaccess_enable() is inadvertently called by the init * thread. */ - init_task.thread_info.ttbr0 = __pa_symbol(reserved_pg_dir); + init_task.thread_info.ttbr0 = phys_to_ttbr(__pa_symbol(reserved_pg_dir)); #endif if (boot_args[1] || boot_args[2] || boot_args[3]) { diff --git a/arch/arm64/kernel/signal.c b/arch/arm64/kernel/signal.c index 6237486ff6bb..f8192f4ae0b8 100644 --- a/arch/arm64/kernel/signal.c +++ b/arch/arm64/kernel/signal.c @@ -911,6 +911,19 @@ static void do_signal(struct pt_regs *regs) restore_saved_sigmask(); } +static bool cpu_affinity_invalid(struct pt_regs *regs) +{ + if (!compat_user_mode(regs)) + return false; + + /* + * We're preemptible, but a reschedule will cause us to check the + * affinity again. + */ + return !cpumask_test_cpu(raw_smp_processor_id(), + system_32bit_el0_cpumask()); +} + asmlinkage void do_notify_resume(struct pt_regs *regs, unsigned long thread_flags) { @@ -938,6 +951,19 @@ asmlinkage void do_notify_resume(struct pt_regs *regs, if (thread_flags & _TIF_NOTIFY_RESUME) { tracehook_notify_resume(regs); rseq_handle_notify_resume(NULL, regs); + + /* + * If we reschedule after checking the affinity + * then we must ensure that TIF_NOTIFY_RESUME + * is set so that we check the affinity again. + * Since tracehook_notify_resume() clears the + * flag, ensure that the compiler doesn't move + * it after the affinity check. + */ + barrier(); + + if (cpu_affinity_invalid(regs)) + force_sig(SIGKILL); } if (thread_flags & _TIF_FOREIGN_FPSTATE) diff --git a/arch/arm64/kernel/smccc-call.S b/arch/arm64/kernel/smccc-call.S index d62447964ed9..d3d37f932b97 100644 --- a/arch/arm64/kernel/smccc-call.S +++ b/arch/arm64/kernel/smccc-call.S @@ -7,8 +7,34 @@ #include #include +#include + +/* + * If we have SMCCC v1.3 and (as is likely) no SVE state in + * the registers then set the SMCCC hint bit to say there's no + * need to preserve it. Do this by directly adjusting the SMCCC + * function value which is already stored in x0 ready to be called. + */ +SYM_FUNC_START(__arm_smccc_sve_check) + + ldr_l x16, smccc_has_sve_hint + cbz x16, 2f + + get_current_task x16 + ldr x16, [x16, #TSK_TI_FLAGS] + tbnz x16, #TIF_FOREIGN_FPSTATE, 1f // Any live FP state? + tbnz x16, #TIF_SVE, 2f // Does that state include SVE? + +1: orr x0, x0, ARM_SMCCC_1_3_SVE_HINT + +2: ret +SYM_FUNC_END(__arm_smccc_sve_check) +EXPORT_SYMBOL(__arm_smccc_sve_check) .macro SMCCC instr +alternative_if ARM64_SVE + bl __arm_smccc_sve_check +alternative_else_nop_endif \instr #0 ldr x4, [sp] stp x0, x1, [x4, #ARM_SMCCC_RES_X0_OFFS] @@ -43,3 +69,60 @@ SYM_FUNC_START(__arm_smccc_hvc) SMCCC hvc SYM_FUNC_END(__arm_smccc_hvc) EXPORT_SYMBOL(__arm_smccc_hvc) + + .macro SMCCC_1_2 instr + /* Save `res` and free a GPR that won't be clobbered */ + stp x1, x19, [sp, #-16]! + + /* Ensure `args` won't be clobbered while loading regs in next step */ + mov x19, x0 + + /* Load the registers x0 - x17 from the struct arm_smccc_1_2_regs */ + ldp x0, x1, [x19, #ARM_SMCCC_1_2_REGS_X0_OFFS] + ldp x2, x3, [x19, #ARM_SMCCC_1_2_REGS_X2_OFFS] + ldp x4, x5, [x19, #ARM_SMCCC_1_2_REGS_X4_OFFS] + ldp x6, x7, [x19, #ARM_SMCCC_1_2_REGS_X6_OFFS] + ldp x8, x9, [x19, #ARM_SMCCC_1_2_REGS_X8_OFFS] + ldp x10, x11, [x19, #ARM_SMCCC_1_2_REGS_X10_OFFS] + ldp x12, x13, [x19, #ARM_SMCCC_1_2_REGS_X12_OFFS] + ldp x14, x15, [x19, #ARM_SMCCC_1_2_REGS_X14_OFFS] + ldp x16, x17, [x19, #ARM_SMCCC_1_2_REGS_X16_OFFS] + + \instr #0 + + /* Load the `res` from the stack */ + ldr x19, [sp] + + /* Store the registers x0 - x17 into the result structure */ + stp x0, x1, [x19, #ARM_SMCCC_1_2_REGS_X0_OFFS] + stp x2, x3, [x19, #ARM_SMCCC_1_2_REGS_X2_OFFS] + stp x4, x5, [x19, #ARM_SMCCC_1_2_REGS_X4_OFFS] + stp x6, x7, [x19, #ARM_SMCCC_1_2_REGS_X6_OFFS] + stp x8, x9, [x19, #ARM_SMCCC_1_2_REGS_X8_OFFS] + stp x10, x11, [x19, #ARM_SMCCC_1_2_REGS_X10_OFFS] + stp x12, x13, [x19, #ARM_SMCCC_1_2_REGS_X12_OFFS] + stp x14, x15, [x19, #ARM_SMCCC_1_2_REGS_X14_OFFS] + stp x16, x17, [x19, #ARM_SMCCC_1_2_REGS_X16_OFFS] + + /* Restore original x19 */ + ldp xzr, x19, [sp], #16 + ret +.endm + +/* + * void arm_smccc_1_2_hvc(const struct arm_smccc_1_2_regs *args, + * struct arm_smccc_1_2_regs *res); + */ +SYM_FUNC_START(arm_smccc_1_2_hvc) + SMCCC_1_2 hvc +SYM_FUNC_END(arm_smccc_1_2_hvc) +EXPORT_SYMBOL(arm_smccc_1_2_hvc) + +/* + * void arm_smccc_1_2_smc(const struct arm_smccc_1_2_regs *args, + * struct arm_smccc_1_2_regs *res); + */ +SYM_FUNC_START(arm_smccc_1_2_smc) + SMCCC_1_2 smc +SYM_FUNC_END(arm_smccc_1_2_smc) +EXPORT_SYMBOL(arm_smccc_1_2_smc) diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index 091018423db2..c0025fbd43ec 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -125,9 +125,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) * page tables. */ secondary_data.task = idle; - secondary_data.stack = task_stack_page(idle) + THREAD_SIZE; update_cpu_boot_status(CPU_MMU_OFF); - __flush_dcache_area(&secondary_data, sizeof(secondary_data)); /* Now bring the CPU into our world */ ret = boot_secondary(cpu, idle); @@ -147,8 +145,6 @@ int __cpu_up(unsigned int cpu, struct task_struct *idle) pr_crit("CPU%u: failed to come online\n", cpu); secondary_data.task = NULL; - secondary_data.stack = NULL; - __flush_dcache_area(&secondary_data, sizeof(secondary_data)); status = READ_ONCE(secondary_data.status); if (status == CPU_MMU_OFF) status = READ_ONCE(__early_cpu_boot_status); @@ -207,10 +203,7 @@ asmlinkage notrace void secondary_start_kernel(void) u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK; struct mm_struct *mm = &init_mm; const struct cpu_operations *ops; - unsigned int cpu; - - cpu = task_cpu(current); - set_my_cpu_offset(per_cpu_offset(cpu)); + unsigned int cpu = smp_processor_id(); /* * All kernel threads share the same mm context; grab a @@ -456,6 +449,11 @@ void __init smp_cpus_done(unsigned int max_cpus) void __init smp_prepare_boot_cpu(void) { + /* + * The runtime per-cpu areas have been allocated by + * setup_per_cpu_areas(), and CPU0's boot time per-cpu area will be + * freed shortly, so we must move over to the runtime per-cpu area. + */ set_my_cpu_offset(per_cpu_offset(smp_processor_id())); cpuinfo_store_boot_cpu(); diff --git a/arch/arm64/kernel/smp_spin_table.c b/arch/arm64/kernel/smp_spin_table.c index c45a83512805..7e1624ecab3c 100644 --- a/arch/arm64/kernel/smp_spin_table.c +++ b/arch/arm64/kernel/smp_spin_table.c @@ -36,7 +36,7 @@ static void write_pen_release(u64 val) unsigned long size = sizeof(secondary_holding_pen_release); secondary_holding_pen_release = val; - __flush_dcache_area(start, size); + dcache_clean_inval_poc((unsigned long)start, (unsigned long)start + size); } @@ -90,8 +90,9 @@ static int smp_spin_table_cpu_prepare(unsigned int cpu) * the boot protocol. */ writeq_relaxed(pa_holding_pen, release_addr); - __flush_dcache_area((__force void *)release_addr, - sizeof(*release_addr)); + dcache_clean_inval_poc((__force unsigned long)release_addr, + (__force unsigned long)release_addr + + sizeof(*release_addr)); /* * Send an event to wake up the secondary CPU. diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c index e978840ec0df..cca6e4abc0ee 100644 --- a/arch/arm64/kernel/stacktrace.c +++ b/arch/arm64/kernel/stacktrace.c @@ -68,13 +68,17 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) unsigned long fp = frame->fp; struct stack_info info; - if (fp & 0xf) - return -EINVAL; - if (!tsk) tsk = current; - if (!on_accessible_stack(tsk, fp, &info)) + /* Final frame; nothing to unwind */ + if (fp == (unsigned long)task_pt_regs(tsk)->stackframe) + return -ENOENT; + + if (fp & 0x7) + return -EINVAL; + + if (!on_accessible_stack(tsk, fp, 16, &info)) return -EINVAL; if (test_bit(info.type, frame->stacks_done)) @@ -128,12 +132,6 @@ int notrace unwind_frame(struct task_struct *tsk, struct stackframe *frame) frame->pc = ptrauth_strip_insn_pac(frame->pc); - /* - * This is a terminal record, so we have finished unwinding. - */ - if (!frame->fp && !frame->pc) - return -ENOENT; - return 0; } NOKPROBE_SYMBOL(unwind_frame); diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c index e3f72df9509d..938ce6fbee8a 100644 --- a/arch/arm64/kernel/suspend.c +++ b/arch/arm64/kernel/suspend.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -91,6 +92,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) int ret = 0; unsigned long flags; struct sleep_stack_data state; + struct arm_cpuidle_irq_context context; /* Report any MTE async fault before going to suspend */ mte_suspend_enter(); @@ -103,12 +105,18 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) flags = local_daif_save(); /* - * Function graph tracer state gets incosistent when the kernel + * Function graph tracer state gets inconsistent when the kernel * calls functions that never return (aka suspend finishers) hence * disable graph tracing during their execution. */ pause_graph_tracing(); + /* + * Switch to using DAIF.IF instead of PMR in order to reliably + * resume if we're using pseudo-NMIs. + */ + arm_cpuidle_save_irq_context(&context); + if (__cpu_suspend_enter(&state)) { /* Call the suspend finisher */ ret = fn(arg); @@ -126,6 +134,8 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) RCU_NONIDLE(__cpu_suspend_exit()); } + arm_cpuidle_restore_irq_context(&context); + unpause_graph_tracing(); /* diff --git a/arch/arm64/kernel/sys_compat.c b/arch/arm64/kernel/sys_compat.c index 265fe3eb1069..db5159a3055f 100644 --- a/arch/arm64/kernel/sys_compat.c +++ b/arch/arm64/kernel/sys_compat.c @@ -41,7 +41,7 @@ __do_compat_cache_op(unsigned long start, unsigned long end) dsb(ish); } - ret = __flush_cache_user_range(start, start + chunk); + ret = caches_clean_inval_user_pou(start, start + chunk); if (ret) return ret; diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index a05d34f0e82a..b03e383d944a 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -45,11 +46,102 @@ #include #include -static const char *handler[] = { - "Synchronous Abort", - "IRQ", - "FIQ", - "Error" +static bool __kprobes __check_eq(unsigned long pstate) +{ + return (pstate & PSR_Z_BIT) != 0; +} + +static bool __kprobes __check_ne(unsigned long pstate) +{ + return (pstate & PSR_Z_BIT) == 0; +} + +static bool __kprobes __check_cs(unsigned long pstate) +{ + return (pstate & PSR_C_BIT) != 0; +} + +static bool __kprobes __check_cc(unsigned long pstate) +{ + return (pstate & PSR_C_BIT) == 0; +} + +static bool __kprobes __check_mi(unsigned long pstate) +{ + return (pstate & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_pl(unsigned long pstate) +{ + return (pstate & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_vs(unsigned long pstate) +{ + return (pstate & PSR_V_BIT) != 0; +} + +static bool __kprobes __check_vc(unsigned long pstate) +{ + return (pstate & PSR_V_BIT) == 0; +} + +static bool __kprobes __check_hi(unsigned long pstate) +{ + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (pstate & PSR_C_BIT) != 0; +} + +static bool __kprobes __check_ls(unsigned long pstate) +{ + pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ + return (pstate & PSR_C_BIT) == 0; +} + +static bool __kprobes __check_ge(unsigned long pstate) +{ + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (pstate & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_lt(unsigned long pstate) +{ + pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ + return (pstate & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_gt(unsigned long pstate) +{ + /*PSR_N_BIT ^= PSR_V_BIT */ + unsigned long temp = pstate ^ (pstate << 3); + + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ + return (temp & PSR_N_BIT) == 0; +} + +static bool __kprobes __check_le(unsigned long pstate) +{ + /*PSR_N_BIT ^= PSR_V_BIT */ + unsigned long temp = pstate ^ (pstate << 3); + + temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ + return (temp & PSR_N_BIT) != 0; +} + +static bool __kprobes __check_al(unsigned long pstate) +{ + return true; +} + +/* + * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that + * it behaves identically to 0b1110 ("al"). + */ +pstate_check_t * const aarch32_opcode_cond_checks[16] = { + __check_eq, __check_ne, __check_cs, __check_cc, + __check_mi, __check_pl, __check_vs, __check_vc, + __check_hi, __check_ls, __check_ge, __check_lt, + __check_gt, __check_le, __check_al, __check_al }; int show_unhandled_signals = 0; @@ -750,28 +842,9 @@ const char *esr_get_class_string(u32 esr) return esr_class_str[ESR_ELx_EC(esr)]; } -/* - * bad_mode handles the impossible case in the exception vector. This is always - * fatal. - */ -asmlinkage void notrace bad_mode(struct pt_regs *regs, int reason, unsigned int esr) -{ - arm64_enter_nmi(regs); - - console_verbose(); - - pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n", - handler[reason], smp_processor_id(), esr, - esr_get_class_string(esr)); - - __show_regs(regs); - local_daif_mask(); - panic("bad mode"); -} - /* * bad_el0_sync handles unexpected, but potentially recoverable synchronous - * exceptions taken from EL0. Unlike bad_mode, this returns. + * exceptions taken from EL0. */ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) { @@ -789,15 +862,11 @@ void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr) DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack) __aligned(16); -asmlinkage void noinstr handle_bad_stack(struct pt_regs *regs) +void panic_bad_stack(struct pt_regs *regs, unsigned int esr, unsigned long far) { unsigned long tsk_stk = (unsigned long)current->stack; unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr); unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack); - unsigned int esr = read_sysreg(esr_el1); - unsigned long far = read_sysreg(far_el1); - - arm64_enter_nmi(regs); console_verbose(); pr_emerg("Insufficient stack space to handle exception!"); @@ -870,15 +939,11 @@ bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr) } } -asmlinkage void noinstr do_serror(struct pt_regs *regs, unsigned int esr) +void do_serror(struct pt_regs *regs, unsigned int esr) { - arm64_enter_nmi(regs); - /* non-RAS errors are not containable */ if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr)) arm64_serror_panic(regs, esr); - - arm64_exit_nmi(regs); } /* GENERIC_BUG traps */ diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 589921392cb1..989bb5dad2c8 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM) += hyp/ kvm-y := $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o $(KVM)/eventfd.o \ - $(KVM)/vfio.o $(KVM)/irqchip.o \ + $(KVM)/vfio.o $(KVM)/irqchip.o $(KVM)/binary_stats.o \ arm.o mmu.o mmio.o psci.o perf.o hypercalls.o pvtime.o \ inject_fault.o va_layout.o handle_exit.o \ guest.o debug.o reset.o sys_regs.o \ diff --git a/arch/arm64/kvm/arch_timer.c b/arch/arm64/kvm/arch_timer.c index 74e0699661e9..3df67c127489 100644 --- a/arch/arm64/kvm/arch_timer.c +++ b/arch/arm64/kvm/arch_timer.c @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -973,6 +974,135 @@ static int kvm_timer_dying_cpu(unsigned int cpu) return 0; } +static int timer_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) +{ + if (vcpu) + irqd_set_forwarded_to_vcpu(d); + else + irqd_clr_forwarded_to_vcpu(d); + + return 0; +} + +static int timer_irq_set_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, bool val) +{ + if (which != IRQCHIP_STATE_ACTIVE || !irqd_is_forwarded_to_vcpu(d)) + return irq_chip_set_parent_state(d, which, val); + + if (val) + irq_chip_mask_parent(d); + else + irq_chip_unmask_parent(d); + + return 0; +} + +static void timer_irq_eoi(struct irq_data *d) +{ + if (!irqd_is_forwarded_to_vcpu(d)) + irq_chip_eoi_parent(d); +} + +static void timer_irq_ack(struct irq_data *d) +{ + d = d->parent_data; + if (d->chip->irq_ack) + d->chip->irq_ack(d); +} + +static struct irq_chip timer_chip = { + .name = "KVM", + .irq_ack = timer_irq_ack, + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = timer_irq_eoi, + .irq_set_type = irq_chip_set_type_parent, + .irq_set_vcpu_affinity = timer_irq_set_vcpu_affinity, + .irq_set_irqchip_state = timer_irq_set_irqchip_state, +}; + +static int timer_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + irq_hw_number_t hwirq = (uintptr_t)arg; + + return irq_domain_set_hwirq_and_chip(domain, virq, hwirq, + &timer_chip, NULL); +} + +static void timer_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ +} + +static const struct irq_domain_ops timer_domain_ops = { + .alloc = timer_irq_domain_alloc, + .free = timer_irq_domain_free, +}; + +static struct irq_ops arch_timer_irq_ops = { + .get_input_level = kvm_arch_timer_get_input_level, +}; + +static void kvm_irq_fixup_flags(unsigned int virq, u32 *flags) +{ + *flags = irq_get_trigger_type(virq); + if (*flags != IRQF_TRIGGER_HIGH && *flags != IRQF_TRIGGER_LOW) { + kvm_err("Invalid trigger for timer IRQ%d, assuming level low\n", + virq); + *flags = IRQF_TRIGGER_LOW; + } +} + +static int kvm_irq_init(struct arch_timer_kvm_info *info) +{ + struct irq_domain *domain = NULL; + + if (info->virtual_irq <= 0) { + kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n", + info->virtual_irq); + return -ENODEV; + } + + host_vtimer_irq = info->virtual_irq; + kvm_irq_fixup_flags(host_vtimer_irq, &host_vtimer_irq_flags); + + if (kvm_vgic_global_state.no_hw_deactivation) { + struct fwnode_handle *fwnode; + struct irq_data *data; + + fwnode = irq_domain_alloc_named_fwnode("kvm-timer"); + if (!fwnode) + return -ENOMEM; + + /* Assume both vtimer and ptimer in the same parent */ + data = irq_get_irq_data(host_vtimer_irq); + domain = irq_domain_create_hierarchy(data->domain, 0, + NR_KVM_TIMERS, fwnode, + &timer_domain_ops, NULL); + if (!domain) { + irq_domain_free_fwnode(fwnode); + return -ENOMEM; + } + + arch_timer_irq_ops.flags |= VGIC_IRQ_SW_RESAMPLE; + WARN_ON(irq_domain_push_irq(domain, host_vtimer_irq, + (void *)TIMER_VTIMER)); + } + + if (info->physical_irq > 0) { + host_ptimer_irq = info->physical_irq; + kvm_irq_fixup_flags(host_ptimer_irq, &host_ptimer_irq_flags); + + if (domain) + WARN_ON(irq_domain_push_irq(domain, host_ptimer_irq, + (void *)TIMER_PTIMER)); + } + + return 0; +} + int kvm_timer_hyp_init(bool has_gic) { struct arch_timer_kvm_info *info; @@ -986,23 +1116,12 @@ int kvm_timer_hyp_init(bool has_gic) return -ENODEV; } + err = kvm_irq_init(info); + if (err) + return err; + /* First, do the virtual EL1 timer irq */ - if (info->virtual_irq <= 0) { - kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n", - info->virtual_irq); - return -ENODEV; - } - host_vtimer_irq = info->virtual_irq; - - host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq); - if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH && - host_vtimer_irq_flags != IRQF_TRIGGER_LOW) { - kvm_err("Invalid trigger for vtimer IRQ%d, assuming level low\n", - host_vtimer_irq); - host_vtimer_irq_flags = IRQF_TRIGGER_LOW; - } - err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler, "kvm guest vtimer", kvm_get_running_vcpus()); if (err) { @@ -1027,15 +1146,6 @@ int kvm_timer_hyp_init(bool has_gic) /* Now let's do the physical EL1 timer irq */ if (info->physical_irq > 0) { - host_ptimer_irq = info->physical_irq; - host_ptimer_irq_flags = irq_get_trigger_type(host_ptimer_irq); - if (host_ptimer_irq_flags != IRQF_TRIGGER_HIGH && - host_ptimer_irq_flags != IRQF_TRIGGER_LOW) { - kvm_err("Invalid trigger for ptimer IRQ%d, assuming level low\n", - host_ptimer_irq); - host_ptimer_irq_flags = IRQF_TRIGGER_LOW; - } - err = request_percpu_irq(host_ptimer_irq, kvm_arch_timer_handler, "kvm guest ptimer", kvm_get_running_vcpus()); if (err) { @@ -1143,7 +1253,7 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) ret = kvm_vgic_map_phys_irq(vcpu, map.direct_vtimer->host_timer_irq, map.direct_vtimer->irq.irq, - kvm_arch_timer_get_input_level); + &arch_timer_irq_ops); if (ret) return ret; @@ -1151,7 +1261,7 @@ int kvm_timer_enable(struct kvm_vcpu *vcpu) ret = kvm_vgic_map_phys_irq(vcpu, map.direct_ptimer->host_timer_irq, map.direct_ptimer->irq.irq, - kvm_arch_timer_get_input_level); + &arch_timer_irq_ops); } if (ret) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index e720148232a0..e9a2b8f27792 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -93,6 +93,12 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, r = 0; kvm->arch.return_nisv_io_abort_to_user = true; break; + case KVM_CAP_ARM_MTE: + if (!system_supports_mte() || kvm->created_vcpus) + return -EINVAL; + r = 0; + kvm->arch.mte_enabled = true; + break; default: r = -EINVAL; break; @@ -237,6 +243,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) */ r = 1; break; + case KVM_CAP_ARM_MTE: + r = system_supports_mte(); + break; case KVM_CAP_STEAL_TIME: r = kvm_arm_pvtime_supported(); break; @@ -689,9 +698,22 @@ static void check_vcpu_requests(struct kvm_vcpu *vcpu) vgic_v4_load(vcpu); preempt_enable(); } + + if (kvm_check_request(KVM_REQ_RELOAD_PMU, vcpu)) + kvm_pmu_handle_pmcr(vcpu, + __vcpu_sys_reg(vcpu, PMCR_EL0)); } } +static bool vcpu_mode_is_bad_32bit(struct kvm_vcpu *vcpu) +{ + if (likely(!vcpu_mode_is_32bit(vcpu))) + return false; + + return !system_supports_32bit_el0() || + static_branch_unlikely(&arm64_mismatched_32bit_el0); +} + /** * kvm_arch_vcpu_ioctl_run - the main VCPU run function to execute guest code * @vcpu: The VCPU pointer @@ -877,7 +899,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) * with the asymmetric AArch32 case), return to userspace with * a fatal error. */ - if (!system_supports_32bit_el0() && vcpu_mode_is_32bit(vcpu)) { + if (vcpu_mode_is_bad_32bit(vcpu)) { /* * As we have caught the guest red-handed, decide that * it isn't fit for purpose anymore by making the vcpu @@ -1078,7 +1100,7 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) stage2_unmap_vm(vcpu->kvm); else - __flush_icache_all(); + icache_inval_all_pou(); } vcpu_reset_hcr(vcpu); @@ -1350,6 +1372,13 @@ long kvm_arch_vm_ioctl(struct file *filp, return 0; } + case KVM_ARM_MTE_COPY_TAGS: { + struct kvm_arm_copy_mte_tags copy_tags; + + if (copy_from_user(©_tags, argp, sizeof(copy_tags))) + return -EFAULT; + return kvm_vm_ioctl_mte_copy_tags(kvm, ©_tags); + } default: return -EINVAL; } diff --git a/arch/arm64/kvm/guest.c b/arch/arm64/kvm/guest.c index 5cb4a1cd5603..1dfb83578277 100644 --- a/arch/arm64/kvm/guest.c +++ b/arch/arm64/kvm/guest.c @@ -28,20 +28,40 @@ #include "trace.h" -struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT("halt_successful_poll", halt_successful_poll), - VCPU_STAT("halt_attempted_poll", halt_attempted_poll), - VCPU_STAT("halt_poll_invalid", halt_poll_invalid), - VCPU_STAT("halt_wakeup", halt_wakeup), - VCPU_STAT("hvc_exit_stat", hvc_exit_stat), - VCPU_STAT("wfe_exit_stat", wfe_exit_stat), - VCPU_STAT("wfi_exit_stat", wfi_exit_stat), - VCPU_STAT("mmio_exit_user", mmio_exit_user), - VCPU_STAT("mmio_exit_kernel", mmio_exit_kernel), - VCPU_STAT("exits", exits), - VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), - VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), - { NULL } +const struct _kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS() +}; +static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == + sizeof(struct kvm_vm_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + STATS_DESC_COUNTER(VCPU, hvc_exit_stat), + STATS_DESC_COUNTER(VCPU, wfe_exit_stat), + STATS_DESC_COUNTER(VCPU, wfi_exit_stat), + STATS_DESC_COUNTER(VCPU, mmio_exit_user), + STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), + STATS_DESC_COUNTER(VCPU, exits) +}; +static_assert(ARRAY_SIZE(kvm_vcpu_stats_desc) == + sizeof(struct kvm_vcpu_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), }; static bool core_reg_offset_is_vreg(u64 off) @@ -995,3 +1015,89 @@ int kvm_arm_vcpu_arch_has_attr(struct kvm_vcpu *vcpu, return ret; } + +long kvm_vm_ioctl_mte_copy_tags(struct kvm *kvm, + struct kvm_arm_copy_mte_tags *copy_tags) +{ + gpa_t guest_ipa = copy_tags->guest_ipa; + size_t length = copy_tags->length; + void __user *tags = copy_tags->addr; + gpa_t gfn; + bool write = !(copy_tags->flags & KVM_ARM_TAGS_FROM_GUEST); + int ret = 0; + + if (!kvm_has_mte(kvm)) + return -EINVAL; + + if (copy_tags->reserved[0] || copy_tags->reserved[1]) + return -EINVAL; + + if (copy_tags->flags & ~KVM_ARM_TAGS_FROM_GUEST) + return -EINVAL; + + if (length & ~PAGE_MASK || guest_ipa & ~PAGE_MASK) + return -EINVAL; + + gfn = gpa_to_gfn(guest_ipa); + + mutex_lock(&kvm->slots_lock); + + while (length > 0) { + kvm_pfn_t pfn = gfn_to_pfn_prot(kvm, gfn, write, NULL); + void *maddr; + unsigned long num_tags; + struct page *page; + + if (is_error_noslot_pfn(pfn)) { + ret = -EFAULT; + goto out; + } + + page = pfn_to_online_page(pfn); + if (!page) { + /* Reject ZONE_DEVICE memory */ + ret = -EFAULT; + goto out; + } + maddr = page_address(page); + + if (!write) { + if (test_bit(PG_mte_tagged, &page->flags)) + num_tags = mte_copy_tags_to_user(tags, maddr, + MTE_GRANULES_PER_PAGE); + else + /* No tags in memory, so write zeros */ + num_tags = MTE_GRANULES_PER_PAGE - + clear_user(tags, MTE_GRANULES_PER_PAGE); + kvm_release_pfn_clean(pfn); + } else { + num_tags = mte_copy_tags_from_user(maddr, tags, + MTE_GRANULES_PER_PAGE); + + /* + * Set the flag after checking the write + * completed fully + */ + if (num_tags == MTE_GRANULES_PER_PAGE) + set_bit(PG_mte_tagged, &page->flags); + + kvm_release_pfn_dirty(pfn); + } + + if (num_tags != MTE_GRANULES_PER_PAGE) { + ret = -EFAULT; + goto out; + } + + gfn++; + tags += num_tags; + length -= PAGE_SIZE; + } + +out: + mutex_unlock(&kvm->slots_lock); + /* If some data has been copied report the number of bytes copied */ + if (length != copy_tags->length) + return copy_tags->length - length; + return ret; +} diff --git a/arch/arm64/kvm/hyp/entry.S b/arch/arm64/kvm/hyp/entry.S index e831d3dfd50d..435346ea1504 100644 --- a/arch/arm64/kvm/hyp/entry.S +++ b/arch/arm64/kvm/hyp/entry.S @@ -13,6 +13,7 @@ #include #include #include +#include #include .text @@ -51,6 +52,9 @@ alternative_else_nop_endif add x29, x0, #VCPU_CONTEXT + // mte_switch_to_guest(g_ctxt, h_ctxt, tmp1) + mte_switch_to_guest x29, x1, x2 + // Macro ptrauth_switch_to_guest format: // ptrauth_switch_to_guest(guest cxt, tmp1, tmp2, tmp3) // The below macro to restore guest keys is not implemented in C code @@ -142,6 +146,9 @@ SYM_INNER_LABEL(__guest_exit, SYM_L_GLOBAL) // when this feature is enabled for kernel code. ptrauth_switch_to_hyp x1, x2, x3, x4, x5 + // mte_switch_to_hyp(g_ctxt, h_ctxt, reg1) + mte_switch_to_hyp x1, x2, x3 + // Restore hyp's sp_el0 restore_sp_el0 x2, x3 diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index 11541b94b328..0418399e0a20 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -112,7 +112,8 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, new |= (old & PSR_C_BIT); new |= (old & PSR_V_BIT); - // TODO: TCO (if/when ARMv8.5-MemTag is exposed to guests) + if (kvm_has_mte(vcpu->kvm)) + new |= PSR_TCO_BIT; new |= (old & PSR_DIT_BIT); diff --git a/arch/arm64/kvm/hyp/hyp-entry.S b/arch/arm64/kvm/hyp/hyp-entry.S index 5f49df4ffdd8..9aa9b73475c9 100644 --- a/arch/arm64/kvm/hyp/hyp-entry.S +++ b/arch/arm64/kvm/hyp/hyp-entry.S @@ -76,6 +76,7 @@ el1_trap: b __guest_exit el1_irq: +el1_fiq: get_vcpu_ptr x1, x0 mov x0, #ARM_EXCEPTION_IRQ b __guest_exit @@ -131,7 +132,6 @@ SYM_CODE_END(\label) invalid_vector el2t_error_invalid invalid_vector el2h_irq_invalid invalid_vector el2h_fiq_invalid - invalid_vector el1_fiq_invalid .ltorg @@ -179,12 +179,12 @@ SYM_CODE_START(__kvm_hyp_vector) valid_vect el1_sync // Synchronous 64-bit EL1 valid_vect el1_irq // IRQ 64-bit EL1 - invalid_vect el1_fiq_invalid // FIQ 64-bit EL1 + valid_vect el1_fiq // FIQ 64-bit EL1 valid_vect el1_error // Error 64-bit EL1 valid_vect el1_sync // Synchronous 32-bit EL1 valid_vect el1_irq // IRQ 32-bit EL1 - invalid_vect el1_fiq_invalid // FIQ 32-bit EL1 + valid_vect el1_fiq // FIQ 32-bit EL1 valid_vect el1_error // Error 32-bit EL1 SYM_CODE_END(__kvm_hyp_vector) diff --git a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h index cce43bfe158f..de7e14c862e6 100644 --- a/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h +++ b/arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h @@ -14,6 +14,7 @@ #include #include #include +#include static inline void __sysreg_save_common_state(struct kvm_cpu_context *ctxt) { @@ -26,6 +27,16 @@ static inline void __sysreg_save_user_state(struct kvm_cpu_context *ctxt) ctxt_sys_reg(ctxt, TPIDRRO_EL0) = read_sysreg(tpidrro_el0); } +static inline bool ctxt_has_mte(struct kvm_cpu_context *ctxt) +{ + struct kvm_vcpu *vcpu = ctxt->__hyp_running_vcpu; + + if (!vcpu) + vcpu = container_of(ctxt, struct kvm_vcpu, arch.ctxt); + + return kvm_has_mte(kern_hyp_va(vcpu->kvm)); +} + static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) { ctxt_sys_reg(ctxt, CSSELR_EL1) = read_sysreg(csselr_el1); @@ -46,6 +57,11 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) ctxt_sys_reg(ctxt, PAR_EL1) = read_sysreg_par(); ctxt_sys_reg(ctxt, TPIDR_EL1) = read_sysreg(tpidr_el1); + if (ctxt_has_mte(ctxt)) { + ctxt_sys_reg(ctxt, TFSR_EL1) = read_sysreg_el1(SYS_TFSR); + ctxt_sys_reg(ctxt, TFSRE0_EL1) = read_sysreg_s(SYS_TFSRE0_EL1); + } + ctxt_sys_reg(ctxt, SP_EL1) = read_sysreg(sp_el1); ctxt_sys_reg(ctxt, ELR_EL1) = read_sysreg_el1(SYS_ELR); ctxt_sys_reg(ctxt, SPSR_EL1) = read_sysreg_el1(SYS_SPSR); @@ -107,6 +123,11 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt) write_sysreg(ctxt_sys_reg(ctxt, PAR_EL1), par_el1); write_sysreg(ctxt_sys_reg(ctxt, TPIDR_EL1), tpidr_el1); + if (ctxt_has_mte(ctxt)) { + write_sysreg_el1(ctxt_sys_reg(ctxt, TFSR_EL1), SYS_TFSR); + write_sysreg_s(ctxt_sys_reg(ctxt, TFSRE0_EL1), SYS_TFSRE0_EL1); + } + if (!has_vhe() && cpus_have_final_cap(ARM64_WORKAROUND_SPECULATIVE_AT) && ctxt->__hyp_running_vcpu) { diff --git a/arch/arm64/kvm/hyp/include/nvhe/gfp.h b/arch/arm64/kvm/hyp/include/nvhe/gfp.h index 18a4494337bd..fb0f523d1492 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/gfp.h +++ b/arch/arm64/kvm/hyp/include/nvhe/gfp.h @@ -7,7 +7,7 @@ #include #include -#define HYP_NO_ORDER UINT_MAX +#define HYP_NO_ORDER USHRT_MAX struct hyp_pool { /* @@ -19,48 +19,13 @@ struct hyp_pool { struct list_head free_area[MAX_ORDER]; phys_addr_t range_start; phys_addr_t range_end; - unsigned int max_order; + unsigned short max_order; }; -static inline void hyp_page_ref_inc(struct hyp_page *p) -{ - struct hyp_pool *pool = hyp_page_to_pool(p); - - hyp_spin_lock(&pool->lock); - p->refcount++; - hyp_spin_unlock(&pool->lock); -} - -static inline int hyp_page_ref_dec_and_test(struct hyp_page *p) -{ - struct hyp_pool *pool = hyp_page_to_pool(p); - int ret; - - hyp_spin_lock(&pool->lock); - p->refcount--; - ret = (p->refcount == 0); - hyp_spin_unlock(&pool->lock); - - return ret; -} - -static inline void hyp_set_page_refcounted(struct hyp_page *p) -{ - struct hyp_pool *pool = hyp_page_to_pool(p); - - hyp_spin_lock(&pool->lock); - if (p->refcount) { - hyp_spin_unlock(&pool->lock); - BUG(); - } - p->refcount = 1; - hyp_spin_unlock(&pool->lock); -} - /* Allocation */ -void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order); -void hyp_get_page(void *addr); -void hyp_put_page(void *addr); +void *hyp_alloc_pages(struct hyp_pool *pool, unsigned short order); +void hyp_get_page(struct hyp_pool *pool, void *addr); +void hyp_put_page(struct hyp_pool *pool, void *addr); /* Used pages cannot be freed */ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, diff --git a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h index 42d81ec739fa..9c227d87c36d 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mem_protect.h @@ -23,7 +23,7 @@ extern struct host_kvm host_kvm; int __pkvm_prot_finalize(void); int __pkvm_mark_hyp(phys_addr_t start, phys_addr_t end); -int kvm_host_prepare_stage2(void *mem_pgt_pool, void *dev_pgt_pool); +int kvm_host_prepare_stage2(void *pgt_pool_base); void handle_host_mem_abort(struct kvm_cpu_context *host_ctxt); static __always_inline void __load_host_stage2(void) diff --git a/arch/arm64/kvm/hyp/include/nvhe/memory.h b/arch/arm64/kvm/hyp/include/nvhe/memory.h index fd78bde939ee..592b7edb3edb 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/memory.h +++ b/arch/arm64/kvm/hyp/include/nvhe/memory.h @@ -7,12 +7,9 @@ #include -struct hyp_pool; struct hyp_page { - unsigned int refcount; - unsigned int order; - struct hyp_pool *pool; - struct list_head node; + unsigned short refcount; + unsigned short order; }; extern u64 __hyp_vmemmap; diff --git a/arch/arm64/kvm/hyp/include/nvhe/mm.h b/arch/arm64/kvm/hyp/include/nvhe/mm.h index 0095f6289742..8ec3a5a7744b 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/mm.h +++ b/arch/arm64/kvm/hyp/include/nvhe/mm.h @@ -78,19 +78,20 @@ static inline unsigned long hyp_s1_pgtable_pages(void) return res; } -static inline unsigned long host_s2_mem_pgtable_pages(void) +static inline unsigned long host_s2_pgtable_pages(void) { + unsigned long res; + /* * Include an extra 16 pages to safely upper-bound the worst case of * concatenated pgds. */ - return __hyp_pgtable_total_pages() + 16; -} + res = __hyp_pgtable_total_pages() + 16; -static inline unsigned long host_s2_dev_pgtable_pages(void) -{ /* Allow 1 GiB for MMIO mappings */ - return __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); + res += __hyp_pgtable_max_pages(SZ_1G >> PAGE_SHIFT); + + return res; } #endif /* __KVM_HYP_MM_H */ diff --git a/arch/arm64/kvm/hyp/nvhe/cache.S b/arch/arm64/kvm/hyp/nvhe/cache.S index 36cef6915428..958734f4d6b0 100644 --- a/arch/arm64/kvm/hyp/nvhe/cache.S +++ b/arch/arm64/kvm/hyp/nvhe/cache.S @@ -7,7 +7,7 @@ #include #include -SYM_FUNC_START_PI(__flush_dcache_area) +SYM_FUNC_START_PI(dcache_clean_inval_poc) dcache_by_line_op civac, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(__flush_dcache_area) +SYM_FUNC_END_PI(dcache_clean_inval_poc) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 4b60c0056c04..d938ce95d3bd 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -23,8 +23,7 @@ extern unsigned long hyp_nr_cpus; struct host_kvm host_kvm; -static struct hyp_pool host_s2_mem; -static struct hyp_pool host_s2_dev; +static struct hyp_pool host_s2_pool; /* * Copies of the host's CPU features registers holding sanitized values. @@ -36,7 +35,7 @@ static const u8 pkvm_hyp_id = 1; static void *host_s2_zalloc_pages_exact(size_t size) { - return hyp_alloc_pages(&host_s2_mem, get_order(size)); + return hyp_alloc_pages(&host_s2_pool, get_order(size)); } static void *host_s2_zalloc_page(void *pool) @@ -44,20 +43,24 @@ static void *host_s2_zalloc_page(void *pool) return hyp_alloc_pages(pool, 0); } -static int prepare_s2_pools(void *mem_pgt_pool, void *dev_pgt_pool) +static void host_s2_get_page(void *addr) +{ + hyp_get_page(&host_s2_pool, addr); +} + +static void host_s2_put_page(void *addr) +{ + hyp_put_page(&host_s2_pool, addr); +} + +static int prepare_s2_pool(void *pgt_pool_base) { unsigned long nr_pages, pfn; int ret; - pfn = hyp_virt_to_pfn(mem_pgt_pool); - nr_pages = host_s2_mem_pgtable_pages(); - ret = hyp_pool_init(&host_s2_mem, pfn, nr_pages, 0); - if (ret) - return ret; - - pfn = hyp_virt_to_pfn(dev_pgt_pool); - nr_pages = host_s2_dev_pgtable_pages(); - ret = hyp_pool_init(&host_s2_dev, pfn, nr_pages, 0); + pfn = hyp_virt_to_pfn(pgt_pool_base); + nr_pages = host_s2_pgtable_pages(); + ret = hyp_pool_init(&host_s2_pool, pfn, nr_pages, 0); if (ret) return ret; @@ -67,8 +70,8 @@ static int prepare_s2_pools(void *mem_pgt_pool, void *dev_pgt_pool) .phys_to_virt = hyp_phys_to_virt, .virt_to_phys = hyp_virt_to_phys, .page_count = hyp_page_count, - .get_page = hyp_get_page, - .put_page = hyp_put_page, + .get_page = host_s2_get_page, + .put_page = host_s2_put_page, }; return 0; @@ -86,7 +89,7 @@ static void prepare_host_vtcr(void) id_aa64mmfr1_el1_sys_val, phys_shift); } -int kvm_host_prepare_stage2(void *mem_pgt_pool, void *dev_pgt_pool) +int kvm_host_prepare_stage2(void *pgt_pool_base) { struct kvm_s2_mmu *mmu = &host_kvm.arch.mmu; int ret; @@ -94,7 +97,7 @@ int kvm_host_prepare_stage2(void *mem_pgt_pool, void *dev_pgt_pool) prepare_host_vtcr(); hyp_spin_lock_init(&host_kvm.lock); - ret = prepare_s2_pools(mem_pgt_pool, dev_pgt_pool); + ret = prepare_s2_pool(pgt_pool_base); if (ret) return ret; @@ -199,11 +202,10 @@ static bool range_is_memory(u64 start, u64 end) } static inline int __host_stage2_idmap(u64 start, u64 end, - enum kvm_pgtable_prot prot, - struct hyp_pool *pool) + enum kvm_pgtable_prot prot) { return kvm_pgtable_stage2_map(&host_kvm.pgt, start, end - start, start, - prot, pool); + prot, &host_s2_pool); } static int host_stage2_idmap(u64 addr) @@ -211,7 +213,6 @@ static int host_stage2_idmap(u64 addr) enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R | KVM_PGTABLE_PROT_W; struct kvm_mem_range range; bool is_memory = find_mem_range(addr, &range); - struct hyp_pool *pool = is_memory ? &host_s2_mem : &host_s2_dev; int ret; if (is_memory) @@ -222,22 +223,21 @@ static int host_stage2_idmap(u64 addr) if (ret) goto unlock; - ret = __host_stage2_idmap(range.start, range.end, prot, pool); - if (is_memory || ret != -ENOMEM) + ret = __host_stage2_idmap(range.start, range.end, prot); + if (ret != -ENOMEM) goto unlock; /* - * host_s2_mem has been provided with enough pages to cover all of - * memory with page granularity, so we should never hit the ENOMEM case. - * However, it is difficult to know how much of the MMIO range we will - * need to cover upfront, so we may need to 'recycle' the pages if we - * run out. + * The pool has been provided with enough pages to cover all of memory + * with page granularity, but it is difficult to know how much of the + * MMIO range we will need to cover upfront, so we may need to 'recycle' + * the pages if we run out. */ ret = host_stage2_unmap_dev_all(); if (ret) goto unlock; - ret = __host_stage2_idmap(range.start, range.end, prot, pool); + ret = __host_stage2_idmap(range.start, range.end, prot); unlock: hyp_spin_unlock(&host_kvm.lock); @@ -258,7 +258,7 @@ int __pkvm_mark_hyp(phys_addr_t start, phys_addr_t end) hyp_spin_lock(&host_kvm.lock); ret = kvm_pgtable_stage2_set_owner(&host_kvm.pgt, start, end - start, - &host_s2_mem, pkvm_hyp_id); + &host_s2_pool, pkvm_hyp_id); hyp_spin_unlock(&host_kvm.lock); return ret != -EAGAIN ? ret : 0; diff --git a/arch/arm64/kvm/hyp/nvhe/page_alloc.c b/arch/arm64/kvm/hyp/nvhe/page_alloc.c index 237e03bf0cb1..41fc25bdfb34 100644 --- a/arch/arm64/kvm/hyp/nvhe/page_alloc.c +++ b/arch/arm64/kvm/hyp/nvhe/page_alloc.c @@ -32,7 +32,7 @@ u64 __hyp_vmemmap; */ static struct hyp_page *__find_buddy_nocheck(struct hyp_pool *pool, struct hyp_page *p, - unsigned int order) + unsigned short order) { phys_addr_t addr = hyp_page_to_phys(p); @@ -51,21 +51,49 @@ static struct hyp_page *__find_buddy_nocheck(struct hyp_pool *pool, /* Find a buddy page currently available for allocation */ static struct hyp_page *__find_buddy_avail(struct hyp_pool *pool, struct hyp_page *p, - unsigned int order) + unsigned short order) { struct hyp_page *buddy = __find_buddy_nocheck(pool, p, order); - if (!buddy || buddy->order != order || list_empty(&buddy->node)) + if (!buddy || buddy->order != order || buddy->refcount) return NULL; return buddy; } +/* + * Pages that are available for allocation are tracked in free-lists, so we use + * the pages themselves to store the list nodes to avoid wasting space. As the + * allocator always returns zeroed pages (which are zeroed on the hyp_put_page() + * path to optimize allocation speed), we also need to clean-up the list node in + * each page when we take it out of the list. + */ +static inline void page_remove_from_list(struct hyp_page *p) +{ + struct list_head *node = hyp_page_to_virt(p); + + __list_del_entry(node); + memset(node, 0, sizeof(*node)); +} + +static inline void page_add_to_list(struct hyp_page *p, struct list_head *head) +{ + struct list_head *node = hyp_page_to_virt(p); + + INIT_LIST_HEAD(node); + list_add_tail(node, head); +} + +static inline struct hyp_page *node_to_page(struct list_head *node) +{ + return hyp_virt_to_page(node); +} + static void __hyp_attach_page(struct hyp_pool *pool, struct hyp_page *p) { - unsigned int order = p->order; + unsigned short order = p->order; struct hyp_page *buddy; memset(hyp_page_to_virt(p), 0, PAGE_SIZE << p->order); @@ -83,32 +111,23 @@ static void __hyp_attach_page(struct hyp_pool *pool, break; /* Take the buddy out of its list, and coallesce with @p */ - list_del_init(&buddy->node); + page_remove_from_list(buddy); buddy->order = HYP_NO_ORDER; p = min(p, buddy); } /* Mark the new head, and insert it */ p->order = order; - list_add_tail(&p->node, &pool->free_area[order]); -} - -static void hyp_attach_page(struct hyp_page *p) -{ - struct hyp_pool *pool = hyp_page_to_pool(p); - - hyp_spin_lock(&pool->lock); - __hyp_attach_page(pool, p); - hyp_spin_unlock(&pool->lock); + page_add_to_list(p, &pool->free_area[order]); } static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool, struct hyp_page *p, - unsigned int order) + unsigned short order) { struct hyp_page *buddy; - list_del_init(&p->node); + page_remove_from_list(p); while (p->order > order) { /* * The buddy of order n - 1 currently has HYP_NO_ORDER as it @@ -119,30 +138,64 @@ static struct hyp_page *__hyp_extract_page(struct hyp_pool *pool, p->order--; buddy = __find_buddy_nocheck(pool, p, p->order); buddy->order = p->order; - list_add_tail(&buddy->node, &pool->free_area[buddy->order]); + page_add_to_list(buddy, &pool->free_area[buddy->order]); } return p; } -void hyp_put_page(void *addr) +static inline void hyp_page_ref_inc(struct hyp_page *p) { - struct hyp_page *p = hyp_virt_to_page(addr); + BUG_ON(p->refcount == USHRT_MAX); + p->refcount++; +} +static inline int hyp_page_ref_dec_and_test(struct hyp_page *p) +{ + p->refcount--; + return (p->refcount == 0); +} + +static inline void hyp_set_page_refcounted(struct hyp_page *p) +{ + BUG_ON(p->refcount); + p->refcount = 1; +} + +static void __hyp_put_page(struct hyp_pool *pool, struct hyp_page *p) +{ if (hyp_page_ref_dec_and_test(p)) - hyp_attach_page(p); + __hyp_attach_page(pool, p); } -void hyp_get_page(void *addr) +/* + * Changes to the buddy tree and page refcounts must be done with the hyp_pool + * lock held. If a refcount change requires an update to the buddy tree (e.g. + * hyp_put_page()), both operations must be done within the same critical + * section to guarantee transient states (e.g. a page with null refcount but + * not yet attached to a free list) can't be observed by well-behaved readers. + */ +void hyp_put_page(struct hyp_pool *pool, void *addr) { struct hyp_page *p = hyp_virt_to_page(addr); - hyp_page_ref_inc(p); + hyp_spin_lock(&pool->lock); + __hyp_put_page(pool, p); + hyp_spin_unlock(&pool->lock); } -void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order) +void hyp_get_page(struct hyp_pool *pool, void *addr) { - unsigned int i = order; + struct hyp_page *p = hyp_virt_to_page(addr); + + hyp_spin_lock(&pool->lock); + hyp_page_ref_inc(p); + hyp_spin_unlock(&pool->lock); +} + +void *hyp_alloc_pages(struct hyp_pool *pool, unsigned short order) +{ + unsigned short i = order; struct hyp_page *p; hyp_spin_lock(&pool->lock); @@ -156,11 +209,11 @@ void *hyp_alloc_pages(struct hyp_pool *pool, unsigned int order) } /* Extract it from the tree at the right order */ - p = list_first_entry(&pool->free_area[i], struct hyp_page, node); + p = node_to_page(pool->free_area[i].next); p = __hyp_extract_page(pool, p, order); - hyp_spin_unlock(&pool->lock); hyp_set_page_refcounted(p); + hyp_spin_unlock(&pool->lock); return hyp_page_to_virt(p); } @@ -181,15 +234,14 @@ int hyp_pool_init(struct hyp_pool *pool, u64 pfn, unsigned int nr_pages, /* Init the vmemmap portion */ p = hyp_phys_to_page(phys); - memset(p, 0, sizeof(*p) * nr_pages); for (i = 0; i < nr_pages; i++) { - p[i].pool = pool; - INIT_LIST_HEAD(&p[i].node); + p[i].order = 0; + hyp_set_page_refcounted(&p[i]); } /* Attach the unused pages to the buddy tree */ for (i = reserved_pages; i < nr_pages; i++) - __hyp_attach_page(pool, &p[i]); + __hyp_put_page(pool, &p[i]); return 0; } diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c index a3d3a275344e..0b574d106519 100644 --- a/arch/arm64/kvm/hyp/nvhe/setup.c +++ b/arch/arm64/kvm/hyp/nvhe/setup.c @@ -24,8 +24,7 @@ unsigned long hyp_nr_cpus; static void *vmemmap_base; static void *hyp_pgt_base; -static void *host_s2_mem_pgt_base; -static void *host_s2_dev_pgt_base; +static void *host_s2_pgt_base; static struct kvm_pgtable_mm_ops pkvm_pgtable_mm_ops; static int divide_memory_pool(void *virt, unsigned long size) @@ -45,14 +44,9 @@ static int divide_memory_pool(void *virt, unsigned long size) if (!hyp_pgt_base) return -ENOMEM; - nr_pages = host_s2_mem_pgtable_pages(); - host_s2_mem_pgt_base = hyp_early_alloc_contig(nr_pages); - if (!host_s2_mem_pgt_base) - return -ENOMEM; - - nr_pages = host_s2_dev_pgtable_pages(); - host_s2_dev_pgt_base = hyp_early_alloc_contig(nr_pages); - if (!host_s2_dev_pgt_base) + nr_pages = host_s2_pgtable_pages(); + host_s2_pgt_base = hyp_early_alloc_contig(nr_pages); + if (!host_s2_pgt_base) return -ENOMEM; return 0; @@ -134,7 +128,8 @@ static void update_nvhe_init_params(void) for (i = 0; i < hyp_nr_cpus; i++) { params = per_cpu_ptr(&kvm_init_params, i); params->pgd_pa = __hyp_pa(pkvm_pgtable.pgd); - __flush_dcache_area(params, sizeof(*params)); + dcache_clean_inval_poc((unsigned long)params, + (unsigned long)params + sizeof(*params)); } } @@ -143,6 +138,16 @@ static void *hyp_zalloc_hyp_page(void *arg) return hyp_alloc_pages(&hpool, 0); } +static void hpool_get_page(void *addr) +{ + hyp_get_page(&hpool, addr); +} + +static void hpool_put_page(void *addr) +{ + hyp_put_page(&hpool, addr); +} + void __noreturn __pkvm_init_finalise(void) { struct kvm_host_data *host_data = this_cpu_ptr(&kvm_host_data); @@ -158,7 +163,7 @@ void __noreturn __pkvm_init_finalise(void) if (ret) goto out; - ret = kvm_host_prepare_stage2(host_s2_mem_pgt_base, host_s2_dev_pgt_base); + ret = kvm_host_prepare_stage2(host_s2_pgt_base); if (ret) goto out; @@ -166,8 +171,8 @@ void __noreturn __pkvm_init_finalise(void) .zalloc_page = hyp_zalloc_hyp_page, .phys_to_virt = hyp_phys_to_virt, .virt_to_phys = hyp_virt_to_phys, - .get_page = hyp_get_page, - .put_page = hyp_put_page, + .get_page = hpool_get_page, + .put_page = hpool_put_page, }; pkvm_pgtable.mm_ops = &pkvm_pgtable_mm_ops; diff --git a/arch/arm64/kvm/hyp/nvhe/tlb.c b/arch/arm64/kvm/hyp/nvhe/tlb.c index 83dc3b271bc5..38ed0f6f2703 100644 --- a/arch/arm64/kvm/hyp/nvhe/tlb.c +++ b/arch/arm64/kvm/hyp/nvhe/tlb.c @@ -104,7 +104,7 @@ void __kvm_tlb_flush_vmid_ipa(struct kvm_s2_mmu *mmu, * you should be running with VHE enabled. */ if (icache_is_vpipt()) - __flush_icache_all(); + icache_inval_all_pou(); __tlb_switch_to_host(&cxt); } diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c index c37c1dc4feaf..05321f4165e3 100644 --- a/arch/arm64/kvm/hyp/pgtable.c +++ b/arch/arm64/kvm/hyp/pgtable.c @@ -577,12 +577,24 @@ static void stage2_put_pte(kvm_pte_t *ptep, struct kvm_s2_mmu *mmu, u64 addr, mm_ops->put_page(ptep); } +static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte) +{ + u64 memattr = pte & KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR; + return memattr == KVM_S2_MEMATTR(pgt, NORMAL); +} + +static bool stage2_pte_executable(kvm_pte_t pte) +{ + return !(pte & KVM_PTE_LEAF_ATTR_HI_S2_XN); +} + static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, struct stage2_map_data *data) { kvm_pte_t new, old = *ptep; u64 granule = kvm_granule_size(level), phys = data->phys; + struct kvm_pgtable *pgt = data->mmu->pgt; struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; if (!kvm_block_mapping_supported(addr, end, phys, level)) @@ -606,6 +618,14 @@ static int stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, stage2_put_pte(ptep, data->mmu, addr, level, mm_ops); } + /* Perform CMOs before installation of the guest stage-2 PTE */ + if (mm_ops->dcache_clean_inval_poc && stage2_pte_cacheable(pgt, new)) + mm_ops->dcache_clean_inval_poc(kvm_pte_follow(new, mm_ops), + granule); + + if (mm_ops->icache_inval_pou && stage2_pte_executable(new)) + mm_ops->icache_inval_pou(kvm_pte_follow(new, mm_ops), granule); + smp_store_release(ptep, new); if (stage2_pte_is_counted(new)) mm_ops->get_page(ptep); @@ -798,12 +818,6 @@ int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size, return ret; } -static bool stage2_pte_cacheable(struct kvm_pgtable *pgt, kvm_pte_t pte) -{ - u64 memattr = pte & KVM_PTE_LEAF_ATTR_LO_S2_MEMATTR; - return memattr == KVM_S2_MEMATTR(pgt, NORMAL); -} - static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, enum kvm_pgtable_walk_flags flag, void * const arg) @@ -839,8 +853,11 @@ static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, stage2_put_pte(ptep, mmu, addr, level, mm_ops); if (need_flush) { - __flush_dcache_area(kvm_pte_follow(pte, mm_ops), - kvm_granule_size(level)); + kvm_pte_t *pte_follow = kvm_pte_follow(pte, mm_ops); + + dcache_clean_inval_poc((unsigned long)pte_follow, + (unsigned long)pte_follow + + kvm_granule_size(level)); } if (childp) @@ -861,10 +878,11 @@ int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) } struct stage2_attr_data { - kvm_pte_t attr_set; - kvm_pte_t attr_clr; - kvm_pte_t pte; - u32 level; + kvm_pte_t attr_set; + kvm_pte_t attr_clr; + kvm_pte_t pte; + u32 level; + struct kvm_pgtable_mm_ops *mm_ops; }; static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, @@ -873,6 +891,7 @@ static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, { kvm_pte_t pte = *ptep; struct stage2_attr_data *data = arg; + struct kvm_pgtable_mm_ops *mm_ops = data->mm_ops; if (!kvm_pte_valid(pte)) return 0; @@ -887,8 +906,17 @@ static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, * but worst-case the access flag update gets lost and will be * set on the next access instead. */ - if (data->pte != pte) + if (data->pte != pte) { + /* + * Invalidate instruction cache before updating the guest + * stage-2 PTE if we are going to add executable permission. + */ + if (mm_ops->icache_inval_pou && + stage2_pte_executable(pte) && !stage2_pte_executable(*ptep)) + mm_ops->icache_inval_pou(kvm_pte_follow(pte, mm_ops), + kvm_granule_size(level)); WRITE_ONCE(*ptep, pte); + } return 0; } @@ -903,6 +931,7 @@ static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, struct stage2_attr_data data = { .attr_set = attr_set & attr_mask, .attr_clr = attr_clr & attr_mask, + .mm_ops = pgt->mm_ops, }; struct kvm_pgtable_walker walker = { .cb = stage2_attr_walker, @@ -988,11 +1017,15 @@ static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, struct kvm_pgtable *pgt = arg; struct kvm_pgtable_mm_ops *mm_ops = pgt->mm_ops; kvm_pte_t pte = *ptep; + kvm_pte_t *pte_follow; if (!kvm_pte_valid(pte) || !stage2_pte_cacheable(pgt, pte)) return 0; - __flush_dcache_area(kvm_pte_follow(pte, mm_ops), kvm_granule_size(level)); + pte_follow = kvm_pte_follow(pte, mm_ops); + dcache_clean_inval_poc((unsigned long)pte_follow, + (unsigned long)pte_follow + + kvm_granule_size(level)); return 0; } diff --git a/arch/arm64/kvm/hyp/reserved_mem.c b/arch/arm64/kvm/hyp/reserved_mem.c index 83ca23ac259b..d654921dd09b 100644 --- a/arch/arm64/kvm/hyp/reserved_mem.c +++ b/arch/arm64/kvm/hyp/reserved_mem.c @@ -71,8 +71,7 @@ void __init kvm_hyp_reserve(void) } hyp_mem_pages += hyp_s1_pgtable_pages(); - hyp_mem_pages += host_s2_mem_pgtable_pages(); - hyp_mem_pages += host_s2_dev_pgtable_pages(); + hyp_mem_pages += host_s2_pgtable_pages(); /* * The hyp_vmemmap needs to be backed by pages, but these pages diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index c10207fed2f3..57292dc5ce35 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -126,6 +126,16 @@ static void *kvm_host_va(phys_addr_t phys) return __va(phys); } +static void clean_dcache_guest_page(void *va, size_t size) +{ + __clean_dcache_guest_page(va, size); +} + +static void invalidate_icache_guest_page(void *va, size_t size) +{ + __invalidate_icache_guest_page(va, size); +} + /* * Unmapping vs dcache management: * @@ -432,6 +442,8 @@ static struct kvm_pgtable_mm_ops kvm_s2_mm_ops = { .page_count = kvm_host_page_count, .phys_to_virt = kvm_host_va, .virt_to_phys = kvm_host_pa, + .dcache_clean_inval_poc = clean_dcache_guest_page, + .icache_inval_pou = invalidate_icache_guest_page, }; /** @@ -693,16 +705,6 @@ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, kvm_mmu_write_protect_pt_masked(kvm, slot, gfn_offset, mask); } -static void clean_dcache_guest_page(kvm_pfn_t pfn, unsigned long size) -{ - __clean_dcache_guest_page(pfn, size); -} - -static void invalidate_icache_guest_page(kvm_pfn_t pfn, unsigned long size) -{ - __invalidate_icache_guest_page(pfn, size); -} - static void kvm_send_hwpoison_signal(unsigned long address, short lsb) { send_sig_mceerr(BUS_MCEERR_AR, (void __user *)address, lsb, current); @@ -822,6 +824,74 @@ transparent_hugepage_adjust(struct kvm_memory_slot *memslot, return PAGE_SIZE; } +static int get_vma_page_shift(struct vm_area_struct *vma, unsigned long hva) +{ + unsigned long pa; + + if (is_vm_hugetlb_page(vma) && !(vma->vm_flags & VM_PFNMAP)) + return huge_page_shift(hstate_vma(vma)); + + if (!(vma->vm_flags & VM_PFNMAP)) + return PAGE_SHIFT; + + VM_BUG_ON(is_vm_hugetlb_page(vma)); + + pa = (vma->vm_pgoff << PAGE_SHIFT) + (hva - vma->vm_start); + +#ifndef __PAGETABLE_PMD_FOLDED + if ((hva & (PUD_SIZE - 1)) == (pa & (PUD_SIZE - 1)) && + ALIGN_DOWN(hva, PUD_SIZE) >= vma->vm_start && + ALIGN(hva, PUD_SIZE) <= vma->vm_end) + return PUD_SHIFT; +#endif + + if ((hva & (PMD_SIZE - 1)) == (pa & (PMD_SIZE - 1)) && + ALIGN_DOWN(hva, PMD_SIZE) >= vma->vm_start && + ALIGN(hva, PMD_SIZE) <= vma->vm_end) + return PMD_SHIFT; + + return PAGE_SHIFT; +} + +/* + * The page will be mapped in stage 2 as Normal Cacheable, so the VM will be + * able to see the page's tags and therefore they must be initialised first. If + * PG_mte_tagged is set, tags have already been initialised. + * + * The race in the test/set of the PG_mte_tagged flag is handled by: + * - preventing VM_SHARED mappings in a memslot with MTE preventing two VMs + * racing to santise the same page + * - mmap_lock protects between a VM faulting a page in and the VMM performing + * an mprotect() to add VM_MTE + */ +static int sanitise_mte_tags(struct kvm *kvm, kvm_pfn_t pfn, + unsigned long size) +{ + unsigned long i, nr_pages = size >> PAGE_SHIFT; + struct page *page; + + if (!kvm_has_mte(kvm)) + return 0; + + /* + * pfn_to_online_page() is used to reject ZONE_DEVICE pages + * that may not support tags. + */ + page = pfn_to_online_page(pfn); + + if (!page) + return -EFAULT; + + for (i = 0; i < nr_pages; i++, page++) { + if (!test_bit(PG_mte_tagged, &page->flags)) { + mte_clear_page_tags(page_address(page)); + set_bit(PG_mte_tagged, &page->flags); + } + } + + return 0; +} + static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, struct kvm_memory_slot *memslot, unsigned long hva, unsigned long fault_status) @@ -830,6 +900,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, bool write_fault, writable, force_pte = false; bool exec_fault; bool device = false; + bool shared; unsigned long mmu_seq; struct kvm *kvm = vcpu->kvm; struct kvm_mmu_memory_cache *memcache = &vcpu->arch.mmu_page_cache; @@ -853,7 +924,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return -EFAULT; } - /* Let's check if we will get back a huge page backed by hugetlbfs */ + /* + * Let's check if we will get back a huge page backed by hugetlbfs, or + * get block mapping for device MMIO region. + */ mmap_read_lock(current->mm); vma = find_vma_intersection(current->mm, hva, hva + 1); if (unlikely(!vma)) { @@ -862,17 +936,19 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return -EFAULT; } - if (is_vm_hugetlb_page(vma)) - vma_shift = huge_page_shift(hstate_vma(vma)); - else - vma_shift = PAGE_SHIFT; - - if (logging_active || - (vma->vm_flags & VM_PFNMAP)) { + /* + * logging_active is guaranteed to never be true for VM_PFNMAP + * memslots. + */ + if (logging_active) { force_pte = true; vma_shift = PAGE_SHIFT; + } else { + vma_shift = get_vma_page_shift(vma, hva); } + shared = (vma->vm_flags & VM_PFNMAP); + switch (vma_shift) { #ifndef __PAGETABLE_PMD_FOLDED case PUD_SHIFT: @@ -943,8 +1019,17 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, return -EFAULT; if (kvm_is_device_pfn(pfn)) { + /* + * If the page was identified as device early by looking at + * the VMA flags, vma_pagesize is already representing the + * largest quantity we can map. If instead it was mapped + * via gfn_to_pfn_prot(), vma_pagesize is set to PAGE_SIZE + * and must not be upgraded. + * + * In both cases, we don't let transparent_hugepage_adjust() + * change things at the last minute. + */ device = true; - force_pte = true; } else if (logging_active && !write_fault) { /* * Only actually map the page as writable if this was a write @@ -965,19 +1050,25 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa, * If we are not forced to use page mapping, check if we are * backed by a THP and thus use block mapping if possible. */ - if (vma_pagesize == PAGE_SIZE && !force_pte) + if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) vma_pagesize = transparent_hugepage_adjust(memslot, hva, &pfn, &fault_ipa); + + if (fault_status != FSC_PERM && !device && kvm_has_mte(kvm)) { + /* Check the VMM hasn't introduced a new VM_SHARED VMA */ + if (!shared) + ret = sanitise_mte_tags(kvm, pfn, vma_pagesize); + else + ret = -EFAULT; + if (ret) + goto out_unlock; + } + if (writable) prot |= KVM_PGTABLE_PROT_W; - if (fault_status != FSC_PERM && !device) - clean_dcache_guest_page(pfn, vma_pagesize); - - if (exec_fault) { + if (exec_fault) prot |= KVM_PGTABLE_PROT_X; - invalidate_icache_guest_page(pfn, vma_pagesize); - } if (device) prot |= KVM_PGTABLE_PROT_DEVICE; @@ -1168,19 +1259,22 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { kvm_pfn_t pfn = pte_pfn(range->pte); + int ret; if (!kvm->arch.mmu.pgt) return false; WARN_ON(range->end - range->start != 1); - /* - * We've moved a page around, probably through CoW, so let's treat it - * just like a translation fault and clean the cache to the PoC. - */ - clean_dcache_guest_page(pfn, PAGE_SIZE); + ret = sanitise_mte_tags(kvm, pfn, PAGE_SIZE); + if (ret) + return false; /* + * We've moved a page around, probably through CoW, so let's treat + * it just like a translation fault and the map handler will clean + * the cache to the PoC. + * * The MMU notifiers will have unmapped a huge PMD before calling * ->change_pte() (which in turn calls kvm_set_spte_gfn()) and * therefore we never need to clear out a huge PMD through this @@ -1346,7 +1440,6 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, { hva_t hva = mem->userspace_addr; hva_t reg_end = hva + mem->memory_size; - bool writable = !(mem->flags & KVM_MEM_READONLY); int ret = 0; if (change != KVM_MR_CREATE && change != KVM_MR_MOVE && @@ -1363,8 +1456,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, mmap_read_lock(current->mm); /* * A memory region could potentially cover multiple VMAs, and any holes - * between them, so iterate over all of them to find out if we can map - * any of them right now. + * between them, so iterate over all of them. * * +--------------------------------------------+ * +---------------+----------------+ +----------------+ @@ -1375,51 +1467,29 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, */ do { struct vm_area_struct *vma; - hva_t vm_start, vm_end; vma = find_vma_intersection(current->mm, hva, reg_end); if (!vma) break; /* - * Take the intersection of this VMA with the memory region + * VM_SHARED mappings are not allowed with MTE to avoid races + * when updating the PG_mte_tagged page flag, see + * sanitise_mte_tags for more details. */ - vm_start = max(hva, vma->vm_start); - vm_end = min(reg_end, vma->vm_end); + if (kvm_has_mte(kvm) && vma->vm_flags & VM_SHARED) + return -EINVAL; if (vma->vm_flags & VM_PFNMAP) { - gpa_t gpa = mem->guest_phys_addr + - (vm_start - mem->userspace_addr); - phys_addr_t pa; - - pa = (phys_addr_t)vma->vm_pgoff << PAGE_SHIFT; - pa += vm_start - vma->vm_start; - /* IO region dirty page logging not allowed */ if (memslot->flags & KVM_MEM_LOG_DIRTY_PAGES) { ret = -EINVAL; - goto out; - } - - ret = kvm_phys_addr_ioremap(kvm, gpa, pa, - vm_end - vm_start, - writable); - if (ret) break; + } } - hva = vm_end; + hva = min(reg_end, vma->vm_end); } while (hva < reg_end); - if (change == KVM_MR_FLAGS_ONLY) - goto out; - - spin_lock(&kvm->mmu_lock); - if (ret) - unmap_stage2_range(&kvm->arch.mmu, mem->guest_phys_addr, mem->memory_size); - else if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) - stage2_flush_memslot(kvm, memslot); - spin_unlock(&kvm->mmu_lock); -out: mmap_read_unlock(current->mm); return ret; } diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index fd167d4f4215..f33825c995cb 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -578,6 +578,7 @@ void kvm_pmu_handle_pmcr(struct kvm_vcpu *vcpu, u64 val) kvm_pmu_set_counter_value(vcpu, ARMV8_PMU_CYCLE_IDX, 0); if (val & ARMV8_PMU_PMCR_P) { + mask &= ~BIT(ARMV8_PMU_CYCLE_IDX); for_each_set_bit(i, &mask, 32) kvm_pmu_set_counter_value(vcpu, i, 0); } @@ -850,6 +851,9 @@ int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) return -EINVAL; } + /* One-off reload of the PMU on first run */ + kvm_make_request(KVM_REQ_RELOAD_PMU, vcpu); + return 0; } diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index d37ebee085cf..cba7872d69a8 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -176,6 +176,10 @@ static bool vcpu_allowed_register_width(struct kvm_vcpu *vcpu) if (!cpus_have_const_cap(ARM64_HAS_32BIT_EL1) && is32bit) return false; + /* MTE is incompatible with AArch32 */ + if (kvm_has_mte(vcpu->kvm) && is32bit) + return false; + /* Check that the vcpus are either all 32bit or all 64bit */ kvm_for_each_vcpu(i, tmp, vcpu->kvm) { if (vcpu_has_feature(tmp, KVM_ARM_VCPU_EL1_32BIT) != is32bit) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 1a7968ad078c..f6f126eb6ac1 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1047,6 +1047,13 @@ static u64 read_id_reg(const struct kvm_vcpu *vcpu, break; case SYS_ID_AA64PFR1_EL1: val &= ~FEATURE(ID_AA64PFR1_MTE); + if (kvm_has_mte(vcpu->kvm)) { + u64 pfr, mte; + + pfr = read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1); + mte = cpuid_feature_extract_unsigned_field(pfr, ID_AA64PFR1_MTE_SHIFT); + val |= FIELD_PREP(FEATURE(ID_AA64PFR1_MTE), mte); + } break; case SYS_ID_AA64ISAR1_EL1: if (!vcpu_has_ptrauth(vcpu)) @@ -1302,6 +1309,23 @@ static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, return true; } +static unsigned int mte_visibility(const struct kvm_vcpu *vcpu, + const struct sys_reg_desc *rd) +{ + if (kvm_has_mte(vcpu->kvm)) + return 0; + + return REG_HIDDEN; +} + +#define MTE_REG(name) { \ + SYS_DESC(SYS_##name), \ + .access = undef_access, \ + .reset = reset_unknown, \ + .reg = name, \ + .visibility = mte_visibility, \ +} + /* sys_reg_desc initialiser for known cpufeature ID registers */ #define ID_SANITISED(name) { \ SYS_DESC(SYS_##name), \ @@ -1470,8 +1494,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 }, { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 }, - { SYS_DESC(SYS_RGSR_EL1), undef_access }, - { SYS_DESC(SYS_GCR_EL1), undef_access }, + MTE_REG(RGSR_EL1), + MTE_REG(GCR_EL1), { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility }, { SYS_DESC(SYS_TRFCR_EL1), undef_access }, @@ -1498,8 +1522,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi }, { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi }, - { SYS_DESC(SYS_TFSR_EL1), undef_access }, - { SYS_DESC(SYS_TFSRE0_EL1), undef_access }, + MTE_REG(TFSR_EL1), + MTE_REG(TFSRE0_EL1), { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 }, { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 }, diff --git a/arch/arm64/kvm/vgic/vgic-init.c b/arch/arm64/kvm/vgic/vgic-init.c index 58cbda00e56d..340c51d87677 100644 --- a/arch/arm64/kvm/vgic/vgic-init.c +++ b/arch/arm64/kvm/vgic/vgic-init.c @@ -482,6 +482,16 @@ static irqreturn_t vgic_maintenance_handler(int irq, void *data) return IRQ_HANDLED; } +static struct gic_kvm_info *gic_kvm_info; + +void __init vgic_set_kvm_info(const struct gic_kvm_info *info) +{ + BUG_ON(gic_kvm_info != NULL); + gic_kvm_info = kmalloc(sizeof(*info), GFP_KERNEL); + if (gic_kvm_info) + *gic_kvm_info = *info; +} + /** * kvm_vgic_init_cpu_hardware - initialize the GIC VE hardware * @@ -509,18 +519,29 @@ void kvm_vgic_init_cpu_hardware(void) */ int kvm_vgic_hyp_init(void) { - const struct gic_kvm_info *gic_kvm_info; + bool has_mask; int ret; - gic_kvm_info = gic_get_kvm_info(); if (!gic_kvm_info) return -ENODEV; - if (!gic_kvm_info->maint_irq) { + has_mask = !gic_kvm_info->no_maint_irq_mask; + + if (has_mask && !gic_kvm_info->maint_irq) { kvm_err("No vgic maintenance irq\n"); return -ENXIO; } + /* + * If we get one of these oddball non-GICs, taint the kernel, + * as we have no idea of how they *really* behave. + */ + if (gic_kvm_info->no_hw_deactivation) { + kvm_info("Non-architectural vgic, tainting kernel\n"); + add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK); + kvm_vgic_global_state.no_hw_deactivation = true; + } + switch (gic_kvm_info->type) { case GIC_V2: ret = vgic_v2_probe(gic_kvm_info); @@ -536,10 +557,17 @@ int kvm_vgic_hyp_init(void) ret = -ENODEV; } + kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq; + + kfree(gic_kvm_info); + gic_kvm_info = NULL; + if (ret) return ret; - kvm_vgic_global_state.maint_irq = gic_kvm_info->maint_irq; + if (!has_mask) + return 0; + ret = request_percpu_irq(kvm_vgic_global_state.maint_irq, vgic_maintenance_handler, "vgic", kvm_get_running_vcpus()); diff --git a/arch/arm64/kvm/vgic/vgic-v2.c b/arch/arm64/kvm/vgic/vgic-v2.c index 11934c2af2f4..2c580204f1dc 100644 --- a/arch/arm64/kvm/vgic/vgic-v2.c +++ b/arch/arm64/kvm/vgic/vgic-v2.c @@ -108,11 +108,22 @@ void vgic_v2_fold_lr_state(struct kvm_vcpu *vcpu) * If this causes us to lower the level, we have to also clear * the physical active state, since we will otherwise never be * told when the interrupt becomes asserted again. + * + * Another case is when the interrupt requires a helping hand + * on deactivation (no HW deactivation, for example). */ - if (vgic_irq_is_mapped_level(irq) && (val & GICH_LR_PENDING_BIT)) { - irq->line_level = vgic_get_phys_line_level(irq); + if (vgic_irq_is_mapped_level(irq)) { + bool resample = false; - if (!irq->line_level) + if (val & GICH_LR_PENDING_BIT) { + irq->line_level = vgic_get_phys_line_level(irq); + resample = !irq->line_level; + } else if (vgic_irq_needs_resampling(irq) && + !(irq->active || irq->pending_latch)) { + resample = true; + } + + if (resample) vgic_irq_set_phys_active(irq, false); } @@ -152,7 +163,7 @@ void vgic_v2_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr) if (irq->group) val |= GICH_LR_GROUP1; - if (irq->hw) { + if (irq->hw && !vgic_irq_needs_resampling(irq)) { val |= GICH_LR_HW; val |= irq->hwintid << GICH_LR_PHYSID_CPUID_SHIFT; /* diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 41ecf219c333..66004f61cd83 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -101,11 +101,22 @@ void vgic_v3_fold_lr_state(struct kvm_vcpu *vcpu) * If this causes us to lower the level, we have to also clear * the physical active state, since we will otherwise never be * told when the interrupt becomes asserted again. + * + * Another case is when the interrupt requires a helping hand + * on deactivation (no HW deactivation, for example). */ - if (vgic_irq_is_mapped_level(irq) && (val & ICH_LR_PENDING_BIT)) { - irq->line_level = vgic_get_phys_line_level(irq); + if (vgic_irq_is_mapped_level(irq)) { + bool resample = false; - if (!irq->line_level) + if (val & ICH_LR_PENDING_BIT) { + irq->line_level = vgic_get_phys_line_level(irq); + resample = !irq->line_level; + } else if (vgic_irq_needs_resampling(irq) && + !(irq->active || irq->pending_latch)) { + resample = true; + } + + if (resample) vgic_irq_set_phys_active(irq, false); } @@ -136,7 +147,7 @@ void vgic_v3_populate_lr(struct kvm_vcpu *vcpu, struct vgic_irq *irq, int lr) } } - if (irq->hw) { + if (irq->hw && !vgic_irq_needs_resampling(irq)) { val |= ICH_LR_HW; val |= ((u64)irq->hwintid) << ICH_LR_PHYS_ID_SHIFT; /* diff --git a/arch/arm64/kvm/vgic/vgic.c b/arch/arm64/kvm/vgic/vgic.c index 15b666200f0b..111bff47e471 100644 --- a/arch/arm64/kvm/vgic/vgic.c +++ b/arch/arm64/kvm/vgic/vgic.c @@ -182,8 +182,8 @@ bool vgic_get_phys_line_level(struct vgic_irq *irq) BUG_ON(!irq->hw); - if (irq->get_input_level) - return irq->get_input_level(irq->intid); + if (irq->ops && irq->ops->get_input_level) + return irq->ops->get_input_level(irq->intid); WARN_ON(irq_get_irqchip_state(irq->host_irq, IRQCHIP_STATE_PENDING, @@ -480,7 +480,7 @@ int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid, /* @irq->irq_lock must be held */ static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq, unsigned int host_irq, - bool (*get_input_level)(int vindid)) + struct irq_ops *ops) { struct irq_desc *desc; struct irq_data *data; @@ -500,7 +500,7 @@ static int kvm_vgic_map_irq(struct kvm_vcpu *vcpu, struct vgic_irq *irq, irq->hw = true; irq->host_irq = host_irq; irq->hwintid = data->hwirq; - irq->get_input_level = get_input_level; + irq->ops = ops; return 0; } @@ -509,11 +509,11 @@ static inline void kvm_vgic_unmap_irq(struct vgic_irq *irq) { irq->hw = false; irq->hwintid = 0; - irq->get_input_level = NULL; + irq->ops = NULL; } int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq, - u32 vintid, bool (*get_input_level)(int vindid)) + u32 vintid, struct irq_ops *ops) { struct vgic_irq *irq = vgic_get_irq(vcpu->kvm, vcpu, vintid); unsigned long flags; @@ -522,7 +522,7 @@ int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq, BUG_ON(!irq); raw_spin_lock_irqsave(&irq->irq_lock, flags); - ret = kvm_vgic_map_irq(vcpu, irq, host_irq, get_input_level); + ret = kvm_vgic_map_irq(vcpu, irq, host_irq, ops); raw_spin_unlock_irqrestore(&irq->irq_lock, flags); vgic_put_irq(vcpu->kvm, irq); diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile index d31e1169d9b8..6dd56a49790a 100644 --- a/arch/arm64/lib/Makefile +++ b/arch/arm64/lib/Makefile @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 lib-y := clear_user.o delay.o copy_from_user.o \ copy_to_user.o copy_in_user.o copy_page.o \ - clear_page.o csum.o memchr.o memcpy.o memmove.o \ + clear_page.o csum.o insn.o memchr.o memcpy.o \ memset.o memcmp.o strcmp.o strncmp.o strlen.o \ strnlen.o strchr.o strrchr.o tishift.o @@ -18,3 +18,5 @@ obj-$(CONFIG_CRC32) += crc32.o obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o obj-$(CONFIG_ARM64_MTE) += mte.o + +obj-$(CONFIG_KASAN_SW_TAGS) += kasan_sw_tags.o diff --git a/arch/arm64/lib/clear_user.S b/arch/arm64/lib/clear_user.S index af9afcbec92c..a7efb2ad2a1c 100644 --- a/arch/arm64/lib/clear_user.S +++ b/arch/arm64/lib/clear_user.S @@ -1,12 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Based on arch/arm/lib/clear_user.S - * - * Copyright (C) 2012 ARM Ltd. + * Copyright (C) 2021 Arm Ltd. */ -#include -#include +#include #include .text @@ -19,25 +16,33 @@ * * Alignment fixed up by hardware. */ + + .p2align 4 + // Alignment is for the loop, but since the prologue (including BTI) + // is also 16 bytes we can keep any padding outside the function SYM_FUNC_START(__arch_clear_user) - mov x2, x1 // save the size for fixup return + add x2, x0, x1 subs x1, x1, #8 b.mi 2f 1: -user_ldst 9f, sttr, xzr, x0, 8 +USER(9f, sttr xzr, [x0]) + add x0, x0, #8 subs x1, x1, #8 - b.pl 1b -2: adds x1, x1, #4 - b.mi 3f -user_ldst 9f, sttr, wzr, x0, 4 - sub x1, x1, #4 -3: adds x1, x1, #2 - b.mi 4f -user_ldst 9f, sttrh, wzr, x0, 2 - sub x1, x1, #2 -4: adds x1, x1, #1 - b.mi 5f -user_ldst 9f, sttrb, wzr, x0, 0 + b.hi 1b +USER(9f, sttr xzr, [x2, #-8]) + mov x0, #0 + ret + +2: tbz x1, #2, 3f +USER(9f, sttr wzr, [x0]) +USER(8f, sttr wzr, [x2, #-4]) + mov x0, #0 + ret + +3: tbz x1, #1, 4f +USER(9f, sttrh wzr, [x0]) +4: tbz x1, #0, 5f +USER(7f, sttrb wzr, [x2, #-1]) 5: mov x0, #0 ret SYM_FUNC_END(__arch_clear_user) @@ -45,6 +50,8 @@ EXPORT_SYMBOL(__arch_clear_user) .section .fixup,"ax" .align 2 -9: mov x0, x2 // return the original size +7: sub x0, x2, #5 // Adjust for faulting on the final byte... +8: add x0, x0, #4 // ...or the second word of the 4-7 byte case +9: sub x0, x2, x0 ret .previous diff --git a/arch/arm64/kernel/insn.c b/arch/arm64/lib/insn.c similarity index 86% rename from arch/arm64/kernel/insn.c rename to arch/arm64/lib/insn.c index 6c0de2f60ea9..b506a4b1e38c 100644 --- a/arch/arm64/kernel/insn.c +++ b/arch/arm64/lib/insn.c @@ -7,21 +7,14 @@ */ #include #include -#include -#include -#include -#include -#include -#include +#include +#include #include -#include -#include #include -#include +#include #include #include -#include #define AARCH64_INSN_SF_BIT BIT(31) #define AARCH64_INSN_N_BIT BIT(22) @@ -30,7 +23,7 @@ static const int aarch64_insn_encoding_class[] = { AARCH64_INSN_CLS_UNKNOWN, AARCH64_INSN_CLS_UNKNOWN, - AARCH64_INSN_CLS_UNKNOWN, + AARCH64_INSN_CLS_SVE, AARCH64_INSN_CLS_UNKNOWN, AARCH64_INSN_CLS_LDST, AARCH64_INSN_CLS_DP_REG, @@ -83,81 +76,6 @@ bool aarch64_insn_is_branch_imm(u32 insn) aarch64_insn_is_bcond(insn)); } -static DEFINE_RAW_SPINLOCK(patch_lock); - -static bool is_exit_text(unsigned long addr) -{ - /* discarded with init text/data */ - return system_state < SYSTEM_RUNNING && - addr >= (unsigned long)__exittext_begin && - addr < (unsigned long)__exittext_end; -} - -static bool is_image_text(unsigned long addr) -{ - return core_kernel_text(addr) || is_exit_text(addr); -} - -static void __kprobes *patch_map(void *addr, int fixmap) -{ - unsigned long uintaddr = (uintptr_t) addr; - bool image = is_image_text(uintaddr); - struct page *page; - - if (image) - page = phys_to_page(__pa_symbol(addr)); - else if (IS_ENABLED(CONFIG_STRICT_MODULE_RWX)) - page = vmalloc_to_page(addr); - else - return addr; - - BUG_ON(!page); - return (void *)set_fixmap_offset(fixmap, page_to_phys(page) + - (uintaddr & ~PAGE_MASK)); -} - -static void __kprobes patch_unmap(int fixmap) -{ - clear_fixmap(fixmap); -} -/* - * In ARMv8-A, A64 instructions have a fixed length of 32 bits and are always - * little-endian. - */ -int __kprobes aarch64_insn_read(void *addr, u32 *insnp) -{ - int ret; - __le32 val; - - ret = copy_from_kernel_nofault(&val, addr, AARCH64_INSN_SIZE); - if (!ret) - *insnp = le32_to_cpu(val); - - return ret; -} - -static int __kprobes __aarch64_insn_write(void *addr, __le32 insn) -{ - void *waddr = addr; - unsigned long flags = 0; - int ret; - - raw_spin_lock_irqsave(&patch_lock, flags); - waddr = patch_map(addr, FIX_TEXT_POKE0); - - ret = copy_to_kernel_nofault(waddr, &insn, AARCH64_INSN_SIZE); - - patch_unmap(FIX_TEXT_POKE0); - raw_spin_unlock_irqrestore(&patch_lock, flags); - - return ret; -} - -int __kprobes aarch64_insn_write(void *addr, u32 insn) -{ - return __aarch64_insn_write(addr, cpu_to_le32(insn)); -} - bool __kprobes aarch64_insn_uses_literal(u32 insn) { /* ldr/ldrsw (literal), prfm */ @@ -187,67 +105,6 @@ bool __kprobes aarch64_insn_is_branch(u32 insn) aarch64_insn_is_bcond(insn); } -int __kprobes aarch64_insn_patch_text_nosync(void *addr, u32 insn) -{ - u32 *tp = addr; - int ret; - - /* A64 instructions must be word aligned */ - if ((uintptr_t)tp & 0x3) - return -EINVAL; - - ret = aarch64_insn_write(tp, insn); - if (ret == 0) - __flush_icache_range((uintptr_t)tp, - (uintptr_t)tp + AARCH64_INSN_SIZE); - - return ret; -} - -struct aarch64_insn_patch { - void **text_addrs; - u32 *new_insns; - int insn_cnt; - atomic_t cpu_count; -}; - -static int __kprobes aarch64_insn_patch_text_cb(void *arg) -{ - int i, ret = 0; - struct aarch64_insn_patch *pp = arg; - - /* The first CPU becomes master */ - if (atomic_inc_return(&pp->cpu_count) == 1) { - for (i = 0; ret == 0 && i < pp->insn_cnt; i++) - ret = aarch64_insn_patch_text_nosync(pp->text_addrs[i], - pp->new_insns[i]); - /* Notify other processors with an additional increment. */ - atomic_inc(&pp->cpu_count); - } else { - while (atomic_read(&pp->cpu_count) <= num_online_cpus()) - cpu_relax(); - isb(); - } - - return ret; -} - -int __kprobes aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt) -{ - struct aarch64_insn_patch patch = { - .text_addrs = addrs, - .new_insns = insns, - .insn_cnt = cnt, - .cpu_count = ATOMIC_INIT(0), - }; - - if (cnt <= 0) - return -EINVAL; - - return stop_machine_cpuslocked(aarch64_insn_patch_text_cb, &patch, - cpu_online_mask); -} - static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type, u32 *maskp, int *shiftp) { @@ -1432,104 +1289,6 @@ u32 aarch32_insn_mcr_extract_crm(u32 insn) return insn & CRM_MASK; } -static bool __kprobes __check_eq(unsigned long pstate) -{ - return (pstate & PSR_Z_BIT) != 0; -} - -static bool __kprobes __check_ne(unsigned long pstate) -{ - return (pstate & PSR_Z_BIT) == 0; -} - -static bool __kprobes __check_cs(unsigned long pstate) -{ - return (pstate & PSR_C_BIT) != 0; -} - -static bool __kprobes __check_cc(unsigned long pstate) -{ - return (pstate & PSR_C_BIT) == 0; -} - -static bool __kprobes __check_mi(unsigned long pstate) -{ - return (pstate & PSR_N_BIT) != 0; -} - -static bool __kprobes __check_pl(unsigned long pstate) -{ - return (pstate & PSR_N_BIT) == 0; -} - -static bool __kprobes __check_vs(unsigned long pstate) -{ - return (pstate & PSR_V_BIT) != 0; -} - -static bool __kprobes __check_vc(unsigned long pstate) -{ - return (pstate & PSR_V_BIT) == 0; -} - -static bool __kprobes __check_hi(unsigned long pstate) -{ - pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ - return (pstate & PSR_C_BIT) != 0; -} - -static bool __kprobes __check_ls(unsigned long pstate) -{ - pstate &= ~(pstate >> 1); /* PSR_C_BIT &= ~PSR_Z_BIT */ - return (pstate & PSR_C_BIT) == 0; -} - -static bool __kprobes __check_ge(unsigned long pstate) -{ - pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - return (pstate & PSR_N_BIT) == 0; -} - -static bool __kprobes __check_lt(unsigned long pstate) -{ - pstate ^= (pstate << 3); /* PSR_N_BIT ^= PSR_V_BIT */ - return (pstate & PSR_N_BIT) != 0; -} - -static bool __kprobes __check_gt(unsigned long pstate) -{ - /*PSR_N_BIT ^= PSR_V_BIT */ - unsigned long temp = pstate ^ (pstate << 3); - - temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ - return (temp & PSR_N_BIT) == 0; -} - -static bool __kprobes __check_le(unsigned long pstate) -{ - /*PSR_N_BIT ^= PSR_V_BIT */ - unsigned long temp = pstate ^ (pstate << 3); - - temp |= (pstate << 1); /*PSR_N_BIT |= PSR_Z_BIT */ - return (temp & PSR_N_BIT) != 0; -} - -static bool __kprobes __check_al(unsigned long pstate) -{ - return true; -} - -/* - * Note that the ARMv8 ARM calls condition code 0b1111 "nv", but states that - * it behaves identically to 0b1110 ("al"). - */ -pstate_check_t * const aarch32_opcode_cond_checks[16] = { - __check_eq, __check_ne, __check_cs, __check_cc, - __check_mi, __check_pl, __check_vs, __check_vc, - __check_hi, __check_ls, __check_ge, __check_lt, - __check_gt, __check_le, __check_al, __check_al -}; - static bool range_of_ones(u64 val) { /* Doesn't handle full ones or full zeroes */ diff --git a/arch/arm64/lib/kasan_sw_tags.S b/arch/arm64/lib/kasan_sw_tags.S new file mode 100644 index 000000000000..5b04464c045e --- /dev/null +++ b/arch/arm64/lib/kasan_sw_tags.S @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2020 Google LLC + */ + +#include +#include + +/* + * Report a tag mismatch detected by tag-based KASAN. + * + * A compiler-generated thunk calls this with a non-AAPCS calling + * convention. Upon entry to this function, registers are as follows: + * + * x0: fault address (see below for restore) + * x1: fault description (see below for restore) + * x2 to x15: callee-saved + * x16 to x17: safe to clobber + * x18 to x30: callee-saved + * sp: pre-decremented by 256 bytes (see below for restore) + * + * The caller has decremented the SP by 256 bytes, and created a + * structure on the stack as follows: + * + * sp + 0..15: x0 and x1 to be restored + * sp + 16..231: free for use + * sp + 232..247: x29 and x30 (same as in GPRs) + * sp + 248..255: free for use + * + * Note that this is not a struct pt_regs. + * + * To call a regular AAPCS function we must save x2 to x15 (which we can + * store in the gaps), and create a frame record (for which we can use + * x29 and x30 spilled by the caller as those match the GPRs). + * + * The caller expects x0 and x1 to be restored from the structure, and + * for the structure to be removed from the stack (i.e. the SP must be + * incremented by 256 prior to return). + */ +SYM_CODE_START(__hwasan_tag_mismatch) +#ifdef BTI_C + BTI_C +#endif + add x29, sp, #232 + stp x2, x3, [sp, #8 * 2] + stp x4, x5, [sp, #8 * 4] + stp x6, x7, [sp, #8 * 6] + stp x8, x9, [sp, #8 * 8] + stp x10, x11, [sp, #8 * 10] + stp x12, x13, [sp, #8 * 12] + stp x14, x15, [sp, #8 * 14] +#ifndef CONFIG_SHADOW_CALL_STACK + str x18, [sp, #8 * 18] +#endif + + mov x2, x30 + bl kasan_tag_mismatch + + ldp x0, x1, [sp] + ldp x2, x3, [sp, #8 * 2] + ldp x4, x5, [sp, #8 * 4] + ldp x6, x7, [sp, #8 * 6] + ldp x8, x9, [sp, #8 * 8] + ldp x10, x11, [sp, #8 * 10] + ldp x12, x13, [sp, #8 * 12] + ldp x14, x15, [sp, #8 * 14] +#ifndef CONFIG_SHADOW_CALL_STACK + ldr x18, [sp, #8 * 18] +#endif + ldp x29, x30, [sp, #8 * 29] + + /* remove the structure from the stack */ + add sp, sp, #256 + ret +SYM_CODE_END(__hwasan_tag_mismatch) +EXPORT_SYMBOL(__hwasan_tag_mismatch) diff --git a/arch/arm64/lib/memchr.S b/arch/arm64/lib/memchr.S index edf6b970a277..7c2276fdab54 100644 --- a/arch/arm64/lib/memchr.S +++ b/arch/arm64/lib/memchr.S @@ -1,9 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Based on arch/arm/lib/memchr.S - * - * Copyright (C) 1995-2000 Russell King - * Copyright (C) 2013 ARM Ltd. + * Copyright (C) 2021 Arm Ltd. */ #include @@ -19,16 +16,60 @@ * Returns: * x0 - address of first occurrence of 'c' or 0 */ + +#define L(label) .L ## label + +#define REP8_01 0x0101010101010101 +#define REP8_7f 0x7f7f7f7f7f7f7f7f + +#define srcin x0 +#define chrin w1 +#define cntin x2 + +#define result x0 + +#define wordcnt x3 +#define rep01 x4 +#define repchr x5 +#define cur_word x6 +#define cur_byte w6 +#define tmp x7 +#define tmp2 x8 + + .p2align 4 + nop SYM_FUNC_START_WEAK_PI(memchr) - and w1, w1, #0xff -1: subs x2, x2, #1 - b.mi 2f - ldrb w3, [x0], #1 - cmp w3, w1 - b.ne 1b - sub x0, x0, #1 + and chrin, chrin, #0xff + lsr wordcnt, cntin, #3 + cbz wordcnt, L(byte_loop) + mov rep01, #REP8_01 + mul repchr, x1, rep01 + and cntin, cntin, #7 +L(word_loop): + ldr cur_word, [srcin], #8 + sub wordcnt, wordcnt, #1 + eor cur_word, cur_word, repchr + sub tmp, cur_word, rep01 + orr tmp2, cur_word, #REP8_7f + bics tmp, tmp, tmp2 + b.ne L(found_word) + cbnz wordcnt, L(word_loop) +L(byte_loop): + cbz cntin, L(not_found) + ldrb cur_byte, [srcin], #1 + sub cntin, cntin, #1 + cmp cur_byte, chrin + b.ne L(byte_loop) + sub srcin, srcin, #1 ret -2: mov x0, #0 +L(found_word): +CPU_LE( rev tmp, tmp) + clz tmp, tmp + sub tmp, tmp, #64 + add result, srcin, tmp, asr #3 + ret +L(not_found): + mov result, #0 ret SYM_FUNC_END_PI(memchr) EXPORT_SYMBOL_NOKASAN(memchr) diff --git a/arch/arm64/lib/memcmp.S b/arch/arm64/lib/memcmp.S index c0671e793ea9..7d956384222f 100644 --- a/arch/arm64/lib/memcmp.S +++ b/arch/arm64/lib/memcmp.S @@ -1,247 +1,139 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2013-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/memcmp.S */ #include #include -/* -* compare memory areas(when two memory areas' offset are different, -* alignment handled by the hardware) -* -* Parameters: -* x0 - const memory area 1 pointer -* x1 - const memory area 2 pointer -* x2 - the maximal compare byte length -* Returns: -* x0 - a compare result, maybe less than, equal to, or greater than ZERO -*/ +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. + */ + +#define L(label) .L ## label /* Parameters and result. */ -src1 .req x0 -src2 .req x1 -limit .req x2 -result .req x0 +#define src1 x0 +#define src2 x1 +#define limit x2 +#define result w0 /* Internal variables. */ -data1 .req x3 -data1w .req w3 -data2 .req x4 -data2w .req w4 -has_nul .req x5 -diff .req x6 -endloop .req x7 -tmp1 .req x8 -tmp2 .req x9 -tmp3 .req x10 -pos .req x11 -limit_wd .req x12 -mask .req x13 +#define data1 x3 +#define data1w w3 +#define data1h x4 +#define data2 x5 +#define data2w w5 +#define data2h x6 +#define tmp1 x7 +#define tmp2 x8 SYM_FUNC_START_WEAK_PI(memcmp) - cbz limit, .Lret0 - eor tmp1, src1, src2 - tst tmp1, #7 - b.ne .Lmisaligned8 - ands tmp1, src1, #7 - b.ne .Lmutual_align - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ - lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ - /* - * The input source addresses are at alignment boundary. - * Directly compare eight bytes each time. - */ -.Lloop_aligned: - ldr data1, [src1], #8 - ldr data2, [src2], #8 -.Lstart_realigned: - subs limit_wd, limit_wd, #1 - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, cs /* Last Dword or differences. */ - cbz endloop, .Lloop_aligned + subs limit, limit, 8 + b.lo L(less8) - /* Not reached the limit, must have found a diff. */ - tbz limit_wd, #63, .Lnot_limit + ldr data1, [src1], 8 + ldr data2, [src2], 8 + cmp data1, data2 + b.ne L(return) - /* Limit % 8 == 0 => the diff is in the last 8 bytes. */ - ands limit, limit, #7 - b.eq .Lnot_limit - /* - * The remained bytes less than 8. It is needed to extract valid data - * from last eight bytes of the intended memory range. - */ - lsl limit, limit, #3 /* bytes-> bits. */ - mov mask, #~0 -CPU_BE( lsr mask, mask, limit ) -CPU_LE( lsl mask, mask, limit ) - bic data1, data1, mask - bic data2, data2, mask + subs limit, limit, 8 + b.gt L(more16) - orr diff, diff, mask - b .Lnot_limit + ldr data1, [src1, limit] + ldr data2, [src2, limit] + b L(return) -.Lmutual_align: - /* - * Sources are mutually aligned, but are not currently at an - * alignment boundary. Round down the addresses and then mask off - * the bytes that precede the start point. - */ - bic src1, src1, #7 - bic src2, src2, #7 - ldr data1, [src1], #8 - ldr data2, [src2], #8 - /* - * We can not add limit with alignment offset(tmp1) here. Since the - * addition probably make the limit overflown. - */ - sub limit_wd, limit, #1/*limit != 0, so no underflow.*/ - and tmp3, limit_wd, #7 - lsr limit_wd, limit_wd, #3 - add tmp3, tmp3, tmp1 - add limit_wd, limit_wd, tmp3, lsr #3 - add limit, limit, tmp1/* Adjust the limit for the extra. */ +L(more16): + ldr data1, [src1], 8 + ldr data2, [src2], 8 + cmp data1, data2 + bne L(return) - lsl tmp1, tmp1, #3/* Bytes beyond alignment -> bits.*/ - neg tmp1, tmp1/* Bits to alignment -64. */ - mov tmp2, #~0 - /*mask off the non-intended bytes before the start address.*/ -CPU_BE( lsl tmp2, tmp2, tmp1 )/*Big-endian.Early bytes are at MSB*/ - /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp1 ) + /* Jump directly to comparing the last 16 bytes for 32 byte (or less) + strings. */ + subs limit, limit, 16 + b.ls L(last_bytes) - orr data1, data1, tmp2 - orr data2, data2, tmp2 - b .Lstart_realigned + /* We overlap loads between 0-32 bytes at either side of SRC1 when we + try to align, so limit it only to strings larger than 128 bytes. */ + cmp limit, 96 + b.ls L(loop16) - /*src1 and src2 have different alignment offset.*/ -.Lmisaligned8: - cmp limit, #8 - b.lo .Ltiny8proc /*limit < 8: compare byte by byte*/ + /* Align src1 and adjust src2 with bytes not yet done. */ + and tmp1, src1, 15 + add limit, limit, tmp1 + sub src1, src1, tmp1 + sub src2, src2, tmp1 - and tmp1, src1, #7 - neg tmp1, tmp1 - add tmp1, tmp1, #8/*valid length in the first 8 bytes of src1*/ - and tmp2, src2, #7 - neg tmp2, tmp2 - add tmp2, tmp2, #8/*valid length in the first 8 bytes of src2*/ - subs tmp3, tmp1, tmp2 - csel pos, tmp1, tmp2, hi /*Choose the maximum.*/ + /* Loop performing 16 bytes per iteration using aligned src1. + Limit is pre-decremented by 16 and must be larger than zero. + Exit if <= 16 bytes left to do or if the data is not equal. */ + .p2align 4 +L(loop16): + ldp data1, data1h, [src1], 16 + ldp data2, data2h, [src2], 16 + subs limit, limit, 16 + ccmp data1, data2, 0, hi + ccmp data1h, data2h, 0, eq + b.eq L(loop16) - sub limit, limit, pos - /*compare the proceeding bytes in the first 8 byte segment.*/ -.Ltinycmp: - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - subs pos, pos, #1 - ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */ - b.eq .Ltinycmp - cbnz pos, 1f /*diff occurred before the last byte.*/ + cmp data1, data2 + bne L(return) + mov data1, data1h + mov data2, data2h + cmp data1, data2 + bne L(return) + + /* Compare last 1-16 bytes using unaligned access. */ +L(last_bytes): + add src1, src1, limit + add src2, src2, limit + ldp data1, data1h, [src1] + ldp data2, data2h, [src2] + cmp data1, data2 + bne L(return) + mov data1, data1h + mov data2, data2h + cmp data1, data2 + + /* Compare data bytes and set return value to 0, -1 or 1. */ +L(return): +#ifndef __AARCH64EB__ + rev data1, data1 + rev data2, data2 +#endif + cmp data1, data2 +L(ret_eq): + cset result, ne + cneg result, result, lo + ret + + .p2align 4 + /* Compare up to 8 bytes. Limit is [-8..-1]. */ +L(less8): + adds limit, limit, 4 + b.lo L(less4) + ldr data1w, [src1], 4 + ldr data2w, [src2], 4 cmp data1w, data2w - b.eq .Lstart_align -1: - sub result, data1, data2 + b.ne L(return) + sub limit, limit, 4 +L(less4): + adds limit, limit, 4 + beq L(ret_eq) +L(byte_loop): + ldrb data1w, [src1], 1 + ldrb data2w, [src2], 1 + subs limit, limit, 1 + ccmp data1w, data2w, 0, ne /* NZCV = 0b0000. */ + b.eq L(byte_loop) + sub result, data1w, data2w ret -.Lstart_align: - lsr limit_wd, limit, #3 - cbz limit_wd, .Lremain8 - - ands xzr, src1, #7 - b.eq .Lrecal_offset - /*process more leading bytes to make src1 aligned...*/ - add src1, src1, tmp3 /*backwards src1 to alignment boundary*/ - add src2, src2, tmp3 - sub limit, limit, tmp3 - lsr limit_wd, limit, #3 - cbz limit_wd, .Lremain8 - /*load 8 bytes from aligned SRC1..*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - - subs limit_wd, limit_wd, #1 - eor diff, data1, data2 /*Non-zero if differences found.*/ - csinv endloop, diff, xzr, ne - cbnz endloop, .Lunequal_proc - /*How far is the current SRC2 from the alignment boundary...*/ - and tmp3, tmp3, #7 - -.Lrecal_offset:/*src1 is aligned now..*/ - neg pos, tmp3 -.Lloopcmp_proc: - /* - * Divide the eight bytes into two parts. First,backwards the src2 - * to an alignment boundary,load eight bytes and compare from - * the SRC2 alignment boundary. If all 8 bytes are equal,then start - * the second part's comparison. Otherwise finish the comparison. - * This special handle can garantee all the accesses are in the - * thread/task space in avoid to overrange access. - */ - ldr data1, [src1,pos] - ldr data2, [src2,pos] - eor diff, data1, data2 /* Non-zero if differences found. */ - cbnz diff, .Lnot_limit - - /*The second part process*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - eor diff, data1, data2 /* Non-zero if differences found. */ - subs limit_wd, limit_wd, #1 - csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ - cbz endloop, .Lloopcmp_proc -.Lunequal_proc: - cbz diff, .Lremain8 - -/* There is difference occurred in the latest comparison. */ -.Lnot_limit: -/* -* For little endian,reverse the low significant equal bits into MSB,then -* following CLZ can find how many equal bits exist. -*/ -CPU_LE( rev diff, diff ) -CPU_LE( rev data1, data1 ) -CPU_LE( rev data2, data2 ) - - /* - * The MS-non-zero bit of DIFF marks either the first bit - * that is different, or the end of the significant data. - * Shifting left now will bring the critical information into the - * top bits. - */ - clz pos, diff - lsl data1, data1, pos - lsl data2, data2, pos - /* - * We need to zero-extend (char is unsigned) the value and then - * perform a signed subtraction. - */ - lsr data1, data1, #56 - sub result, data1, data2, lsr #56 - ret - -.Lremain8: - /* Limit % 8 == 0 =>. all data are equal.*/ - ands limit, limit, #7 - b.eq .Lret0 - -.Ltiny8proc: - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - subs limit, limit, #1 - - ccmp data1w, data2w, #0, ne /* NZCV = 0b0000. */ - b.eq .Ltiny8proc - sub result, data1, data2 - ret -.Lret0: - mov result, #0 - ret SYM_FUNC_END_PI(memcmp) EXPORT_SYMBOL_NOKASAN(memcmp) diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S index dc8d2a216a6e..b82fd64ee1e1 100644 --- a/arch/arm64/lib/memcpy.S +++ b/arch/arm64/lib/memcpy.S @@ -1,66 +1,252 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2012-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/memcpy.S */ #include #include -#include -/* - * Copy a buffer from src to dest (alignment handled by the hardware) +/* Assumptions: + * + * ARMv8-a, AArch64, unaligned accesses. * - * Parameters: - * x0 - dest - * x1 - src - * x2 - n - * Returns: - * x0 - dest */ - .macro ldrb1 reg, ptr, val - ldrb \reg, [\ptr], \val - .endm - .macro strb1 reg, ptr, val - strb \reg, [\ptr], \val - .endm +#define L(label) .L ## label - .macro ldrh1 reg, ptr, val - ldrh \reg, [\ptr], \val - .endm +#define dstin x0 +#define src x1 +#define count x2 +#define dst x3 +#define srcend x4 +#define dstend x5 +#define A_l x6 +#define A_lw w6 +#define A_h x7 +#define B_l x8 +#define B_lw w8 +#define B_h x9 +#define C_l x10 +#define C_lw w10 +#define C_h x11 +#define D_l x12 +#define D_h x13 +#define E_l x14 +#define E_h x15 +#define F_l x16 +#define F_h x17 +#define G_l count +#define G_h dst +#define H_l src +#define H_h srcend +#define tmp1 x14 - .macro strh1 reg, ptr, val - strh \reg, [\ptr], \val - .endm +/* This implementation handles overlaps and supports both memcpy and memmove + from a single entry point. It uses unaligned accesses and branchless + sequences to keep the code small, simple and improve performance. - .macro ldr1 reg, ptr, val - ldr \reg, [\ptr], \val - .endm + Copies are split into 3 main cases: small copies of up to 32 bytes, medium + copies of up to 128 bytes, and large copies. The overhead of the overlap + check is negligible since it is only required for large copies. - .macro str1 reg, ptr, val - str \reg, [\ptr], \val - .endm - - .macro ldp1 reg1, reg2, ptr, val - ldp \reg1, \reg2, [\ptr], \val - .endm - - .macro stp1 reg1, reg2, ptr, val - stp \reg1, \reg2, [\ptr], \val - .endm + Large copies use a software pipelined loop processing 64 bytes per iteration. + The destination pointer is 16-byte aligned to minimize unaligned accesses. + The loop tail is handled by always copying 64 bytes from the end. +*/ +SYM_FUNC_START_ALIAS(__memmove) +SYM_FUNC_START_WEAK_ALIAS_PI(memmove) SYM_FUNC_START_ALIAS(__memcpy) SYM_FUNC_START_WEAK_PI(memcpy) -#include "copy_template.S" + add srcend, src, count + add dstend, dstin, count + cmp count, 128 + b.hi L(copy_long) + cmp count, 32 + b.hi L(copy32_128) + + /* Small copies: 0..32 bytes. */ + cmp count, 16 + b.lo L(copy16) + ldp A_l, A_h, [src] + ldp D_l, D_h, [srcend, -16] + stp A_l, A_h, [dstin] + stp D_l, D_h, [dstend, -16] ret + + /* Copy 8-15 bytes. */ +L(copy16): + tbz count, 3, L(copy8) + ldr A_l, [src] + ldr A_h, [srcend, -8] + str A_l, [dstin] + str A_h, [dstend, -8] + ret + + .p2align 3 + /* Copy 4-7 bytes. */ +L(copy8): + tbz count, 2, L(copy4) + ldr A_lw, [src] + ldr B_lw, [srcend, -4] + str A_lw, [dstin] + str B_lw, [dstend, -4] + ret + + /* Copy 0..3 bytes using a branchless sequence. */ +L(copy4): + cbz count, L(copy0) + lsr tmp1, count, 1 + ldrb A_lw, [src] + ldrb C_lw, [srcend, -1] + ldrb B_lw, [src, tmp1] + strb A_lw, [dstin] + strb B_lw, [dstin, tmp1] + strb C_lw, [dstend, -1] +L(copy0): + ret + + .p2align 4 + /* Medium copies: 33..128 bytes. */ +L(copy32_128): + ldp A_l, A_h, [src] + ldp B_l, B_h, [src, 16] + ldp C_l, C_h, [srcend, -32] + ldp D_l, D_h, [srcend, -16] + cmp count, 64 + b.hi L(copy128) + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy 65..128 bytes. */ +L(copy128): + ldp E_l, E_h, [src, 32] + ldp F_l, F_h, [src, 48] + cmp count, 96 + b.ls L(copy96) + ldp G_l, G_h, [srcend, -64] + ldp H_l, H_h, [srcend, -48] + stp G_l, G_h, [dstend, -64] + stp H_l, H_h, [dstend, -48] +L(copy96): + stp A_l, A_h, [dstin] + stp B_l, B_h, [dstin, 16] + stp E_l, E_h, [dstin, 32] + stp F_l, F_h, [dstin, 48] + stp C_l, C_h, [dstend, -32] + stp D_l, D_h, [dstend, -16] + ret + + .p2align 4 + /* Copy more than 128 bytes. */ +L(copy_long): + /* Use backwards copy if there is an overlap. */ + sub tmp1, dstin, src + cbz tmp1, L(copy0) + cmp tmp1, count + b.lo L(copy_long_backwards) + + /* Copy 16 bytes and then align dst to 16-byte alignment. */ + + ldp D_l, D_h, [src] + and tmp1, dstin, 15 + bic dst, dstin, 15 + sub src, src, tmp1 + add count, count, tmp1 /* Count is now 16 too large. */ + ldp A_l, A_h, [src, 16] + stp D_l, D_h, [dstin] + ldp B_l, B_h, [src, 32] + ldp C_l, C_h, [src, 48] + ldp D_l, D_h, [src, 64]! + subs count, count, 128 + 16 /* Test and readjust count. */ + b.ls L(copy64_from_end) + +L(loop64): + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [src, 16] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [src, 32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [src, 48] + stp D_l, D_h, [dst, 64]! + ldp D_l, D_h, [src, 64]! + subs count, count, 64 + b.hi L(loop64) + + /* Write the last iteration and copy 64 bytes from the end. */ +L(copy64_from_end): + ldp E_l, E_h, [srcend, -64] + stp A_l, A_h, [dst, 16] + ldp A_l, A_h, [srcend, -48] + stp B_l, B_h, [dst, 32] + ldp B_l, B_h, [srcend, -32] + stp C_l, C_h, [dst, 48] + ldp C_l, C_h, [srcend, -16] + stp D_l, D_h, [dst, 64] + stp E_l, E_h, [dstend, -64] + stp A_l, A_h, [dstend, -48] + stp B_l, B_h, [dstend, -32] + stp C_l, C_h, [dstend, -16] + ret + + .p2align 4 + + /* Large backwards copy for overlapping copies. + Copy 16 bytes and then align dst to 16-byte alignment. */ +L(copy_long_backwards): + ldp D_l, D_h, [srcend, -16] + and tmp1, dstend, 15 + sub srcend, srcend, tmp1 + sub count, count, tmp1 + ldp A_l, A_h, [srcend, -16] + stp D_l, D_h, [dstend, -16] + ldp B_l, B_h, [srcend, -32] + ldp C_l, C_h, [srcend, -48] + ldp D_l, D_h, [srcend, -64]! + sub dstend, dstend, tmp1 + subs count, count, 128 + b.ls L(copy64_from_start) + +L(loop64_backwards): + stp A_l, A_h, [dstend, -16] + ldp A_l, A_h, [srcend, -16] + stp B_l, B_h, [dstend, -32] + ldp B_l, B_h, [srcend, -32] + stp C_l, C_h, [dstend, -48] + ldp C_l, C_h, [srcend, -48] + stp D_l, D_h, [dstend, -64]! + ldp D_l, D_h, [srcend, -64]! + subs count, count, 64 + b.hi L(loop64_backwards) + + /* Write the last iteration and copy 64 bytes from the start. */ +L(copy64_from_start): + ldp G_l, G_h, [src, 48] + stp A_l, A_h, [dstend, -16] + ldp A_l, A_h, [src, 32] + stp B_l, B_h, [dstend, -32] + ldp B_l, B_h, [src, 16] + stp C_l, C_h, [dstend, -48] + ldp C_l, C_h, [src] + stp D_l, D_h, [dstend, -64] + stp G_l, G_h, [dstin, 48] + stp A_l, A_h, [dstin, 32] + stp B_l, B_h, [dstin, 16] + stp C_l, C_h, [dstin] + ret + SYM_FUNC_END_PI(memcpy) EXPORT_SYMBOL(memcpy) SYM_FUNC_END_ALIAS(__memcpy) EXPORT_SYMBOL(__memcpy) +SYM_FUNC_END_ALIAS_PI(memmove) +EXPORT_SYMBOL(memmove) +SYM_FUNC_END_ALIAS(__memmove) +EXPORT_SYMBOL(__memmove) diff --git a/arch/arm64/lib/memmove.S b/arch/arm64/lib/memmove.S deleted file mode 100644 index 1035dce4bdaf..000000000000 --- a/arch/arm64/lib/memmove.S +++ /dev/null @@ -1,189 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. - * - * This code is based on glibc cortex strings work originally authored by Linaro - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ - */ - -#include -#include -#include - -/* - * Move a buffer from src to test (alignment handled by the hardware). - * If dest <= src, call memcpy, otherwise copy in reverse order. - * - * Parameters: - * x0 - dest - * x1 - src - * x2 - n - * Returns: - * x0 - dest - */ -dstin .req x0 -src .req x1 -count .req x2 -tmp1 .req x3 -tmp1w .req w3 -tmp2 .req x4 -tmp2w .req w4 -tmp3 .req x5 -tmp3w .req w5 -dst .req x6 - -A_l .req x7 -A_h .req x8 -B_l .req x9 -B_h .req x10 -C_l .req x11 -C_h .req x12 -D_l .req x13 -D_h .req x14 - -SYM_FUNC_START_ALIAS(__memmove) -SYM_FUNC_START_WEAK_PI(memmove) - cmp dstin, src - b.lo __memcpy - add tmp1, src, count - cmp dstin, tmp1 - b.hs __memcpy /* No overlap. */ - - add dst, dstin, count - add src, src, count - cmp count, #16 - b.lo .Ltail15 /*probably non-alignment accesses.*/ - - ands tmp2, src, #15 /* Bytes to reach alignment. */ - b.eq .LSrcAligned - sub count, count, tmp2 - /* - * process the aligned offset length to make the src aligned firstly. - * those extra instructions' cost is acceptable. It also make the - * coming accesses are based on aligned address. - */ - tbz tmp2, #0, 1f - ldrb tmp1w, [src, #-1]! - strb tmp1w, [dst, #-1]! -1: - tbz tmp2, #1, 2f - ldrh tmp1w, [src, #-2]! - strh tmp1w, [dst, #-2]! -2: - tbz tmp2, #2, 3f - ldr tmp1w, [src, #-4]! - str tmp1w, [dst, #-4]! -3: - tbz tmp2, #3, .LSrcAligned - ldr tmp1, [src, #-8]! - str tmp1, [dst, #-8]! - -.LSrcAligned: - cmp count, #64 - b.ge .Lcpy_over64 - - /* - * Deal with small copies quickly by dropping straight into the - * exit block. - */ -.Ltail63: - /* - * Copy up to 48 bytes of data. At this point we only need the - * bottom 6 bits of count to be accurate. - */ - ands tmp1, count, #0x30 - b.eq .Ltail15 - cmp tmp1w, #0x20 - b.eq 1f - b.lt 2f - ldp A_l, A_h, [src, #-16]! - stp A_l, A_h, [dst, #-16]! -1: - ldp A_l, A_h, [src, #-16]! - stp A_l, A_h, [dst, #-16]! -2: - ldp A_l, A_h, [src, #-16]! - stp A_l, A_h, [dst, #-16]! - -.Ltail15: - tbz count, #3, 1f - ldr tmp1, [src, #-8]! - str tmp1, [dst, #-8]! -1: - tbz count, #2, 2f - ldr tmp1w, [src, #-4]! - str tmp1w, [dst, #-4]! -2: - tbz count, #1, 3f - ldrh tmp1w, [src, #-2]! - strh tmp1w, [dst, #-2]! -3: - tbz count, #0, .Lexitfunc - ldrb tmp1w, [src, #-1] - strb tmp1w, [dst, #-1] - -.Lexitfunc: - ret - -.Lcpy_over64: - subs count, count, #128 - b.ge .Lcpy_body_large - /* - * Less than 128 bytes to copy, so handle 64 bytes here and then jump - * to the tail. - */ - ldp A_l, A_h, [src, #-16] - stp A_l, A_h, [dst, #-16] - ldp B_l, B_h, [src, #-32] - ldp C_l, C_h, [src, #-48] - stp B_l, B_h, [dst, #-32] - stp C_l, C_h, [dst, #-48] - ldp D_l, D_h, [src, #-64]! - stp D_l, D_h, [dst, #-64]! - - tst count, #0x3f - b.ne .Ltail63 - ret - - /* - * Critical loop. Start at a new cache line boundary. Assuming - * 64 bytes per line this ensures the entire loop is in one line. - */ - .p2align L1_CACHE_SHIFT -.Lcpy_body_large: - /* pre-load 64 bytes data. */ - ldp A_l, A_h, [src, #-16] - ldp B_l, B_h, [src, #-32] - ldp C_l, C_h, [src, #-48] - ldp D_l, D_h, [src, #-64]! -1: - /* - * interlace the load of next 64 bytes data block with store of the last - * loaded 64 bytes data. - */ - stp A_l, A_h, [dst, #-16] - ldp A_l, A_h, [src, #-16] - stp B_l, B_h, [dst, #-32] - ldp B_l, B_h, [src, #-32] - stp C_l, C_h, [dst, #-48] - ldp C_l, C_h, [src, #-48] - stp D_l, D_h, [dst, #-64]! - ldp D_l, D_h, [src, #-64]! - subs count, count, #64 - b.ge 1b - stp A_l, A_h, [dst, #-16] - stp B_l, B_h, [dst, #-32] - stp C_l, C_h, [dst, #-48] - stp D_l, D_h, [dst, #-64]! - - tst count, #0x3f - b.ne .Ltail63 - ret -SYM_FUNC_END_PI(memmove) -EXPORT_SYMBOL(memmove) -SYM_FUNC_END_ALIAS(__memmove) -EXPORT_SYMBOL(__memmove) diff --git a/arch/arm64/lib/mte.S b/arch/arm64/lib/mte.S index 351537c12f36..e83643b3995f 100644 --- a/arch/arm64/lib/mte.S +++ b/arch/arm64/lib/mte.S @@ -36,6 +36,26 @@ SYM_FUNC_START(mte_clear_page_tags) ret SYM_FUNC_END(mte_clear_page_tags) +/* + * Zero the page and tags at the same time + * + * Parameters: + * x0 - address to the beginning of the page + */ +SYM_FUNC_START(mte_zero_clear_page_tags) + mrs x1, dczid_el0 + and w1, w1, #0xf + mov x2, #4 + lsl x1, x2, x1 + and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag + +1: dc gzva, x0 + add x0, x0, x1 + tst x0, #(PAGE_SIZE - 1) + b.ne 1b + ret +SYM_FUNC_END(mte_zero_clear_page_tags) + /* * Copy the tags from the source page to the destination one * x0 - address of the destination page diff --git a/arch/arm64/lib/strcmp.S b/arch/arm64/lib/strcmp.S index 4e79566726c8..d7bee210a798 100644 --- a/arch/arm64/lib/strcmp.S +++ b/arch/arm64/lib/strcmp.S @@ -1,84 +1,123 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2012-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/afd6244a1f8d9229/string/aarch64/strcmp.S */ #include #include -/* - * compare two strings +/* Assumptions: * - * Parameters: - * x0 - const string 1 pointer - * x1 - const string 2 pointer - * Returns: - * x0 - an integer less than, equal to, or greater than zero - * if s1 is found, respectively, to be less than, to match, - * or be greater than s2. + * ARMv8-a, AArch64 */ +#define L(label) .L ## label + #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 /* Parameters and result. */ -src1 .req x0 -src2 .req x1 -result .req x0 +#define src1 x0 +#define src2 x1 +#define result x0 /* Internal variables. */ -data1 .req x2 -data1w .req w2 -data2 .req x3 -data2w .req w3 -has_nul .req x4 -diff .req x5 -syndrome .req x6 -tmp1 .req x7 -tmp2 .req x8 -tmp3 .req x9 -zeroones .req x10 -pos .req x11 +#define data1 x2 +#define data1w w2 +#define data2 x3 +#define data2w w3 +#define has_nul x4 +#define diff x5 +#define syndrome x6 +#define tmp1 x7 +#define tmp2 x8 +#define tmp3 x9 +#define zeroones x10 +#define pos x11 + /* Start of performance-critical section -- one 64B cache line. */ + .align 6 SYM_FUNC_START_WEAK_PI(strcmp) eor tmp1, src1, src2 mov zeroones, #REP8_01 tst tmp1, #7 - b.ne .Lmisaligned8 + b.ne L(misaligned8) ands tmp1, src1, #7 - b.ne .Lmutual_align - - /* - * NUL detection works on the principle that (X - 1) & (~X) & 0x80 - * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - * can be done in parallel across the entire word. - */ -.Lloop_aligned: + b.ne L(mutual_align) + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ +L(loop_aligned): ldr data1, [src1], #8 ldr data2, [src2], #8 -.Lstart_realigned: +L(start_realigned): sub tmp1, data1, zeroones orr tmp2, data1, #REP8_7f eor diff, data1, data2 /* Non-zero if differences found. */ bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ orr syndrome, diff, has_nul - cbz syndrome, .Lloop_aligned - b .Lcal_cmpresult + cbz syndrome, L(loop_aligned) + /* End of performance-critical section -- one 64B cache line. */ -.Lmutual_align: - /* - * Sources are mutually aligned, but are not currently at an - * alignment boundary. Round down the addresses and then mask off - * the bytes that preceed the start point. - */ +L(end): +#ifndef __AARCH64EB__ + rev syndrome, syndrome + rev data1, data1 + /* The MS-non-zero bit of the syndrome marks either the first bit + that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ + clz pos, syndrome + rev data2, data2 + lsl data1, data1, pos + lsl data2, data2, pos + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, #56 + sub result, data1, data2, lsr #56 + ret +#else + /* For big-endian we cannot use the trick with the syndrome value + as carry-propagation can corrupt the upper bits if the trailing + bytes in the string contain 0x01. */ + /* However, if there is no NUL byte in the dword, we can generate + the result directly. We can't just subtract the bytes as the + MSB might be significant. */ + cbnz has_nul, 1f + cmp data1, data2 + cset result, ne + cneg result, result, lo + ret +1: + /* Re-compute the NUL-byte detection, using a byte-reversed value. */ + rev tmp3, data1 + sub tmp1, tmp3, zeroones + orr tmp2, tmp3, #REP8_7f + bic has_nul, tmp1, tmp2 + rev has_nul, has_nul + orr syndrome, diff, has_nul + clz pos, syndrome + /* The MS-non-zero bit of the syndrome marks either the first bit + that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ + lsl data1, data1, pos + lsl data2, data2, pos + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, #56 + sub result, data1, data2, lsr #56 + ret +#endif + +L(mutual_align): + /* Sources are mutually aligned, but are not currently at an + alignment boundary. Round down the addresses and then mask off + the bytes that preceed the start point. */ bic src1, src1, #7 bic src2, src2, #7 lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ @@ -86,138 +125,52 @@ SYM_FUNC_START_WEAK_PI(strcmp) neg tmp1, tmp1 /* Bits to alignment -64. */ ldr data2, [src2], #8 mov tmp2, #~0 +#ifdef __AARCH64EB__ /* Big-endian. Early bytes are at MSB. */ -CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ + lsl tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ +#else /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ - + lsr tmp2, tmp2, tmp1 /* Shift (tmp1 & 63). */ +#endif orr data1, data1, tmp2 orr data2, data2, tmp2 - b .Lstart_realigned + b L(start_realigned) -.Lmisaligned8: - /* - * Get the align offset length to compare per byte first. - * After this process, one string's address will be aligned. - */ - and tmp1, src1, #7 - neg tmp1, tmp1 - add tmp1, tmp1, #8 - and tmp2, src2, #7 - neg tmp2, tmp2 - add tmp2, tmp2, #8 - subs tmp3, tmp1, tmp2 - csel pos, tmp1, tmp2, hi /*Choose the maximum. */ -.Ltinycmp: +L(misaligned8): + /* Align SRC1 to 8 bytes and then compare 8 bytes at a time, always + checking to make sure that we don't access beyond page boundary in + SRC2. */ + tst src1, #7 + b.eq L(loop_misaligned) +L(do_misaligned): ldrb data1w, [src1], #1 ldrb data2w, [src2], #1 - subs pos, pos, #1 - ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ - ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ - b.eq .Ltinycmp - cbnz pos, 1f /*find the null or unequal...*/ cmp data1w, #1 - ccmp data1w, data2w, #0, cs - b.eq .Lstart_align /*the last bytes are equal....*/ -1: + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ + b.ne L(done) + tst src1, #7 + b.ne L(do_misaligned) + +L(loop_misaligned): + /* Test if we are within the last dword of the end of a 4K page. If + yes then jump back to the misaligned loop to copy a byte at a time. */ + and tmp1, src2, #0xff8 + eor tmp1, tmp1, #0xff8 + cbz tmp1, L(do_misaligned) + ldr data1, [src1], #8 + ldr data2, [src2], #8 + + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + eor diff, data1, data2 /* Non-zero if differences found. */ + bic has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + orr syndrome, diff, has_nul + cbz syndrome, L(loop_misaligned) + b L(end) + +L(done): sub result, data1, data2 ret -.Lstart_align: - ands xzr, src1, #7 - b.eq .Lrecal_offset - /*process more leading bytes to make str1 aligned...*/ - add src1, src1, tmp3 - add src2, src2, tmp3 - /*load 8 bytes from aligned str1 and non-aligned str2..*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul, tmp1, tmp2 - eor diff, data1, data2 /* Non-zero if differences found. */ - orr syndrome, diff, has_nul - cbnz syndrome, .Lcal_cmpresult - /*How far is the current str2 from the alignment boundary...*/ - and tmp3, tmp3, #7 -.Lrecal_offset: - neg pos, tmp3 -.Lloopcmp_proc: - /* - * Divide the eight bytes into two parts. First,backwards the src2 - * to an alignment boundary,load eight bytes from the SRC2 alignment - * boundary,then compare with the relative bytes from SRC1. - * If all 8 bytes are equal,then start the second part's comparison. - * Otherwise finish the comparison. - * This special handle can garantee all the accesses are in the - * thread/task space in avoid to overrange access. - */ - ldr data1, [src1,pos] - ldr data2, [src2,pos] - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul, tmp1, tmp2 - eor diff, data1, data2 /* Non-zero if differences found. */ - orr syndrome, diff, has_nul - cbnz syndrome, .Lcal_cmpresult - - /*The second part process*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bic has_nul, tmp1, tmp2 - eor diff, data1, data2 /* Non-zero if differences found. */ - orr syndrome, diff, has_nul - cbz syndrome, .Lloopcmp_proc - -.Lcal_cmpresult: - /* - * reversed the byte-order as big-endian,then CLZ can find the most - * significant zero bits. - */ -CPU_LE( rev syndrome, syndrome ) -CPU_LE( rev data1, data1 ) -CPU_LE( rev data2, data2 ) - - /* - * For big-endian we cannot use the trick with the syndrome value - * as carry-propagation can corrupt the upper bits if the trailing - * bytes in the string contain 0x01. - * However, if there is no NUL byte in the dword, we can generate - * the result directly. We cannot just subtract the bytes as the - * MSB might be significant. - */ -CPU_BE( cbnz has_nul, 1f ) -CPU_BE( cmp data1, data2 ) -CPU_BE( cset result, ne ) -CPU_BE( cneg result, result, lo ) -CPU_BE( ret ) -CPU_BE( 1: ) - /*Re-compute the NUL-byte detection, using a byte-reversed value. */ -CPU_BE( rev tmp3, data1 ) -CPU_BE( sub tmp1, tmp3, zeroones ) -CPU_BE( orr tmp2, tmp3, #REP8_7f ) -CPU_BE( bic has_nul, tmp1, tmp2 ) -CPU_BE( rev has_nul, has_nul ) -CPU_BE( orr syndrome, diff, has_nul ) - - clz pos, syndrome - /* - * The MS-non-zero bit of the syndrome marks either the first bit - * that is different, or the top bit of the first zero byte. - * Shifting left now will bring the critical information into the - * top bits. - */ - lsl data1, data1, pos - lsl data2, data2, pos - /* - * But we need to zero-extend (char is unsigned) the value and then - * perform a signed 32-bit subtraction. - */ - lsr data1, data1, #56 - sub result, data1, data2, lsr #56 - ret SYM_FUNC_END_PI(strcmp) EXPORT_SYMBOL_NOKASAN(strcmp) diff --git a/arch/arm64/lib/strlen.S b/arch/arm64/lib/strlen.S index ee3ed882dd79..35fbdb7d6e1a 100644 --- a/arch/arm64/lib/strlen.S +++ b/arch/arm64/lib/strlen.S @@ -1,115 +1,203 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2013-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/98e4d6a5c13c8e54/string/aarch64/strlen.S */ #include #include -/* - * calculate the length of a string +/* Assumptions: * - * Parameters: - * x0 - const string pointer - * Returns: - * x0 - the return length of specific string + * ARMv8-a, AArch64, unaligned accesses, min page size 4k. */ +#define L(label) .L ## label + /* Arguments and results. */ -srcin .req x0 -len .req x0 +#define srcin x0 +#define len x0 /* Locals and temporaries. */ -src .req x1 -data1 .req x2 -data2 .req x3 -data2a .req x4 -has_nul1 .req x5 -has_nul2 .req x6 -tmp1 .req x7 -tmp2 .req x8 -tmp3 .req x9 -tmp4 .req x10 -zeroones .req x11 -pos .req x12 +#define src x1 +#define data1 x2 +#define data2 x3 +#define has_nul1 x4 +#define has_nul2 x5 +#define tmp1 x4 +#define tmp2 x5 +#define tmp3 x6 +#define tmp4 x7 +#define zeroones x8 + + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. A faster check + (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives + false hits for characters 129..255. */ #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 +#define MIN_PAGE_SIZE 4096 + + /* Since strings are short on average, we check the first 16 bytes + of the string for a NUL character. In order to do an unaligned ldp + safely we have to do a page cross check first. If there is a NUL + byte we calculate the length from the 2 8-byte words using + conditional select to reduce branch mispredictions (it is unlikely + strlen will be repeatedly called on strings with the same length). + + If the string is longer than 16 bytes, we align src so don't need + further page cross checks, and process 32 bytes per iteration + using the fast NUL check. If we encounter non-ASCII characters, + fallback to a second loop using the full NUL check. + + If the page cross check fails, we read 16 bytes from an aligned + address, remove any characters before the string, and continue + in the main loop using aligned loads. Since strings crossing a + page in the first 16 bytes are rare (probability of + 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized. + + AArch64 systems have a minimum page size of 4k. We don't bother + checking for larger page sizes - the cost of setting up the correct + page size is just not worth the extra gain from a small reduction in + the cases taking the slow path. Note that we only care about + whether the first fetch, which may be misaligned, crosses a page + boundary. */ + SYM_FUNC_START_WEAK_PI(strlen) - mov zeroones, #REP8_01 - bic src, srcin, #15 - ands tmp1, srcin, #15 - b.ne .Lmisaligned - /* - * NUL detection works on the principle that (X - 1) & (~X) & 0x80 - * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - * can be done in parallel across the entire word. - */ - /* - * The inner loop deals with two Dwords at a time. This has a - * slightly higher start-up cost, but we should win quite quickly, - * especially on cores with a high number of issue slots per - * cycle, as we get much better parallelism out of the operations. - */ -.Lloop: - ldp data1, data2, [src], #16 -.Lrealigned: + and tmp1, srcin, MIN_PAGE_SIZE - 1 + mov zeroones, REP8_01 + cmp tmp1, MIN_PAGE_SIZE - 16 + b.gt L(page_cross) + ldp data1, data2, [srcin] +#ifdef __AARCH64EB__ + /* For big-endian, carry propagation (if the final byte in the + string is 0x01) means we cannot use has_nul1/2 directly. + Since we expect strings to be small and early-exit, + byte-swap the data now so has_null1/2 will be correct. */ + rev data1, data1 + rev data2, data2 +#endif sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f + orr tmp2, data1, REP8_7f sub tmp3, data2, zeroones - orr tmp4, data2, #REP8_7f - bic has_nul1, tmp1, tmp2 - bics has_nul2, tmp3, tmp4 - ccmp has_nul1, #0, #0, eq /* NZCV = 0000 */ - b.eq .Lloop + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(main_loop_entry) - sub len, src, srcin - cbz has_nul1, .Lnul_in_data2 -CPU_BE( mov data2, data1 ) /*prepare data to re-calculate the syndrome*/ - sub len, len, #8 - mov has_nul2, has_nul1 -.Lnul_in_data2: - /* - * For big-endian, carry propagation (if the final byte in the - * string is 0x01) means we cannot use has_nul directly. The - * easiest way to get the correct byte is to byte-swap the data - * and calculate the syndrome a second time. - */ -CPU_BE( rev data2, data2 ) -CPU_BE( sub tmp1, data2, zeroones ) -CPU_BE( orr tmp2, data2, #REP8_7f ) -CPU_BE( bic has_nul2, tmp1, tmp2 ) - - sub len, len, #8 - rev has_nul2, has_nul2 - clz pos, has_nul2 - add len, len, pos, lsr #3 /* Bits to bytes. */ + /* Enter with C = has_nul1 == 0. */ + csel has_nul1, has_nul1, has_nul2, cc + mov len, 8 + rev has_nul1, has_nul1 + clz tmp1, has_nul1 + csel len, xzr, len, cc + add len, len, tmp1, lsr 3 ret -.Lmisaligned: - cmp tmp1, #8 - neg tmp1, tmp1 - ldp data1, data2, [src], #16 - lsl tmp1, tmp1, #3 /* Bytes beyond alignment -> bits. */ - mov tmp2, #~0 - /* Big-endian. Early bytes are at MSB. */ -CPU_BE( lsl tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ - /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp1 ) /* Shift (tmp1 & 63). */ + /* The inner loop processes 32 bytes per iteration and uses the fast + NUL check. If we encounter non-ASCII characters, use a second + loop with the accurate NUL check. */ + .p2align 4 +L(main_loop_entry): + bic src, srcin, 15 + sub src, src, 16 +L(main_loop): + ldp data1, data2, [src, 32]! +L(page_cross_entry): + sub tmp1, data1, zeroones + sub tmp3, data2, zeroones + orr tmp2, tmp1, tmp3 + tst tmp2, zeroones, lsl 7 + bne 1f + ldp data1, data2, [src, 16] + sub tmp1, data1, zeroones + sub tmp3, data2, zeroones + orr tmp2, tmp1, tmp3 + tst tmp2, zeroones, lsl 7 + beq L(main_loop) + add src, src, 16 +1: + /* The fast check failed, so do the slower, accurate NUL check. */ + orr tmp2, data1, REP8_7f + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(nonascii_loop) + + /* Enter with C = has_nul1 == 0. */ +L(tail): +#ifdef __AARCH64EB__ + /* For big-endian, carry propagation (if the final byte in the + string is 0x01) means we cannot use has_nul1/2 directly. The + easiest way to get the correct byte is to byte-swap the data + and calculate the syndrome a second time. */ + csel data1, data1, data2, cc + rev data1, data1 + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + bic has_nul1, tmp1, tmp2 +#else + csel has_nul1, has_nul1, has_nul2, cc +#endif + sub len, src, srcin + rev has_nul1, has_nul1 + add tmp2, len, 8 + clz tmp1, has_nul1 + csel len, len, tmp2, cc + add len, len, tmp1, lsr 3 + ret + +L(nonascii_loop): + ldp data1, data2, [src, 16]! + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + bne L(tail) + ldp data1, data2, [src, 16]! + sub tmp1, data1, zeroones + orr tmp2, data1, REP8_7f + sub tmp3, data2, zeroones + orr tmp4, data2, REP8_7f + bics has_nul1, tmp1, tmp2 + bic has_nul2, tmp3, tmp4 + ccmp has_nul2, 0, 0, eq + beq L(nonascii_loop) + b L(tail) + + /* Load 16 bytes from [srcin & ~15] and force the bytes that precede + srcin to 0x7f, so we ignore any NUL bytes before the string. + Then continue in the aligned loop. */ +L(page_cross): + bic src, srcin, 15 + ldp data1, data2, [src] + lsl tmp1, srcin, 3 + mov tmp4, -1 +#ifdef __AARCH64EB__ + /* Big-endian. Early bytes are at MSB. */ + lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ +#else + /* Little-endian. Early bytes are at LSB. */ + lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */ +#endif + orr tmp1, tmp1, REP8_80 + orn data1, data1, tmp1 + orn tmp2, data2, tmp1 + tst srcin, 8 + csel data1, data1, tmp4, eq + csel data2, data2, tmp2, eq + b L(page_cross_entry) - orr data1, data1, tmp2 - orr data2a, data2, tmp2 - csinv data1, data1, xzr, le - csel data2, data2, data2a, le - b .Lrealigned SYM_FUNC_END_PI(strlen) EXPORT_SYMBOL_NOKASAN(strlen) diff --git a/arch/arm64/lib/strncmp.S b/arch/arm64/lib/strncmp.S index 2a7ee949ed47..48d44f7fddb1 100644 --- a/arch/arm64/lib/strncmp.S +++ b/arch/arm64/lib/strncmp.S @@ -1,299 +1,261 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * Copyright (C) 2013 ARM Ltd. - * Copyright (C) 2013 Linaro. + * Copyright (c) 2013-2021, Arm Limited. * - * This code is based on glibc cortex strings work originally authored by Linaro - * be found @ - * - * http://bazaar.launchpad.net/~linaro-toolchain-dev/cortex-strings/trunk/ - * files/head:/src/aarch64/ + * Adapted from the original at: + * https://github.com/ARM-software/optimized-routines/blob/e823e3abf5f89ecb/string/aarch64/strncmp.S */ #include #include -/* - * compare two strings +/* Assumptions: * - * Parameters: - * x0 - const string 1 pointer - * x1 - const string 2 pointer - * x2 - the maximal length to be compared - * Returns: - * x0 - an integer less than, equal to, or greater than zero if s1 is found, - * respectively, to be less than, to match, or be greater than s2. + * ARMv8-a, AArch64 */ +#define L(label) .L ## label + #define REP8_01 0x0101010101010101 #define REP8_7f 0x7f7f7f7f7f7f7f7f #define REP8_80 0x8080808080808080 /* Parameters and result. */ -src1 .req x0 -src2 .req x1 -limit .req x2 -result .req x0 +#define src1 x0 +#define src2 x1 +#define limit x2 +#define result x0 /* Internal variables. */ -data1 .req x3 -data1w .req w3 -data2 .req x4 -data2w .req w4 -has_nul .req x5 -diff .req x6 -syndrome .req x7 -tmp1 .req x8 -tmp2 .req x9 -tmp3 .req x10 -zeroones .req x11 -pos .req x12 -limit_wd .req x13 -mask .req x14 -endloop .req x15 +#define data1 x3 +#define data1w w3 +#define data2 x4 +#define data2w w4 +#define has_nul x5 +#define diff x6 +#define syndrome x7 +#define tmp1 x8 +#define tmp2 x9 +#define tmp3 x10 +#define zeroones x11 +#define pos x12 +#define limit_wd x13 +#define mask x14 +#define endloop x15 +#define count mask SYM_FUNC_START_WEAK_PI(strncmp) - cbz limit, .Lret0 + cbz limit, L(ret0) eor tmp1, src1, src2 mov zeroones, #REP8_01 tst tmp1, #7 - b.ne .Lmisaligned8 - ands tmp1, src1, #7 - b.ne .Lmutual_align + and count, src1, #7 + b.ne L(misaligned8) + cbnz count, L(mutual_align) /* Calculate the number of full and partial words -1. */ - /* - * when limit is mulitply of 8, if not sub 1, - * the judgement of last dword will wrong. - */ - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ - lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ + sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ + lsr limit_wd, limit_wd, #3 /* Convert to Dwords. */ - /* - * NUL detection works on the principle that (X - 1) & (~X) & 0x80 - * (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and - * can be done in parallel across the entire word. - */ -.Lloop_aligned: + /* NUL detection works on the principle that (X - 1) & (~X) & 0x80 + (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and + can be done in parallel across the entire word. */ + .p2align 4 +L(loop_aligned): ldr data1, [src1], #8 ldr data2, [src2], #8 -.Lstart_realigned: +L(start_realigned): subs limit_wd, limit_wd, #1 sub tmp1, data1, zeroones orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, pl /* Last Dword or differences.*/ - bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + eor diff, data1, data2 /* Non-zero if differences found. */ + csinv endloop, diff, xzr, pl /* Last Dword or differences. */ + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ ccmp endloop, #0, #0, eq - b.eq .Lloop_aligned + b.eq L(loop_aligned) + /* End of main loop */ - /*Not reached the limit, must have found the end or a diff. */ - tbz limit_wd, #63, .Lnot_limit + /* Not reached the limit, must have found the end or a diff. */ + tbz limit_wd, #63, L(not_limit) /* Limit % 8 == 0 => all bytes significant. */ ands limit, limit, #7 - b.eq .Lnot_limit + b.eq L(not_limit) - lsl limit, limit, #3 /* Bits -> bytes. */ + lsl limit, limit, #3 /* Bits -> bytes. */ mov mask, #~0 -CPU_BE( lsr mask, mask, limit ) -CPU_LE( lsl mask, mask, limit ) +#ifdef __AARCH64EB__ + lsr mask, mask, limit +#else + lsl mask, mask, limit +#endif bic data1, data1, mask bic data2, data2, mask /* Make sure that the NUL byte is marked in the syndrome. */ orr has_nul, has_nul, mask -.Lnot_limit: +L(not_limit): orr syndrome, diff, has_nul - b .Lcal_cmpresult -.Lmutual_align: - /* - * Sources are mutually aligned, but are not currently at an - * alignment boundary. Round down the addresses and then mask off - * the bytes that precede the start point. - * We also need to adjust the limit calculations, but without - * overflowing if the limit is near ULONG_MAX. - */ - bic src1, src1, #7 - bic src2, src2, #7 - ldr data1, [src1], #8 - neg tmp3, tmp1, lsl #3 /* 64 - bits(bytes beyond align). */ - ldr data2, [src2], #8 - mov tmp2, #~0 - sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ - /* Big-endian. Early bytes are at MSB. */ -CPU_BE( lsl tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */ - /* Little-endian. Early bytes are at LSB. */ -CPU_LE( lsr tmp2, tmp2, tmp3 ) /* Shift (tmp1 & 63). */ - - and tmp3, limit_wd, #7 - lsr limit_wd, limit_wd, #3 - /* Adjust the limit. Only low 3 bits used, so overflow irrelevant.*/ - add limit, limit, tmp1 - add tmp3, tmp3, tmp1 - orr data1, data1, tmp2 - orr data2, data2, tmp2 - add limit_wd, limit_wd, tmp3, lsr #3 - b .Lstart_realigned - -/*when src1 offset is not equal to src2 offset...*/ -.Lmisaligned8: - cmp limit, #8 - b.lo .Ltiny8proc /*limit < 8... */ - /* - * Get the align offset length to compare per byte first. - * After this process, one string's address will be aligned.*/ - and tmp1, src1, #7 - neg tmp1, tmp1 - add tmp1, tmp1, #8 - and tmp2, src2, #7 - neg tmp2, tmp2 - add tmp2, tmp2, #8 - subs tmp3, tmp1, tmp2 - csel pos, tmp1, tmp2, hi /*Choose the maximum. */ - /* - * Here, limit is not less than 8, so directly run .Ltinycmp - * without checking the limit.*/ - sub limit, limit, pos -.Ltinycmp: - ldrb data1w, [src1], #1 - ldrb data2w, [src2], #1 - subs pos, pos, #1 - ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ - ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ - b.eq .Ltinycmp - cbnz pos, 1f /*find the null or unequal...*/ - cmp data1w, #1 - ccmp data1w, data2w, #0, cs - b.eq .Lstart_align /*the last bytes are equal....*/ -1: - sub result, data1, data2 - ret - -.Lstart_align: - lsr limit_wd, limit, #3 - cbz limit_wd, .Lremain8 - /*process more leading bytes to make str1 aligned...*/ - ands xzr, src1, #7 - b.eq .Lrecal_offset - add src1, src1, tmp3 /*tmp3 is positive in this branch.*/ - add src2, src2, tmp3 - ldr data1, [src1], #8 - ldr data2, [src2], #8 - - sub limit, limit, tmp3 - lsr limit_wd, limit, #3 - subs limit_wd, limit_wd, #1 - - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ - bics has_nul, tmp1, tmp2 - ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/ - b.ne .Lunequal_proc - /*How far is the current str2 from the alignment boundary...*/ - and tmp3, tmp3, #7 -.Lrecal_offset: - neg pos, tmp3 -.Lloopcmp_proc: - /* - * Divide the eight bytes into two parts. First,backwards the src2 - * to an alignment boundary,load eight bytes from the SRC2 alignment - * boundary,then compare with the relative bytes from SRC1. - * If all 8 bytes are equal,then start the second part's comparison. - * Otherwise finish the comparison. - * This special handle can garantee all the accesses are in the - * thread/task space in avoid to overrange access. - */ - ldr data1, [src1,pos] - ldr data2, [src2,pos] - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, eq - cbnz endloop, .Lunequal_proc - - /*The second part process*/ - ldr data1, [src1], #8 - ldr data2, [src2], #8 - subs limit_wd, limit_wd, #1 - sub tmp1, data1, zeroones - orr tmp2, data1, #REP8_7f - eor diff, data1, data2 /* Non-zero if differences found. */ - csinv endloop, diff, xzr, ne/*if limit_wd is 0,will finish the cmp*/ - bics has_nul, tmp1, tmp2 - ccmp endloop, #0, #0, eq /*has_null is ZERO: no null byte*/ - b.eq .Lloopcmp_proc - -.Lunequal_proc: - orr syndrome, diff, has_nul - cbz syndrome, .Lremain8 -.Lcal_cmpresult: - /* - * reversed the byte-order as big-endian,then CLZ can find the most - * significant zero bits. - */ -CPU_LE( rev syndrome, syndrome ) -CPU_LE( rev data1, data1 ) -CPU_LE( rev data2, data2 ) - /* - * For big-endian we cannot use the trick with the syndrome value - * as carry-propagation can corrupt the upper bits if the trailing - * bytes in the string contain 0x01. - * However, if there is no NUL byte in the dword, we can generate - * the result directly. We can't just subtract the bytes as the - * MSB might be significant. - */ -CPU_BE( cbnz has_nul, 1f ) -CPU_BE( cmp data1, data2 ) -CPU_BE( cset result, ne ) -CPU_BE( cneg result, result, lo ) -CPU_BE( ret ) -CPU_BE( 1: ) - /* Re-compute the NUL-byte detection, using a byte-reversed value.*/ -CPU_BE( rev tmp3, data1 ) -CPU_BE( sub tmp1, tmp3, zeroones ) -CPU_BE( orr tmp2, tmp3, #REP8_7f ) -CPU_BE( bic has_nul, tmp1, tmp2 ) -CPU_BE( rev has_nul, has_nul ) -CPU_BE( orr syndrome, diff, has_nul ) - /* - * The MS-non-zero bit of the syndrome marks either the first bit - * that is different, or the top bit of the first zero byte. - * Shifting left now will bring the critical information into the - * top bits. - */ +#ifndef __AARCH64EB__ + rev syndrome, syndrome + rev data1, data1 + /* The MS-non-zero bit of the syndrome marks either the first bit + that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ clz pos, syndrome + rev data2, data2 lsl data1, data1, pos lsl data2, data2, pos - /* - * But we need to zero-extend (char is unsigned) the value and then - * perform a signed 32-bit subtraction. - */ + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ lsr data1, data1, #56 sub result, data1, data2, lsr #56 ret +#else + /* For big-endian we cannot use the trick with the syndrome value + as carry-propagation can corrupt the upper bits if the trailing + bytes in the string contain 0x01. */ + /* However, if there is no NUL byte in the dword, we can generate + the result directly. We can't just subtract the bytes as the + MSB might be significant. */ + cbnz has_nul, 1f + cmp data1, data2 + cset result, ne + cneg result, result, lo + ret +1: + /* Re-compute the NUL-byte detection, using a byte-reversed value. */ + rev tmp3, data1 + sub tmp1, tmp3, zeroones + orr tmp2, tmp3, #REP8_7f + bic has_nul, tmp1, tmp2 + rev has_nul, has_nul + orr syndrome, diff, has_nul + clz pos, syndrome + /* The MS-non-zero bit of the syndrome marks either the first bit + that is different, or the top bit of the first zero byte. + Shifting left now will bring the critical information into the + top bits. */ + lsl data1, data1, pos + lsl data2, data2, pos + /* But we need to zero-extend (char is unsigned) the value and then + perform a signed 32-bit subtraction. */ + lsr data1, data1, #56 + sub result, data1, data2, lsr #56 + ret +#endif -.Lremain8: - /* Limit % 8 == 0 => all bytes significant. */ - ands limit, limit, #7 - b.eq .Lret0 -.Ltiny8proc: +L(mutual_align): + /* Sources are mutually aligned, but are not currently at an + alignment boundary. Round down the addresses and then mask off + the bytes that precede the start point. + We also need to adjust the limit calculations, but without + overflowing if the limit is near ULONG_MAX. */ + bic src1, src1, #7 + bic src2, src2, #7 + ldr data1, [src1], #8 + neg tmp3, count, lsl #3 /* 64 - bits(bytes beyond align). */ + ldr data2, [src2], #8 + mov tmp2, #~0 + sub limit_wd, limit, #1 /* limit != 0, so no underflow. */ +#ifdef __AARCH64EB__ + /* Big-endian. Early bytes are at MSB. */ + lsl tmp2, tmp2, tmp3 /* Shift (count & 63). */ +#else + /* Little-endian. Early bytes are at LSB. */ + lsr tmp2, tmp2, tmp3 /* Shift (count & 63). */ +#endif + and tmp3, limit_wd, #7 + lsr limit_wd, limit_wd, #3 + /* Adjust the limit. Only low 3 bits used, so overflow irrelevant. */ + add limit, limit, count + add tmp3, tmp3, count + orr data1, data1, tmp2 + orr data2, data2, tmp2 + add limit_wd, limit_wd, tmp3, lsr #3 + b L(start_realigned) + + .p2align 4 + /* Don't bother with dwords for up to 16 bytes. */ +L(misaligned8): + cmp limit, #16 + b.hs L(try_misaligned_words) + +L(byte_loop): + /* Perhaps we can do better than this. */ ldrb data1w, [src1], #1 ldrb data2w, [src2], #1 subs limit, limit, #1 - - ccmp data1w, #1, #0, ne /* NZCV = 0b0000. */ - ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ - b.eq .Ltiny8proc + ccmp data1w, #1, #0, hi /* NZCV = 0b0000. */ + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ + b.eq L(byte_loop) +L(done): sub result, data1, data2 ret + /* Align the SRC1 to a dword by doing a bytewise compare and then do + the dword loop. */ +L(try_misaligned_words): + lsr limit_wd, limit, #3 + cbz count, L(do_misaligned) -.Lret0: + neg count, count + and count, count, #7 + sub limit, limit, count + lsr limit_wd, limit, #3 + +L(page_end_loop): + ldrb data1w, [src1], #1 + ldrb data2w, [src2], #1 + cmp data1w, #1 + ccmp data1w, data2w, #0, cs /* NZCV = 0b0000. */ + b.ne L(done) + subs count, count, #1 + b.hi L(page_end_loop) + +L(do_misaligned): + /* Prepare ourselves for the next page crossing. Unlike the aligned + loop, we fetch 1 less dword because we risk crossing bounds on + SRC2. */ + mov count, #8 + subs limit_wd, limit_wd, #1 + b.lo L(done_loop) +L(loop_misaligned): + and tmp2, src2, #0xff8 + eor tmp2, tmp2, #0xff8 + cbz tmp2, L(page_end_loop) + + ldr data1, [src1], #8 + ldr data2, [src2], #8 + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + eor diff, data1, data2 /* Non-zero if differences found. */ + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + ccmp diff, #0, #0, eq + b.ne L(not_limit) + subs limit_wd, limit_wd, #1 + b.pl L(loop_misaligned) + +L(done_loop): + /* We found a difference or a NULL before the limit was reached. */ + and limit, limit, #7 + cbz limit, L(not_limit) + /* Read the last word. */ + sub src1, src1, 8 + sub src2, src2, 8 + ldr data1, [src1, limit] + ldr data2, [src2, limit] + sub tmp1, data1, zeroones + orr tmp2, data1, #REP8_7f + eor diff, data1, data2 /* Non-zero if differences found. */ + bics has_nul, tmp1, tmp2 /* Non-zero if NUL terminator. */ + ccmp diff, #0, #0, eq + b.ne L(not_limit) + +L(ret0): mov result, #0 ret + SYM_FUNC_END_PI(strncmp) EXPORT_SYMBOL_NOKASAN(strncmp) diff --git a/arch/arm64/lib/uaccess_flushcache.c b/arch/arm64/lib/uaccess_flushcache.c index c83bb5a4aad2..baee22961bdb 100644 --- a/arch/arm64/lib/uaccess_flushcache.c +++ b/arch/arm64/lib/uaccess_flushcache.c @@ -15,7 +15,7 @@ void memcpy_flushcache(void *dst, const void *src, size_t cnt) * barrier to order the cache maintenance against the memcpy. */ memcpy(dst, src, cnt); - __clean_dcache_area_pop(dst, cnt); + dcache_clean_pop((unsigned long)dst, (unsigned long)dst + cnt); } EXPORT_SYMBOL_GPL(memcpy_flushcache); @@ -33,6 +33,6 @@ unsigned long __copy_user_flushcache(void *to, const void __user *from, rc = raw_copy_from_user(to, from, n); /* See above */ - __clean_dcache_area_pop(to, n - rc); + dcache_clean_pop((unsigned long)to, (unsigned long)to + n - rc); return rc; } diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 2d881f34dd9d..5051b3c1a4f1 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -15,7 +15,7 @@ #include /* - * flush_icache_range(start,end) + * caches_clean_inval_pou_macro(start,end) [fixup] * * Ensure that the I and D caches are coherent within specified region. * This is typically used when code has been written to a memory region, @@ -23,70 +23,54 @@ * * - start - virtual start address of region * - end - virtual end address of region + * - fixup - optional label to branch to on user fault */ -SYM_FUNC_START(__flush_icache_range) - /* FALLTHROUGH */ - -/* - * __flush_cache_user_range(start,end) - * - * Ensure that the I and D caches are coherent within specified region. - * This is typically used when code has been written to a memory region, - * and will be executed. - * - * - start - virtual start address of region - * - end - virtual end address of region - */ -SYM_FUNC_START(__flush_cache_user_range) - uaccess_ttbr0_enable x2, x3, x4 +.macro caches_clean_inval_pou_macro, fixup alternative_if ARM64_HAS_CACHE_IDC - dsb ishst - b 7f + dsb ishst + b .Ldc_skip_\@ alternative_else_nop_endif - dcache_line_size x2, x3 - sub x3, x2, #1 - bic x4, x0, x3 -1: -user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE - add x4, x4, x2 - cmp x4, x1 - b.lo 1b - dsb ish - -7: + mov x2, x0 + mov x3, x1 + dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup +.Ldc_skip_\@: alternative_if ARM64_HAS_CACHE_DIC isb - b 8f + b .Lic_skip_\@ alternative_else_nop_endif - invalidate_icache_by_line x0, x1, x2, x3, 9f -8: mov x0, #0 -1: - uaccess_ttbr0_disable x1, x2 - ret -9: - mov x0, #-EFAULT - b 1b -SYM_FUNC_END(__flush_icache_range) -SYM_FUNC_END(__flush_cache_user_range) + invalidate_icache_by_line x0, x1, x2, x3, \fixup +.Lic_skip_\@: +.endm /* - * invalidate_icache_range(start,end) + * caches_clean_inval_pou(start,end) * - * Ensure that the I cache is invalid within specified region. + * Ensure that the I and D caches are coherent within specified region. + * This is typically used when code has been written to a memory region, + * and will be executed. * * - start - virtual start address of region * - end - virtual end address of region */ -SYM_FUNC_START(invalidate_icache_range) -alternative_if ARM64_HAS_CACHE_DIC - mov x0, xzr - isb +SYM_FUNC_START(caches_clean_inval_pou) + caches_clean_inval_pou_macro ret -alternative_else_nop_endif +SYM_FUNC_END(caches_clean_inval_pou) +/* + * caches_clean_inval_user_pou(start,end) + * + * Ensure that the I and D caches are coherent within specified region. + * This is typically used when code has been written to a memory region, + * and will be executed. + * + * - start - virtual start address of region + * - end - virtual end address of region + */ +SYM_FUNC_START(caches_clean_inval_user_pou) uaccess_ttbr0_enable x2, x3, x4 - invalidate_icache_by_line x0, x1, x2, x3, 2f + caches_clean_inval_pou_macro 2f mov x0, xzr 1: uaccess_ttbr0_disable x1, x2 @@ -94,60 +78,77 @@ alternative_else_nop_endif 2: mov x0, #-EFAULT b 1b -SYM_FUNC_END(invalidate_icache_range) +SYM_FUNC_END(caches_clean_inval_user_pou) /* - * __flush_dcache_area(kaddr, size) + * icache_inval_pou(start,end) * - * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * Ensure that the I cache is invalid within specified region. + * + * - start - virtual start address of region + * - end - virtual end address of region + */ +SYM_FUNC_START(icache_inval_pou) +alternative_if ARM64_HAS_CACHE_DIC + isb + ret +alternative_else_nop_endif + + invalidate_icache_by_line x0, x1, x2, x3 + ret +SYM_FUNC_END(icache_inval_pou) + +/* + * dcache_clean_inval_poc(start, end) + * + * Ensure that any D-cache lines for the interval [start, end) * are cleaned and invalidated to the PoC. * - * - kaddr - kernel address - * - size - size in question + * - start - virtual start address of region + * - end - virtual end address of region */ -SYM_FUNC_START_PI(__flush_dcache_area) +SYM_FUNC_START_PI(dcache_clean_inval_poc) dcache_by_line_op civac, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(__flush_dcache_area) +SYM_FUNC_END_PI(dcache_clean_inval_poc) /* - * __clean_dcache_area_pou(kaddr, size) + * dcache_clean_pou(start, end) * - * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * Ensure that any D-cache lines for the interval [start, end) * are cleaned to the PoU. * - * - kaddr - kernel address - * - size - size in question + * - start - virtual start address of region + * - end - virtual end address of region */ -SYM_FUNC_START(__clean_dcache_area_pou) +SYM_FUNC_START(dcache_clean_pou) alternative_if ARM64_HAS_CACHE_IDC dsb ishst ret alternative_else_nop_endif dcache_by_line_op cvau, ish, x0, x1, x2, x3 ret -SYM_FUNC_END(__clean_dcache_area_pou) +SYM_FUNC_END(dcache_clean_pou) /* - * __inval_dcache_area(kaddr, size) + * dcache_inval_poc(start, end) * - * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * Ensure that any D-cache lines for the interval [start, end) * are invalidated. Any partial lines at the ends of the interval are * also cleaned to PoC to prevent data loss. * - * - kaddr - kernel address - * - size - size in question + * - start - kernel start address of region + * - end - kernel end address of region */ SYM_FUNC_START_LOCAL(__dma_inv_area) -SYM_FUNC_START_PI(__inval_dcache_area) +SYM_FUNC_START_PI(dcache_inval_poc) /* FALLTHROUGH */ /* - * __dma_inv_area(start, size) + * __dma_inv_area(start, end) * - start - virtual start address of region - * - size - size in question + * - end - virtual end address of region */ - add x1, x1, x0 dcache_line_size x2, x3 sub x3, x2, #1 tst x1, x3 // end cache line aligned? @@ -165,48 +166,48 @@ SYM_FUNC_START_PI(__inval_dcache_area) b.lo 2b dsb sy ret -SYM_FUNC_END_PI(__inval_dcache_area) +SYM_FUNC_END_PI(dcache_inval_poc) SYM_FUNC_END(__dma_inv_area) /* - * __clean_dcache_area_poc(kaddr, size) + * dcache_clean_poc(start, end) * - * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * Ensure that any D-cache lines for the interval [start, end) * are cleaned to the PoC. * - * - kaddr - kernel address - * - size - size in question + * - start - virtual start address of region + * - end - virtual end address of region */ SYM_FUNC_START_LOCAL(__dma_clean_area) -SYM_FUNC_START_PI(__clean_dcache_area_poc) +SYM_FUNC_START_PI(dcache_clean_poc) /* FALLTHROUGH */ /* - * __dma_clean_area(start, size) + * __dma_clean_area(start, end) * - start - virtual start address of region - * - size - size in question + * - end - virtual end address of region */ dcache_by_line_op cvac, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(__clean_dcache_area_poc) +SYM_FUNC_END_PI(dcache_clean_poc) SYM_FUNC_END(__dma_clean_area) /* - * __clean_dcache_area_pop(kaddr, size) + * dcache_clean_pop(start, end) * - * Ensure that any D-cache lines for the interval [kaddr, kaddr+size) + * Ensure that any D-cache lines for the interval [start, end) * are cleaned to the PoP. * - * - kaddr - kernel address - * - size - size in question + * - start - virtual start address of region + * - end - virtual end address of region */ -SYM_FUNC_START_PI(__clean_dcache_area_pop) +SYM_FUNC_START_PI(dcache_clean_pop) alternative_if_not ARM64_HAS_DCPOP - b __clean_dcache_area_poc + b dcache_clean_poc alternative_else_nop_endif dcache_by_line_op cvap, sy, x0, x1, x2, x3 ret -SYM_FUNC_END_PI(__clean_dcache_area_pop) +SYM_FUNC_END_PI(dcache_clean_pop) /* * __dma_flush_area(start, size) @@ -217,6 +218,7 @@ SYM_FUNC_END_PI(__clean_dcache_area_pop) * - size - size in question */ SYM_FUNC_START_PI(__dma_flush_area) + add x1, x0, x1 dcache_by_line_op civac, sy, x0, x1, x2, x3 ret SYM_FUNC_END_PI(__dma_flush_area) @@ -228,6 +230,7 @@ SYM_FUNC_END_PI(__dma_flush_area) * - dir - DMA direction */ SYM_FUNC_START_PI(__dma_map_area) + add x1, x0, x1 cmp w2, #DMA_FROM_DEVICE b.eq __dma_inv_area b __dma_clean_area @@ -240,6 +243,7 @@ SYM_FUNC_END_PI(__dma_map_area) * - dir - DMA direction */ SYM_FUNC_START_PI(__dma_unmap_area) + add x1, x0, x1 cmp w2, #DMA_TO_DEVICE b.ne __dma_inv_area ret diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c index 001737a8f309..cd72576ae2b7 100644 --- a/arch/arm64/mm/context.c +++ b/arch/arm64/mm/context.c @@ -402,14 +402,12 @@ static int asids_init(void) { asid_bits = get_cpu_asid_bits(); atomic64_set(&asid_generation, ASID_FIRST_VERSION); - asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS), sizeof(*asid_map), - GFP_KERNEL); + asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); if (!asid_map) panic("Failed to allocate bitmap for %lu ASIDs\n", NUM_USER_ASIDS); - pinned_asid_map = kcalloc(BITS_TO_LONGS(NUM_USER_ASIDS), - sizeof(*pinned_asid_map), GFP_KERNEL); + pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); nr_pinned_asids = 0; /* diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 871c82ab0a30..349c488765ca 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -99,6 +99,8 @@ static void mem_abort_decode(unsigned int esr) pr_alert(" EA = %lu, S1PTW = %lu\n", (esr & ESR_ELx_EA) >> ESR_ELx_EA_SHIFT, (esr & ESR_ELx_S1PTW) >> ESR_ELx_S1PTW_SHIFT); + pr_alert(" FSC = 0x%02x: %s\n", (esr & ESR_ELx_FSC), + esr_to_fault_info(esr)->name); if (esr_is_data_abort(esr)) data_abort_decode(esr); @@ -232,13 +234,17 @@ static bool is_el1_instruction_abort(unsigned int esr) return ESR_ELx_EC(esr) == ESR_ELx_EC_IABT_CUR; } +static bool is_el1_data_abort(unsigned int esr) +{ + return ESR_ELx_EC(esr) == ESR_ELx_EC_DABT_CUR; +} + static inline bool is_el1_permission_fault(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - unsigned int ec = ESR_ELx_EC(esr); unsigned int fsc_type = esr & ESR_ELx_FSC_TYPE; - if (ec != ESR_ELx_EC_DABT_CUR && ec != ESR_ELx_EC_IABT_CUR) + if (!is_el1_data_abort(esr) && !is_el1_instruction_abort(esr)) return false; if (fsc_type == ESR_ELx_FSC_PERM) @@ -258,7 +264,7 @@ static bool __kprobes is_spurious_el1_translation_fault(unsigned long addr, unsigned long flags; u64 par, dfsc; - if (ESR_ELx_EC(esr) != ESR_ELx_EC_DABT_CUR || + if (!is_el1_data_abort(esr) || (esr & ESR_ELx_FSC_TYPE) != ESR_ELx_FSC_FAULT) return false; @@ -346,10 +352,9 @@ static void do_tag_recovery(unsigned long addr, unsigned int esr, static bool is_el1_mte_sync_tag_check_fault(unsigned int esr) { - unsigned int ec = ESR_ELx_EC(esr); unsigned int fsc = esr & ESR_ELx_FSC; - if (ec != ESR_ELx_EC_DABT_CUR) + if (!is_el1_data_abort(esr)) return false; if (fsc == ESR_ELx_FSC_MTE) @@ -504,7 +509,7 @@ static vm_fault_t __do_page_fault(struct mm_struct *mm, unsigned long addr, */ if (!(vma->vm_flags & vm_flags)) return VM_FAULT_BADACCESS; - return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags, regs); + return handle_mm_fault(vma, addr, mm_flags, regs); } static bool is_el0_instruction_abort(unsigned int esr) @@ -836,13 +841,6 @@ void do_mem_abort(unsigned long far, unsigned int esr, struct pt_regs *regs) } NOKPROBE_SYMBOL(do_mem_abort); -void do_el0_irq_bp_hardening(void) -{ - /* PC has already been checked in entry.S */ - arm64_apply_bp_hardening(); -} -NOKPROBE_SYMBOL(do_el0_irq_bp_hardening); - void do_sp_pc_abort(unsigned long addr, unsigned int esr, struct pt_regs *regs) { arm64_notify_die("SP/PC alignment exception", regs, SIGBUS, BUS_ADRALN, @@ -921,3 +919,29 @@ void do_debug_exception(unsigned long addr_if_watchpoint, unsigned int esr, debug_exception_exit(regs); } NOKPROBE_SYMBOL(do_debug_exception); + +/* + * Used during anonymous page fault handling. + */ +struct page *alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, + unsigned long vaddr) +{ + gfp_t flags = GFP_HIGHUSER_MOVABLE | __GFP_ZERO; + + /* + * If the page is mapped with PROT_MTE, initialise the tags at the + * point of allocation and page zeroing as this is usually faster than + * separate DC ZVA and STGM. + */ + if (vma->vm_flags & VM_MTE) + flags |= __GFP_ZEROTAGS; + + return alloc_page_vma(flags, vma, vaddr); +} + +void tag_clear_highpage(struct page *page) +{ + mte_zero_clear_page_tags(page_address(page)); + page_kasan_tag_reset(page); + set_bit(PG_mte_tagged, &page->flags); +} diff --git a/arch/arm64/mm/flush.c b/arch/arm64/mm/flush.c index 6d44c028d1c9..2aaf950b906c 100644 --- a/arch/arm64/mm/flush.c +++ b/arch/arm64/mm/flush.c @@ -14,28 +14,25 @@ #include #include -void sync_icache_aliases(void *kaddr, unsigned long len) +void sync_icache_aliases(unsigned long start, unsigned long end) { - unsigned long addr = (unsigned long)kaddr; - if (icache_is_aliasing()) { - __clean_dcache_area_pou(kaddr, len); - __flush_icache_all(); + dcache_clean_pou(start, end); + icache_inval_all_pou(); } else { /* * Don't issue kick_all_cpus_sync() after I-cache invalidation * for user mappings. */ - __flush_icache_range(addr, addr + len); + caches_clean_inval_pou(start, end); } } -static void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, - unsigned long uaddr, void *kaddr, - unsigned long len) +static void flush_ptrace_access(struct vm_area_struct *vma, unsigned long start, + unsigned long end) { if (vma->vm_flags & VM_EXEC) - sync_icache_aliases(kaddr, len); + sync_icache_aliases(start, end); } /* @@ -48,7 +45,7 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, unsigned long len) { memcpy(dst, src, len); - flush_ptrace_access(vma, page, uaddr, dst, len); + flush_ptrace_access(vma, (unsigned long)dst, (unsigned long)dst + len); } void __sync_icache_dcache(pte_t pte) @@ -56,7 +53,9 @@ void __sync_icache_dcache(pte_t pte) struct page *page = pte_page(pte); if (!test_bit(PG_dcache_clean, &page->flags)) { - sync_icache_aliases(page_address(page), page_size(page)); + sync_icache_aliases((unsigned long)page_address(page), + (unsigned long)page_address(page) + + page_size(page)); set_bit(PG_dcache_clean, &page->flags); } } @@ -77,20 +76,20 @@ EXPORT_SYMBOL(flush_dcache_page); /* * Additional functions defined in assembly. */ -EXPORT_SYMBOL(__flush_icache_range); +EXPORT_SYMBOL(caches_clean_inval_pou); #ifdef CONFIG_ARCH_HAS_PMEM_API void arch_wb_cache_pmem(void *addr, size_t size) { /* Ensure order against any prior non-cacheable writes */ dmb(osh); - __clean_dcache_area_pop(addr, size); + dcache_clean_pop((unsigned long)addr, (unsigned long)addr + size); } EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); void arch_invalidate_pmem(void *addr, size_t size) { - __inval_dcache_area(addr, size); + dcache_inval_poc((unsigned long)addr, (unsigned long)addr + size); } EXPORT_SYMBOL_GPL(arch_invalidate_pmem); #endif diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index e55409caaee3..6e1ca044ca90 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -499,6 +499,13 @@ void __init mem_init(void) BUILD_BUG_ON(TASK_SIZE_32 > DEFAULT_MAP_WINDOW_64); #endif + /* + * Selected page table levels should match when derived from + * scratch using the virtual address range and page size. + */ + BUILD_BUG_ON(ARM64_HW_PGTABLE_LEVELS(CONFIG_ARM64_VA_BITS) != + CONFIG_PGTABLE_LEVELS); + if (PAGE_SIZE >= 16384 && get_num_physpages() <= 128) { extern int sysctl_overcommit_memory; /* diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c index 89b66ef43a0f..0b28cc218091 100644 --- a/arch/arm64/mm/mmu.c +++ b/arch/arm64/mm/mmu.c @@ -228,7 +228,7 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end, next = pmd_addr_end(addr, end); /* try section mapping first */ - if (((addr | next | phys) & ~SECTION_MASK) == 0 && + if (((addr | next | phys) & ~PMD_MASK) == 0 && (flags & NO_BLOCK_MAPPINGS) == 0) { pmd_set_huge(pmdp, phys, prot); @@ -1114,14 +1114,14 @@ static void free_empty_tables(unsigned long addr, unsigned long end, } #endif -#if !ARM64_SWAPPER_USES_SECTION_MAPS +#if !ARM64_KERNEL_USES_PMD_MAPS int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap) { WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); return vmemmap_populate_basepages(start, end, node, altmap); } -#else /* !ARM64_SWAPPER_USES_SECTION_MAPS */ +#else /* !ARM64_KERNEL_USES_PMD_MAPS */ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, struct vmem_altmap *altmap) { @@ -1166,17 +1166,18 @@ int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node, return 0; } -#endif /* !ARM64_SWAPPER_USES_SECTION_MAPS */ +#endif /* !ARM64_KERNEL_USES_PMD_MAPS */ + +#ifdef CONFIG_MEMORY_HOTPLUG void vmemmap_free(unsigned long start, unsigned long end, struct vmem_altmap *altmap) { -#ifdef CONFIG_MEMORY_HOTPLUG WARN_ON((start < VMEMMAP_START) || (end > VMEMMAP_END)); unmap_hotplug_range(start, end, true, altmap); free_empty_tables(start, end, VMEMMAP_START, VMEMMAP_END); -#endif } +#endif /* CONFIG_MEMORY_HOTPLUG */ static inline pud_t *fixmap_pud(unsigned long addr) { diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index 97d7bcd8d4f2..35936c5ae1ce 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -46,9 +46,13 @@ #endif #ifdef CONFIG_KASAN_HW_TAGS -#define TCR_KASAN_HW_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1 +#define TCR_MTE_FLAGS SYS_TCR_EL1_TCMA1 | TCR_TBI1 | TCR_TBID1 #else -#define TCR_KASAN_HW_FLAGS 0 +/* + * The mte_zero_clear_page_tags() implementation uses DC GZVA, which relies on + * TBI being enabled at EL1. + */ +#define TCR_MTE_FLAGS TCR_TBI1 | TCR_TBID1 #endif /* @@ -58,10 +62,8 @@ #define MAIR_EL1_SET \ (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) | \ - MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) | \ MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) | \ - MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) | \ MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) #ifdef CONFIG_CPU_PM @@ -83,11 +85,7 @@ SYM_FUNC_START(cpu_do_suspend) mrs x9, mdscr_el1 mrs x10, oslsr_el1 mrs x11, sctlr_el1 -alternative_if_not ARM64_HAS_VIRT_HOST_EXTN - mrs x12, tpidr_el1 -alternative_else - mrs x12, tpidr_el2 -alternative_endif + get_this_cpu_offset x12 mrs x13, sp_el0 stp x2, x3, [x0] stp x4, x5, [x0, #16] @@ -145,11 +143,7 @@ SYM_FUNC_START(cpu_do_resume) msr mdscr_el1, x10 msr sctlr_el1, x12 -alternative_if_not ARM64_HAS_VIRT_HOST_EXTN - msr tpidr_el1, x13 -alternative_else - msr tpidr_el2, x13 -alternative_endif + set_this_cpu_offset x13 msr sp_el0, x14 /* * Restore oslsr_el1 by writing oslar_el1 @@ -464,7 +458,7 @@ SYM_FUNC_START(__cpu_setup) msr_s SYS_TFSRE0_EL1, xzr /* set the TCR_EL1 bits */ - mov_q x10, TCR_KASAN_HW_FLAGS + mov_q x10, TCR_MTE_FLAGS orr tcr, tcr, x10 1: #endif diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index a1937dfff31c..1c403536c9bb 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -157,10 +157,6 @@ static const struct prot_bits pte_bits[] = { .mask = PTE_ATTRINDX_MASK, .val = PTE_ATTRINDX(MT_DEVICE_nGnRE), .set = "DEVICE/nGnRE", - }, { - .mask = PTE_ATTRINDX_MASK, - .val = PTE_ATTRINDX(MT_DEVICE_GRE), - .set = "DEVICE/GRE", }, { .mask = PTE_ATTRINDX_MASK, .val = PTE_ATTRINDX(MT_NORMAL_NC), diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index f7b194878a99..dd5000da18b8 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include "bpf_jit.h" diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 21fbdda7086e..49305c2e6dfd 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -3,7 +3,8 @@ # Internal CPU capabilities constants, keep this list sorted BTI -HAS_32BIT_EL0 +# Unreliable: use system_supports_32bit_el0() instead. +HAS_32BIT_EL0_DO_NOT_USE HAS_32BIT_EL1 HAS_ADDRESS_AUTH HAS_ADDRESS_AUTH_ARCH diff --git a/arch/ia64/include/asm/page.h b/arch/ia64/include/asm/page.h index f4dc81fa7146..1b990466d540 100644 --- a/arch/ia64/include/asm/page.h +++ b/arch/ia64/include/asm/page.h @@ -82,16 +82,16 @@ do { \ } while (0) -#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ ({ \ struct page *page = alloc_page_vma( \ - GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr); \ + GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr); \ if (page) \ flush_dcache_page(page); \ page; \ }) -#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE #define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) diff --git a/arch/m68k/Kconfig.machine b/arch/m68k/Kconfig.machine index 4d59ec2f5b8d..d964c1f27399 100644 --- a/arch/m68k/Kconfig.machine +++ b/arch/m68k/Kconfig.machine @@ -25,6 +25,9 @@ config ATARI this kernel on an Atari, say Y here and browse the material available in ; otherwise say N. +config ATARI_KBD_CORE + bool + config MAC bool "Macintosh support" depends on MMU diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile index 82620f14124d..c54055a3d284 100644 --- a/arch/m68k/Makefile +++ b/arch/m68k/Makefile @@ -66,8 +66,7 @@ KBUILD_CFLAGS += $(cpuflags-y) KBUILD_CFLAGS += -pipe -ffreestanding ifdef CONFIG_MMU -# without -fno-strength-reduce the 53c7xx.c driver fails ;-( -KBUILD_CFLAGS += -fno-strength-reduce -ffixed-a2 +KBUILD_CFLAGS += -ffixed-a2 else # we can use a m68k-linux-gcc toolchain with these in place KBUILD_CPPFLAGS += -DUTS_SYSNAME=\"uClinux\" diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 4fe26d54627e..0a2cacf7be08 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -85,7 +85,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -207,6 +206,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -253,7 +253,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -563,6 +562,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -627,6 +627,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 8d4ddcebe7b8..4dc6dcfaf28a 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -81,7 +81,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -203,6 +202,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -249,7 +249,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -519,6 +518,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -583,6 +583,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 21b2990fe9af..23d910a692ab 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -88,7 +88,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -210,6 +209,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -256,7 +256,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -541,6 +540,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -605,6 +605,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index c3f3f462e6ce..2c3f42833846 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -78,7 +78,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -200,6 +199,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -246,7 +246,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -512,6 +511,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -576,6 +576,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index 8c908fc5c191..5b1898d4b249 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -80,7 +80,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -202,6 +201,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -248,7 +248,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -521,6 +520,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -585,6 +585,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index b03300df13fc..9606ccd8dafa 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -79,7 +79,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -201,6 +200,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -247,7 +247,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -543,6 +542,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -607,6 +607,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index e2c8368e2231..3175ba5007e1 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -99,7 +99,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -221,6 +220,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -267,7 +267,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -628,6 +627,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -692,6 +692,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index c7442f9dd469..793085f00c99 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -77,7 +77,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -199,6 +198,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -245,7 +245,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -511,6 +510,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -575,6 +575,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index 233b82ea103a..56fbac7943b2 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -78,7 +78,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -200,6 +199,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -246,7 +246,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -512,6 +511,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -576,6 +576,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 514e2e8cddbd..0e15431b65e2 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -79,7 +79,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -201,6 +200,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -247,7 +247,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -530,6 +529,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -594,6 +594,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 73293a0b3dc8..3490a05f29b8 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -75,7 +75,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -197,6 +196,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -243,7 +243,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -514,6 +513,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -577,6 +577,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index bca8a6f3e92f..4e92c8c332fc 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -75,7 +75,6 @@ CONFIG_IPV6_VTI=m CONFIG_IPV6_GRE=m CONFIG_NETFILTER=y CONFIG_NF_CONNTRACK=m -CONFIG_NF_LOG_NETDEV=m CONFIG_NF_CONNTRACK_ZONES=y # CONFIG_NF_CONNTRACK_PROCFS is not set # CONFIG_NF_CT_PROTO_DCCP is not set @@ -197,6 +196,7 @@ CONFIG_NFT_FIB_IPV4=m CONFIG_NF_TABLES_ARP=y CONFIG_NF_FLOW_TABLE_IPV4=m CONFIG_NF_LOG_ARP=m +CONFIG_NF_LOG_IPV4=m CONFIG_IP_NF_IPTABLES=m CONFIG_IP_NF_MATCH_AH=m CONFIG_IP_NF_MATCH_ECN=m @@ -243,7 +243,6 @@ CONFIG_IP6_NF_TARGET_NPT=m CONFIG_NF_TABLES_BRIDGE=m CONFIG_NFT_BRIDGE_META=m CONFIG_NFT_BRIDGE_REJECT=m -CONFIG_NF_LOG_BRIDGE=m CONFIG_NF_CONNTRACK_BRIDGE=m CONFIG_BRIDGE_NF_EBTABLES=m CONFIG_BRIDGE_EBT_BROUTE=m @@ -513,6 +512,7 @@ CONFIG_CRYPTO_TEST=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m +CONFIG_CRYPTO_ECDSA=m CONFIG_CRYPTO_ECRDSA=m CONFIG_CRYPTO_SM2=m CONFIG_CRYPTO_CURVE25519=m @@ -577,6 +577,7 @@ CONFIG_KUNIT_ALL_TESTS=m CONFIG_TEST_LIST_SORT=m CONFIG_TEST_MIN_HEAP=m CONFIG_TEST_SORT=m +CONFIG_TEST_DIV64=m CONFIG_REED_SOLOMON_TEST=m CONFIG_ATOMIC64_SELFTEST=m CONFIG_ASYNC_RAID6_TEST=m diff --git a/arch/m68k/include/asm/page_no.h b/arch/m68k/include/asm/page_no.h index 8d0f862ee9d7..c9d0d84158a4 100644 --- a/arch/m68k/include/asm/page_no.h +++ b/arch/m68k/include/asm/page_no.h @@ -13,9 +13,9 @@ extern unsigned long memory_end; #define clear_user_page(page, vaddr, pg) clear_page(page) #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) -#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ - alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) -#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE #define __pa(vaddr) ((unsigned long)(vaddr)) #define __va(paddr) ((void *)((unsigned long)(paddr))) diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c index 1c1b875fadc1..2e192a5df949 100644 --- a/arch/m68k/kernel/dma.c +++ b/arch/m68k/kernel/dma.c @@ -34,9 +34,6 @@ pgprot_t pgprot_dmacoherent(pgprot_t prot) return prot; } #else - -#include - void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index fca4547d580f..696f6b009377 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -109,10 +109,11 @@ static inline bool kvm_is_error_hva(unsigned long addr) } struct kvm_vm_stat { - ulong remote_tlb_flush; + struct kvm_vm_stat_generic generic; }; struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; u64 wait_exits; u64 cache_exits; u64 signal_exits; @@ -142,12 +143,6 @@ struct kvm_vcpu_stat { #ifdef CONFIG_CPU_LOONGSON64 u64 vz_cpucfg_exits; #endif - u64 halt_successful_poll; - u64 halt_attempted_poll; - u64 halt_poll_success_ns; - u64 halt_poll_fail_ns; - u64 halt_poll_invalid; - u64 halt_wakeup; }; struct kvm_arch_memory_slot { diff --git a/arch/mips/kvm/Makefile b/arch/mips/kvm/Makefile index 30cc060857c7..c67250a956b8 100644 --- a/arch/mips/kvm/Makefile +++ b/arch/mips/kvm/Makefile @@ -2,7 +2,7 @@ # Makefile for KVM support for MIPS # -common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o eventfd.o) +common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o eventfd.o binary_stats.o) EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 4d4af97dcc88..af9dd029a4e1 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -38,43 +38,63 @@ #define VECTORSPACING 0x100 /* for EI/VI mode */ #endif -struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT("wait", wait_exits), - VCPU_STAT("cache", cache_exits), - VCPU_STAT("signal", signal_exits), - VCPU_STAT("interrupt", int_exits), - VCPU_STAT("cop_unusable", cop_unusable_exits), - VCPU_STAT("tlbmod", tlbmod_exits), - VCPU_STAT("tlbmiss_ld", tlbmiss_ld_exits), - VCPU_STAT("tlbmiss_st", tlbmiss_st_exits), - VCPU_STAT("addrerr_st", addrerr_st_exits), - VCPU_STAT("addrerr_ld", addrerr_ld_exits), - VCPU_STAT("syscall", syscall_exits), - VCPU_STAT("resvd_inst", resvd_inst_exits), - VCPU_STAT("break_inst", break_inst_exits), - VCPU_STAT("trap_inst", trap_inst_exits), - VCPU_STAT("msa_fpe", msa_fpe_exits), - VCPU_STAT("fpe", fpe_exits), - VCPU_STAT("msa_disabled", msa_disabled_exits), - VCPU_STAT("flush_dcache", flush_dcache_exits), - VCPU_STAT("vz_gpsi", vz_gpsi_exits), - VCPU_STAT("vz_gsfc", vz_gsfc_exits), - VCPU_STAT("vz_hc", vz_hc_exits), - VCPU_STAT("vz_grr", vz_grr_exits), - VCPU_STAT("vz_gva", vz_gva_exits), - VCPU_STAT("vz_ghfc", vz_ghfc_exits), - VCPU_STAT("vz_gpa", vz_gpa_exits), - VCPU_STAT("vz_resvd", vz_resvd_exits), +const struct _kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS() +}; +static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == + sizeof(struct kvm_vm_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + STATS_DESC_COUNTER(VCPU, wait_exits), + STATS_DESC_COUNTER(VCPU, cache_exits), + STATS_DESC_COUNTER(VCPU, signal_exits), + STATS_DESC_COUNTER(VCPU, int_exits), + STATS_DESC_COUNTER(VCPU, cop_unusable_exits), + STATS_DESC_COUNTER(VCPU, tlbmod_exits), + STATS_DESC_COUNTER(VCPU, tlbmiss_ld_exits), + STATS_DESC_COUNTER(VCPU, tlbmiss_st_exits), + STATS_DESC_COUNTER(VCPU, addrerr_st_exits), + STATS_DESC_COUNTER(VCPU, addrerr_ld_exits), + STATS_DESC_COUNTER(VCPU, syscall_exits), + STATS_DESC_COUNTER(VCPU, resvd_inst_exits), + STATS_DESC_COUNTER(VCPU, break_inst_exits), + STATS_DESC_COUNTER(VCPU, trap_inst_exits), + STATS_DESC_COUNTER(VCPU, msa_fpe_exits), + STATS_DESC_COUNTER(VCPU, fpe_exits), + STATS_DESC_COUNTER(VCPU, msa_disabled_exits), + STATS_DESC_COUNTER(VCPU, flush_dcache_exits), + STATS_DESC_COUNTER(VCPU, vz_gpsi_exits), + STATS_DESC_COUNTER(VCPU, vz_gsfc_exits), + STATS_DESC_COUNTER(VCPU, vz_hc_exits), + STATS_DESC_COUNTER(VCPU, vz_grr_exits), + STATS_DESC_COUNTER(VCPU, vz_gva_exits), + STATS_DESC_COUNTER(VCPU, vz_ghfc_exits), + STATS_DESC_COUNTER(VCPU, vz_gpa_exits), + STATS_DESC_COUNTER(VCPU, vz_resvd_exits), #ifdef CONFIG_CPU_LOONGSON64 - VCPU_STAT("vz_cpucfg", vz_cpucfg_exits), + STATS_DESC_COUNTER(VCPU, vz_cpucfg_exits), #endif - VCPU_STAT("halt_successful_poll", halt_successful_poll), - VCPU_STAT("halt_attempted_poll", halt_attempted_poll), - VCPU_STAT("halt_poll_invalid", halt_poll_invalid), - VCPU_STAT("halt_wakeup", halt_wakeup), - VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), - VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), - {NULL} +}; +static_assert(ARRAY_SIZE(kvm_vcpu_stats_desc) == + sizeof(struct kvm_vcpu_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), }; bool kvm_trace_guest_mode_change; diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h index 1c7b75834e04..02ee6f5ac9fe 100644 --- a/arch/powerpc/include/asm/asm-prototypes.h +++ b/arch/powerpc/include/asm/asm-prototypes.h @@ -120,6 +120,7 @@ extern s32 patch__call_flush_branch_caches3; extern s32 patch__flush_count_cache_return; extern s32 patch__flush_link_stack_return; extern s32 patch__call_kvm_flush_link_stack; +extern s32 patch__call_kvm_flush_link_stack_p9; extern s32 patch__memset_nocache, patch__memcpy_nocache; extern long flush_branch_caches; @@ -140,7 +141,7 @@ void kvmhv_load_host_pmu(void); void kvmhv_save_guest_pmu(struct kvm_vcpu *vcpu, bool pmu_in_use); void kvmhv_load_guest_pmu(struct kvm_vcpu *vcpu); -int __kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu); +void kvmppc_p9_enter_guest(struct kvm_vcpu *vcpu); long kvmppc_h_set_dabr(struct kvm_vcpu *vcpu, unsigned long dabr); long kvmppc_h_set_xdabr(struct kvm_vcpu *vcpu, unsigned long dabr, diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h index eace8c3f7b0a..c02f42d1031e 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu.h +++ b/arch/powerpc/include/asm/book3s/64/mmu.h @@ -19,6 +19,7 @@ struct mmu_psize_def { int penc[MMU_PAGE_COUNT]; /* HPTE encoding */ unsigned int tlbiel; /* tlbiel supported for that page size */ unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */ + unsigned long h_rpt_pgsize; /* H_RPT_INVALIDATE page size encoding */ union { unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */ unsigned long ap; /* Ap encoding used by PowerISA 3.0 */ diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index 8b33601cdb9d..a46fd37ad552 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -4,6 +4,10 @@ #include +#define RIC_FLUSH_TLB 0 +#define RIC_FLUSH_PWC 1 +#define RIC_FLUSH_ALL 2 + struct vm_area_struct; struct mm_struct; struct mmu_gather; diff --git a/arch/powerpc/include/asm/cputhreads.h b/arch/powerpc/include/asm/cputhreads.h index 98c8bd155bf9..b167186aaee4 100644 --- a/arch/powerpc/include/asm/cputhreads.h +++ b/arch/powerpc/include/asm/cputhreads.h @@ -98,6 +98,36 @@ static inline int cpu_last_thread_sibling(int cpu) return cpu | (threads_per_core - 1); } +/* + * tlb_thread_siblings are siblings which share a TLB. This is not + * architected, is not something a hypervisor could emulate and a future + * CPU may change behaviour even in compat mode, so this should only be + * used on PowerNV, and only with care. + */ +static inline int cpu_first_tlb_thread_sibling(int cpu) +{ + if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8)) + return cpu & ~0x6; /* Big Core */ + else + return cpu_first_thread_sibling(cpu); +} + +static inline int cpu_last_tlb_thread_sibling(int cpu) +{ + if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8)) + return cpu | 0x6; /* Big Core */ + else + return cpu_last_thread_sibling(cpu); +} + +static inline int cpu_tlb_thread_sibling_step(void) +{ + if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8)) + return 2; /* Big Core */ + else + return 1; +} + static inline u32 get_tensr(void) { #ifdef CONFIG_BOOKE diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index c1a8aac01cf9..bb6f78fcf981 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -35,6 +35,19 @@ /* PACA save area size in u64 units (exgen, exmc, etc) */ #define EX_SIZE 10 +/* PACA save area offsets */ +#define EX_R9 0 +#define EX_R10 8 +#define EX_R11 16 +#define EX_R12 24 +#define EX_R13 32 +#define EX_DAR 40 +#define EX_DSISR 48 +#define EX_CCR 52 +#define EX_CFAR 56 +#define EX_PPR 64 +#define EX_CTR 72 + /* * maximum recursive depth of MCE exceptions */ diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index e3b29eda8074..7e4b2cef40c2 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -413,9 +413,9 @@ #define H_RPTI_TYPE_NESTED 0x0001 /* Invalidate nested guest partition-scope */ #define H_RPTI_TYPE_TLB 0x0002 /* Invalidate TLB */ #define H_RPTI_TYPE_PWC 0x0004 /* Invalidate Page Walk Cache */ -/* Invalidate Process Table Entries if H_RPTI_TYPE_NESTED is clear */ +/* Invalidate caching of Process Table Entries if H_RPTI_TYPE_NESTED is clear */ #define H_RPTI_TYPE_PRT 0x0008 -/* Invalidate Partition Table Entries if H_RPTI_TYPE_NESTED is set */ +/* Invalidate caching of Partition Table Entries if H_RPTI_TYPE_NESTED is set */ #define H_RPTI_TYPE_PAT 0x0008 #define H_RPTI_TYPE_ALL (H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | \ H_RPTI_TYPE_PRT) diff --git a/arch/powerpc/include/asm/kvm_asm.h b/arch/powerpc/include/asm/kvm_asm.h index a3633560493b..fbbf3cec92e9 100644 --- a/arch/powerpc/include/asm/kvm_asm.h +++ b/arch/powerpc/include/asm/kvm_asm.h @@ -147,6 +147,7 @@ #define KVM_GUEST_MODE_SKIP 2 #define KVM_GUEST_MODE_GUEST_HV 3 #define KVM_GUEST_MODE_HOST_HV 4 +#define KVM_GUEST_MODE_HV_P9 5 /* ISA >= v3.0 path */ #define KVM_INST_FETCH_FAILED -1 diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index e6b53c6e21e3..caaa0f592d8e 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -307,6 +307,9 @@ void kvmhv_set_ptbl_entry(unsigned int lpid, u64 dw0, u64 dw1); void kvmhv_release_all_nested(struct kvm *kvm); long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu); long kvmhv_do_nested_tlbie(struct kvm_vcpu *vcpu); +long do_h_rpt_invalidate_pat(struct kvm_vcpu *vcpu, unsigned long lpid, + unsigned long type, unsigned long pg_sizes, + unsigned long start, unsigned long end); int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr); void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr); diff --git a/arch/powerpc/include/asm/kvm_book3s_64.h b/arch/powerpc/include/asm/kvm_book3s_64.h index 9bb9bb370b53..eaf3a562bf1e 100644 --- a/arch/powerpc/include/asm/kvm_book3s_64.h +++ b/arch/powerpc/include/asm/kvm_book3s_64.h @@ -153,9 +153,17 @@ static inline bool kvmhv_vcpu_is_radix(struct kvm_vcpu *vcpu) return radix; } +int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr); + #define KVM_DEFAULT_HPT_ORDER 24 /* 16MB HPT by default */ #endif +/* + * Invalid HDSISR value which is used to indicate when HW has not set the reg. + * Used to work around an errata. + */ +#define HDSISR_CANARY 0x7fff + /* * We use a lock bit in HPTE dword 0 to synchronize updates and * accesses to each HPTE, and another bit to indicate non-present diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 7f2e90db2050..9f52f282b1aa 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -81,12 +81,13 @@ struct kvmppc_book3s_shadow_vcpu; struct kvm_nested_guest; struct kvm_vm_stat { - ulong remote_tlb_flush; - ulong num_2M_pages; - ulong num_1G_pages; + struct kvm_vm_stat_generic generic; + u64 num_2M_pages; + u64 num_1G_pages; }; struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; u64 sum_exits; u64 mmio_exits; u64 signal_exits; @@ -102,14 +103,8 @@ struct kvm_vcpu_stat { u64 emulated_inst_exits; u64 dec_exits; u64 ext_intr_exits; - u64 halt_poll_success_ns; - u64 halt_poll_fail_ns; u64 halt_wait_ns; - u64 halt_successful_poll; - u64 halt_attempted_poll; u64 halt_successful_wait; - u64 halt_poll_invalid; - u64 halt_wakeup; u64 dbell_exits; u64 gdbell_exits; u64 ld; @@ -298,7 +293,6 @@ struct kvm_arch { u8 fwnmi_enabled; u8 secure_guest; u8 svm_enabled; - bool threads_indep; bool nested_enable; bool dawr1_enabled; pgd_t *pgtable; @@ -684,7 +678,12 @@ struct kvm_vcpu_arch { ulong fault_dar; u32 fault_dsisr; unsigned long intr_msr; - ulong fault_gpa; /* guest real address of page fault (POWER9) */ + /* + * POWER9 and later: fault_gpa contains the guest real address of page + * fault for a radix guest, or segment descriptor (equivalent to result + * from slbmfev of SLB entry that translated the EA) for hash guests. + */ + ulong fault_gpa; #endif #ifdef CONFIG_BOOKE diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 5bf8ae9bb2cc..2d88944f9f34 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -129,6 +129,7 @@ extern void kvmppc_core_vcpu_put(struct kvm_vcpu *vcpu); extern int kvmppc_core_prepare_to_enter(struct kvm_vcpu *vcpu); extern int kvmppc_core_pending_dec(struct kvm_vcpu *vcpu); extern void kvmppc_core_queue_machine_check(struct kvm_vcpu *vcpu, ulong flags); +extern void kvmppc_core_queue_syscall(struct kvm_vcpu *vcpu); extern void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags); extern void kvmppc_core_queue_fpunavail(struct kvm_vcpu *vcpu); extern void kvmppc_core_queue_vec_unavail(struct kvm_vcpu *vcpu); @@ -606,6 +607,7 @@ extern void kvmppc_free_pimap(struct kvm *kvm); extern int kvmppc_xics_rm_complete(struct kvm_vcpu *vcpu, u32 hcall); extern void kvmppc_xics_free_icp(struct kvm_vcpu *vcpu); extern int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd); +extern int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req); extern u64 kvmppc_xics_get_icp(struct kvm_vcpu *vcpu); extern int kvmppc_xics_set_icp(struct kvm_vcpu *vcpu, u64 icpval); extern int kvmppc_xics_connect_vcpu(struct kvm_device *dev, @@ -638,6 +640,8 @@ static inline int kvmppc_xics_enabled(struct kvm_vcpu *vcpu) static inline void kvmppc_xics_free_icp(struct kvm_vcpu *vcpu) { } static inline int kvmppc_xics_hcall(struct kvm_vcpu *vcpu, u32 cmd) { return 0; } +static inline int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req) + { return 0; } #endif #ifdef CONFIG_KVM_XIVE @@ -655,8 +659,6 @@ extern int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority); extern int kvmppc_xive_int_on(struct kvm *kvm, u32 irq); extern int kvmppc_xive_int_off(struct kvm *kvm, u32 irq); -extern void kvmppc_xive_init_module(void); -extern void kvmppc_xive_exit_module(void); extern int kvmppc_xive_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu, u32 cpu); @@ -671,6 +673,8 @@ extern int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval); extern int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, bool line_status); extern void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu); +extern void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu); +extern void kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu); static inline int kvmppc_xive_enabled(struct kvm_vcpu *vcpu) { @@ -680,8 +684,6 @@ static inline int kvmppc_xive_enabled(struct kvm_vcpu *vcpu) extern int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu, u32 cpu); extern void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu); -extern void kvmppc_xive_native_init_module(void); -extern void kvmppc_xive_native_exit_module(void); extern int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val); extern int kvmppc_xive_native_set_vp(struct kvm_vcpu *vcpu, @@ -695,8 +697,6 @@ static inline int kvmppc_xive_get_xive(struct kvm *kvm, u32 irq, u32 *server, u32 *priority) { return -1; } static inline int kvmppc_xive_int_on(struct kvm *kvm, u32 irq) { return -1; } static inline int kvmppc_xive_int_off(struct kvm *kvm, u32 irq) { return -1; } -static inline void kvmppc_xive_init_module(void) { } -static inline void kvmppc_xive_exit_module(void) { } static inline int kvmppc_xive_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu, u32 cpu) { return -EBUSY; } @@ -711,14 +711,14 @@ static inline int kvmppc_xive_set_icp(struct kvm_vcpu *vcpu, u64 icpval) { retur static inline int kvmppc_xive_set_irq(struct kvm *kvm, int irq_source_id, u32 irq, int level, bool line_status) { return -ENODEV; } static inline void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) { } +static inline void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu) { } +static inline void kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu) { } static inline int kvmppc_xive_enabled(struct kvm_vcpu *vcpu) { return 0; } static inline int kvmppc_xive_native_connect_vcpu(struct kvm_device *dev, struct kvm_vcpu *vcpu, u32 cpu) { return -EBUSY; } static inline void kvmppc_xive_native_cleanup_vcpu(struct kvm_vcpu *vcpu) { } -static inline void kvmppc_xive_native_init_module(void) { } -static inline void kvmppc_xive_native_exit_module(void) { } static inline int kvmppc_xive_native_get_vp(struct kvm_vcpu *vcpu, union kvmppc_one_reg *val) { return 0; } @@ -754,7 +754,7 @@ long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu, unsigned long tce_value, unsigned long npages); long int kvmppc_rm_h_confer(struct kvm_vcpu *vcpu, int target, unsigned int yield_count); -long kvmppc_h_random(struct kvm_vcpu *vcpu); +long kvmppc_rm_h_random(struct kvm_vcpu *vcpu); void kvmhv_commence_exit(int trap); void kvmppc_realmode_machine_check(struct kvm_vcpu *vcpu); void kvmppc_subcore_enter_guest(void); diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index 4bc45d3ed8b0..db186c539d37 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -122,12 +122,6 @@ static inline bool need_extra_context(struct mm_struct *mm, unsigned long ea) } #endif -#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && defined(CONFIG_PPC_RADIX_MMU) -extern void radix_kvm_prefetch_workaround(struct mm_struct *mm); -#else -static inline void radix_kvm_prefetch_workaround(struct mm_struct *mm) { } -#endif - extern void switch_cop(struct mm_struct *next); extern int use_cop(unsigned long acop, struct mm_struct *mm); extern void drop_cop(unsigned long acop, struct mm_struct *mm); @@ -222,6 +216,18 @@ static inline void mm_context_add_copro(struct mm_struct *mm) { } static inline void mm_context_remove_copro(struct mm_struct *mm) { } #endif +#if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE) && defined(CONFIG_PPC_RADIX_MMU) +void do_h_rpt_invalidate_prt(unsigned long pid, unsigned long lpid, + unsigned long type, unsigned long pg_sizes, + unsigned long start, unsigned long end); +#else +static inline void do_h_rpt_invalidate_prt(unsigned long pid, + unsigned long lpid, + unsigned long type, + unsigned long pg_sizes, + unsigned long start, + unsigned long end) { } +#endif extern void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk); diff --git a/arch/powerpc/include/asm/time.h b/arch/powerpc/include/asm/time.h index 8dd3cdb25338..8c2c3dd4ddba 100644 --- a/arch/powerpc/include/asm/time.h +++ b/arch/powerpc/include/asm/time.h @@ -97,6 +97,18 @@ extern void div128_by_32(u64 dividend_high, u64 dividend_low, extern void secondary_cpu_time_init(void); extern void __init time_init(void); +#ifdef CONFIG_PPC64 +static inline unsigned long test_irq_work_pending(void) +{ + unsigned long x; + + asm volatile("lbz %0,%1(13)" + : "=r" (x) + : "i" (offsetof(struct paca_struct, irq_work_pending))); + return x; +} +#endif + DECLARE_PER_CPU(u64, decrementers_next_tb); /* Convert timebase ticks to nanoseconds */ diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 28af4efb4587..aa267d173ded 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -534,7 +534,6 @@ int main(void) OFFSET(VCPU_SLB_NR, kvm_vcpu, arch.slb_nr); OFFSET(VCPU_FAULT_DSISR, kvm_vcpu, arch.fault_dsisr); OFFSET(VCPU_FAULT_DAR, kvm_vcpu, arch.fault_dar); - OFFSET(VCPU_FAULT_GPA, kvm_vcpu, arch.fault_gpa); OFFSET(VCPU_INTR_MSR, kvm_vcpu, arch.intr_msr); OFFSET(VCPU_LAST_INST, kvm_vcpu, arch.last_inst); OFFSET(VCPU_TRAP, kvm_vcpu, arch.trap); diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index fa8e52a0239e..f7fc6e078d4e 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -21,22 +21,6 @@ #include #include -/* PACA save area offsets (exgen, exmc, etc) */ -#define EX_R9 0 -#define EX_R10 8 -#define EX_R11 16 -#define EX_R12 24 -#define EX_R13 32 -#define EX_DAR 40 -#define EX_DSISR 48 -#define EX_CCR 52 -#define EX_CFAR 56 -#define EX_PPR 64 -#define EX_CTR 72 -.if EX_SIZE != 10 - .error "EX_SIZE is wrong" -.endif - /* * Following are fixed section helper macros. * @@ -133,7 +117,6 @@ name: #define IBRANCH_TO_COMMON .L_IBRANCH_TO_COMMON_\name\() /* ENTRY branch to common */ #define IREALMODE_COMMON .L_IREALMODE_COMMON_\name\() /* Common runs in realmode */ #define IMASK .L_IMASK_\name\() /* IRQ soft-mask bit */ -#define IKVM_SKIP .L_IKVM_SKIP_\name\() /* Generate KVM skip handler */ #define IKVM_REAL .L_IKVM_REAL_\name\() /* Real entry tests KVM */ #define __IKVM_REAL(name) .L_IKVM_REAL_ ## name #define IKVM_VIRT .L_IKVM_VIRT_\name\() /* Virt entry tests KVM */ @@ -190,9 +173,6 @@ do_define_int n .ifndef IMASK IMASK=0 .endif - .ifndef IKVM_SKIP - IKVM_SKIP=0 - .endif .ifndef IKVM_REAL IKVM_REAL=0 .endif @@ -207,8 +187,6 @@ do_define_int n .endif .endm -#ifdef CONFIG_KVM_BOOK3S_64_HANDLER -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE /* * All interrupts which set HSRR registers, as well as SRESET and MCE and * syscall when invoked with "sc 1" switch to MSR[HV]=1 (HVMODE) to be taken, @@ -238,88 +216,28 @@ do_define_int n /* * If an interrupt is taken while a guest is running, it is immediately routed - * to KVM to handle. If both HV and PR KVM arepossible, KVM interrupts go first - * to kvmppc_interrupt_hv, which handles the PR guest case. + * to KVM to handle. */ -#define kvmppc_interrupt kvmppc_interrupt_hv -#else -#define kvmppc_interrupt kvmppc_interrupt_pr -#endif -.macro KVMTEST name +.macro KVMTEST name handler +#ifdef CONFIG_KVM_BOOK3S_64_HANDLER lbz r10,HSTATE_IN_GUEST(r13) cmpwi r10,0 - bne \name\()_kvm -.endm - -.macro GEN_KVM name - .balign IFETCH_ALIGN_BYTES -\name\()_kvm: - - .if IKVM_SKIP - cmpwi r10,KVM_GUEST_MODE_SKIP - beq 89f - .else -BEGIN_FTR_SECTION - ld r10,IAREA+EX_CFAR(r13) - std r10,HSTATE_CFAR(r13) -END_FTR_SECTION_IFSET(CPU_FTR_CFAR) - .endif - - ld r10,IAREA+EX_CTR(r13) - mtctr r10 -BEGIN_FTR_SECTION - ld r10,IAREA+EX_PPR(r13) - std r10,HSTATE_PPR(r13) -END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) - ld r11,IAREA+EX_R11(r13) - ld r12,IAREA+EX_R12(r13) - std r12,HSTATE_SCRATCH0(r13) - sldi r12,r9,32 - ld r9,IAREA+EX_R9(r13) - ld r10,IAREA+EX_R10(r13) /* HSRR variants have the 0x2 bit added to their trap number */ .if IHSRR_IF_HVMODE BEGIN_FTR_SECTION - ori r12,r12,(IVEC + 0x2) + li r10,(IVEC + 0x2) FTR_SECTION_ELSE - ori r12,r12,(IVEC) + li r10,(IVEC) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) .elseif IHSRR - ori r12,r12,(IVEC+ 0x2) + li r10,(IVEC + 0x2) .else - ori r12,r12,(IVEC) + li r10,(IVEC) .endif - b kvmppc_interrupt - - .if IKVM_SKIP -89: mtocrf 0x80,r9 - ld r10,IAREA+EX_CTR(r13) - mtctr r10 - ld r9,IAREA+EX_R9(r13) - ld r10,IAREA+EX_R10(r13) - ld r11,IAREA+EX_R11(r13) - ld r12,IAREA+EX_R12(r13) - .if IHSRR_IF_HVMODE - BEGIN_FTR_SECTION - b kvmppc_skip_Hinterrupt - FTR_SECTION_ELSE - b kvmppc_skip_interrupt - ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) - .elseif IHSRR - b kvmppc_skip_Hinterrupt - .else - b kvmppc_skip_interrupt - .endif - .endif -.endm - -#else -.macro KVMTEST name -.endm -.macro GEN_KVM name -.endm + bne \handler #endif +.endm /* * This is the BOOK3S interrupt entry code macro. @@ -461,7 +379,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR) DEFINE_FIXED_SYMBOL(\name\()_common_real) \name\()_common_real: .if IKVM_REAL - KVMTEST \name + KVMTEST \name kvm_interrupt .endif ld r10,PACAKMSR(r13) /* get MSR value for kernel */ @@ -484,7 +402,7 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real) DEFINE_FIXED_SYMBOL(\name\()_common_virt) \name\()_common_virt: .if IKVM_VIRT - KVMTEST \name + KVMTEST \name kvm_interrupt 1: .endif .endif /* IVIRT */ @@ -498,7 +416,7 @@ DEFINE_FIXED_SYMBOL(\name\()_common_virt) DEFINE_FIXED_SYMBOL(\name\()_common_real) \name\()_common_real: .if IKVM_REAL - KVMTEST \name + KVMTEST \name kvm_interrupt .endif .endm @@ -1000,8 +918,6 @@ EXC_COMMON_BEGIN(system_reset_common) EXCEPTION_RESTORE_REGS RFI_TO_USER_OR_KERNEL - GEN_KVM system_reset - /** * Interrupt 0x200 - Machine Check Interrupt (MCE). @@ -1070,7 +986,6 @@ INT_DEFINE_BEGIN(machine_check) ISET_RI=0 IDAR=1 IDSISR=1 - IKVM_SKIP=1 IKVM_REAL=1 INT_DEFINE_END(machine_check) @@ -1166,7 +1081,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) /* * Check if we are coming from guest. If yes, then run the normal * exception handler which will take the - * machine_check_kvm->kvmppc_interrupt branch to deliver the MC event + * machine_check_kvm->kvm_interrupt branch to deliver the MC event * to guest. */ lbz r11,HSTATE_IN_GUEST(r13) @@ -1236,8 +1151,6 @@ EXC_COMMON_BEGIN(machine_check_common) bl machine_check_exception b interrupt_return - GEN_KVM machine_check - #ifdef CONFIG_PPC_P7_NAP /* @@ -1342,7 +1255,6 @@ INT_DEFINE_BEGIN(data_access) IVEC=0x300 IDAR=1 IDSISR=1 - IKVM_SKIP=1 IKVM_REAL=1 INT_DEFINE_END(data_access) @@ -1373,8 +1285,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) REST_NVGPRS(r1) b interrupt_return - GEN_KVM data_access - /** * Interrupt 0x380 - Data Segment Interrupt (DSLB). @@ -1396,7 +1306,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) INT_DEFINE_BEGIN(data_access_slb) IVEC=0x380 IDAR=1 - IKVM_SKIP=1 IKVM_REAL=1 INT_DEFINE_END(data_access_slb) @@ -1425,8 +1334,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) bl do_bad_slb_fault b interrupt_return - GEN_KVM data_access_slb - /** * Interrupt 0x400 - Instruction Storage Interrupt (ISI). @@ -1463,8 +1370,6 @@ MMU_FTR_SECTION_ELSE ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) b interrupt_return - GEN_KVM instruction_access - /** * Interrupt 0x480 - Instruction Segment Interrupt (ISLB). @@ -1509,8 +1414,6 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX) bl do_bad_slb_fault b interrupt_return - GEN_KVM instruction_access_slb - /** * Interrupt 0x500 - External Interrupt. @@ -1555,8 +1458,6 @@ EXC_COMMON_BEGIN(hardware_interrupt_common) bl do_IRQ b interrupt_return - GEN_KVM hardware_interrupt - /** * Interrupt 0x600 - Alignment Interrupt @@ -1584,8 +1485,6 @@ EXC_COMMON_BEGIN(alignment_common) REST_NVGPRS(r1) /* instruction emulation may change GPRs */ b interrupt_return - GEN_KVM alignment - /** * Interrupt 0x700 - Program Interrupt (program check). @@ -1693,8 +1592,6 @@ EXC_COMMON_BEGIN(program_check_common) REST_NVGPRS(r1) /* instruction emulation may change GPRs */ b interrupt_return - GEN_KVM program_check - /* * Interrupt 0x800 - Floating-Point Unavailable Interrupt. @@ -1744,8 +1641,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM) b interrupt_return #endif - GEN_KVM fp_unavailable - /** * Interrupt 0x900 - Decrementer Interrupt. @@ -1784,8 +1679,6 @@ EXC_COMMON_BEGIN(decrementer_common) bl timer_interrupt b interrupt_return - GEN_KVM decrementer - /** * Interrupt 0x980 - Hypervisor Decrementer Interrupt. @@ -1831,8 +1724,6 @@ EXC_COMMON_BEGIN(hdecrementer_common) ld r13,PACA_EXGEN+EX_R13(r13) HRFI_TO_KERNEL - GEN_KVM hdecrementer - /** * Interrupt 0xa00 - Directed Privileged Doorbell Interrupt. @@ -1872,8 +1763,6 @@ EXC_COMMON_BEGIN(doorbell_super_common) #endif b interrupt_return - GEN_KVM doorbell_super - EXC_REAL_NONE(0xb00, 0x100) EXC_VIRT_NONE(0x4b00, 0x100) @@ -1923,7 +1812,7 @@ INT_DEFINE_END(system_call) GET_PACA(r13) std r10,PACA_EXGEN+EX_R10(r13) INTERRUPT_TO_KERNEL - KVMTEST system_call /* uses r10, branch to system_call_kvm */ + KVMTEST system_call kvm_hcall /* uses r10, branch to kvm_hcall */ mfctr r9 #else mr r9,r13 @@ -1979,14 +1868,16 @@ EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100) EXC_VIRT_END(system_call, 0x4c00, 0x100) #ifdef CONFIG_KVM_BOOK3S_64_HANDLER -TRAMP_REAL_BEGIN(system_call_kvm) - /* - * This is a hcall, so register convention is as above, with these - * differences: - * r13 = PACA - * ctr = orig r13 - * orig r10 saved in PACA - */ +TRAMP_REAL_BEGIN(kvm_hcall) + std r9,PACA_EXGEN+EX_R9(r13) + std r11,PACA_EXGEN+EX_R11(r13) + std r12,PACA_EXGEN+EX_R12(r13) + mfcr r9 + mfctr r10 + std r10,PACA_EXGEN+EX_R13(r13) + li r10,0 + std r10,PACA_EXGEN+EX_CFAR(r13) + std r10,PACA_EXGEN+EX_CTR(r13) /* * Save the PPR (on systems that support it) before changing to * HMT_MEDIUM. That allows the KVM code to save that value into the @@ -1994,31 +1885,24 @@ TRAMP_REAL_BEGIN(system_call_kvm) */ BEGIN_FTR_SECTION mfspr r10,SPRN_PPR - std r10,HSTATE_PPR(r13) + std r10,PACA_EXGEN+EX_PPR(r13) END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) + HMT_MEDIUM - mfctr r10 - SET_SCRATCH0(r10) - mfcr r10 - std r12,HSTATE_SCRATCH0(r13) - sldi r12,r10,32 - ori r12,r12,0xc00 + #ifdef CONFIG_RELOCATABLE /* - * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives + * Requires __LOAD_FAR_HANDLER beause kvmppc_hcall lives * outside the head section. */ - __LOAD_FAR_HANDLER(r10, kvmppc_interrupt) + __LOAD_FAR_HANDLER(r10, kvmppc_hcall) mtctr r10 - ld r10,PACA_EXGEN+EX_R10(r13) bctr #else - ld r10,PACA_EXGEN+EX_R10(r13) - b kvmppc_interrupt + b kvmppc_hcall #endif #endif - /** * Interrupt 0xd00 - Trace Interrupt. * This is a synchronous interrupt in response to instruction step or @@ -2043,8 +1927,6 @@ EXC_COMMON_BEGIN(single_step_common) bl single_step_exception b interrupt_return - GEN_KVM single_step - /** * Interrupt 0xe00 - Hypervisor Data Storage Interrupt (HDSI). @@ -2063,7 +1945,6 @@ INT_DEFINE_BEGIN(h_data_storage) IHSRR=1 IDAR=1 IDSISR=1 - IKVM_SKIP=1 IKVM_REAL=1 IKVM_VIRT=1 INT_DEFINE_END(h_data_storage) @@ -2084,8 +1965,6 @@ MMU_FTR_SECTION_ELSE ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX) b interrupt_return - GEN_KVM h_data_storage - /** * Interrupt 0xe20 - Hypervisor Instruction Storage Interrupt (HISI). @@ -2111,8 +1990,6 @@ EXC_COMMON_BEGIN(h_instr_storage_common) bl unknown_exception b interrupt_return - GEN_KVM h_instr_storage - /** * Interrupt 0xe40 - Hypervisor Emulation Assistance Interrupt. @@ -2137,8 +2014,6 @@ EXC_COMMON_BEGIN(emulation_assist_common) REST_NVGPRS(r1) /* instruction emulation may change GPRs */ b interrupt_return - GEN_KVM emulation_assist - /** * Interrupt 0xe60 - Hypervisor Maintenance Interrupt (HMI). @@ -2210,16 +2085,12 @@ EXC_COMMON_BEGIN(hmi_exception_early_common) EXCEPTION_RESTORE_REGS hsrr=1 GEN_INT_ENTRY hmi_exception, virt=0 - GEN_KVM hmi_exception_early - EXC_COMMON_BEGIN(hmi_exception_common) GEN_COMMON hmi_exception addi r3,r1,STACK_FRAME_OVERHEAD bl handle_hmi_exception b interrupt_return - GEN_KVM hmi_exception - /** * Interrupt 0xe80 - Directed Hypervisor Doorbell Interrupt. @@ -2250,8 +2121,6 @@ EXC_COMMON_BEGIN(h_doorbell_common) #endif b interrupt_return - GEN_KVM h_doorbell - /** * Interrupt 0xea0 - Hypervisor Virtualization Interrupt. @@ -2278,8 +2147,6 @@ EXC_COMMON_BEGIN(h_virt_irq_common) bl do_IRQ b interrupt_return - GEN_KVM h_virt_irq - EXC_REAL_NONE(0xec0, 0x20) EXC_VIRT_NONE(0x4ec0, 0x20) @@ -2323,8 +2190,6 @@ EXC_COMMON_BEGIN(performance_monitor_common) bl performance_monitor_exception b interrupt_return - GEN_KVM performance_monitor - /** * Interrupt 0xf20 - Vector Unavailable Interrupt. @@ -2374,8 +2239,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) bl altivec_unavailable_exception b interrupt_return - GEN_KVM altivec_unavailable - /** * Interrupt 0xf40 - VSX Unavailable Interrupt. @@ -2424,8 +2287,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX) bl vsx_unavailable_exception b interrupt_return - GEN_KVM vsx_unavailable - /** * Interrupt 0xf60 - Facility Unavailable Interrupt. @@ -2454,8 +2315,6 @@ EXC_COMMON_BEGIN(facility_unavailable_common) REST_NVGPRS(r1) /* instruction emulation may change GPRs */ b interrupt_return - GEN_KVM facility_unavailable - /** * Interrupt 0xf60 - Hypervisor Facility Unavailable Interrupt. @@ -2484,8 +2343,6 @@ EXC_COMMON_BEGIN(h_facility_unavailable_common) REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */ b interrupt_return - GEN_KVM h_facility_unavailable - EXC_REAL_NONE(0xfa0, 0x20) EXC_VIRT_NONE(0x4fa0, 0x20) @@ -2515,8 +2372,6 @@ EXC_COMMON_BEGIN(cbe_system_error_common) bl cbe_system_error_exception b interrupt_return - GEN_KVM cbe_system_error - #else /* CONFIG_CBE_RAS */ EXC_REAL_NONE(0x1200, 0x100) EXC_VIRT_NONE(0x5200, 0x100) @@ -2548,8 +2403,6 @@ EXC_COMMON_BEGIN(instruction_breakpoint_common) bl instruction_breakpoint_exception b interrupt_return - GEN_KVM instruction_breakpoint - EXC_REAL_NONE(0x1400, 0x100) EXC_VIRT_NONE(0x5400, 0x100) @@ -2670,8 +2523,6 @@ EXC_COMMON_BEGIN(denorm_exception_common) bl unknown_exception b interrupt_return - GEN_KVM denorm_exception - #ifdef CONFIG_CBE_RAS INT_DEFINE_BEGIN(cbe_maintenance) @@ -2689,8 +2540,6 @@ EXC_COMMON_BEGIN(cbe_maintenance_common) bl cbe_maintenance_exception b interrupt_return - GEN_KVM cbe_maintenance - #else /* CONFIG_CBE_RAS */ EXC_REAL_NONE(0x1600, 0x100) EXC_VIRT_NONE(0x5600, 0x100) @@ -2721,8 +2570,6 @@ EXC_COMMON_BEGIN(altivec_assist_common) #endif b interrupt_return - GEN_KVM altivec_assist - #ifdef CONFIG_CBE_RAS INT_DEFINE_BEGIN(cbe_thermal) @@ -2740,8 +2587,6 @@ EXC_COMMON_BEGIN(cbe_thermal_common) bl cbe_thermal_exception b interrupt_return - GEN_KVM cbe_thermal - #else /* CONFIG_CBE_RAS */ EXC_REAL_NONE(0x1800, 0x100) EXC_VIRT_NONE(0x5800, 0x100) @@ -2994,6 +2839,15 @@ TRAMP_REAL_BEGIN(rfscv_flush_fallback) USE_TEXT_SECTION() +#ifdef CONFIG_KVM_BOOK3S_64_HANDLER +kvm_interrupt: + /* + * The conditional branch in KVMTEST can't reach all the way, + * make a stub. + */ + b kvmppc_interrupt +#endif + _GLOBAL(do_uaccess_flush) UACCESS_FLUSH_FIXUP_SECTION nop @@ -3009,32 +2863,6 @@ EXPORT_SYMBOL(do_uaccess_flush) MASKED_INTERRUPT MASKED_INTERRUPT hsrr=1 -#ifdef CONFIG_KVM_BOOK3S_64_HANDLER -kvmppc_skip_interrupt: - /* - * Here all GPRs are unchanged from when the interrupt happened - * except for r13, which is saved in SPRG_SCRATCH0. - */ - mfspr r13, SPRN_SRR0 - addi r13, r13, 4 - mtspr SPRN_SRR0, r13 - GET_SCRATCH0(r13) - RFI_TO_KERNEL - b . - -kvmppc_skip_Hinterrupt: - /* - * Here all GPRs are unchanged from when the interrupt happened - * except for r13, which is saved in SPRG_SCRATCH0. - */ - mfspr r13, SPRN_HSRR0 - addi r13, r13, 4 - mtspr SPRN_HSRR0, r13 - GET_SCRATCH0(r13) - HRFI_TO_KERNEL - b . -#endif - /* * Relocation-on interrupts: A subset of the interrupts can be delivered * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c index 0fdfcdd9d880..c17d1c9362b5 100644 --- a/arch/powerpc/kernel/security.c +++ b/arch/powerpc/kernel/security.c @@ -432,16 +432,19 @@ device_initcall(stf_barrier_debugfs_init); static void update_branch_cache_flush(void) { - u32 *site; + u32 *site, __maybe_unused *site2; #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE site = &patch__call_kvm_flush_link_stack; + site2 = &patch__call_kvm_flush_link_stack_p9; // This controls the branch from guest_exit_cont to kvm_flush_link_stack if (link_stack_flush_type == BRANCH_CACHE_FLUSH_NONE) { patch_instruction_site(site, ppc_inst(PPC_INST_NOP)); + patch_instruction_site(site2, ppc_inst(PPC_INST_NOP)); } else { // Could use HW flush, but that could also flush count cache patch_branch_site(site, (u64)&kvm_flush_link_stack, BRANCH_SET_LINK); + patch_branch_site(site2, (u64)&kvm_flush_link_stack, BRANCH_SET_LINK); } #endif diff --git a/arch/powerpc/kernel/time.c b/arch/powerpc/kernel/time.c index b67d93a609a2..da995c5fb97d 100644 --- a/arch/powerpc/kernel/time.c +++ b/arch/powerpc/kernel/time.c @@ -508,16 +508,6 @@ EXPORT_SYMBOL(profile_pc); * 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable... */ #ifdef CONFIG_PPC64 -static inline unsigned long test_irq_work_pending(void) -{ - unsigned long x; - - asm volatile("lbz %0,%1(13)" - : "=r" (x) - : "i" (offsetof(struct paca_struct, irq_work_pending))); - return x; -} - static inline void set_irq_work_pending_flag(void) { asm volatile("stb %0,%1(13)" : : diff --git a/arch/powerpc/kvm/Makefile b/arch/powerpc/kvm/Makefile index 2bfeaa13befb..583c14ef596e 100644 --- a/arch/powerpc/kvm/Makefile +++ b/arch/powerpc/kvm/Makefile @@ -6,7 +6,7 @@ ccflags-y := -Ivirt/kvm -Iarch/powerpc/kvm KVM := ../../../virt/kvm -common-objs-y = $(KVM)/kvm_main.o $(KVM)/eventfd.o +common-objs-y = $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/binary_stats.o common-objs-$(CONFIG_KVM_VFIO) += $(KVM)/vfio.o common-objs-$(CONFIG_KVM_MMIO) += $(KVM)/coalesced_mmio.o @@ -57,6 +57,7 @@ kvm-pr-y := \ book3s_32_mmu.o kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HANDLER) += \ + book3s_64_entry.o \ tm.o ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE @@ -86,6 +87,7 @@ kvm-book3s_64-builtin-tm-objs-$(CONFIG_PPC_TRANSACTIONAL_MEM) += \ ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE kvm-book3s_64-builtin-objs-$(CONFIG_KVM_BOOK3S_64_HANDLER) += \ book3s_hv_hmi.o \ + book3s_hv_p9_entry.o \ book3s_hv_rmhandlers.o \ book3s_hv_rm_mmu.o \ book3s_hv_ras.o \ diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index 2b691f4d1f26..79833f78d1da 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -38,37 +38,66 @@ /* #define EXIT_DEBUG */ -struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT("exits", sum_exits), - VCPU_STAT("mmio", mmio_exits), - VCPU_STAT("sig", signal_exits), - VCPU_STAT("sysc", syscall_exits), - VCPU_STAT("inst_emu", emulated_inst_exits), - VCPU_STAT("dec", dec_exits), - VCPU_STAT("ext_intr", ext_intr_exits), - VCPU_STAT("queue_intr", queue_intr), - VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), - VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), - VCPU_STAT("halt_wait_ns", halt_wait_ns), - VCPU_STAT("halt_successful_poll", halt_successful_poll), - VCPU_STAT("halt_attempted_poll", halt_attempted_poll), - VCPU_STAT("halt_successful_wait", halt_successful_wait), - VCPU_STAT("halt_poll_invalid", halt_poll_invalid), - VCPU_STAT("halt_wakeup", halt_wakeup), - VCPU_STAT("pf_storage", pf_storage), - VCPU_STAT("sp_storage", sp_storage), - VCPU_STAT("pf_instruc", pf_instruc), - VCPU_STAT("sp_instruc", sp_instruc), - VCPU_STAT("ld", ld), - VCPU_STAT("ld_slow", ld_slow), - VCPU_STAT("st", st), - VCPU_STAT("st_slow", st_slow), - VCPU_STAT("pthru_all", pthru_all), - VCPU_STAT("pthru_host", pthru_host), - VCPU_STAT("pthru_bad_aff", pthru_bad_aff), - VM_STAT("largepages_2M", num_2M_pages, .mode = 0444), - VM_STAT("largepages_1G", num_1G_pages, .mode = 0444), - { NULL } +const struct _kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS(), + STATS_DESC_ICOUNTER(VM, num_2M_pages), + STATS_DESC_ICOUNTER(VM, num_1G_pages) +}; +static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == + sizeof(struct kvm_vm_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + STATS_DESC_COUNTER(VCPU, sum_exits), + STATS_DESC_COUNTER(VCPU, mmio_exits), + STATS_DESC_COUNTER(VCPU, signal_exits), + STATS_DESC_COUNTER(VCPU, light_exits), + STATS_DESC_COUNTER(VCPU, itlb_real_miss_exits), + STATS_DESC_COUNTER(VCPU, itlb_virt_miss_exits), + STATS_DESC_COUNTER(VCPU, dtlb_real_miss_exits), + STATS_DESC_COUNTER(VCPU, dtlb_virt_miss_exits), + STATS_DESC_COUNTER(VCPU, syscall_exits), + STATS_DESC_COUNTER(VCPU, isi_exits), + STATS_DESC_COUNTER(VCPU, dsi_exits), + STATS_DESC_COUNTER(VCPU, emulated_inst_exits), + STATS_DESC_COUNTER(VCPU, dec_exits), + STATS_DESC_COUNTER(VCPU, ext_intr_exits), + STATS_DESC_TIME_NSEC(VCPU, halt_wait_ns), + STATS_DESC_COUNTER(VCPU, halt_successful_wait), + STATS_DESC_COUNTER(VCPU, dbell_exits), + STATS_DESC_COUNTER(VCPU, gdbell_exits), + STATS_DESC_COUNTER(VCPU, ld), + STATS_DESC_COUNTER(VCPU, st), + STATS_DESC_COUNTER(VCPU, pf_storage), + STATS_DESC_COUNTER(VCPU, pf_instruc), + STATS_DESC_COUNTER(VCPU, sp_storage), + STATS_DESC_COUNTER(VCPU, sp_instruc), + STATS_DESC_COUNTER(VCPU, queue_intr), + STATS_DESC_COUNTER(VCPU, ld_slow), + STATS_DESC_COUNTER(VCPU, st_slow), + STATS_DESC_COUNTER(VCPU, pthru_all), + STATS_DESC_COUNTER(VCPU, pthru_host), + STATS_DESC_COUNTER(VCPU, pthru_bad_aff) +}; +static_assert(ARRAY_SIZE(kvm_vcpu_stats_desc) == + sizeof(struct kvm_vcpu_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), }; static inline void kvmppc_update_int_pending(struct kvm_vcpu *vcpu, @@ -171,6 +200,12 @@ void kvmppc_core_queue_machine_check(struct kvm_vcpu *vcpu, ulong flags) } EXPORT_SYMBOL_GPL(kvmppc_core_queue_machine_check); +void kvmppc_core_queue_syscall(struct kvm_vcpu *vcpu) +{ + kvmppc_inject_interrupt(vcpu, BOOK3S_INTERRUPT_SYSCALL, 0); +} +EXPORT_SYMBOL(kvmppc_core_queue_syscall); + void kvmppc_core_queue_program(struct kvm_vcpu *vcpu, ulong flags) { /* might as well deliver this straight away */ @@ -1044,13 +1079,10 @@ static int kvmppc_book3s_init(void) #ifdef CONFIG_KVM_XICS #ifdef CONFIG_KVM_XIVE if (xics_on_xive()) { - kvmppc_xive_init_module(); kvm_register_device_ops(&kvm_xive_ops, KVM_DEV_TYPE_XICS); - if (kvmppc_xive_native_supported()) { - kvmppc_xive_native_init_module(); + if (kvmppc_xive_native_supported()) kvm_register_device_ops(&kvm_xive_native_ops, KVM_DEV_TYPE_XIVE); - } } else #endif kvm_register_device_ops(&kvm_xics_ops, KVM_DEV_TYPE_XICS); @@ -1060,12 +1092,6 @@ static int kvmppc_book3s_init(void) static void kvmppc_book3s_exit(void) { -#ifdef CONFIG_KVM_XICS - if (xics_on_xive()) { - kvmppc_xive_exit_module(); - kvmppc_xive_native_exit_module(); - } -#endif #ifdef CONFIG_KVM_BOOK3S_32_HANDLER kvmppc_book3s_exit_pr(); #endif diff --git a/arch/powerpc/kvm/book3s_64_entry.S b/arch/powerpc/kvm/book3s_64_entry.S new file mode 100644 index 000000000000..983b8c18bc31 --- /dev/null +++ b/arch/powerpc/kvm/book3s_64_entry.S @@ -0,0 +1,416 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * These are branched to from interrupt handlers in exception-64s.S which set + * IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero. + */ + +/* + * This is a hcall, so register convention is as + * Documentation/powerpc/papr_hcalls.rst. + * + * This may also be a syscall from PR-KVM userspace that is to be + * reflected to the PR guest kernel, so registers may be set up for + * a system call rather than hcall. We don't currently clobber + * anything here, but the 0xc00 handler has already clobbered CTR + * and CR0, so PR-KVM can not support a guest kernel that preserves + * those registers across its system calls. + * + * The state of registers is as kvmppc_interrupt, except CFAR is not + * saved, R13 is not in SCRATCH0, and R10 does not contain the trap. + */ +.global kvmppc_hcall +.balign IFETCH_ALIGN_BYTES +kvmppc_hcall: +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + lbz r10,HSTATE_IN_GUEST(r13) + cmpwi r10,KVM_GUEST_MODE_HV_P9 + beq kvmppc_p9_exit_hcall +#endif + ld r10,PACA_EXGEN+EX_R13(r13) + SET_SCRATCH0(r10) + li r10,0xc00 + /* Now we look like kvmppc_interrupt */ + li r11,PACA_EXGEN + b .Lgot_save_area + +/* + * KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that + * call convention: + * + * guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area + * guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts + * guest R13 also saved in SCRATCH0 + * R13 = PACA + * R11 = (H)SRR0 + * R12 = (H)SRR1 + * R9 = guest CR + * PPR is set to medium + * + * With the addition for KVM: + * R10 = trap vector + */ +.global kvmppc_interrupt +.balign IFETCH_ALIGN_BYTES +kvmppc_interrupt: +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + std r10,HSTATE_SCRATCH0(r13) + lbz r10,HSTATE_IN_GUEST(r13) + cmpwi r10,KVM_GUEST_MODE_HV_P9 + beq kvmppc_p9_exit_interrupt + ld r10,HSTATE_SCRATCH0(r13) +#endif + li r11,PACA_EXGEN + cmpdi r10,0x200 + bgt+ .Lgot_save_area + li r11,PACA_EXMC + beq .Lgot_save_area + li r11,PACA_EXNMI +.Lgot_save_area: + add r11,r11,r13 +BEGIN_FTR_SECTION + ld r12,EX_CFAR(r11) + std r12,HSTATE_CFAR(r13) +END_FTR_SECTION_IFSET(CPU_FTR_CFAR) + ld r12,EX_CTR(r11) + mtctr r12 +BEGIN_FTR_SECTION + ld r12,EX_PPR(r11) + std r12,HSTATE_PPR(r13) +END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) + ld r12,EX_R12(r11) + std r12,HSTATE_SCRATCH0(r13) + sldi r12,r9,32 + or r12,r12,r10 + ld r9,EX_R9(r11) + ld r10,EX_R10(r11) + ld r11,EX_R11(r11) + + /* + * Hcalls and other interrupts come here after normalising register + * contents and save locations: + * + * R12 = (guest CR << 32) | interrupt vector + * R13 = PACA + * guest R12 saved in shadow HSTATE_SCRATCH0 + * guest R13 saved in SPRN_SCRATCH0 + */ + std r9,HSTATE_SCRATCH2(r13) + lbz r9,HSTATE_IN_GUEST(r13) + cmpwi r9,KVM_GUEST_MODE_SKIP + beq- .Lmaybe_skip +.Lno_skip: +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE + cmpwi r9,KVM_GUEST_MODE_GUEST + beq kvmppc_interrupt_pr +#endif + b kvmppc_interrupt_hv +#else + b kvmppc_interrupt_pr +#endif + +/* + * "Skip" interrupts are part of a trick KVM uses a with hash guests to load + * the faulting instruction in guest memory from the the hypervisor without + * walking page tables. + * + * When the guest takes a fault that requires the hypervisor to load the + * instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1 + * and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets + * MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions + * but loads and stores will access the guest context. This is used to load + * the faulting instruction using the faulting guest effective address. + * + * However the guest context may not be able to translate, or it may cause a + * machine check or other issue, which results in a fault in the host + * (even with KVM-HV). + * + * These faults come here because KVM_GUEST_MODE_SKIP was set, so if they + * are (or are likely) caused by that load, the instruction is skipped by + * just returning with the PC advanced +4, where it is noticed the load did + * not execute and it goes to the slow path which walks the page tables to + * read guest memory. + */ +.Lmaybe_skip: + cmpwi r12,BOOK3S_INTERRUPT_MACHINE_CHECK + beq 1f + cmpwi r12,BOOK3S_INTERRUPT_DATA_STORAGE + beq 1f + cmpwi r12,BOOK3S_INTERRUPT_DATA_SEGMENT + beq 1f +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + /* HSRR interrupts get 2 added to interrupt number */ + cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2 + beq 2f +#endif + b .Lno_skip +1: mfspr r9,SPRN_SRR0 + addi r9,r9,4 + mtspr SPRN_SRR0,r9 + ld r12,HSTATE_SCRATCH0(r13) + ld r9,HSTATE_SCRATCH2(r13) + GET_SCRATCH0(r13) + RFI_TO_KERNEL +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE +2: mfspr r9,SPRN_HSRR0 + addi r9,r9,4 + mtspr SPRN_HSRR0,r9 + ld r12,HSTATE_SCRATCH0(r13) + ld r9,HSTATE_SCRATCH2(r13) + GET_SCRATCH0(r13) + HRFI_TO_KERNEL +#endif + +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + +/* Stack frame offsets for kvmppc_p9_enter_guest */ +#define SFS (144 + STACK_FRAME_MIN_SIZE) +#define STACK_SLOT_NVGPRS (SFS - 144) /* 18 gprs */ + +/* + * void kvmppc_p9_enter_guest(struct vcpu *vcpu); + * + * Enter the guest on a ISAv3.0 or later system. + */ +.balign IFETCH_ALIGN_BYTES +_GLOBAL(kvmppc_p9_enter_guest) +EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest) + mflr r0 + std r0,PPC_LR_STKOFF(r1) + stdu r1,-SFS(r1) + + std r1,HSTATE_HOST_R1(r13) + + mfcr r4 + stw r4,SFS+8(r1) + + reg = 14 + .rept 18 + std reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) + reg = reg + 1 + .endr + + ld r4,VCPU_LR(r3) + mtlr r4 + ld r4,VCPU_CTR(r3) + mtctr r4 + ld r4,VCPU_XER(r3) + mtspr SPRN_XER,r4 + + ld r1,VCPU_CR(r3) + +BEGIN_FTR_SECTION + ld r4,VCPU_CFAR(r3) + mtspr SPRN_CFAR,r4 +END_FTR_SECTION_IFSET(CPU_FTR_CFAR) +BEGIN_FTR_SECTION + ld r4,VCPU_PPR(r3) + mtspr SPRN_PPR,r4 +END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) + + reg = 4 + .rept 28 + ld reg,__VCPU_GPR(reg)(r3) + reg = reg + 1 + .endr + + ld r4,VCPU_KVM(r3) + lbz r4,KVM_SECURE_GUEST(r4) + cmpdi r4,0 + ld r4,VCPU_GPR(R4)(r3) + bne .Lret_to_ultra + + mtcr r1 + + ld r0,VCPU_GPR(R0)(r3) + ld r1,VCPU_GPR(R1)(r3) + ld r2,VCPU_GPR(R2)(r3) + ld r3,VCPU_GPR(R3)(r3) + + HRFI_TO_GUEST + b . + + /* + * Use UV_RETURN ultracall to return control back to the Ultravisor + * after processing an hypercall or interrupt that was forwarded + * (a.k.a. reflected) to the Hypervisor. + * + * All registers have already been reloaded except the ucall requires: + * R0 = hcall result + * R2 = SRR1, so UV can detect a synthesized interrupt (if any) + * R3 = UV_RETURN + */ +.Lret_to_ultra: + mtcr r1 + ld r1,VCPU_GPR(R1)(r3) + + ld r0,VCPU_GPR(R3)(r3) + mfspr r2,SPRN_SRR1 + LOAD_REG_IMMEDIATE(r3, UV_RETURN) + sc 2 + +/* + * kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from + * above if the interrupt was taken for a guest that was entered via + * kvmppc_p9_enter_guest(). + * + * The exit code recovers the host stack and vcpu pointer, saves all guest GPRs + * and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re- + * establishes the host stack and registers to return from the + * kvmppc_p9_enter_guest() function, which saves CTR and other guest registers + * (SPRs and FP, VEC, etc). + */ +.balign IFETCH_ALIGN_BYTES +kvmppc_p9_exit_hcall: + mfspr r11,SPRN_SRR0 + mfspr r12,SPRN_SRR1 + li r10,0xc00 + std r10,HSTATE_SCRATCH0(r13) + +.balign IFETCH_ALIGN_BYTES +kvmppc_p9_exit_interrupt: + /* + * If set to KVM_GUEST_MODE_HV_P9 but we're still in the + * hypervisor, that means we can't return from the entry stack. + */ + rldicl. r10,r12,64-MSR_HV_LG,63 + bne- kvmppc_p9_bad_interrupt + + std r1,HSTATE_SCRATCH1(r13) + std r3,HSTATE_SCRATCH2(r13) + ld r1,HSTATE_HOST_R1(r13) + ld r3,HSTATE_KVM_VCPU(r13) + + std r9,VCPU_CR(r3) + +1: + std r11,VCPU_PC(r3) + std r12,VCPU_MSR(r3) + + reg = 14 + .rept 18 + std reg,__VCPU_GPR(reg)(r3) + reg = reg + 1 + .endr + + /* r1, r3, r9-r13 are saved to vcpu by C code */ + std r0,VCPU_GPR(R0)(r3) + std r2,VCPU_GPR(R2)(r3) + reg = 4 + .rept 5 + std reg,__VCPU_GPR(reg)(r3) + reg = reg + 1 + .endr + + ld r2,PACATOC(r13) + + mflr r4 + std r4,VCPU_LR(r3) + mfspr r4,SPRN_XER + std r4,VCPU_XER(r3) + + reg = 14 + .rept 18 + ld reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) + reg = reg + 1 + .endr + + lwz r4,SFS+8(r1) + mtcr r4 + + /* + * Flush the link stack here, before executing the first blr on the + * way out of the guest. + * + * The link stack won't match coming out of the guest anyway so the + * only cost is the flush itself. The call clobbers r0. + */ +1: nop + patch_site 1b patch__call_kvm_flush_link_stack_p9 + + addi r1,r1,SFS + ld r0,PPC_LR_STKOFF(r1) + mtlr r0 + blr + +/* + * Took an interrupt somewhere right before HRFID to guest, so registers are + * in a bad way. Return things hopefully enough to run host virtual code and + * run the Linux interrupt handler (SRESET or MCE) to print something useful. + * + * We could be really clever and save all host registers in known locations + * before setting HSTATE_IN_GUEST, then restoring them all here, and setting + * return address to a fixup that sets them up again. But that's a lot of + * effort for a small bit of code. Lots of other things to do first. + */ +kvmppc_p9_bad_interrupt: +BEGIN_MMU_FTR_SECTION + /* + * Hash host doesn't try to recover MMU (requires host SLB reload) + */ + b . +END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) + /* + * Clean up guest registers to give host a chance to run. + */ + li r10,0 + mtspr SPRN_AMR,r10 + mtspr SPRN_IAMR,r10 + mtspr SPRN_CIABR,r10 + mtspr SPRN_DAWRX0,r10 +BEGIN_FTR_SECTION + mtspr SPRN_DAWRX1,r10 +END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) + mtspr SPRN_PID,r10 + + /* + * Switch to host MMU mode + */ + ld r10, HSTATE_KVM_VCPU(r13) + ld r10, VCPU_KVM(r10) + lwz r10, KVM_HOST_LPID(r10) + mtspr SPRN_LPID,r10 + + ld r10, HSTATE_KVM_VCPU(r13) + ld r10, VCPU_KVM(r10) + ld r10, KVM_HOST_LPCR(r10) + mtspr SPRN_LPCR,r10 + + /* + * Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear + * MSR_RI in r12 ([H]SRR1) so the handler won't try to return. + */ + li r10,KVM_GUEST_MODE_NONE + stb r10,HSTATE_IN_GUEST(r13) + li r10,MSR_RI + andc r12,r12,r10 + + /* + * Go back to interrupt handler. MCE and SRESET have their specific + * PACA save area so they should be used directly. They set up their + * own stack. The other handlers all use EXGEN. They will use the + * guest r1 if it looks like a kernel stack, so just load the + * emergency stack and go to program check for all other interrupts. + */ + ld r10,HSTATE_SCRATCH0(r13) + cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK + beq machine_check_common + + cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET + beq system_reset_common + + b . +#endif diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index d909c069363e..b5905ae4377c 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -21,6 +21,7 @@ #include #include #include +#include /* * Supported radix tree geometry. @@ -318,9 +319,19 @@ void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr, } psi = shift_to_mmu_psize(pshift); - rb = addr | (mmu_get_ap(psi) << PPC_BITLSHIFT(58)); - rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(0, 0, 1), - lpid, rb); + + if (!firmware_has_feature(FW_FEATURE_RPT_INVALIDATE)) { + rb = addr | (mmu_get_ap(psi) << PPC_BITLSHIFT(58)); + rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(0, 0, 1), + lpid, rb); + } else { + rc = pseries_rpt_invalidate(lpid, H_RPTI_TARGET_CMMU, + H_RPTI_TYPE_NESTED | + H_RPTI_TYPE_TLB, + psize_to_rpti_pgsize(psi), + addr, addr + psize); + } + if (rc) pr_err("KVM: TLB page invalidation hcall failed, rc=%ld\n", rc); } @@ -334,8 +345,14 @@ static void kvmppc_radix_flush_pwc(struct kvm *kvm, unsigned int lpid) return; } - rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(1, 0, 1), - lpid, TLBIEL_INVAL_SET_LPID); + if (!firmware_has_feature(FW_FEATURE_RPT_INVALIDATE)) + rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(1, 0, 1), + lpid, TLBIEL_INVAL_SET_LPID); + else + rc = pseries_rpt_invalidate(lpid, H_RPTI_TARGET_CMMU, + H_RPTI_TYPE_NESTED | + H_RPTI_TYPE_PWC, H_RPTI_PAGE_ALL, + 0, -1UL); if (rc) pr_err("KVM: TLB PWC invalidation hcall failed, rc=%ld\n", rc); } diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c index 083a4e037718..dc6591548f0c 100644 --- a/arch/powerpc/kvm/book3s_64_vio_hv.c +++ b/arch/powerpc/kvm/book3s_64_vio_hv.c @@ -391,10 +391,6 @@ long kvmppc_rm_h_put_tce(struct kvm_vcpu *vcpu, unsigned long liobn, /* udbg_printf("H_PUT_TCE(): liobn=0x%lx ioba=0x%lx, tce=0x%lx\n", */ /* liobn, ioba, tce); */ - /* For radix, we might be in virtual mode, so punt */ - if (kvm_is_radix(vcpu->kvm)) - return H_TOO_HARD; - stt = kvmppc_find_table(vcpu->kvm, liobn); if (!stt) return H_TOO_HARD; @@ -489,10 +485,6 @@ long kvmppc_rm_h_put_tce_indirect(struct kvm_vcpu *vcpu, bool prereg = false; struct kvmppc_spapr_tce_iommu_table *stit; - /* For radix, we might be in virtual mode, so punt */ - if (kvm_is_radix(vcpu->kvm)) - return H_TOO_HARD; - /* * used to check for invalidations in progress */ @@ -602,10 +594,6 @@ long kvmppc_rm_h_stuff_tce(struct kvm_vcpu *vcpu, long i, ret; struct kvmppc_spapr_tce_iommu_table *stit; - /* For radix, we might be in virtual mode, so punt */ - if (kvm_is_radix(vcpu->kvm)) - return H_TOO_HARD; - stt = kvmppc_find_table(vcpu->kvm, liobn); if (!stt) return H_TOO_HARD; diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index bc0813644666..cd544a46183e 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -76,6 +76,7 @@ #include #include #include +#include #include "book3s.h" @@ -103,13 +104,9 @@ static int target_smt_mode; module_param(target_smt_mode, int, 0644); MODULE_PARM_DESC(target_smt_mode, "Target threads per core (0 = max)"); -static bool indep_threads_mode = true; -module_param(indep_threads_mode, bool, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(indep_threads_mode, "Independent-threads mode (only on POWER9)"); - static bool one_vm_per_core; module_param(one_vm_per_core, bool, S_IRUGO | S_IWUSR); -MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires indep_threads_mode=N)"); +MODULE_PARM_DESC(one_vm_per_core, "Only run vCPUs from the same VM on a core (requires POWER8 or older)"); #ifdef CONFIG_KVM_XICS static const struct kernel_param_ops module_param_ops = { @@ -134,9 +131,6 @@ static inline bool nesting_enabled(struct kvm *kvm) return kvm->arch.nested_enable && kvm_is_radix(kvm); } -/* If set, the threads on each CPU core have to be in the same MMU mode */ -static bool no_mixing_hpt_and_radix __read_mostly; - static int kvmppc_hv_setup_htab_rma(struct kvm_vcpu *vcpu); /* @@ -236,7 +230,7 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) waitp = kvm_arch_vcpu_get_wait(vcpu); if (rcuwait_wake_up(waitp)) - ++vcpu->stat.halt_wakeup; + ++vcpu->stat.generic.halt_wakeup; cpu = READ_ONCE(vcpu->arch.thread_cpu); if (cpu >= 0 && kvmppc_ipi_thread(cpu)) @@ -807,7 +801,8 @@ static int kvmppc_h_set_mode(struct kvm_vcpu *vcpu, unsigned long mflags, * KVM does not support mflags=2 (AIL=2) and AIL=1 is reserved. * Keep this in synch with kvmppc_filter_guest_lpcr_hv. */ - if (mflags != 0 && mflags != 3) + if (cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG) && + kvmhv_vcpu_is_radix(vcpu) && mflags == 3) return H_UNSUPPORTED_FLAG_START; return H_TOO_HARD; default: @@ -899,6 +894,10 @@ static int kvm_arch_vcpu_yield_to(struct kvm_vcpu *target) * H_SUCCESS if the source vcore wasn't idle (e.g. if it may * have useful work to do and should not confer) so we don't * recheck that here. + * + * In the case of the P9 single vcpu per vcore case, the real + * mode handler is not called but no other threads are in the + * source vcore. */ spin_lock(&vcore->lock); @@ -924,8 +923,71 @@ static int kvmppc_get_yield_count(struct kvm_vcpu *vcpu) return yield_count; } +/* + * H_RPT_INVALIDATE hcall handler for nested guests. + * + * Handles only nested process-scoped invalidation requests in L0. + */ +static int kvmppc_nested_h_rpt_invalidate(struct kvm_vcpu *vcpu) +{ + unsigned long type = kvmppc_get_gpr(vcpu, 6); + unsigned long pid, pg_sizes, start, end; + + /* + * The partition-scoped invalidations aren't handled here in L0. + */ + if (type & H_RPTI_TYPE_NESTED) + return RESUME_HOST; + + pid = kvmppc_get_gpr(vcpu, 4); + pg_sizes = kvmppc_get_gpr(vcpu, 7); + start = kvmppc_get_gpr(vcpu, 8); + end = kvmppc_get_gpr(vcpu, 9); + + do_h_rpt_invalidate_prt(pid, vcpu->arch.nested->shadow_lpid, + type, pg_sizes, start, end); + + kvmppc_set_gpr(vcpu, 3, H_SUCCESS); + return RESUME_GUEST; +} + +static long kvmppc_h_rpt_invalidate(struct kvm_vcpu *vcpu, + unsigned long id, unsigned long target, + unsigned long type, unsigned long pg_sizes, + unsigned long start, unsigned long end) +{ + if (!kvm_is_radix(vcpu->kvm)) + return H_UNSUPPORTED; + + if (end < start) + return H_P5; + + /* + * Partition-scoped invalidation for nested guests. + */ + if (type & H_RPTI_TYPE_NESTED) { + if (!nesting_enabled(vcpu->kvm)) + return H_FUNCTION; + + /* Support only cores as target */ + if (target != H_RPTI_TARGET_CMMU) + return H_P2; + + return do_h_rpt_invalidate_pat(vcpu, id, type, pg_sizes, + start, end); + } + + /* + * Process-scoped invalidation for L1 guests. + */ + do_h_rpt_invalidate_prt(id, vcpu->kvm->arch.lpid, + type, pg_sizes, start, end); + return H_SUCCESS; +} + int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) { + struct kvm *kvm = vcpu->kvm; unsigned long req = kvmppc_get_gpr(vcpu, 3); unsigned long target, ret = H_SUCCESS; int yield_count; @@ -937,11 +999,57 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) return RESUME_HOST; switch (req) { + case H_REMOVE: + ret = kvmppc_h_remove(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5), + kvmppc_get_gpr(vcpu, 6)); + if (ret == H_TOO_HARD) + return RESUME_HOST; + break; + case H_ENTER: + ret = kvmppc_h_enter(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5), + kvmppc_get_gpr(vcpu, 6), + kvmppc_get_gpr(vcpu, 7)); + if (ret == H_TOO_HARD) + return RESUME_HOST; + break; + case H_READ: + ret = kvmppc_h_read(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5)); + if (ret == H_TOO_HARD) + return RESUME_HOST; + break; + case H_CLEAR_MOD: + ret = kvmppc_h_clear_mod(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5)); + if (ret == H_TOO_HARD) + return RESUME_HOST; + break; + case H_CLEAR_REF: + ret = kvmppc_h_clear_ref(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5)); + if (ret == H_TOO_HARD) + return RESUME_HOST; + break; + case H_PROTECT: + ret = kvmppc_h_protect(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5), + kvmppc_get_gpr(vcpu, 6)); + if (ret == H_TOO_HARD) + return RESUME_HOST; + break; + case H_BULK_REMOVE: + ret = kvmppc_h_bulk_remove(vcpu); + if (ret == H_TOO_HARD) + return RESUME_HOST; + break; + case H_CEDE: break; case H_PROD: target = kvmppc_get_gpr(vcpu, 4); - tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); + tvcpu = kvmppc_find_vcpu(kvm, target); if (!tvcpu) { ret = H_PARAMETER; break; @@ -955,7 +1063,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) target = kvmppc_get_gpr(vcpu, 4); if (target == -1) break; - tvcpu = kvmppc_find_vcpu(vcpu->kvm, target); + tvcpu = kvmppc_find_vcpu(kvm, target); if (!tvcpu) { ret = H_PARAMETER; break; @@ -971,12 +1079,12 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) kvmppc_get_gpr(vcpu, 6)); break; case H_RTAS: - if (list_empty(&vcpu->kvm->arch.rtas_tokens)) + if (list_empty(&kvm->arch.rtas_tokens)) return RESUME_HOST; - idx = srcu_read_lock(&vcpu->kvm->srcu); + idx = srcu_read_lock(&kvm->srcu); rc = kvmppc_rtas_hcall(vcpu); - srcu_read_unlock(&vcpu->kvm->srcu, idx); + srcu_read_unlock(&kvm->srcu, idx); if (rc == -ENOENT) return RESUME_HOST; @@ -1060,15 +1168,23 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) if (!powernv_get_random_long(&vcpu->arch.regs.gpr[4])) ret = H_HARDWARE; break; + case H_RPT_INVALIDATE: + ret = kvmppc_h_rpt_invalidate(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5), + kvmppc_get_gpr(vcpu, 6), + kvmppc_get_gpr(vcpu, 7), + kvmppc_get_gpr(vcpu, 8), + kvmppc_get_gpr(vcpu, 9)); + break; case H_SET_PARTITION_TABLE: ret = H_FUNCTION; - if (nesting_enabled(vcpu->kvm)) + if (nesting_enabled(kvm)) ret = kvmhv_set_partition_table(vcpu); break; case H_ENTER_NESTED: ret = H_FUNCTION; - if (!nesting_enabled(vcpu->kvm)) + if (!nesting_enabled(kvm)) break; ret = kvmhv_enter_nested_guest(vcpu); if (ret == H_INTERRUPT) { @@ -1083,12 +1199,12 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) break; case H_TLB_INVALIDATE: ret = H_FUNCTION; - if (nesting_enabled(vcpu->kvm)) + if (nesting_enabled(kvm)) ret = kvmhv_do_nested_tlbie(vcpu); break; case H_COPY_TOFROM_GUEST: ret = H_FUNCTION; - if (nesting_enabled(vcpu->kvm)) + if (nesting_enabled(kvm)) ret = kvmhv_copy_tofrom_guest_nested(vcpu); break; case H_PAGE_INIT: @@ -1099,7 +1215,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) case H_SVM_PAGE_IN: ret = H_UNSUPPORTED; if (kvmppc_get_srr1(vcpu) & MSR_S) - ret = kvmppc_h_svm_page_in(vcpu->kvm, + ret = kvmppc_h_svm_page_in(kvm, kvmppc_get_gpr(vcpu, 4), kvmppc_get_gpr(vcpu, 5), kvmppc_get_gpr(vcpu, 6)); @@ -1107,7 +1223,7 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) case H_SVM_PAGE_OUT: ret = H_UNSUPPORTED; if (kvmppc_get_srr1(vcpu) & MSR_S) - ret = kvmppc_h_svm_page_out(vcpu->kvm, + ret = kvmppc_h_svm_page_out(kvm, kvmppc_get_gpr(vcpu, 4), kvmppc_get_gpr(vcpu, 5), kvmppc_get_gpr(vcpu, 6)); @@ -1115,12 +1231,12 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) case H_SVM_INIT_START: ret = H_UNSUPPORTED; if (kvmppc_get_srr1(vcpu) & MSR_S) - ret = kvmppc_h_svm_init_start(vcpu->kvm); + ret = kvmppc_h_svm_init_start(kvm); break; case H_SVM_INIT_DONE: ret = H_UNSUPPORTED; if (kvmppc_get_srr1(vcpu) & MSR_S) - ret = kvmppc_h_svm_init_done(vcpu->kvm); + ret = kvmppc_h_svm_init_done(kvm); break; case H_SVM_INIT_ABORT: /* @@ -1130,24 +1246,26 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) * Instead the kvm->arch.secure_guest flag is checked inside * kvmppc_h_svm_init_abort(). */ - ret = kvmppc_h_svm_init_abort(vcpu->kvm); + ret = kvmppc_h_svm_init_abort(kvm); break; default: return RESUME_HOST; } + WARN_ON_ONCE(ret == H_TOO_HARD); kvmppc_set_gpr(vcpu, 3, ret); vcpu->arch.hcall_needed = 0; return RESUME_GUEST; } /* - * Handle H_CEDE in the nested virtualization case where we haven't - * called the real-mode hcall handlers in book3s_hv_rmhandlers.S. + * Handle H_CEDE in the P9 path where we don't call the real-mode hcall + * handlers in book3s_hv_rmhandlers.S. + * * This has to be done early, not in kvmppc_pseries_do_hcall(), so * that the cede logic in kvmppc_run_single_vcpu() works properly. */ -static void kvmppc_nested_cede(struct kvm_vcpu *vcpu) +static void kvmppc_cede(struct kvm_vcpu *vcpu) { vcpu->arch.shregs.msr |= MSR_EE; vcpu->arch.ceded = 1; @@ -1178,6 +1296,7 @@ static int kvmppc_hcall_impl_hv(unsigned long cmd) case H_XIRR_X: #endif case H_PAGE_INIT: + case H_RPT_INVALIDATE: return 1; } @@ -1400,13 +1519,39 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, } case BOOK3S_INTERRUPT_SYSCALL: { - /* hcall - punt to userspace */ int i; - /* hypercall with MSR_PR has already been handled in rmode, - * and never reaches here. - */ + if (unlikely(vcpu->arch.shregs.msr & MSR_PR)) { + /* + * Guest userspace executed sc 1. This can only be + * reached by the P9 path because the old path + * handles this case in realmode hcall handlers. + */ + if (!kvmhv_vcpu_is_radix(vcpu)) { + /* + * A guest could be running PR KVM, so this + * may be a PR KVM hcall. It must be reflected + * to the guest kernel as a sc interrupt. + */ + kvmppc_core_queue_syscall(vcpu); + } else { + /* + * Radix guests can not run PR KVM or nested HV + * hash guests which might run PR KVM, so this + * is always a privilege fault. Send a program + * check to guest kernel. + */ + kvmppc_core_queue_program(vcpu, SRR1_PROGPRIV); + } + r = RESUME_GUEST; + break; + } + /* + * hcall - gather args and set exit_reason. This will next be + * handled by kvmppc_pseries_do_hcall which may be able to deal + * with it and resume guest, or may punt to userspace. + */ run->papr_hcall.nr = kvmppc_get_gpr(vcpu, 3); for (i = 0; i < 9; ++i) run->papr_hcall.args[i] = kvmppc_get_gpr(vcpu, 4 + i); @@ -1419,20 +1564,102 @@ static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, * We get these next two if the guest accesses a page which it thinks * it has mapped but which is not actually present, either because * it is for an emulated I/O device or because the corresonding - * host page has been paged out. Any other HDSI/HISI interrupts - * have been handled already. + * host page has been paged out. + * + * Any other HDSI/HISI interrupts have been handled already for P7/8 + * guests. For POWER9 hash guests not using rmhandlers, basic hash + * fault handling is done here. */ - case BOOK3S_INTERRUPT_H_DATA_STORAGE: - r = RESUME_PAGE_FAULT; + case BOOK3S_INTERRUPT_H_DATA_STORAGE: { + unsigned long vsid; + long err; + + if (vcpu->arch.fault_dsisr == HDSISR_CANARY) { + r = RESUME_GUEST; /* Just retry if it's the canary */ + break; + } + + if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) { + /* + * Radix doesn't require anything, and pre-ISAv3.0 hash + * already attempted to handle this in rmhandlers. The + * hash fault handling below is v3 only (it uses ASDR + * via fault_gpa). + */ + r = RESUME_PAGE_FAULT; + break; + } + + if (!(vcpu->arch.fault_dsisr & (DSISR_NOHPTE | DSISR_PROTFAULT))) { + kvmppc_core_queue_data_storage(vcpu, + vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); + r = RESUME_GUEST; + break; + } + + if (!(vcpu->arch.shregs.msr & MSR_DR)) + vsid = vcpu->kvm->arch.vrma_slb_v; + else + vsid = vcpu->arch.fault_gpa; + + err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar, + vsid, vcpu->arch.fault_dsisr, true); + if (err == 0) { + r = RESUME_GUEST; + } else if (err == -1 || err == -2) { + r = RESUME_PAGE_FAULT; + } else { + kvmppc_core_queue_data_storage(vcpu, + vcpu->arch.fault_dar, err); + r = RESUME_GUEST; + } break; - case BOOK3S_INTERRUPT_H_INST_STORAGE: + } + case BOOK3S_INTERRUPT_H_INST_STORAGE: { + unsigned long vsid; + long err; + vcpu->arch.fault_dar = kvmppc_get_pc(vcpu); vcpu->arch.fault_dsisr = vcpu->arch.shregs.msr & DSISR_SRR1_MATCH_64S; - if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE) - vcpu->arch.fault_dsisr |= DSISR_ISSTORE; - r = RESUME_PAGE_FAULT; + if (kvm_is_radix(vcpu->kvm) || !cpu_has_feature(CPU_FTR_ARCH_300)) { + /* + * Radix doesn't require anything, and pre-ISAv3.0 hash + * already attempted to handle this in rmhandlers. The + * hash fault handling below is v3 only (it uses ASDR + * via fault_gpa). + */ + if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE) + vcpu->arch.fault_dsisr |= DSISR_ISSTORE; + r = RESUME_PAGE_FAULT; + break; + } + + if (!(vcpu->arch.fault_dsisr & SRR1_ISI_NOPT)) { + kvmppc_core_queue_inst_storage(vcpu, + vcpu->arch.fault_dsisr); + r = RESUME_GUEST; + break; + } + + if (!(vcpu->arch.shregs.msr & MSR_IR)) + vsid = vcpu->kvm->arch.vrma_slb_v; + else + vsid = vcpu->arch.fault_gpa; + + err = kvmppc_hpte_hv_fault(vcpu, vcpu->arch.fault_dar, + vsid, vcpu->arch.fault_dsisr, false); + if (err == 0) { + r = RESUME_GUEST; + } else if (err == -1) { + r = RESUME_PAGE_FAULT; + } else { + kvmppc_core_queue_inst_storage(vcpu, err); + r = RESUME_GUEST; + } break; + } + /* * This occurs if the guest executes an illegal instruction. * If the guest debug is disabled, generate a program interrupt @@ -1593,6 +1820,23 @@ static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu) if (!xics_on_xive()) kvmppc_xics_rm_complete(vcpu, 0); break; + case BOOK3S_INTERRUPT_SYSCALL: + { + unsigned long req = kvmppc_get_gpr(vcpu, 3); + + /* + * The H_RPT_INVALIDATE hcalls issued by nested + * guests for process-scoped invalidations when + * GTSE=0, are handled here in L0. + */ + if (req == H_RPT_INVALIDATE) { + r = kvmppc_nested_h_rpt_invalidate(vcpu); + break; + } + + r = RESUME_HOST; + break; + } default: r = RESUME_HOST; break; @@ -1654,6 +1898,14 @@ unsigned long kvmppc_filter_lpcr_hv(struct kvm *kvm, unsigned long lpcr) lpcr &= ~LPCR_AIL; if ((lpcr & LPCR_AIL) != LPCR_AIL_3) lpcr &= ~LPCR_AIL; /* LPCR[AIL]=1/2 is disallowed */ + /* + * On some POWER9s we force AIL off for radix guests to prevent + * executing in MSR[HV]=1 mode with the MMU enabled and PIDR set to + * guest, which can result in Q0 translations with LPID=0 PID=PIDR to + * be cached, which the host TLB management does not expect. + */ + if (kvm_is_radix(kvm) && cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) + lpcr &= ~LPCR_AIL; /* * On POWER9, allow userspace to enable large decrementer for the @@ -2233,7 +2485,7 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, */ static int threads_per_vcore(struct kvm *kvm) { - if (kvm->arch.threads_indep) + if (cpu_has_feature(CPU_FTR_ARCH_300)) return 1; return threads_per_subcore; } @@ -2657,7 +2909,7 @@ static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu) cpumask_t *cpu_in_guest; int i; - cpu = cpu_first_thread_sibling(cpu); + cpu = cpu_first_tlb_thread_sibling(cpu); if (nested) { cpumask_set_cpu(cpu, &nested->need_tlb_flush); cpu_in_guest = &nested->cpu_in_guest; @@ -2671,9 +2923,10 @@ static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu) * the other side is the first smp_mb() in kvmppc_run_core(). */ smp_mb(); - for (i = 0; i < threads_per_core; ++i) - if (cpumask_test_cpu(cpu + i, cpu_in_guest)) - smp_call_function_single(cpu + i, do_nothing, NULL, 1); + for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu); + i += cpu_tlb_thread_sibling_step()) + if (cpumask_test_cpu(i, cpu_in_guest)) + smp_call_function_single(i, do_nothing, NULL, 1); } static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu) @@ -2704,8 +2957,8 @@ static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu) */ if (prev_cpu != pcpu) { if (prev_cpu >= 0 && - cpu_first_thread_sibling(prev_cpu) != - cpu_first_thread_sibling(pcpu)) + cpu_first_tlb_thread_sibling(prev_cpu) != + cpu_first_tlb_thread_sibling(pcpu)) radix_flush_cpu(kvm, prev_cpu, vcpu); if (nested) nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu; @@ -2967,9 +3220,6 @@ static void prepare_threads(struct kvmppc_vcore *vc) for_each_runnable_thread(i, vcpu, vc) { if (signal_pending(vcpu->arch.run_task)) vcpu->arch.ret = -EINTR; - else if (no_mixing_hpt_and_radix && - kvm_is_radix(vc->kvm) != radix_enabled()) - vcpu->arch.ret = -EINVAL; else if (vcpu->arch.vpa.update_pending || vcpu->arch.slb_shadow.update_pending || vcpu->arch.dtl.update_pending) @@ -3176,6 +3426,9 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) int trap; bool is_power8; + if (WARN_ON_ONCE(cpu_has_feature(CPU_FTR_ARCH_300))) + return; + /* * Remove from the list any threads that have a signal pending * or need a VPA update done @@ -3203,9 +3456,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) * Make sure we are running on primary threads, and that secondary * threads are offline. Also check if the number of threads in this * guest are greater than the current system threads per guest. - * On POWER9, we need to be not in independent-threads mode if - * this is a HPT guest on a radix host machine where the - * CPU threads may not be in different MMU modes. */ if ((controlled_threads > 1) && ((vc->num_threads > threads_per_subcore) || !on_primary_thread())) { @@ -3229,18 +3479,6 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) if (vc->num_threads < target_threads) collect_piggybacks(&core_info, target_threads); - /* - * On radix, arrange for TLB flushing if necessary. - * This has to be done before disabling interrupts since - * it uses smp_call_function(). - */ - pcpu = smp_processor_id(); - if (kvm_is_radix(vc->kvm)) { - for (sub = 0; sub < core_info.n_subcores; ++sub) - for_each_runnable_thread(i, vcpu, core_info.vc[sub]) - kvmppc_prepare_radix_vcpu(vcpu, pcpu); - } - /* * Hard-disable interrupts, and check resched flag and signals. * If we need to reschedule or deliver a signal, clean up @@ -3273,8 +3511,7 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) cmd_bit = stat_bit = 0; split = core_info.n_subcores; sip = NULL; - is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S) - && !cpu_has_feature(CPU_FTR_ARCH_300); + is_power8 = cpu_has_feature(CPU_FTR_ARCH_207S); if (split > 1) { sip = &split_info; @@ -3478,184 +3715,113 @@ static noinline void kvmppc_run_core(struct kvmppc_vcore *vc) trace_kvmppc_run_core(vc, 1); } -/* - * Load up hypervisor-mode registers on P9. - */ -static int kvmhv_load_hv_regs_and_go(struct kvm_vcpu *vcpu, u64 time_limit, - unsigned long lpcr) +static void load_spr_state(struct kvm_vcpu *vcpu) { - struct kvmppc_vcore *vc = vcpu->arch.vcore; - s64 hdec; - u64 tb, purr, spurr; - int trap; - unsigned long host_hfscr = mfspr(SPRN_HFSCR); - unsigned long host_ciabr = mfspr(SPRN_CIABR); - unsigned long host_dawr0 = mfspr(SPRN_DAWR0); - unsigned long host_dawrx0 = mfspr(SPRN_DAWRX0); - unsigned long host_psscr = mfspr(SPRN_PSSCR); - unsigned long host_pidr = mfspr(SPRN_PID); - unsigned long host_dawr1 = 0; - unsigned long host_dawrx1 = 0; - - if (cpu_has_feature(CPU_FTR_DAWR1)) { - host_dawr1 = mfspr(SPRN_DAWR1); - host_dawrx1 = mfspr(SPRN_DAWRX1); - } + mtspr(SPRN_DSCR, vcpu->arch.dscr); + mtspr(SPRN_IAMR, vcpu->arch.iamr); + mtspr(SPRN_PSPB, vcpu->arch.pspb); + mtspr(SPRN_FSCR, vcpu->arch.fscr); + mtspr(SPRN_TAR, vcpu->arch.tar); + mtspr(SPRN_EBBHR, vcpu->arch.ebbhr); + mtspr(SPRN_EBBRR, vcpu->arch.ebbrr); + mtspr(SPRN_BESCR, vcpu->arch.bescr); + mtspr(SPRN_WORT, vcpu->arch.wort); + mtspr(SPRN_TIDR, vcpu->arch.tid); + mtspr(SPRN_AMR, vcpu->arch.amr); + mtspr(SPRN_UAMOR, vcpu->arch.uamor); /* - * P8 and P9 suppress the HDEC exception when LPCR[HDICE] = 0, - * so set HDICE before writing HDEC. + * DAR, DSISR, and for nested HV, SPRGs must be set with MSR[RI] + * clear (or hstate set appropriately to catch those registers + * being clobbered if we take a MCE or SRESET), so those are done + * later. */ - mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr | LPCR_HDICE); - isync(); - hdec = time_limit - mftb(); - if (hdec < 0) { - mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr); - isync(); - return BOOK3S_INTERRUPT_HV_DECREMENTER; - } - mtspr(SPRN_HDEC, hdec); + if (!(vcpu->arch.ctrl & 1)) + mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1); +} - if (vc->tb_offset) { - u64 new_tb = mftb() + vc->tb_offset; - mtspr(SPRN_TBU40, new_tb); - tb = mftb(); - if ((tb & 0xffffff) < (new_tb & 0xffffff)) - mtspr(SPRN_TBU40, new_tb + 0x1000000); - vc->tb_offset_applied = vc->tb_offset; - } +static void store_spr_state(struct kvm_vcpu *vcpu) +{ + vcpu->arch.ctrl = mfspr(SPRN_CTRLF); - if (vc->pcr) - mtspr(SPRN_PCR, vc->pcr | PCR_MASK); - mtspr(SPRN_DPDES, vc->dpdes); - mtspr(SPRN_VTB, vc->vtb); - - local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR); - local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR); - mtspr(SPRN_PURR, vcpu->arch.purr); - mtspr(SPRN_SPURR, vcpu->arch.spurr); - - if (dawr_enabled()) { - mtspr(SPRN_DAWR0, vcpu->arch.dawr0); - mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0); - if (cpu_has_feature(CPU_FTR_DAWR1)) { - mtspr(SPRN_DAWR1, vcpu->arch.dawr1); - mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1); - } - } - mtspr(SPRN_CIABR, vcpu->arch.ciabr); - mtspr(SPRN_IC, vcpu->arch.ic); - mtspr(SPRN_PID, vcpu->arch.pid); - - mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC | - (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); - - mtspr(SPRN_HFSCR, vcpu->arch.hfscr); - - mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0); - mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1); - mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2); - mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3); - - mtspr(SPRN_AMOR, ~0UL); - - mtspr(SPRN_LPCR, lpcr); - isync(); - - kvmppc_xive_push_vcpu(vcpu); - - mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0); - mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1); - - trap = __kvmhv_vcpu_entry_p9(vcpu); - - /* Advance host PURR/SPURR by the amount used by guest */ - purr = mfspr(SPRN_PURR); - spurr = mfspr(SPRN_SPURR); - mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr + - purr - vcpu->arch.purr); - mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr + - spurr - vcpu->arch.spurr); - vcpu->arch.purr = purr; - vcpu->arch.spurr = spurr; - - vcpu->arch.ic = mfspr(SPRN_IC); - vcpu->arch.pid = mfspr(SPRN_PID); - vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS; - - vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0); - vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1); - vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2); - vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3); - - /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */ - mtspr(SPRN_PSSCR, host_psscr | - (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); - mtspr(SPRN_HFSCR, host_hfscr); - mtspr(SPRN_CIABR, host_ciabr); - mtspr(SPRN_DAWR0, host_dawr0); - mtspr(SPRN_DAWRX0, host_dawrx0); - if (cpu_has_feature(CPU_FTR_DAWR1)) { - mtspr(SPRN_DAWR1, host_dawr1); - mtspr(SPRN_DAWRX1, host_dawrx1); - } - mtspr(SPRN_PID, host_pidr); - - /* - * Since this is radix, do a eieio; tlbsync; ptesync sequence in - * case we interrupted the guest between a tlbie and a ptesync. - */ - asm volatile("eieio; tlbsync; ptesync"); - - /* - * cp_abort is required if the processor supports local copy-paste - * to clear the copy buffer that was under control of the guest. - */ - if (cpu_has_feature(CPU_FTR_ARCH_31)) - asm volatile(PPC_CP_ABORT); - - mtspr(SPRN_LPID, vcpu->kvm->arch.host_lpid); /* restore host LPID */ - isync(); - - vc->dpdes = mfspr(SPRN_DPDES); - vc->vtb = mfspr(SPRN_VTB); - mtspr(SPRN_DPDES, 0); - if (vc->pcr) - mtspr(SPRN_PCR, PCR_MASK); - - if (vc->tb_offset_applied) { - u64 new_tb = mftb() - vc->tb_offset_applied; - mtspr(SPRN_TBU40, new_tb); - tb = mftb(); - if ((tb & 0xffffff) < (new_tb & 0xffffff)) - mtspr(SPRN_TBU40, new_tb + 0x1000000); - vc->tb_offset_applied = 0; - } - - mtspr(SPRN_HDEC, 0x7fffffff); - mtspr(SPRN_LPCR, vcpu->kvm->arch.host_lpcr); - - return trap; + vcpu->arch.iamr = mfspr(SPRN_IAMR); + vcpu->arch.pspb = mfspr(SPRN_PSPB); + vcpu->arch.fscr = mfspr(SPRN_FSCR); + vcpu->arch.tar = mfspr(SPRN_TAR); + vcpu->arch.ebbhr = mfspr(SPRN_EBBHR); + vcpu->arch.ebbrr = mfspr(SPRN_EBBRR); + vcpu->arch.bescr = mfspr(SPRN_BESCR); + vcpu->arch.wort = mfspr(SPRN_WORT); + vcpu->arch.tid = mfspr(SPRN_TIDR); + vcpu->arch.amr = mfspr(SPRN_AMR); + vcpu->arch.uamor = mfspr(SPRN_UAMOR); + vcpu->arch.dscr = mfspr(SPRN_DSCR); } /* - * Virtual-mode guest entry for POWER9 and later when the host and - * guest are both using the radix MMU. The LPIDR has already been set. + * Privileged (non-hypervisor) host registers to save. + */ +struct p9_host_os_sprs { + unsigned long dscr; + unsigned long tidr; + unsigned long iamr; + unsigned long amr; + unsigned long fscr; +}; + +static void save_p9_host_os_sprs(struct p9_host_os_sprs *host_os_sprs) +{ + host_os_sprs->dscr = mfspr(SPRN_DSCR); + host_os_sprs->tidr = mfspr(SPRN_TIDR); + host_os_sprs->iamr = mfspr(SPRN_IAMR); + host_os_sprs->amr = mfspr(SPRN_AMR); + host_os_sprs->fscr = mfspr(SPRN_FSCR); +} + +/* vcpu guest regs must already be saved */ +static void restore_p9_host_os_sprs(struct kvm_vcpu *vcpu, + struct p9_host_os_sprs *host_os_sprs) +{ + mtspr(SPRN_PSPB, 0); + mtspr(SPRN_WORT, 0); + mtspr(SPRN_UAMOR, 0); + + mtspr(SPRN_DSCR, host_os_sprs->dscr); + mtspr(SPRN_TIDR, host_os_sprs->tidr); + mtspr(SPRN_IAMR, host_os_sprs->iamr); + + if (host_os_sprs->amr != vcpu->arch.amr) + mtspr(SPRN_AMR, host_os_sprs->amr); + + if (host_os_sprs->fscr != vcpu->arch.fscr) + mtspr(SPRN_FSCR, host_os_sprs->fscr); + + /* Save guest CTRL register, set runlatch to 1 */ + if (!(vcpu->arch.ctrl & 1)) + mtspr(SPRN_CTRLT, 1); +} + +static inline bool hcall_is_xics(unsigned long req) +{ + return req == H_EOI || req == H_CPPR || req == H_IPI || + req == H_IPOLL || req == H_XIRR || req == H_XIRR_X; +} + +/* + * Guest entry for POWER9 and later CPUs. */ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr) { struct kvmppc_vcore *vc = vcpu->arch.vcore; - unsigned long host_dscr = mfspr(SPRN_DSCR); - unsigned long host_tidr = mfspr(SPRN_TIDR); - unsigned long host_iamr = mfspr(SPRN_IAMR); - unsigned long host_amr = mfspr(SPRN_AMR); - unsigned long host_fscr = mfspr(SPRN_FSCR); + struct p9_host_os_sprs host_os_sprs; s64 dec; u64 tb; int trap, save_pmu; + WARN_ON_ONCE(vcpu->arch.ceded); + dec = mfspr(SPRN_DEC); tb = mftb(); if (dec < 0) @@ -3664,7 +3830,7 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, if (local_paca->kvm_hstate.dec_expires < time_limit) time_limit = local_paca->kvm_hstate.dec_expires; - vcpu->arch.ceded = 0; + save_p9_host_os_sprs(&host_os_sprs); kvmhv_save_host_pmu(); /* saves it to PACA kvm_hstate */ @@ -3693,24 +3859,20 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, #endif mtspr(SPRN_VRSAVE, vcpu->arch.vrsave); - mtspr(SPRN_DSCR, vcpu->arch.dscr); - mtspr(SPRN_IAMR, vcpu->arch.iamr); - mtspr(SPRN_PSPB, vcpu->arch.pspb); - mtspr(SPRN_FSCR, vcpu->arch.fscr); - mtspr(SPRN_TAR, vcpu->arch.tar); - mtspr(SPRN_EBBHR, vcpu->arch.ebbhr); - mtspr(SPRN_EBBRR, vcpu->arch.ebbrr); - mtspr(SPRN_BESCR, vcpu->arch.bescr); - mtspr(SPRN_WORT, vcpu->arch.wort); - mtspr(SPRN_TIDR, vcpu->arch.tid); - mtspr(SPRN_DAR, vcpu->arch.shregs.dar); - mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); - mtspr(SPRN_AMR, vcpu->arch.amr); - mtspr(SPRN_UAMOR, vcpu->arch.uamor); - - if (!(vcpu->arch.ctrl & 1)) - mtspr(SPRN_CTRLT, mfspr(SPRN_CTRLF) & ~1); + load_spr_state(vcpu); + /* + * When setting DEC, we must always deal with irq_work_raise via NMI vs + * setting DEC. The problem occurs right as we switch into guest mode + * if a NMI hits and sets pending work and sets DEC, then that will + * apply to the guest and not bring us back to the host. + * + * irq_work_raise could check a flag (or possibly LPCR[HDICE] for + * example) and set HDEC to 1? That wouldn't solve the nested hv + * case which needs to abort the hcall or zero the time limit. + * + * XXX: Another day's problem. + */ mtspr(SPRN_DEC, vcpu->arch.dec_expires - mftb()); if (kvmhv_on_pseries()) { @@ -3718,7 +3880,7 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, * We need to save and restore the guest visible part of the * psscr (i.e. using SPRN_PSSCR_PR) since the hypervisor * doesn't do this for us. Note only required if pseries since - * this is done in kvmhv_load_hv_regs_and_go() below otherwise. + * this is done in kvmhv_vcpu_entry_p9() below otherwise. */ unsigned long host_psscr; /* call our hypervisor to load up HV regs and go */ @@ -3738,6 +3900,8 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, hvregs.vcpu_token = vcpu->vcpu_id; } hvregs.hdec_expiry = time_limit; + mtspr(SPRN_DAR, vcpu->arch.shregs.dar); + mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); trap = plpar_hcall_norets(H_ENTER_NESTED, __pa(&hvregs), __pa(&vcpu->arch.regs)); kvmhv_restore_hv_return_state(vcpu, &hvregs); @@ -3750,15 +3914,41 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, /* H_CEDE has to be handled now, not later */ if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested && kvmppc_get_gpr(vcpu, 3) == H_CEDE) { - kvmppc_nested_cede(vcpu); + kvmppc_cede(vcpu); kvmppc_set_gpr(vcpu, 3, 0); trap = 0; } } else { - trap = kvmhv_load_hv_regs_and_go(vcpu, time_limit, lpcr); + kvmppc_xive_push_vcpu(vcpu); + trap = kvmhv_vcpu_entry_p9(vcpu, time_limit, lpcr); + if (trap == BOOK3S_INTERRUPT_SYSCALL && !vcpu->arch.nested && + !(vcpu->arch.shregs.msr & MSR_PR)) { + unsigned long req = kvmppc_get_gpr(vcpu, 3); + + /* H_CEDE has to be handled now, not later */ + if (req == H_CEDE) { + kvmppc_cede(vcpu); + kvmppc_xive_rearm_escalation(vcpu); /* may un-cede */ + kvmppc_set_gpr(vcpu, 3, 0); + trap = 0; + + /* XICS hcalls must be handled before xive is pulled */ + } else if (hcall_is_xics(req)) { + int ret; + + ret = kvmppc_xive_xics_hcall(vcpu, req); + if (ret != H_TOO_HARD) { + kvmppc_set_gpr(vcpu, 3, ret); + trap = 0; + } + } + } + kvmppc_xive_pull_vcpu(vcpu); + + if (kvm_is_radix(vcpu->kvm)) + vcpu->arch.slb_max = 0; } - vcpu->arch.slb_max = 0; dec = mfspr(SPRN_DEC); if (!(lpcr & LPCR_LD)) /* Sign extend if not using large decrementer */ dec = (s32) dec; @@ -3766,36 +3956,10 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, vcpu->arch.dec_expires = dec + tb; vcpu->cpu = -1; vcpu->arch.thread_cpu = -1; - /* Save guest CTRL register, set runlatch to 1 */ - vcpu->arch.ctrl = mfspr(SPRN_CTRLF); - if (!(vcpu->arch.ctrl & 1)) - mtspr(SPRN_CTRLT, vcpu->arch.ctrl | 1); - vcpu->arch.iamr = mfspr(SPRN_IAMR); - vcpu->arch.pspb = mfspr(SPRN_PSPB); - vcpu->arch.fscr = mfspr(SPRN_FSCR); - vcpu->arch.tar = mfspr(SPRN_TAR); - vcpu->arch.ebbhr = mfspr(SPRN_EBBHR); - vcpu->arch.ebbrr = mfspr(SPRN_EBBRR); - vcpu->arch.bescr = mfspr(SPRN_BESCR); - vcpu->arch.wort = mfspr(SPRN_WORT); - vcpu->arch.tid = mfspr(SPRN_TIDR); - vcpu->arch.amr = mfspr(SPRN_AMR); - vcpu->arch.uamor = mfspr(SPRN_UAMOR); - vcpu->arch.dscr = mfspr(SPRN_DSCR); + store_spr_state(vcpu); - mtspr(SPRN_PSPB, 0); - mtspr(SPRN_WORT, 0); - mtspr(SPRN_UAMOR, 0); - mtspr(SPRN_DSCR, host_dscr); - mtspr(SPRN_TIDR, host_tidr); - mtspr(SPRN_IAMR, host_iamr); - - if (host_amr != vcpu->arch.amr) - mtspr(SPRN_AMR, host_amr); - - if (host_fscr != vcpu->arch.fscr) - mtspr(SPRN_FSCR, host_fscr); + restore_p9_host_os_sprs(vcpu, &host_os_sprs); msr_check_and_set(MSR_FP | MSR_VEC | MSR_VSX); store_fp_state(&vcpu->arch.fp); @@ -3825,6 +3989,9 @@ static int kvmhv_p9_guest_entry(struct kvm_vcpu *vcpu, u64 time_limit, vc->in_guest = 0; mtspr(SPRN_DEC, local_paca->kvm_hstate.dec_expires - mftb()); + /* We may have raced with new irq work */ + if (test_irq_work_pending()) + set_dec(1); mtspr(SPRN_SPRG_VDSO_WRITE, local_paca->sprg_vdso); kvmhv_load_host_pmu(); @@ -3925,7 +4092,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) cur = start_poll = ktime_get(); if (vc->halt_poll_ns) { ktime_t stop = ktime_add_ns(start_poll, vc->halt_poll_ns); - ++vc->runner->stat.halt_attempted_poll; + ++vc->runner->stat.generic.halt_attempted_poll; vc->vcore_state = VCORE_POLLING; spin_unlock(&vc->lock); @@ -3942,7 +4109,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) vc->vcore_state = VCORE_INACTIVE; if (!do_sleep) { - ++vc->runner->stat.halt_successful_poll; + ++vc->runner->stat.generic.halt_successful_poll; goto out; } } @@ -3954,7 +4121,7 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) do_sleep = 0; /* If we polled, count this as a successful poll */ if (vc->halt_poll_ns) - ++vc->runner->stat.halt_successful_poll; + ++vc->runner->stat.generic.halt_successful_poll; goto out; } @@ -3981,13 +4148,13 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) ktime_to_ns(cur) - ktime_to_ns(start_wait); /* Attribute failed poll time */ if (vc->halt_poll_ns) - vc->runner->stat.halt_poll_fail_ns += + vc->runner->stat.generic.halt_poll_fail_ns += ktime_to_ns(start_wait) - ktime_to_ns(start_poll); } else { /* Attribute successful poll time */ if (vc->halt_poll_ns) - vc->runner->stat.halt_poll_success_ns += + vc->runner->stat.generic.halt_poll_success_ns += ktime_to_ns(cur) - ktime_to_ns(start_poll); } @@ -4014,7 +4181,6 @@ static void kvmppc_vcore_blocked(struct kvmppc_vcore *vc) /* * This never fails for a radix guest, as none of the operations it does * for a radix guest can fail or have a way to report failure. - * kvmhv_run_single_vcpu() relies on this fact. */ static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu) { @@ -4170,7 +4336,7 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, { struct kvm_run *run = vcpu->run; int trap, r, pcpu; - int srcu_idx, lpid; + int srcu_idx; struct kvmppc_vcore *vc; struct kvm *kvm = vcpu->kvm; struct kvm_nested_guest *nested = vcpu->arch.nested; @@ -4193,8 +4359,15 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, vc->runner = vcpu; /* See if the MMU is ready to go */ - if (!kvm->arch.mmu_ready) - kvmhv_setup_mmu(vcpu); + if (!kvm->arch.mmu_ready) { + r = kvmhv_setup_mmu(vcpu); + if (r) { + run->exit_reason = KVM_EXIT_FAIL_ENTRY; + run->fail_entry.hardware_entry_failure_reason = 0; + vcpu->arch.ret = r; + return r; + } + } if (need_resched()) cond_resched(); @@ -4207,7 +4380,8 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, preempt_disable(); pcpu = smp_processor_id(); vc->pcpu = pcpu; - kvmppc_prepare_radix_vcpu(vcpu, pcpu); + if (kvm_is_radix(kvm)) + kvmppc_prepare_radix_vcpu(vcpu, pcpu); local_irq_disable(); hard_irq_disable(); @@ -4244,13 +4418,6 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, vc->vcore_state = VCORE_RUNNING; trace_kvmppc_run_core(vc, 0); - if (cpu_has_feature(CPU_FTR_HVMODE)) { - lpid = nested ? nested->shadow_lpid : kvm->arch.lpid; - mtspr(SPRN_LPID, lpid); - isync(); - kvmppc_check_need_tlb_flush(kvm, pcpu, nested); - } - guest_enter_irqoff(); srcu_idx = srcu_read_lock(&kvm->srcu); @@ -4269,11 +4436,6 @@ int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, srcu_read_unlock(&kvm->srcu, srcu_idx); - if (cpu_has_feature(CPU_FTR_HVMODE)) { - mtspr(SPRN_LPID, kvm->arch.host_lpid); - isync(); - } - set_irq_happened(trap); kvmppc_set_host_core(pcpu); @@ -4419,19 +4581,23 @@ static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) vcpu->arch.state = KVMPPC_VCPU_BUSY_IN_HOST; do { - /* - * The TLB prefetch bug fixup is only in the kvmppc_run_vcpu - * path, which also handles hash and dependent threads mode. - */ - if (kvm->arch.threads_indep && kvm_is_radix(kvm) && - !cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) + if (cpu_has_feature(CPU_FTR_ARCH_300)) r = kvmhv_run_single_vcpu(vcpu, ~(u64)0, vcpu->arch.vcore->lpcr); else r = kvmppc_run_vcpu(vcpu); - if (run->exit_reason == KVM_EXIT_PAPR_HCALL && - !(vcpu->arch.shregs.msr & MSR_PR)) { + if (run->exit_reason == KVM_EXIT_PAPR_HCALL) { + if (WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_PR)) { + /* + * These should have been caught reflected + * into the guest by now. Final sanity check: + * don't allow userspace to execute hcalls in + * the hypervisor. + */ + r = RESUME_GUEST; + continue; + } trace_kvm_hcall_enter(vcpu); r = kvmppc_pseries_do_hcall(vcpu); trace_kvm_hcall_exit(vcpu, r); @@ -5038,18 +5204,8 @@ static int kvmppc_core_init_vm_hv(struct kvm *kvm) /* * Track that we now have a HV mode VM active. This blocks secondary * CPU threads from coming online. - * On POWER9, we only need to do this if the "indep_threads_mode" - * module parameter has been set to N. */ - if (cpu_has_feature(CPU_FTR_ARCH_300)) { - if (!indep_threads_mode && !cpu_has_feature(CPU_FTR_HVMODE)) { - pr_warn("KVM: Ignoring indep_threads_mode=N in nested hypervisor\n"); - kvm->arch.threads_indep = true; - } else { - kvm->arch.threads_indep = indep_threads_mode; - } - } - if (!kvm->arch.threads_indep) + if (!cpu_has_feature(CPU_FTR_ARCH_300)) kvm_hv_vm_activated(); /* @@ -5090,7 +5246,7 @@ static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) { debugfs_remove_recursive(kvm->arch.debugfs_dir); - if (!kvm->arch.threads_indep) + if (!cpu_has_feature(CPU_FTR_ARCH_300)) kvm_hv_vm_deactivated(); kvmppc_free_vcores(kvm); @@ -5511,7 +5667,9 @@ static int kvmhv_enable_nested(struct kvm *kvm) { if (!nested) return -EPERM; - if (!cpu_has_feature(CPU_FTR_ARCH_300) || no_mixing_hpt_and_radix) + if (!cpu_has_feature(CPU_FTR_ARCH_300)) + return -ENODEV; + if (!radix_enabled()) return -ENODEV; /* kvm == NULL means the caller is testing if the capability exists */ @@ -5674,11 +5832,25 @@ static int kvmhv_enable_dawr1(struct kvm *kvm) static bool kvmppc_hash_v3_possible(void) { - if (radix_enabled() && no_mixing_hpt_and_radix) + if (!cpu_has_feature(CPU_FTR_ARCH_300)) return false; - return cpu_has_feature(CPU_FTR_ARCH_300) && - cpu_has_feature(CPU_FTR_HVMODE); + if (!cpu_has_feature(CPU_FTR_HVMODE)) + return false; + + /* + * POWER9 chips before version 2.02 can't have some threads in + * HPT mode and some in radix mode on the same core. + */ + if (radix_enabled()) { + unsigned int pvr = mfspr(SPRN_PVR); + if ((pvr >> 16) == PVR_POWER9 && + (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) || + ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101))) + return false; + } + + return true; } static struct kvmppc_ops kvm_ops_hv = { @@ -5822,18 +5994,6 @@ static int kvmppc_book3s_init_hv(void) if (kvmppc_radix_possible()) r = kvmppc_radix_init(); - /* - * POWER9 chips before version 2.02 can't have some threads in - * HPT mode and some in radix mode on the same core. - */ - if (cpu_has_feature(CPU_FTR_ARCH_300)) { - unsigned int pvr = mfspr(SPRN_PVR); - if ((pvr >> 16) == PVR_POWER9 && - (((pvr & 0xe000) == 0 && (pvr & 0xfff) < 0x202) || - ((pvr & 0xe000) == 0x2000 && (pvr & 0xfff) < 0x101))) - no_mixing_hpt_and_radix = true; - } - r = kvmppc_uvmem_init(); if (r < 0) pr_err("KVM-HV: kvmppc_uvmem_init failed %d\n", r); diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c index 7a0e33a9c980..be8ef1c5b1bf 100644 --- a/arch/powerpc/kvm/book3s_hv_builtin.c +++ b/arch/powerpc/kvm/book3s_hv_builtin.c @@ -34,21 +34,6 @@ #include "book3s_xics.h" #include "book3s_xive.h" -/* - * The XIVE module will populate these when it loads - */ -unsigned long (*__xive_vm_h_xirr)(struct kvm_vcpu *vcpu); -unsigned long (*__xive_vm_h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server); -int (*__xive_vm_h_ipi)(struct kvm_vcpu *vcpu, unsigned long server, - unsigned long mfrr); -int (*__xive_vm_h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr); -int (*__xive_vm_h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr); -EXPORT_SYMBOL_GPL(__xive_vm_h_xirr); -EXPORT_SYMBOL_GPL(__xive_vm_h_ipoll); -EXPORT_SYMBOL_GPL(__xive_vm_h_ipi); -EXPORT_SYMBOL_GPL(__xive_vm_h_cppr); -EXPORT_SYMBOL_GPL(__xive_vm_h_eoi); - /* * Hash page table alignment on newer cpus(CPU_FTR_ARCH_206) * should be power of 2. @@ -196,16 +181,9 @@ int kvmppc_hwrng_present(void) } EXPORT_SYMBOL_GPL(kvmppc_hwrng_present); -long kvmppc_h_random(struct kvm_vcpu *vcpu) +long kvmppc_rm_h_random(struct kvm_vcpu *vcpu) { - int r; - - /* Only need to do the expensive mfmsr() on radix */ - if (kvm_is_radix(vcpu->kvm) && (mfmsr() & MSR_IR)) - r = powernv_get_random_long(&vcpu->arch.regs.gpr[4]); - else - r = powernv_get_random_real_mode(&vcpu->arch.regs.gpr[4]); - if (r) + if (powernv_get_random_real_mode(&vcpu->arch.regs.gpr[4])) return H_SUCCESS; return H_HARDWARE; @@ -221,15 +199,6 @@ void kvmhv_rm_send_ipi(int cpu) void __iomem *xics_phys; unsigned long msg = PPC_DBELL_TYPE(PPC_DBELL_SERVER); - /* For a nested hypervisor, use the XICS via hcall */ - if (kvmhv_on_pseries()) { - unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; - - plpar_hcall_raw(H_IPI, retbuf, get_hard_smp_processor_id(cpu), - IPI_PRIORITY); - return; - } - /* On POWER9 we can use msgsnd for any destination cpu. */ if (cpu_has_feature(CPU_FTR_ARCH_300)) { msg |= get_hard_smp_processor_id(cpu); @@ -442,19 +411,12 @@ static long kvmppc_read_one_intr(bool *again) return 1; /* Now read the interrupt from the ICP */ - if (kvmhv_on_pseries()) { - unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; - - rc = plpar_hcall_raw(H_XIRR, retbuf, 0xFF); - xirr = cpu_to_be32(retbuf[0]); - } else { - xics_phys = local_paca->kvm_hstate.xics_phys; - rc = 0; - if (!xics_phys) - rc = opal_int_get_xirr(&xirr, false); - else - xirr = __raw_rm_readl(xics_phys + XICS_XIRR); - } + xics_phys = local_paca->kvm_hstate.xics_phys; + rc = 0; + if (!xics_phys) + rc = opal_int_get_xirr(&xirr, false); + else + xirr = __raw_rm_readl(xics_phys + XICS_XIRR); if (rc < 0) return 1; @@ -483,13 +445,7 @@ static long kvmppc_read_one_intr(bool *again) */ if (xisr == XICS_IPI) { rc = 0; - if (kvmhv_on_pseries()) { - unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; - - plpar_hcall_raw(H_IPI, retbuf, - hard_smp_processor_id(), 0xff); - plpar_hcall_raw(H_EOI, retbuf, h_xirr); - } else if (xics_phys) { + if (xics_phys) { __raw_rm_writeb(0xff, xics_phys + XICS_MFRR); __raw_rm_writel(xirr, xics_phys + XICS_XIRR); } else { @@ -515,13 +471,7 @@ static long kvmppc_read_one_intr(bool *again) /* We raced with the host, * we need to resend that IPI, bummer */ - if (kvmhv_on_pseries()) { - unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; - - plpar_hcall_raw(H_IPI, retbuf, - hard_smp_processor_id(), - IPI_PRIORITY); - } else if (xics_phys) + if (xics_phys) __raw_rm_writeb(IPI_PRIORITY, xics_phys + XICS_MFRR); else @@ -541,22 +491,13 @@ static long kvmppc_read_one_intr(bool *again) } #ifdef CONFIG_KVM_XICS -static inline bool is_rm(void) -{ - return !(mfmsr() & MSR_DR); -} - unsigned long kvmppc_rm_h_xirr(struct kvm_vcpu *vcpu) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xics_on_xive()) { - if (is_rm()) - return xive_rm_h_xirr(vcpu); - if (unlikely(!__xive_vm_h_xirr)) - return H_NOT_AVAILABLE; - return __xive_vm_h_xirr(vcpu); - } else + if (xics_on_xive()) + return xive_rm_h_xirr(vcpu); + else return xics_rm_h_xirr(vcpu); } @@ -565,13 +506,9 @@ unsigned long kvmppc_rm_h_xirr_x(struct kvm_vcpu *vcpu) if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; vcpu->arch.regs.gpr[5] = get_tb(); - if (xics_on_xive()) { - if (is_rm()) - return xive_rm_h_xirr(vcpu); - if (unlikely(!__xive_vm_h_xirr)) - return H_NOT_AVAILABLE; - return __xive_vm_h_xirr(vcpu); - } else + if (xics_on_xive()) + return xive_rm_h_xirr(vcpu); + else return xics_rm_h_xirr(vcpu); } @@ -579,13 +516,9 @@ unsigned long kvmppc_rm_h_ipoll(struct kvm_vcpu *vcpu, unsigned long server) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xics_on_xive()) { - if (is_rm()) - return xive_rm_h_ipoll(vcpu, server); - if (unlikely(!__xive_vm_h_ipoll)) - return H_NOT_AVAILABLE; - return __xive_vm_h_ipoll(vcpu, server); - } else + if (xics_on_xive()) + return xive_rm_h_ipoll(vcpu, server); + else return H_TOO_HARD; } @@ -594,13 +527,9 @@ int kvmppc_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xics_on_xive()) { - if (is_rm()) - return xive_rm_h_ipi(vcpu, server, mfrr); - if (unlikely(!__xive_vm_h_ipi)) - return H_NOT_AVAILABLE; - return __xive_vm_h_ipi(vcpu, server, mfrr); - } else + if (xics_on_xive()) + return xive_rm_h_ipi(vcpu, server, mfrr); + else return xics_rm_h_ipi(vcpu, server, mfrr); } @@ -608,13 +537,9 @@ int kvmppc_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xics_on_xive()) { - if (is_rm()) - return xive_rm_h_cppr(vcpu, cppr); - if (unlikely(!__xive_vm_h_cppr)) - return H_NOT_AVAILABLE; - return __xive_vm_h_cppr(vcpu, cppr); - } else + if (xics_on_xive()) + return xive_rm_h_cppr(vcpu, cppr); + else return xics_rm_h_cppr(vcpu, cppr); } @@ -622,13 +547,9 @@ int kvmppc_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) { if (!kvmppc_xics_enabled(vcpu)) return H_TOO_HARD; - if (xics_on_xive()) { - if (is_rm()) - return xive_rm_h_eoi(vcpu, xirr); - if (unlikely(!__xive_vm_h_eoi)) - return H_NOT_AVAILABLE; - return __xive_vm_h_eoi(vcpu, xirr); - } else + if (xics_on_xive()) + return xive_rm_h_eoi(vcpu, xirr); + else return xics_rm_h_eoi(vcpu, xirr); } #endif /* CONFIG_KVM_XICS */ @@ -800,7 +721,7 @@ void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu, * Thus we make all 4 threads use the same bit. */ if (cpu_has_feature(CPU_FTR_ARCH_300)) - pcpu = cpu_first_thread_sibling(pcpu); + pcpu = cpu_first_tlb_thread_sibling(pcpu); if (nested) need_tlb_flush = &nested->need_tlb_flush; diff --git a/arch/powerpc/kvm/book3s_hv_interrupts.S b/arch/powerpc/kvm/book3s_hv_interrupts.S index 327417d79eac..4444f83cb133 100644 --- a/arch/powerpc/kvm/book3s_hv_interrupts.S +++ b/arch/powerpc/kvm/book3s_hv_interrupts.S @@ -58,7 +58,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) /* * Put whatever is in the decrementer into the * hypervisor decrementer. - * Because of a hardware deviation in P8 and P9, + * Because of a hardware deviation in P8, * we need to set LPCR[HDICE] before writing HDEC. */ ld r5, HSTATE_KVM_VCORE(r13) @@ -67,15 +67,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) ori r8, r9, LPCR_HDICE mtspr SPRN_LPCR, r8 isync - andis. r0, r9, LPCR_LD@h mfspr r8,SPRN_DEC mftb r7 -BEGIN_FTR_SECTION - /* On POWER9, don't sign-extend if host LPCR[LD] bit is set */ - bne 32f -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) extsw r8,r8 -32: mtspr SPRN_HDEC,r8 + mtspr SPRN_HDEC,r8 add r8,r8,r7 std r8,HSTATE_DECEXP(r13) diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c index 60724f674421..8543ad538b0c 100644 --- a/arch/powerpc/kvm/book3s_hv_nested.c +++ b/arch/powerpc/kvm/book3s_hv_nested.c @@ -19,6 +19,7 @@ #include #include #include +#include static struct patb_entry *pseries_partition_tb; @@ -53,7 +54,8 @@ void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr) hr->dawrx1 = vcpu->arch.dawrx1; } -static void byteswap_pt_regs(struct pt_regs *regs) +/* Use noinline_for_stack due to https://bugs.llvm.org/show_bug.cgi?id=49610 */ +static noinline_for_stack void byteswap_pt_regs(struct pt_regs *regs) { unsigned long *addr = (unsigned long *) regs; @@ -467,8 +469,15 @@ static void kvmhv_flush_lpid(unsigned int lpid) return; } - rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(2, 0, 1), - lpid, TLBIEL_INVAL_SET_LPID); + if (!firmware_has_feature(FW_FEATURE_RPT_INVALIDATE)) + rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(2, 0, 1), + lpid, TLBIEL_INVAL_SET_LPID); + else + rc = pseries_rpt_invalidate(lpid, H_RPTI_TARGET_CMMU, + H_RPTI_TYPE_NESTED | + H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC | + H_RPTI_TYPE_PAT, + H_RPTI_PAGE_ALL, 0, -1UL); if (rc) pr_err("KVM: TLB LPID invalidation hcall failed, rc=%ld\n", rc); } @@ -1214,6 +1223,113 @@ long kvmhv_do_nested_tlbie(struct kvm_vcpu *vcpu) return H_SUCCESS; } +static long do_tlb_invalidate_nested_all(struct kvm_vcpu *vcpu, + unsigned long lpid, unsigned long ric) +{ + struct kvm *kvm = vcpu->kvm; + struct kvm_nested_guest *gp; + + gp = kvmhv_get_nested(kvm, lpid, false); + if (gp) { + kvmhv_emulate_tlbie_lpid(vcpu, gp, ric); + kvmhv_put_nested(gp); + } + return H_SUCCESS; +} + +/* + * Number of pages above which we invalidate the entire LPID rather than + * flush individual pages. + */ +static unsigned long tlb_range_flush_page_ceiling __read_mostly = 33; + +static long do_tlb_invalidate_nested_tlb(struct kvm_vcpu *vcpu, + unsigned long lpid, + unsigned long pg_sizes, + unsigned long start, + unsigned long end) +{ + int ret = H_P4; + unsigned long addr, nr_pages; + struct mmu_psize_def *def; + unsigned long psize, ap, page_size; + bool flush_lpid; + + for (psize = 0; psize < MMU_PAGE_COUNT; psize++) { + def = &mmu_psize_defs[psize]; + if (!(pg_sizes & def->h_rpt_pgsize)) + continue; + + nr_pages = (end - start) >> def->shift; + flush_lpid = nr_pages > tlb_range_flush_page_ceiling; + if (flush_lpid) + return do_tlb_invalidate_nested_all(vcpu, lpid, + RIC_FLUSH_TLB); + addr = start; + ap = mmu_get_ap(psize); + page_size = 1UL << def->shift; + do { + ret = kvmhv_emulate_tlbie_tlb_addr(vcpu, lpid, ap, + get_epn(addr)); + if (ret) + return H_P4; + addr += page_size; + } while (addr < end); + } + return ret; +} + +/* + * Performs partition-scoped invalidations for nested guests + * as part of H_RPT_INVALIDATE hcall. + */ +long do_h_rpt_invalidate_pat(struct kvm_vcpu *vcpu, unsigned long lpid, + unsigned long type, unsigned long pg_sizes, + unsigned long start, unsigned long end) +{ + /* + * If L2 lpid isn't valid, we need to return H_PARAMETER. + * + * However, nested KVM issues a L2 lpid flush call when creating + * partition table entries for L2. This happens even before the + * corresponding shadow lpid is created in HV which happens in + * H_ENTER_NESTED call. Since we can't differentiate this case from + * the invalid case, we ignore such flush requests and return success. + */ + if (!kvmhv_find_nested(vcpu->kvm, lpid)) + return H_SUCCESS; + + /* + * A flush all request can be handled by a full lpid flush only. + */ + if ((type & H_RPTI_TYPE_NESTED_ALL) == H_RPTI_TYPE_NESTED_ALL) + return do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_ALL); + + /* + * We don't need to handle a PWC flush like process table here, + * because intermediate partition scoped table in nested guest doesn't + * really have PWC. Only level we have PWC is in L0 and for nested + * invalidate at L0 we always do kvm_flush_lpid() which does + * radix__flush_all_lpid(). For range invalidate at any level, we + * are not removing the higher level page tables and hence there is + * no PWC invalidate needed. + * + * if (type & H_RPTI_TYPE_PWC) { + * ret = do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_PWC); + * if (ret) + * return H_P4; + * } + */ + + if (start == 0 && end == -1) + return do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_TLB); + + if (type & H_RPTI_TYPE_TLB) + return do_tlb_invalidate_nested_tlb(vcpu, lpid, pg_sizes, + start, end); + return H_SUCCESS; +} + /* Used to convert a nested guest real address to a L1 guest real address */ static int kvmhv_translate_addr_nested(struct kvm_vcpu *vcpu, struct kvm_nested_guest *gp, diff --git a/arch/powerpc/kvm/book3s_hv_p9_entry.c b/arch/powerpc/kvm/book3s_hv_p9_entry.c new file mode 100644 index 000000000000..83f592eadcd2 --- /dev/null +++ b/arch/powerpc/kvm/book3s_hv_p9_entry.c @@ -0,0 +1,508 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING +static void __start_timing(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) +{ + struct kvmppc_vcore *vc = vcpu->arch.vcore; + u64 tb = mftb() - vc->tb_offset_applied; + + vcpu->arch.cur_activity = next; + vcpu->arch.cur_tb_start = tb; +} + +static void __accumulate_time(struct kvm_vcpu *vcpu, struct kvmhv_tb_accumulator *next) +{ + struct kvmppc_vcore *vc = vcpu->arch.vcore; + struct kvmhv_tb_accumulator *curr; + u64 tb = mftb() - vc->tb_offset_applied; + u64 prev_tb; + u64 delta; + u64 seq; + + curr = vcpu->arch.cur_activity; + vcpu->arch.cur_activity = next; + prev_tb = vcpu->arch.cur_tb_start; + vcpu->arch.cur_tb_start = tb; + + if (!curr) + return; + + delta = tb - prev_tb; + + seq = curr->seqcount; + curr->seqcount = seq + 1; + smp_wmb(); + curr->tb_total += delta; + if (seq == 0 || delta < curr->tb_min) + curr->tb_min = delta; + if (delta > curr->tb_max) + curr->tb_max = delta; + smp_wmb(); + curr->seqcount = seq + 2; +} + +#define start_timing(vcpu, next) __start_timing(vcpu, next) +#define end_timing(vcpu) __start_timing(vcpu, NULL) +#define accumulate_time(vcpu, next) __accumulate_time(vcpu, next) +#else +#define start_timing(vcpu, next) do {} while (0) +#define end_timing(vcpu) do {} while (0) +#define accumulate_time(vcpu, next) do {} while (0) +#endif + +static inline void mfslb(unsigned int idx, u64 *slbee, u64 *slbev) +{ + asm volatile("slbmfev %0,%1" : "=r" (*slbev) : "r" (idx)); + asm volatile("slbmfee %0,%1" : "=r" (*slbee) : "r" (idx)); +} + +static inline void mtslb(u64 slbee, u64 slbev) +{ + asm volatile("slbmte %0,%1" :: "r" (slbev), "r" (slbee)); +} + +static inline void clear_slb_entry(unsigned int idx) +{ + mtslb(idx, 0); +} + +static inline void slb_clear_invalidate_partition(void) +{ + clear_slb_entry(0); + asm volatile(PPC_SLBIA(6)); +} + +/* + * Malicious or buggy radix guests may have inserted SLB entries + * (only 0..3 because radix always runs with UPRT=1), so these must + * be cleared here to avoid side-channels. slbmte is used rather + * than slbia, as it won't clear cached translations. + */ +static void radix_clear_slb(void) +{ + int i; + + for (i = 0; i < 4; i++) + clear_slb_entry(i); +} + +static void switch_mmu_to_guest_radix(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) +{ + struct kvm_nested_guest *nested = vcpu->arch.nested; + u32 lpid; + + lpid = nested ? nested->shadow_lpid : kvm->arch.lpid; + + /* + * All the isync()s are overkill but trivially follow the ISA + * requirements. Some can likely be replaced with justification + * comment for why they are not needed. + */ + isync(); + mtspr(SPRN_LPID, lpid); + isync(); + mtspr(SPRN_LPCR, lpcr); + isync(); + mtspr(SPRN_PID, vcpu->arch.pid); + isync(); +} + +static void switch_mmu_to_guest_hpt(struct kvm *kvm, struct kvm_vcpu *vcpu, u64 lpcr) +{ + u32 lpid; + int i; + + lpid = kvm->arch.lpid; + + mtspr(SPRN_LPID, lpid); + mtspr(SPRN_LPCR, lpcr); + mtspr(SPRN_PID, vcpu->arch.pid); + + for (i = 0; i < vcpu->arch.slb_max; i++) + mtslb(vcpu->arch.slb[i].orige, vcpu->arch.slb[i].origv); + + isync(); +} + +static void switch_mmu_to_host(struct kvm *kvm, u32 pid) +{ + isync(); + mtspr(SPRN_PID, pid); + isync(); + mtspr(SPRN_LPID, kvm->arch.host_lpid); + isync(); + mtspr(SPRN_LPCR, kvm->arch.host_lpcr); + isync(); + + if (!radix_enabled()) + slb_restore_bolted_realmode(); +} + +static void save_clear_host_mmu(struct kvm *kvm) +{ + if (!radix_enabled()) { + /* + * Hash host could save and restore host SLB entries to + * reduce SLB fault overheads of VM exits, but for now the + * existing code clears all entries and restores just the + * bolted ones when switching back to host. + */ + slb_clear_invalidate_partition(); + } +} + +static void save_clear_guest_mmu(struct kvm *kvm, struct kvm_vcpu *vcpu) +{ + if (kvm_is_radix(kvm)) { + radix_clear_slb(); + } else { + int i; + int nr = 0; + + /* + * This must run before switching to host (radix host can't + * access all SLBs). + */ + for (i = 0; i < vcpu->arch.slb_nr; i++) { + u64 slbee, slbev; + mfslb(i, &slbee, &slbev); + if (slbee & SLB_ESID_V) { + vcpu->arch.slb[nr].orige = slbee | i; + vcpu->arch.slb[nr].origv = slbev; + nr++; + } + } + vcpu->arch.slb_max = nr; + slb_clear_invalidate_partition(); + } +} + +int kvmhv_vcpu_entry_p9(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr) +{ + struct kvm *kvm = vcpu->kvm; + struct kvm_nested_guest *nested = vcpu->arch.nested; + struct kvmppc_vcore *vc = vcpu->arch.vcore; + s64 hdec; + u64 tb, purr, spurr; + u64 *exsave; + bool ri_set; + int trap; + unsigned long msr; + unsigned long host_hfscr; + unsigned long host_ciabr; + unsigned long host_dawr0; + unsigned long host_dawrx0; + unsigned long host_psscr; + unsigned long host_pidr; + unsigned long host_dawr1; + unsigned long host_dawrx1; + + hdec = time_limit - mftb(); + if (hdec < 0) + return BOOK3S_INTERRUPT_HV_DECREMENTER; + + WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_HV); + WARN_ON_ONCE(!(vcpu->arch.shregs.msr & MSR_ME)); + + start_timing(vcpu, &vcpu->arch.rm_entry); + + vcpu->arch.ceded = 0; + + if (vc->tb_offset) { + u64 new_tb = mftb() + vc->tb_offset; + mtspr(SPRN_TBU40, new_tb); + tb = mftb(); + if ((tb & 0xffffff) < (new_tb & 0xffffff)) + mtspr(SPRN_TBU40, new_tb + 0x1000000); + vc->tb_offset_applied = vc->tb_offset; + } + + msr = mfmsr(); + + host_hfscr = mfspr(SPRN_HFSCR); + host_ciabr = mfspr(SPRN_CIABR); + host_dawr0 = mfspr(SPRN_DAWR0); + host_dawrx0 = mfspr(SPRN_DAWRX0); + host_psscr = mfspr(SPRN_PSSCR); + host_pidr = mfspr(SPRN_PID); + if (cpu_has_feature(CPU_FTR_DAWR1)) { + host_dawr1 = mfspr(SPRN_DAWR1); + host_dawrx1 = mfspr(SPRN_DAWRX1); + } + + if (vc->pcr) + mtspr(SPRN_PCR, vc->pcr | PCR_MASK); + mtspr(SPRN_DPDES, vc->dpdes); + mtspr(SPRN_VTB, vc->vtb); + + local_paca->kvm_hstate.host_purr = mfspr(SPRN_PURR); + local_paca->kvm_hstate.host_spurr = mfspr(SPRN_SPURR); + mtspr(SPRN_PURR, vcpu->arch.purr); + mtspr(SPRN_SPURR, vcpu->arch.spurr); + + if (dawr_enabled()) { + mtspr(SPRN_DAWR0, vcpu->arch.dawr0); + mtspr(SPRN_DAWRX0, vcpu->arch.dawrx0); + if (cpu_has_feature(CPU_FTR_DAWR1)) { + mtspr(SPRN_DAWR1, vcpu->arch.dawr1); + mtspr(SPRN_DAWRX1, vcpu->arch.dawrx1); + } + } + mtspr(SPRN_CIABR, vcpu->arch.ciabr); + mtspr(SPRN_IC, vcpu->arch.ic); + + mtspr(SPRN_PSSCR, vcpu->arch.psscr | PSSCR_EC | + (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); + + mtspr(SPRN_HFSCR, vcpu->arch.hfscr); + + mtspr(SPRN_HSRR0, vcpu->arch.regs.nip); + mtspr(SPRN_HSRR1, (vcpu->arch.shregs.msr & ~MSR_HV) | MSR_ME); + + /* + * On POWER9 DD2.1 and below, sometimes on a Hypervisor Data Storage + * Interrupt (HDSI) the HDSISR is not be updated at all. + * + * To work around this we put a canary value into the HDSISR before + * returning to a guest and then check for this canary when we take a + * HDSI. If we find the canary on a HDSI, we know the hardware didn't + * update the HDSISR. In this case we return to the guest to retake the + * HDSI which should correctly update the HDSISR the second time HDSI + * entry. + * + * Just do this on all p9 processors for now. + */ + mtspr(SPRN_HDSISR, HDSISR_CANARY); + + mtspr(SPRN_SPRG0, vcpu->arch.shregs.sprg0); + mtspr(SPRN_SPRG1, vcpu->arch.shregs.sprg1); + mtspr(SPRN_SPRG2, vcpu->arch.shregs.sprg2); + mtspr(SPRN_SPRG3, vcpu->arch.shregs.sprg3); + + mtspr(SPRN_AMOR, ~0UL); + + local_paca->kvm_hstate.in_guest = KVM_GUEST_MODE_HV_P9; + + /* + * Hash host, hash guest, or radix guest with prefetch bug, all have + * to disable the MMU before switching to guest MMU state. + */ + if (!radix_enabled() || !kvm_is_radix(kvm) || + cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) + __mtmsrd(msr & ~(MSR_IR|MSR_DR|MSR_RI), 0); + + save_clear_host_mmu(kvm); + + if (kvm_is_radix(kvm)) { + switch_mmu_to_guest_radix(kvm, vcpu, lpcr); + if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) + __mtmsrd(0, 1); /* clear RI */ + + } else { + switch_mmu_to_guest_hpt(kvm, vcpu, lpcr); + } + + /* TLBIEL uses LPID=LPIDR, so run this after setting guest LPID */ + kvmppc_check_need_tlb_flush(kvm, vc->pcpu, nested); + + /* + * P9 suppresses the HDEC exception when LPCR[HDICE] = 0, + * so set guest LPCR (with HDICE) before writing HDEC. + */ + mtspr(SPRN_HDEC, hdec); + + mtspr(SPRN_DAR, vcpu->arch.shregs.dar); + mtspr(SPRN_DSISR, vcpu->arch.shregs.dsisr); + mtspr(SPRN_SRR0, vcpu->arch.shregs.srr0); + mtspr(SPRN_SRR1, vcpu->arch.shregs.srr1); + + accumulate_time(vcpu, &vcpu->arch.guest_time); + + kvmppc_p9_enter_guest(vcpu); + + accumulate_time(vcpu, &vcpu->arch.rm_intr); + + /* XXX: Could get these from r11/12 and paca exsave instead */ + vcpu->arch.shregs.srr0 = mfspr(SPRN_SRR0); + vcpu->arch.shregs.srr1 = mfspr(SPRN_SRR1); + vcpu->arch.shregs.dar = mfspr(SPRN_DAR); + vcpu->arch.shregs.dsisr = mfspr(SPRN_DSISR); + + /* 0x2 bit for HSRR is only used by PR and P7/8 HV paths, clear it */ + trap = local_paca->kvm_hstate.scratch0 & ~0x2; + + /* HSRR interrupts leave MSR[RI] unchanged, SRR interrupts clear it. */ + ri_set = false; + if (likely(trap > BOOK3S_INTERRUPT_MACHINE_CHECK)) { + if (trap != BOOK3S_INTERRUPT_SYSCALL && + (vcpu->arch.shregs.msr & MSR_RI)) + ri_set = true; + exsave = local_paca->exgen; + } else if (trap == BOOK3S_INTERRUPT_SYSTEM_RESET) { + exsave = local_paca->exnmi; + } else { /* trap == 0x200 */ + exsave = local_paca->exmc; + } + + vcpu->arch.regs.gpr[1] = local_paca->kvm_hstate.scratch1; + vcpu->arch.regs.gpr[3] = local_paca->kvm_hstate.scratch2; + + /* + * Only set RI after reading machine check regs (DAR, DSISR, SRR0/1) + * and hstate scratch (which we need to move into exsave to make + * re-entrant vs SRESET/MCE) + */ + if (ri_set) { + if (unlikely(!(mfmsr() & MSR_RI))) { + __mtmsrd(MSR_RI, 1); + WARN_ON_ONCE(1); + } + } else { + WARN_ON_ONCE(mfmsr() & MSR_RI); + __mtmsrd(MSR_RI, 1); + } + + vcpu->arch.regs.gpr[9] = exsave[EX_R9/sizeof(u64)]; + vcpu->arch.regs.gpr[10] = exsave[EX_R10/sizeof(u64)]; + vcpu->arch.regs.gpr[11] = exsave[EX_R11/sizeof(u64)]; + vcpu->arch.regs.gpr[12] = exsave[EX_R12/sizeof(u64)]; + vcpu->arch.regs.gpr[13] = exsave[EX_R13/sizeof(u64)]; + vcpu->arch.ppr = exsave[EX_PPR/sizeof(u64)]; + vcpu->arch.cfar = exsave[EX_CFAR/sizeof(u64)]; + vcpu->arch.regs.ctr = exsave[EX_CTR/sizeof(u64)]; + + vcpu->arch.last_inst = KVM_INST_FETCH_FAILED; + + if (unlikely(trap == BOOK3S_INTERRUPT_MACHINE_CHECK)) { + vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; + vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; + kvmppc_realmode_machine_check(vcpu); + + } else if (unlikely(trap == BOOK3S_INTERRUPT_HMI)) { + kvmppc_realmode_hmi_handler(); + + } else if (trap == BOOK3S_INTERRUPT_H_EMUL_ASSIST) { + vcpu->arch.emul_inst = mfspr(SPRN_HEIR); + + } else if (trap == BOOK3S_INTERRUPT_H_DATA_STORAGE) { + vcpu->arch.fault_dar = exsave[EX_DAR/sizeof(u64)]; + vcpu->arch.fault_dsisr = exsave[EX_DSISR/sizeof(u64)]; + vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); + + } else if (trap == BOOK3S_INTERRUPT_H_INST_STORAGE) { + vcpu->arch.fault_gpa = mfspr(SPRN_ASDR); + + } else if (trap == BOOK3S_INTERRUPT_H_FAC_UNAVAIL) { + vcpu->arch.hfscr = mfspr(SPRN_HFSCR); + +#ifdef CONFIG_PPC_TRANSACTIONAL_MEM + /* + * Softpatch interrupt for transactional memory emulation cases + * on POWER9 DD2.2. This is early in the guest exit path - we + * haven't saved registers or done a treclaim yet. + */ + } else if (trap == BOOK3S_INTERRUPT_HV_SOFTPATCH) { + vcpu->arch.emul_inst = mfspr(SPRN_HEIR); + + /* + * The cases we want to handle here are those where the guest + * is in real suspend mode and is trying to transition to + * transactional mode. + */ + if (local_paca->kvm_hstate.fake_suspend && + (vcpu->arch.shregs.msr & MSR_TS_S)) { + if (kvmhv_p9_tm_emulation_early(vcpu)) { + /* Prevent it being handled again. */ + trap = 0; + } + } +#endif + } + + accumulate_time(vcpu, &vcpu->arch.rm_exit); + + /* Advance host PURR/SPURR by the amount used by guest */ + purr = mfspr(SPRN_PURR); + spurr = mfspr(SPRN_SPURR); + mtspr(SPRN_PURR, local_paca->kvm_hstate.host_purr + + purr - vcpu->arch.purr); + mtspr(SPRN_SPURR, local_paca->kvm_hstate.host_spurr + + spurr - vcpu->arch.spurr); + vcpu->arch.purr = purr; + vcpu->arch.spurr = spurr; + + vcpu->arch.ic = mfspr(SPRN_IC); + vcpu->arch.pid = mfspr(SPRN_PID); + vcpu->arch.psscr = mfspr(SPRN_PSSCR) & PSSCR_GUEST_VIS; + + vcpu->arch.shregs.sprg0 = mfspr(SPRN_SPRG0); + vcpu->arch.shregs.sprg1 = mfspr(SPRN_SPRG1); + vcpu->arch.shregs.sprg2 = mfspr(SPRN_SPRG2); + vcpu->arch.shregs.sprg3 = mfspr(SPRN_SPRG3); + + /* Preserve PSSCR[FAKE_SUSPEND] until we've called kvmppc_save_tm_hv */ + mtspr(SPRN_PSSCR, host_psscr | + (local_paca->kvm_hstate.fake_suspend << PSSCR_FAKE_SUSPEND_LG)); + mtspr(SPRN_HFSCR, host_hfscr); + mtspr(SPRN_CIABR, host_ciabr); + mtspr(SPRN_DAWR0, host_dawr0); + mtspr(SPRN_DAWRX0, host_dawrx0); + if (cpu_has_feature(CPU_FTR_DAWR1)) { + mtspr(SPRN_DAWR1, host_dawr1); + mtspr(SPRN_DAWRX1, host_dawrx1); + } + + if (kvm_is_radix(kvm)) { + /* + * Since this is radix, do a eieio; tlbsync; ptesync sequence + * in case we interrupted the guest between a tlbie and a + * ptesync. + */ + asm volatile("eieio; tlbsync; ptesync"); + } + + /* + * cp_abort is required if the processor supports local copy-paste + * to clear the copy buffer that was under control of the guest. + */ + if (cpu_has_feature(CPU_FTR_ARCH_31)) + asm volatile(PPC_CP_ABORT); + + vc->dpdes = mfspr(SPRN_DPDES); + vc->vtb = mfspr(SPRN_VTB); + mtspr(SPRN_DPDES, 0); + if (vc->pcr) + mtspr(SPRN_PCR, PCR_MASK); + + if (vc->tb_offset_applied) { + u64 new_tb = mftb() - vc->tb_offset_applied; + mtspr(SPRN_TBU40, new_tb); + tb = mftb(); + if ((tb & 0xffffff) < (new_tb & 0xffffff)) + mtspr(SPRN_TBU40, new_tb + 0x1000000); + vc->tb_offset_applied = 0; + } + + mtspr(SPRN_HDEC, 0x7fffffff); + + save_clear_guest_mmu(kvm, vcpu); + switch_mmu_to_host(kvm, host_pidr); + local_paca->kvm_hstate.in_guest = KVM_GUEST_MODE_NONE; + + /* + * If we are in real mode, only switch MMU on after the MMU is + * switched to host, to avoid the P9_RADIX_PREFETCH_BUG. + */ + __mtmsrd(msr, 0); + + end_timing(vcpu); + + return trap; +} +EXPORT_SYMBOL_GPL(kvmhv_vcpu_entry_p9); diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 7a0f12404e0e..632b2545072b 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -46,6 +46,10 @@ static int global_invalidates(struct kvm *kvm) else global = 1; + /* LPID has been switched to host if in virt mode so can't do local */ + if (!global && (mfmsr() & (MSR_IR|MSR_DR))) + global = 1; + if (!global) { /* any other core might now have stale TLB entries... */ smp_wmb(); @@ -56,7 +60,7 @@ static int global_invalidates(struct kvm *kvm) * so use the bit for the first thread to represent the core. */ if (cpu_has_feature(CPU_FTR_ARCH_300)) - cpu = cpu_first_thread_sibling(cpu); + cpu = cpu_first_tlb_thread_sibling(cpu); cpumask_clear_cpu(cpu, &kvm->arch.need_tlb_flush); } @@ -398,6 +402,7 @@ long kvmppc_h_enter(struct kvm_vcpu *vcpu, unsigned long flags, vcpu->arch.pgdir, true, &vcpu->arch.regs.gpr[4]); } +EXPORT_SYMBOL_GPL(kvmppc_h_enter); #ifdef __BIG_ENDIAN__ #define LOCK_TOKEN (*(u32 *)(&get_paca()->lock_token)) @@ -542,6 +547,7 @@ long kvmppc_h_remove(struct kvm_vcpu *vcpu, unsigned long flags, return kvmppc_do_h_remove(vcpu->kvm, flags, pte_index, avpn, &vcpu->arch.regs.gpr[4]); } +EXPORT_SYMBOL_GPL(kvmppc_h_remove); long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) { @@ -660,6 +666,7 @@ long kvmppc_h_bulk_remove(struct kvm_vcpu *vcpu) return ret; } +EXPORT_SYMBOL_GPL(kvmppc_h_bulk_remove); long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, unsigned long pte_index, unsigned long avpn) @@ -730,6 +737,7 @@ long kvmppc_h_protect(struct kvm_vcpu *vcpu, unsigned long flags, return H_SUCCESS; } +EXPORT_SYMBOL_GPL(kvmppc_h_protect); long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, unsigned long pte_index) @@ -770,6 +778,7 @@ long kvmppc_h_read(struct kvm_vcpu *vcpu, unsigned long flags, } return H_SUCCESS; } +EXPORT_SYMBOL_GPL(kvmppc_h_read); long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags, unsigned long pte_index) @@ -818,6 +827,7 @@ long kvmppc_h_clear_ref(struct kvm_vcpu *vcpu, unsigned long flags, unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); return ret; } +EXPORT_SYMBOL_GPL(kvmppc_h_clear_ref); long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags, unsigned long pte_index) @@ -865,6 +875,7 @@ long kvmppc_h_clear_mod(struct kvm_vcpu *vcpu, unsigned long flags, unlock_hpte(hpte, v & ~HPTE_V_HVLOCK); return ret; } +EXPORT_SYMBOL_GPL(kvmppc_h_clear_mod); static int kvmppc_get_hpa(struct kvm_vcpu *vcpu, unsigned long mmu_seq, unsigned long gpa, int writing, unsigned long *hpa, @@ -1283,3 +1294,4 @@ long kvmppc_hpte_hv_fault(struct kvm_vcpu *vcpu, unsigned long addr, return -1; /* send fault up to host kernel mode */ } +EXPORT_SYMBOL_GPL(kvmppc_hpte_hv_fault); diff --git a/arch/powerpc/kvm/book3s_hv_rm_xics.c b/arch/powerpc/kvm/book3s_hv_rm_xics.c index c2c9c733f359..0a11ec88a0ae 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_xics.c +++ b/arch/powerpc/kvm/book3s_hv_rm_xics.c @@ -141,13 +141,6 @@ static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, return; } - if (xive_enabled() && kvmhv_on_pseries()) { - /* No XICS access or hypercalls available, too hard */ - this_icp->rm_action |= XICS_RM_KICK_VCPU; - this_icp->rm_kick_target = vcpu; - return; - } - /* * Check if the core is loaded, * if not, find an available host core to post to wake the VCPU, @@ -771,14 +764,6 @@ static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again) void __iomem *xics_phys; int64_t rc; - if (kvmhv_on_pseries()) { - unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; - - iosync(); - plpar_hcall_raw(H_EOI, retbuf, hwirq); - return; - } - rc = pnv_opal_pci_msi_eoi(c, hwirq); if (rc) diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 004f0d4e665f..8dd437d7a2c6 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -25,18 +25,10 @@ #include #include #include -#include #include #include #include #include -#include - -/* Sign-extend HDEC if not on POWER9 */ -#define EXTEND_HDEC(reg) \ -BEGIN_FTR_SECTION; \ - extsw reg, reg; \ -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) /* Values in HSTATE_NAPPING(r13) */ #define NAPPING_CEDE 1 @@ -44,9 +36,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) #define NAPPING_UNSPLIT 3 /* Stack frame offsets for kvmppc_hv_entry */ -#define SFS 208 +#define SFS 160 #define STACK_SLOT_TRAP (SFS-4) -#define STACK_SLOT_SHORT_PATH (SFS-8) #define STACK_SLOT_TID (SFS-16) #define STACK_SLOT_PSSCR (SFS-24) #define STACK_SLOT_PID (SFS-32) @@ -57,11 +48,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) #define STACK_SLOT_HFSCR (SFS-72) #define STACK_SLOT_AMR (SFS-80) #define STACK_SLOT_UAMOR (SFS-88) -#define STACK_SLOT_DAWR1 (SFS-96) -#define STACK_SLOT_DAWRX1 (SFS-104) -#define STACK_SLOT_FSCR (SFS-112) -/* the following is used by the P9 short path */ -#define STACK_SLOT_NVGPRS (SFS-152) /* 18 gprs */ +#define STACK_SLOT_FSCR (SFS-96) /* * Call kvmppc_hv_entry in real mode. @@ -137,15 +124,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) /* Return the trap number on this thread as the return value */ mr r3, r12 - /* - * If we came back from the guest via a relocation-on interrupt, - * we will be in virtual mode at this point, which makes it a - * little easier to get back to the caller. - */ - mfmsr r0 - andi. r0, r0, MSR_IR /* in real mode? */ - bne .Lvirt_return - /* RFI into the highmem handler */ mfmsr r6 li r0, MSR_RI @@ -155,11 +133,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) mtsrr1 r7 RFI_TO_KERNEL - /* Virtual-mode return */ -.Lvirt_return: - mtlr r8 - blr - kvmppc_primary_no_guest: /* We handle this much like a ceded vcpu */ /* put the HDEC into the DEC, since HDEC interrupts don't wake us */ @@ -246,7 +219,7 @@ kvm_novcpu_wakeup: /* See if our timeslice has expired (HDEC is negative) */ mfspr r0, SPRN_HDEC - EXTEND_HDEC(r0) + extsw r0, r0 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER cmpdi r0, 0 blt kvm_novcpu_exit @@ -348,10 +321,8 @@ kvm_secondary_got_guest: lbz r4, HSTATE_PTID(r13) cmpwi r4, 0 bne 63f - LOAD_REG_ADDR(r6, decrementer_max) - ld r6, 0(r6) + lis r6,0x7fff /* MAX_INT@h */ mtspr SPRN_HDEC, r6 -BEGIN_FTR_SECTION /* and set per-LPAR registers, if doing dynamic micro-threading */ ld r6, HSTATE_SPLIT_MODE(r13) cmpdi r6, 0 @@ -363,7 +334,6 @@ BEGIN_FTR_SECTION ld r0, KVM_SPLIT_LDBAR(r6) mtspr SPRN_LDBAR, r0 isync -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) 63: /* Order load of vcpu after load of vcore */ lwsync @@ -434,7 +404,6 @@ kvm_no_guest: blr 53: -BEGIN_FTR_SECTION HMT_LOW ld r5, HSTATE_KVM_VCORE(r13) cmpdi r5, 0 @@ -449,14 +418,6 @@ BEGIN_FTR_SECTION b kvm_unsplit_nap 60: HMT_MEDIUM b kvm_secondary_got_guest -FTR_SECTION_ELSE - HMT_LOW - ld r5, HSTATE_KVM_VCORE(r13) - cmpdi r5, 0 - beq kvm_no_guest - HMT_MEDIUM - b kvm_secondary_got_guest -ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 54: li r0, KVM_HWTHREAD_IN_KVM stb r0, HSTATE_HWTHREAD_STATE(r13) @@ -582,13 +543,11 @@ kvmppc_hv_entry: bne 10f lwz r7,KVM_LPID(r9) -BEGIN_FTR_SECTION ld r6,KVM_SDR1(r9) li r0,LPID_RSVD /* switch to reserved LPID */ mtspr SPRN_LPID,r0 ptesync mtspr SPRN_SDR1,r6 /* switch to partition page table */ -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) mtspr SPRN_LPID,r7 isync @@ -668,16 +627,6 @@ kvmppc_got_guest: mtspr SPRN_SPURR,r8 /* Save host values of some registers */ -BEGIN_FTR_SECTION - mfspr r5, SPRN_TIDR - mfspr r6, SPRN_PSSCR - mfspr r7, SPRN_PID - std r5, STACK_SLOT_TID(r1) - std r6, STACK_SLOT_PSSCR(r1) - std r7, STACK_SLOT_PID(r1) - mfspr r5, SPRN_HFSCR - std r5, STACK_SLOT_HFSCR(r1) -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) BEGIN_FTR_SECTION mfspr r5, SPRN_CIABR mfspr r6, SPRN_DAWR0 @@ -690,12 +639,6 @@ BEGIN_FTR_SECTION mfspr r5, SPRN_FSCR std r5, STACK_SLOT_FSCR(r1) END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) -BEGIN_FTR_SECTION - mfspr r6, SPRN_DAWR1 - mfspr r7, SPRN_DAWRX1 - std r6, STACK_SLOT_DAWR1(r1) - std r7, STACK_SLOT_DAWRX1(r1) -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S | CPU_FTR_DAWR1) mfspr r5, SPRN_AMR std r5, STACK_SLOT_AMR(r1) @@ -713,13 +656,9 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) #ifdef CONFIG_PPC_TRANSACTIONAL_MEM -/* - * Branch around the call if both CPU_FTR_TM and - * CPU_FTR_P9_TM_HV_ASSIST are off. - */ BEGIN_FTR_SECTION b 91f -END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) +END_FTR_SECTION_IFCLR(CPU_FTR_TM) /* * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) */ @@ -786,12 +725,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) ld r6, VCPU_DAWRX0(r4) mtspr SPRN_DAWR0, r5 mtspr SPRN_DAWRX0, r6 -BEGIN_FTR_SECTION - ld r5, VCPU_DAWR1(r4) - ld r6, VCPU_DAWRX1(r4) - mtspr SPRN_DAWR1, r5 - mtspr SPRN_DAWRX1, r6 -END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) 1: ld r7, VCPU_CIABR(r4) ld r8, VCPU_TAR(r4) @@ -809,7 +742,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) mtspr SPRN_BESCR, r6 mtspr SPRN_PID, r7 mtspr SPRN_WORT, r8 -BEGIN_FTR_SECTION /* POWER8-only registers */ ld r5, VCPU_TCSCR(r4) ld r6, VCPU_ACOP(r4) @@ -820,18 +752,6 @@ BEGIN_FTR_SECTION mtspr SPRN_CSIGR, r7 mtspr SPRN_TACR, r8 nop -FTR_SECTION_ELSE - /* POWER9-only registers */ - ld r5, VCPU_TID(r4) - ld r6, VCPU_PSSCR(r4) - lbz r8, HSTATE_FAKE_SUSPEND(r13) - oris r6, r6, PSSCR_EC@h /* This makes stop trap to HV */ - rldimi r6, r8, PSSCR_FAKE_SUSPEND_LG, 63 - PSSCR_FAKE_SUSPEND_LG - ld r7, VCPU_HFSCR(r4) - mtspr SPRN_TIDR, r5 - mtspr SPRN_PSSCR, r6 - mtspr SPRN_HFSCR, r7 -ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) 8: ld r5, VCPU_SPRG0(r4) @@ -901,23 +821,15 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) /* Check if HDEC expires soon */ mfspr r3, SPRN_HDEC - EXTEND_HDEC(r3) + extsw r3, r3 cmpdi r3, 512 /* 1 microsecond */ blt hdec_soon - ld r6, VCPU_KVM(r4) - lbz r0, KVM_RADIX(r6) - cmpwi r0, 0 - bne 9f - - /* For hash guest, clear out and reload the SLB */ -BEGIN_MMU_FTR_SECTION - /* Radix host won't have populated the SLB, so no need to clear */ + /* Clear out and reload the SLB */ li r6, 0 slbmte r6, r6 PPC_SLBIA(6) ptesync -END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) /* Load up guest SLB entries (N.B. slb_max will be 0 for radix) */ lwz r5,VCPU_SLB_MAX(r4) @@ -932,96 +844,9 @@ END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX) bdnz 1b 9: -#ifdef CONFIG_KVM_XICS - /* We are entering the guest on that thread, push VCPU to XIVE */ - ld r11, VCPU_XIVE_SAVED_STATE(r4) - li r9, TM_QW1_OS - lwz r8, VCPU_XIVE_CAM_WORD(r4) - cmpwi r8, 0 - beq no_xive - li r7, TM_QW1_OS + TM_WORD2 - mfmsr r0 - andi. r0, r0, MSR_DR /* in real mode? */ - beq 2f - ld r10, HSTATE_XIVE_TIMA_VIRT(r13) - cmpldi cr1, r10, 0 - beq cr1, no_xive - eieio - stdx r11,r9,r10 - stwx r8,r7,r10 - b 3f -2: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) - cmpldi cr1, r10, 0 - beq cr1, no_xive - eieio - stdcix r11,r9,r10 - stwcix r8,r7,r10 -3: li r9, 1 - stb r9, VCPU_XIVE_PUSHED(r4) - eieio - - /* - * We clear the irq_pending flag. There is a small chance of a - * race vs. the escalation interrupt happening on another - * processor setting it again, but the only consequence is to - * cause a spurrious wakeup on the next H_CEDE which is not an - * issue. - */ - li r0,0 - stb r0, VCPU_IRQ_PENDING(r4) - - /* - * In single escalation mode, if the escalation interrupt is - * on, we mask it. - */ - lbz r0, VCPU_XIVE_ESC_ON(r4) - cmpwi cr1, r0,0 - beq cr1, 1f - li r9, XIVE_ESB_SET_PQ_01 - beq 4f /* in real mode? */ - ld r10, VCPU_XIVE_ESC_VADDR(r4) - ldx r0, r10, r9 - b 5f -4: ld r10, VCPU_XIVE_ESC_RADDR(r4) - ldcix r0, r10, r9 -5: sync - - /* We have a possible subtle race here: The escalation interrupt might - * have fired and be on its way to the host queue while we mask it, - * and if we unmask it early enough (re-cede right away), there is - * a theorical possibility that it fires again, thus landing in the - * target queue more than once which is a big no-no. - * - * Fortunately, solving this is rather easy. If the above load setting - * PQ to 01 returns a previous value where P is set, then we know the - * escalation interrupt is somewhere on its way to the host. In that - * case we simply don't clear the xive_esc_on flag below. It will be - * eventually cleared by the handler for the escalation interrupt. - * - * Then, when doing a cede, we check that flag again before re-enabling - * the escalation interrupt, and if set, we abort the cede. - */ - andi. r0, r0, XIVE_ESB_VAL_P - bne- 1f - - /* Now P is 0, we can clear the flag */ - li r0, 0 - stb r0, VCPU_XIVE_ESC_ON(r4) -1: -no_xive: -#endif /* CONFIG_KVM_XICS */ - - li r0, 0 - stw r0, STACK_SLOT_SHORT_PATH(r1) - deliver_guest_interrupt: /* r4 = vcpu, r13 = paca */ /* Check if we can deliver an external or decrementer interrupt now */ ld r0, VCPU_PENDING_EXC(r4) -BEGIN_FTR_SECTION - /* On POWER9, also check for emulated doorbell interrupt */ - lbz r3, VCPU_DBELL_REQ(r4) - or r0, r0, r3 -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) cmpdi r0, 0 beq 71f mr r3, r4 @@ -1033,7 +858,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) mtspr SPRN_SRR0, r6 mtspr SPRN_SRR1, r7 -fast_guest_entry_c: ld r10, VCPU_PC(r4) ld r11, VCPU_MSR(r4) /* r11 = vcpu->arch.msr & ~MSR_HV */ @@ -1095,18 +919,8 @@ BEGIN_FTR_SECTION mtspr SPRN_PPR, r0 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) -/* Move canary into DSISR to check for later */ -BEGIN_FTR_SECTION - li r0, 0x7fff - mtspr SPRN_HDSISR, r0 -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) - - ld r6, VCPU_KVM(r4) - lbz r7, KVM_SECURE_GUEST(r6) - cmpdi r7, 0 ld r6, VCPU_GPR(R6)(r4) ld r7, VCPU_GPR(R7)(r4) - bne ret_to_ultra ld r0, VCPU_CR(r4) mtcr r0 @@ -1117,117 +931,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) ld r4, VCPU_GPR(R4)(r4) HRFI_TO_GUEST b . -/* - * Use UV_RETURN ultracall to return control back to the Ultravisor after - * processing an hypercall or interrupt that was forwarded (a.k.a. reflected) - * to the Hypervisor. - * - * All registers have already been loaded, except: - * R0 = hcall result - * R2 = SRR1, so UV can detect a synthesized interrupt (if any) - * R3 = UV_RETURN - */ -ret_to_ultra: - ld r0, VCPU_CR(r4) - mtcr r0 - - ld r0, VCPU_GPR(R3)(r4) - mfspr r2, SPRN_SRR1 - li r3, 0 - ori r3, r3, UV_RETURN - ld r4, VCPU_GPR(R4)(r4) - sc 2 - -/* - * Enter the guest on a P9 or later system where we have exactly - * one vcpu per vcore and we don't need to go to real mode - * (which implies that host and guest are both using radix MMU mode). - * r3 = vcpu pointer - * Most SPRs and all the VSRs have been loaded already. - */ -_GLOBAL(__kvmhv_vcpu_entry_p9) -EXPORT_SYMBOL_GPL(__kvmhv_vcpu_entry_p9) - mflr r0 - std r0, PPC_LR_STKOFF(r1) - stdu r1, -SFS(r1) - - li r0, 1 - stw r0, STACK_SLOT_SHORT_PATH(r1) - - std r3, HSTATE_KVM_VCPU(r13) - mfcr r4 - stw r4, SFS+8(r1) - - std r1, HSTATE_HOST_R1(r13) - - reg = 14 - .rept 18 - std reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) - reg = reg + 1 - .endr - - reg = 14 - .rept 18 - ld reg, __VCPU_GPR(reg)(r3) - reg = reg + 1 - .endr - - mfmsr r10 - std r10, HSTATE_HOST_MSR(r13) - - mr r4, r3 - b fast_guest_entry_c -guest_exit_short_path: - /* - * Malicious or buggy radix guests may have inserted SLB entries - * (only 0..3 because radix always runs with UPRT=1), so these must - * be cleared here to avoid side-channels. slbmte is used rather - * than slbia, as it won't clear cached translations. - */ - li r0,0 - slbmte r0,r0 - li r4,1 - slbmte r0,r4 - li r4,2 - slbmte r0,r4 - li r4,3 - slbmte r0,r4 - - li r0, KVM_GUEST_MODE_NONE - stb r0, HSTATE_IN_GUEST(r13) - - reg = 14 - .rept 18 - std reg, __VCPU_GPR(reg)(r9) - reg = reg + 1 - .endr - - reg = 14 - .rept 18 - ld reg, STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1) - reg = reg + 1 - .endr - - lwz r4, SFS+8(r1) - mtcr r4 - - mr r3, r12 /* trap number */ - - addi r1, r1, SFS - ld r0, PPC_LR_STKOFF(r1) - mtlr r0 - - /* If we are in real mode, do a rfid to get back to the caller */ - mfmsr r4 - andi. r5, r4, MSR_IR - bnelr - rldicl r5, r4, 64 - MSR_TS_S_LG, 62 /* extract TS field */ - mtspr SPRN_SRR0, r0 - ld r10, HSTATE_HOST_MSR(r13) - rldimi r10, r5, MSR_TS_S_LG, 63 - MSR_TS_T_LG - mtspr SPRN_SRR1, r10 - RFI_TO_KERNEL - b . secondary_too_late: li r12, 0 @@ -1268,21 +971,16 @@ hdec_soon: kvmppc_interrupt_hv: /* * Register contents: + * R9 = HSTATE_IN_GUEST * R12 = (guest CR << 32) | interrupt vector * R13 = PACA * guest R12 saved in shadow VCPU SCRATCH0 * guest R13 saved in SPRN_SCRATCH0 + * guest R9 saved in HSTATE_SCRATCH2 */ - std r9, HSTATE_SCRATCH2(r13) - lbz r9, HSTATE_IN_GUEST(r13) - cmpwi r9, KVM_GUEST_MODE_HOST_HV - beq kvmppc_bad_host_intr -#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE - cmpwi r9, KVM_GUEST_MODE_GUEST - ld r9, HSTATE_SCRATCH2(r13) - beq kvmppc_interrupt_pr -#endif /* We're now back in the host but in guest MMU context */ + cmpwi r9,KVM_GUEST_MODE_HOST_HV + beq kvmppc_bad_host_intr li r9, KVM_GUEST_MODE_HOST_HV stb r9, HSTATE_IN_GUEST(r13) @@ -1400,7 +1098,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER bne 2f mfspr r3,SPRN_HDEC - EXTEND_HDEC(r3) + extsw r3, r3 cmpdi r3,0 mr r4,r9 bge fast_guest_return @@ -1412,14 +1110,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) /* Hypervisor doorbell - exit only if host IPI flag set */ cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL bne 3f -BEGIN_FTR_SECTION - PPC_MSGSYNC - lwsync - /* always exit if we're running a nested guest */ - ld r0, VCPU_NESTED(r9) - cmpdi r0, 0 - bne guest_exit_cont -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) lbz r0, HSTATE_HOST_IPI(r13) cmpwi r0, 0 beq maybe_reenter_guest @@ -1449,62 +1139,16 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ mr r4, r9 bl kvmhv_accumulate_time #endif -#ifdef CONFIG_KVM_XICS - /* We are exiting, pull the VP from the XIVE */ - lbz r0, VCPU_XIVE_PUSHED(r9) - cmpwi cr0, r0, 0 - beq 1f - li r7, TM_SPC_PULL_OS_CTX - li r6, TM_QW1_OS - mfmsr r0 - andi. r0, r0, MSR_DR /* in real mode? */ - beq 2f - ld r10, HSTATE_XIVE_TIMA_VIRT(r13) - cmpldi cr0, r10, 0 - beq 1f - /* First load to pull the context, we ignore the value */ - eieio - lwzx r11, r7, r10 - /* Second load to recover the context state (Words 0 and 1) */ - ldx r11, r6, r10 - b 3f -2: ld r10, HSTATE_XIVE_TIMA_PHYS(r13) - cmpldi cr0, r10, 0 - beq 1f - /* First load to pull the context, we ignore the value */ - eieio - lwzcix r11, r7, r10 - /* Second load to recover the context state (Words 0 and 1) */ - ldcix r11, r6, r10 -3: std r11, VCPU_XIVE_SAVED_STATE(r9) - /* Fixup some of the state for the next load */ - li r10, 0 - li r0, 0xff - stb r10, VCPU_XIVE_PUSHED(r9) - stb r10, (VCPU_XIVE_SAVED_STATE+3)(r9) - stb r0, (VCPU_XIVE_SAVED_STATE+4)(r9) - eieio -1: -#endif /* CONFIG_KVM_XICS */ /* * Possibly flush the link stack here, before we do a blr in - * guest_exit_short_path. + * kvmhv_switch_to_host. */ 1: nop patch_site 1b patch__call_kvm_flush_link_stack - /* If we came in through the P9 short path, go back out to C now */ - lwz r0, STACK_SLOT_SHORT_PATH(r1) - cmpwi r0, 0 - bne guest_exit_short_path - /* For hash guest, read the guest SLB and save it away */ - ld r5, VCPU_KVM(r9) - lbz r0, KVM_RADIX(r5) li r5, 0 - cmpwi r0, 0 - bne 0f /* for radix, save 0 entries */ lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */ mtctr r0 li r6,0 @@ -1528,9 +1172,6 @@ guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */ stw r5,VCPU_SLB_MAX(r9) /* load host SLB entries */ -BEGIN_MMU_FTR_SECTION - b guest_bypass -END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) ld r8,PACA_SLBSHADOWPTR(r13) .rept SLB_NUM_BOLTED @@ -1543,21 +1184,6 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) slbmte r6,r5 1: addi r8,r8,16 .endr - b guest_bypass - -0: /* - * Sanitise radix guest SLB, see guest_exit_short_path comment. - * We clear vcpu->arch.slb_max to match earlier behaviour. - */ - li r0,0 - stw r0,VCPU_SLB_MAX(r9) - slbmte r0,r0 - li r4,1 - slbmte r0,r4 - li r4,2 - slbmte r0,r4 - li r4,3 - slbmte r0,r4 guest_bypass: stw r12, STACK_SLOT_TRAP(r1) @@ -1567,12 +1193,6 @@ guest_bypass: ld r3, HSTATE_KVM_VCORE(r13) mfspr r5,SPRN_DEC mftb r6 - /* On P9, if the guest has large decr enabled, don't sign extend */ -BEGIN_FTR_SECTION - ld r4, VCORE_LPCR(r3) - andis. r4, r4, LPCR_LD@h - bne 16f -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) extsw r5,r5 16: add r5,r5,r6 /* r5 is a guest timebase value here, convert to host TB */ @@ -1646,7 +1266,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S) std r6, VCPU_BESCR(r9) stw r7, VCPU_GUEST_PID(r9) std r8, VCPU_WORT(r9) -BEGIN_FTR_SECTION mfspr r5, SPRN_TCSCR mfspr r6, SPRN_ACOP mfspr r7, SPRN_CSIGR @@ -1655,17 +1274,6 @@ BEGIN_FTR_SECTION std r6, VCPU_ACOP(r9) std r7, VCPU_CSIGR(r9) std r8, VCPU_TACR(r9) -FTR_SECTION_ELSE - mfspr r5, SPRN_TIDR - mfspr r6, SPRN_PSSCR - std r5, VCPU_TID(r9) - rldicl r6, r6, 4, 50 /* r6 &= PSSCR_GUEST_VIS */ - rotldi r6, r6, 60 - std r6, VCPU_PSSCR(r9) - /* Restore host HFSCR value */ - ld r7, STACK_SLOT_HFSCR(r1) - mtspr SPRN_HFSCR, r7 -ALT_FTR_SECTION_END_IFCLR(CPU_FTR_ARCH_300) BEGIN_FTR_SECTION ld r5, STACK_SLOT_FSCR(r1) mtspr SPRN_FSCR, r5 @@ -1677,13 +1285,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) li r0, 0 mtspr SPRN_PSPB, r0 mtspr SPRN_WORT, r0 -BEGIN_FTR_SECTION mtspr SPRN_TCSCR, r0 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */ li r0, 1 sldi r0, r0, 31 mtspr SPRN_MMCRS, r0 -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) /* Save and restore AMR, IAMR and UAMOR before turning on the MMU */ ld r8, STACK_SLOT_IAMR(r1) @@ -1740,13 +1346,9 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) bl kvmppc_save_fp #ifdef CONFIG_PPC_TRANSACTIONAL_MEM -/* - * Branch around the call if both CPU_FTR_TM and - * CPU_FTR_P9_TM_HV_ASSIST are off. - */ BEGIN_FTR_SECTION b 91f -END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) +END_FTR_SECTION_IFCLR(CPU_FTR_TM) /* * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) */ @@ -1792,80 +1394,6 @@ BEGIN_FTR_SECTION mtspr SPRN_DAWR0, r6 mtspr SPRN_DAWRX0, r7 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) -BEGIN_FTR_SECTION - ld r6, STACK_SLOT_DAWR1(r1) - ld r7, STACK_SLOT_DAWRX1(r1) - mtspr SPRN_DAWR1, r6 - mtspr SPRN_DAWRX1, r7 -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S | CPU_FTR_DAWR1) -BEGIN_FTR_SECTION - ld r5, STACK_SLOT_TID(r1) - ld r6, STACK_SLOT_PSSCR(r1) - ld r7, STACK_SLOT_PID(r1) - mtspr SPRN_TIDR, r5 - mtspr SPRN_PSSCR, r6 - mtspr SPRN_PID, r7 -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) - -#ifdef CONFIG_PPC_RADIX_MMU - /* - * Are we running hash or radix ? - */ - ld r5, VCPU_KVM(r9) - lbz r0, KVM_RADIX(r5) - cmpwi cr2, r0, 0 - beq cr2, 2f - - /* - * Radix: do eieio; tlbsync; ptesync sequence in case we - * interrupted the guest between a tlbie and a ptesync. - */ - eieio - tlbsync - ptesync - -BEGIN_FTR_SECTION - /* Radix: Handle the case where the guest used an illegal PID */ - LOAD_REG_ADDR(r4, mmu_base_pid) - lwz r3, VCPU_GUEST_PID(r9) - lwz r5, 0(r4) - cmpw cr0,r3,r5 - blt 2f - - /* - * Illegal PID, the HW might have prefetched and cached in the TLB - * some translations for the LPID 0 / guest PID combination which - * Linux doesn't know about, so we need to flush that PID out of - * the TLB. First we need to set LPIDR to 0 so tlbiel applies to - * the right context. - */ - li r0,0 - mtspr SPRN_LPID,r0 - isync - - /* Then do a congruence class local flush */ - ld r6,VCPU_KVM(r9) - lwz r0,KVM_TLB_SETS(r6) - mtctr r0 - li r7,0x400 /* IS field = 0b01 */ - ptesync - sldi r0,r3,32 /* RS has PID */ -1: PPC_TLBIEL(7,0,2,1,1) /* RIC=2, PRS=1, R=1 */ - addi r7,r7,0x1000 - bdnz 1b - ptesync -END_FTR_SECTION_IFSET(CPU_FTR_P9_RADIX_PREFETCH_BUG) - -2: -#endif /* CONFIG_PPC_RADIX_MMU */ - - /* - * cp_abort is required if the processor supports local copy-paste - * to clear the copy buffer that was under control of the guest. - */ -BEGIN_FTR_SECTION - PPC_CP_ABORT -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_31) /* * POWER7/POWER8 guest -> host partition switch code. @@ -1902,13 +1430,11 @@ kvmhv_switch_to_host: /* Primary thread switches back to host partition */ lwz r7,KVM_HOST_LPID(r4) -BEGIN_FTR_SECTION ld r6,KVM_HOST_SDR1(r4) li r8,LPID_RSVD /* switch to reserved LPID */ mtspr SPRN_LPID,r8 ptesync mtspr SPRN_SDR1,r6 /* switch to host page table */ -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) mtspr SPRN_LPID,r7 isync @@ -2117,26 +1643,13 @@ kvmppc_tm_emul: * reflect the HDSI to the guest as a DSI. */ kvmppc_hdsi: - ld r3, VCPU_KVM(r9) - lbz r0, KVM_RADIX(r3) mfspr r4, SPRN_HDAR mfspr r6, SPRN_HDSISR -BEGIN_FTR_SECTION - /* Look for DSISR canary. If we find it, retry instruction */ - cmpdi r6, 0x7fff - beq 6f -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) - cmpwi r0, 0 - bne .Lradix_hdsi /* on radix, just save DAR/DSISR/ASDR */ /* HPTE not found fault or protection fault? */ andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h beq 1f /* if not, send it to the guest */ andi. r0, r11, MSR_DR /* data relocation enabled? */ beq 3f -BEGIN_FTR_SECTION - mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ - b 4f -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) clrrdi r0, r4, 28 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ li r0, BOOK3S_INTERRUPT_DATA_SEGMENT @@ -2204,31 +1717,15 @@ fast_interrupt_c_return: stb r0, HSTATE_IN_GUEST(r13) b guest_exit_cont -.Lradix_hdsi: - std r4, VCPU_FAULT_DAR(r9) - stw r6, VCPU_FAULT_DSISR(r9) -.Lradix_hisi: - mfspr r5, SPRN_ASDR - std r5, VCPU_FAULT_GPA(r9) - b guest_exit_cont - /* * Similarly for an HISI, reflect it to the guest as an ISI unless * it is an HPTE not found fault for a page that we have paged out. */ kvmppc_hisi: - ld r3, VCPU_KVM(r9) - lbz r0, KVM_RADIX(r3) - cmpwi r0, 0 - bne .Lradix_hisi /* for radix, just save ASDR */ andis. r0, r11, SRR1_ISI_NOPT@h beq 1f andi. r0, r11, MSR_IR /* instruction relocation enabled? */ beq 3f -BEGIN_FTR_SECTION - mfspr r5, SPRN_ASDR /* on POWER9, use ASDR to get VSID */ - b 4f -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) clrrdi r0, r10, 28 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */ li r0, BOOK3S_INTERRUPT_INST_SEGMENT @@ -2276,10 +1773,6 @@ hcall_try_real_mode: andi. r0,r11,MSR_PR /* sc 1 from userspace - reflect to guest syscall */ bne sc_1_fast_return - /* sc 1 from nested guest - give it to L1 to handle */ - ld r0, VCPU_NESTED(r9) - cmpdi r0, 0 - bne guest_exit_cont clrrdi r3,r3,2 cmpldi r3,hcall_real_table_end - hcall_real_table bge guest_exit_cont @@ -2544,7 +2037,7 @@ hcall_real_table: #else .long 0 /* 0x2fc - H_XIRR_X*/ #endif - .long DOTSYM(kvmppc_h_random) - hcall_real_table + .long DOTSYM(kvmppc_rm_h_random) - hcall_real_table .globl hcall_real_table_end hcall_real_table_end: @@ -2675,13 +2168,9 @@ _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */ bl kvmppc_save_fp #ifdef CONFIG_PPC_TRANSACTIONAL_MEM -/* - * Branch around the call if both CPU_FTR_TM and - * CPU_FTR_P9_TM_HV_ASSIST are off. - */ BEGIN_FTR_SECTION b 91f -END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) +END_FTR_SECTION_IFCLR(CPU_FTR_TM) /* * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) */ @@ -2701,15 +2190,8 @@ END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) mfspr r3, SPRN_DEC mfspr r4, SPRN_HDEC mftb r5 -BEGIN_FTR_SECTION - /* On P9 check whether the guest has large decrementer mode enabled */ - ld r6, HSTATE_KVM_VCORE(r13) - ld r6, VCORE_LPCR(r6) - andis. r6, r6, LPCR_LD@h - bne 68f -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) extsw r3, r3 -68: EXTEND_HDEC(r4) + extsw r4, r4 cmpd r3, r4 ble 67f mtspr SPRN_DEC, r4 @@ -2754,28 +2236,11 @@ BEGIN_FTR_SECTION END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) kvm_nap_sequence: /* desired LPCR value in r5 */ -BEGIN_FTR_SECTION - /* - * PSSCR bits: exit criterion = 1 (wakeup based on LPCR at sreset) - * enable state loss = 1 (allow SMT mode switch) - * requested level = 0 (just stop dispatching) - */ - lis r3, (PSSCR_EC | PSSCR_ESL)@h - /* Set LPCR_PECE_HVEE bit to enable wakeup by HV interrupts */ - li r4, LPCR_PECE_HVEE@higher - sldi r4, r4, 32 - or r5, r5, r4 -FTR_SECTION_ELSE li r3, PNV_THREAD_NAP -ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) mtspr SPRN_LPCR,r5 isync -BEGIN_FTR_SECTION - bl isa300_idle_stop_mayloss -FTR_SECTION_ELSE bl isa206_idle_insn_mayloss -ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) mfspr r0, SPRN_CTRLF ori r0, r0, 1 @@ -2794,10 +2259,8 @@ ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_300) beq kvm_end_cede cmpwi r0, NAPPING_NOVCPU beq kvm_novcpu_wakeup -BEGIN_FTR_SECTION cmpwi r0, NAPPING_UNSPLIT beq kvm_unsplit_wakeup -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) twi 31,0,0 /* Nap state must not be zero */ 33: mr r4, r3 @@ -2817,13 +2280,9 @@ kvm_end_cede: #endif #ifdef CONFIG_PPC_TRANSACTIONAL_MEM -/* - * Branch around the call if both CPU_FTR_TM and - * CPU_FTR_P9_TM_HV_ASSIST are off. - */ BEGIN_FTR_SECTION b 91f -END_FTR_SECTION(CPU_FTR_TM | CPU_FTR_P9_TM_HV_ASSIST, 0) +END_FTR_SECTION_IFCLR(CPU_FTR_TM) /* * NOTE THAT THIS TRASHES ALL NON-VOLATILE REGISTERS (but not CR) */ @@ -2913,47 +2372,7 @@ kvm_cede_prodded: /* we've ceded but we want to give control to the host */ kvm_cede_exit: ld r9, HSTATE_KVM_VCPU(r13) -#ifdef CONFIG_KVM_XICS - /* are we using XIVE with single escalation? */ - ld r10, VCPU_XIVE_ESC_VADDR(r9) - cmpdi r10, 0 - beq 3f - li r6, XIVE_ESB_SET_PQ_00 - /* - * If we still have a pending escalation, abort the cede, - * and we must set PQ to 10 rather than 00 so that we don't - * potentially end up with two entries for the escalation - * interrupt in the XIVE interrupt queue. In that case - * we also don't want to set xive_esc_on to 1 here in - * case we race with xive_esc_irq(). - */ - lbz r5, VCPU_XIVE_ESC_ON(r9) - cmpwi r5, 0 - beq 4f - li r0, 0 - stb r0, VCPU_CEDED(r9) - /* - * The escalation interrupts are special as we don't EOI them. - * There is no need to use the load-after-store ordering offset - * to set PQ to 10 as we won't use StoreEOI. - */ - li r6, XIVE_ESB_SET_PQ_10 - b 5f -4: li r0, 1 - stb r0, VCPU_XIVE_ESC_ON(r9) - /* make sure store to xive_esc_on is seen before xive_esc_irq runs */ - sync -5: /* Enable XIVE escalation */ - mfmsr r0 - andi. r0, r0, MSR_DR /* in real mode? */ - beq 1f - ldx r0, r10, r6 - b 2f -1: ld r10, VCPU_XIVE_ESC_RADDR(r9) - ldcix r0, r10, r6 -2: sync -#endif /* CONFIG_KVM_XICS */ -3: b guest_exit_cont + b guest_exit_cont /* Try to do machine check recovery in real mode */ machine_check_realmode: @@ -3030,10 +2449,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) PPC_MSGCLR(6) /* see if it's a host IPI */ li r3, 1 -BEGIN_FTR_SECTION - PPC_MSGSYNC - lwsync -END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) lbz r0, HSTATE_HOST_IPI(r13) cmpwi r0, 0 bnelr @@ -3342,73 +2757,12 @@ kvmppc_bad_host_intr: std r3, STACK_FRAME_OVERHEAD-16(r1) /* - * On POWER9 do a minimal restore of the MMU and call C code, - * which will print a message and panic. * XXX On POWER7 and POWER8, we just spin here since we don't * know what the other threads are doing (and we don't want to * coordinate with them) - but at least we now have register state * in memory that we might be able to look at from another CPU. */ -BEGIN_FTR_SECTION b . -END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) - ld r9, HSTATE_KVM_VCPU(r13) - ld r10, VCPU_KVM(r9) - - li r0, 0 - mtspr SPRN_AMR, r0 - mtspr SPRN_IAMR, r0 - mtspr SPRN_CIABR, r0 - mtspr SPRN_DAWRX0, r0 -BEGIN_FTR_SECTION - mtspr SPRN_DAWRX1, r0 -END_FTR_SECTION_IFSET(CPU_FTR_DAWR1) - - /* Clear hash and radix guest SLB, see guest_exit_short_path comment. */ - slbmte r0, r0 - PPC_SLBIA(6) - -BEGIN_MMU_FTR_SECTION - b 4f -END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) - - ptesync - ld r8, PACA_SLBSHADOWPTR(r13) - .rept SLB_NUM_BOLTED - li r3, SLBSHADOW_SAVEAREA - LDX_BE r5, r8, r3 - addi r3, r3, 8 - LDX_BE r6, r8, r3 - andis. r7, r5, SLB_ESID_V@h - beq 3f - slbmte r6, r5 -3: addi r8, r8, 16 - .endr - -4: lwz r7, KVM_HOST_LPID(r10) - mtspr SPRN_LPID, r7 - mtspr SPRN_PID, r0 - ld r8, KVM_HOST_LPCR(r10) - mtspr SPRN_LPCR, r8 - isync - li r0, KVM_GUEST_MODE_NONE - stb r0, HSTATE_IN_GUEST(r13) - - /* - * Turn on the MMU and jump to C code - */ - bcl 20, 31, .+4 -5: mflr r3 - addi r3, r3, 9f - 5b - li r4, -1 - rldimi r3, r4, 62, 0 /* ensure 0xc000000000000000 bits are set */ - ld r4, PACAKMSR(r13) - mtspr SPRN_SRR0, r3 - mtspr SPRN_SRR1, r4 - RFI_TO_KERNEL -9: addi r3, r1, STACK_FRAME_OVERHEAD - bl kvmppc_bad_interrupt - b 9b /* * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index d7733b07f489..71bcb0140461 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -493,7 +493,7 @@ static void kvmppc_set_msr_pr(struct kvm_vcpu *vcpu, u64 msr) if (!vcpu->arch.pending_exceptions) { kvm_vcpu_block(vcpu); kvm_clear_request(KVM_REQ_UNHALT, vcpu); - vcpu->stat.halt_wakeup++; + vcpu->stat.generic.halt_wakeup++; /* Unset POW bit after we woke up */ msr &= ~MSR_POW; diff --git a/arch/powerpc/kvm/book3s_pr_papr.c b/arch/powerpc/kvm/book3s_pr_papr.c index 031c8015864a..ac14239f3424 100644 --- a/arch/powerpc/kvm/book3s_pr_papr.c +++ b/arch/powerpc/kvm/book3s_pr_papr.c @@ -378,7 +378,7 @@ int kvmppc_h_pr(struct kvm_vcpu *vcpu, unsigned long cmd) kvmppc_set_msr_fast(vcpu, kvmppc_get_msr(vcpu) | MSR_EE); kvm_vcpu_block(vcpu); kvm_clear_request(KVM_REQ_UNHALT, vcpu); - vcpu->stat.halt_wakeup++; + vcpu->stat.generic.halt_wakeup++; return EMULATE_DONE; case H_LOGICAL_CI_LOAD: return kvmppc_h_pr_logical_ci_load(vcpu); diff --git a/arch/powerpc/kvm/book3s_segment.S b/arch/powerpc/kvm/book3s_segment.S index 1f492aa4c8d6..202046a83fc1 100644 --- a/arch/powerpc/kvm/book3s_segment.S +++ b/arch/powerpc/kvm/book3s_segment.S @@ -164,12 +164,15 @@ kvmppc_interrupt_pr: /* 64-bit entry. Register usage at this point: * * SPRG_SCRATCH0 = guest R13 + * R9 = HSTATE_IN_GUEST * R12 = (guest CR << 32) | exit handler id * R13 = PACA * HSTATE.SCRATCH0 = guest R12 + * HSTATE.SCRATCH2 = guest R9 */ #ifdef CONFIG_PPC64 /* Match 32-bit entry */ + ld r9,HSTATE_SCRATCH2(r13) rotldi r12, r12, 32 /* Flip R12 halves for stw */ stw r12, HSTATE_SCRATCH1(r13) /* CR is now in the low half */ srdi r12, r12, 32 /* shift trap into low half */ diff --git a/arch/powerpc/kvm/book3s_xive.c b/arch/powerpc/kvm/book3s_xive.c index e7219b6f5f9a..9268d386b128 100644 --- a/arch/powerpc/kvm/book3s_xive.c +++ b/arch/powerpc/kvm/book3s_xive.c @@ -127,6 +127,71 @@ void kvmppc_xive_push_vcpu(struct kvm_vcpu *vcpu) } EXPORT_SYMBOL_GPL(kvmppc_xive_push_vcpu); +/* + * Pull a vcpu's context from the XIVE on guest exit. + * This assumes we are in virtual mode (MMU on) + */ +void kvmppc_xive_pull_vcpu(struct kvm_vcpu *vcpu) +{ + void __iomem *tima = local_paca->kvm_hstate.xive_tima_virt; + + if (!vcpu->arch.xive_pushed) + return; + + /* + * Should not have been pushed if there is no tima + */ + if (WARN_ON(!tima)) + return; + + eieio(); + /* First load to pull the context, we ignore the value */ + __raw_readl(tima + TM_SPC_PULL_OS_CTX); + /* Second load to recover the context state (Words 0 and 1) */ + vcpu->arch.xive_saved_state.w01 = __raw_readq(tima + TM_QW1_OS); + + /* Fixup some of the state for the next load */ + vcpu->arch.xive_saved_state.lsmfb = 0; + vcpu->arch.xive_saved_state.ack = 0xff; + vcpu->arch.xive_pushed = 0; + eieio(); +} +EXPORT_SYMBOL_GPL(kvmppc_xive_pull_vcpu); + +void kvmppc_xive_rearm_escalation(struct kvm_vcpu *vcpu) +{ + void __iomem *esc_vaddr = (void __iomem *)vcpu->arch.xive_esc_vaddr; + + if (!esc_vaddr) + return; + + /* we are using XIVE with single escalation */ + + if (vcpu->arch.xive_esc_on) { + /* + * If we still have a pending escalation, abort the cede, + * and we must set PQ to 10 rather than 00 so that we don't + * potentially end up with two entries for the escalation + * interrupt in the XIVE interrupt queue. In that case + * we also don't want to set xive_esc_on to 1 here in + * case we race with xive_esc_irq(). + */ + vcpu->arch.ceded = 0; + /* + * The escalation interrupts are special as we don't EOI them. + * There is no need to use the load-after-store ordering offset + * to set PQ to 10 as we won't use StoreEOI. + */ + __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_10); + } else { + vcpu->arch.xive_esc_on = true; + mb(); + __raw_readq(esc_vaddr + XIVE_ESB_SET_PQ_00); + } + mb(); +} +EXPORT_SYMBOL_GPL(kvmppc_xive_rearm_escalation); + /* * This is a simple trigger for a generic XIVE IRQ. This must * only be called for interrupts that support a trigger page @@ -2075,6 +2140,36 @@ static int kvmppc_xive_create(struct kvm_device *dev, u32 type) return 0; } +int kvmppc_xive_xics_hcall(struct kvm_vcpu *vcpu, u32 req) +{ + struct kvmppc_vcore *vc = vcpu->arch.vcore; + + /* The VM should have configured XICS mode before doing XICS hcalls. */ + if (!kvmppc_xics_enabled(vcpu)) + return H_TOO_HARD; + + switch (req) { + case H_XIRR: + return xive_vm_h_xirr(vcpu); + case H_CPPR: + return xive_vm_h_cppr(vcpu, kvmppc_get_gpr(vcpu, 4)); + case H_EOI: + return xive_vm_h_eoi(vcpu, kvmppc_get_gpr(vcpu, 4)); + case H_IPI: + return xive_vm_h_ipi(vcpu, kvmppc_get_gpr(vcpu, 4), + kvmppc_get_gpr(vcpu, 5)); + case H_IPOLL: + return xive_vm_h_ipoll(vcpu, kvmppc_get_gpr(vcpu, 4)); + case H_XIRR_X: + xive_vm_h_xirr(vcpu); + kvmppc_set_gpr(vcpu, 5, get_tb() + vc->tb_offset); + return H_SUCCESS; + } + + return H_UNSUPPORTED; +} +EXPORT_SYMBOL_GPL(kvmppc_xive_xics_hcall); + int kvmppc_xive_debug_show_queues(struct seq_file *m, struct kvm_vcpu *vcpu) { struct kvmppc_xive_vcpu *xc = vcpu->arch.xive_vcpu; @@ -2257,21 +2352,3 @@ struct kvm_device_ops kvm_xive_ops = { .get_attr = xive_get_attr, .has_attr = xive_has_attr, }; - -void kvmppc_xive_init_module(void) -{ - __xive_vm_h_xirr = xive_vm_h_xirr; - __xive_vm_h_ipoll = xive_vm_h_ipoll; - __xive_vm_h_ipi = xive_vm_h_ipi; - __xive_vm_h_cppr = xive_vm_h_cppr; - __xive_vm_h_eoi = xive_vm_h_eoi; -} - -void kvmppc_xive_exit_module(void) -{ - __xive_vm_h_xirr = NULL; - __xive_vm_h_ipoll = NULL; - __xive_vm_h_ipi = NULL; - __xive_vm_h_cppr = NULL; - __xive_vm_h_eoi = NULL; -} diff --git a/arch/powerpc/kvm/book3s_xive.h b/arch/powerpc/kvm/book3s_xive.h index 86c24a4ad809..afe9eeac6d56 100644 --- a/arch/powerpc/kvm/book3s_xive.h +++ b/arch/powerpc/kvm/book3s_xive.h @@ -289,13 +289,6 @@ extern int xive_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, extern int xive_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr); extern int xive_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr); -extern unsigned long (*__xive_vm_h_xirr)(struct kvm_vcpu *vcpu); -extern unsigned long (*__xive_vm_h_ipoll)(struct kvm_vcpu *vcpu, unsigned long server); -extern int (*__xive_vm_h_ipi)(struct kvm_vcpu *vcpu, unsigned long server, - unsigned long mfrr); -extern int (*__xive_vm_h_cppr)(struct kvm_vcpu *vcpu, unsigned long cppr); -extern int (*__xive_vm_h_eoi)(struct kvm_vcpu *vcpu, unsigned long xirr); - /* * Common Xive routines for XICS-over-XIVE and XIVE native */ diff --git a/arch/powerpc/kvm/book3s_xive_native.c b/arch/powerpc/kvm/book3s_xive_native.c index 76800c84f2a3..1253666dd4d8 100644 --- a/arch/powerpc/kvm/book3s_xive_native.c +++ b/arch/powerpc/kvm/book3s_xive_native.c @@ -1281,13 +1281,3 @@ struct kvm_device_ops kvm_xive_native_ops = { .has_attr = kvmppc_xive_native_has_attr, .mmap = kvmppc_xive_native_mmap, }; - -void kvmppc_xive_native_init_module(void) -{ - ; -} - -void kvmppc_xive_native_exit_module(void) -{ - ; -} diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 7d5fe43f85c4..551b30d84aee 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -36,29 +36,59 @@ unsigned long kvmppc_booke_handlers; -struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT("mmio", mmio_exits), - VCPU_STAT("sig", signal_exits), - VCPU_STAT("itlb_r", itlb_real_miss_exits), - VCPU_STAT("itlb_v", itlb_virt_miss_exits), - VCPU_STAT("dtlb_r", dtlb_real_miss_exits), - VCPU_STAT("dtlb_v", dtlb_virt_miss_exits), - VCPU_STAT("sysc", syscall_exits), - VCPU_STAT("isi", isi_exits), - VCPU_STAT("dsi", dsi_exits), - VCPU_STAT("inst_emu", emulated_inst_exits), - VCPU_STAT("dec", dec_exits), - VCPU_STAT("ext_intr", ext_intr_exits), - VCPU_STAT("halt_successful_poll", halt_successful_poll), - VCPU_STAT("halt_attempted_poll", halt_attempted_poll), - VCPU_STAT("halt_poll_invalid", halt_poll_invalid), - VCPU_STAT("halt_wakeup", halt_wakeup), - VCPU_STAT("doorbell", dbell_exits), - VCPU_STAT("guest doorbell", gdbell_exits), - VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), - VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), - VM_STAT("remote_tlb_flush", remote_tlb_flush), - { NULL } +const struct _kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS(), + STATS_DESC_ICOUNTER(VM, num_2M_pages), + STATS_DESC_ICOUNTER(VM, num_1G_pages) +}; +static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == + sizeof(struct kvm_vm_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + STATS_DESC_COUNTER(VCPU, sum_exits), + STATS_DESC_COUNTER(VCPU, mmio_exits), + STATS_DESC_COUNTER(VCPU, signal_exits), + STATS_DESC_COUNTER(VCPU, light_exits), + STATS_DESC_COUNTER(VCPU, itlb_real_miss_exits), + STATS_DESC_COUNTER(VCPU, itlb_virt_miss_exits), + STATS_DESC_COUNTER(VCPU, dtlb_real_miss_exits), + STATS_DESC_COUNTER(VCPU, dtlb_virt_miss_exits), + STATS_DESC_COUNTER(VCPU, syscall_exits), + STATS_DESC_COUNTER(VCPU, isi_exits), + STATS_DESC_COUNTER(VCPU, dsi_exits), + STATS_DESC_COUNTER(VCPU, emulated_inst_exits), + STATS_DESC_COUNTER(VCPU, dec_exits), + STATS_DESC_COUNTER(VCPU, ext_intr_exits), + STATS_DESC_TIME_NSEC(VCPU, halt_wait_ns), + STATS_DESC_COUNTER(VCPU, halt_successful_wait), + STATS_DESC_COUNTER(VCPU, dbell_exits), + STATS_DESC_COUNTER(VCPU, gdbell_exits), + STATS_DESC_COUNTER(VCPU, ld), + STATS_DESC_COUNTER(VCPU, st), + STATS_DESC_COUNTER(VCPU, pthru_all), + STATS_DESC_COUNTER(VCPU, pthru_host), + STATS_DESC_COUNTER(VCPU, pthru_bad_aff) +}; +static_assert(ARRAY_SIZE(kvm_vcpu_stats_desc) == + sizeof(struct kvm_vcpu_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), }; /* TODO: use vcpu_printf() */ diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index a2a68a958fa0..be33b5321a76 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -682,6 +682,9 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 && !kvmppc_hv_ops->enable_dawr1(NULL)); break; + case KVM_CAP_PPC_RPT_INVALIDATE: + r = 1; + break; #endif default: r = 0; diff --git a/arch/powerpc/mm/book3s64/radix_pgtable.c b/arch/powerpc/mm/book3s64/radix_pgtable.c index 5fef8db3b463..6e3495221ab7 100644 --- a/arch/powerpc/mm/book3s64/radix_pgtable.c +++ b/arch/powerpc/mm/book3s64/radix_pgtable.c @@ -357,30 +357,19 @@ static void __init radix_init_pgtable(void) } /* Find out how many PID bits are supported */ - if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) { - if (!mmu_pid_bits) - mmu_pid_bits = 20; - mmu_base_pid = 1; - } else if (cpu_has_feature(CPU_FTR_HVMODE)) { - if (!mmu_pid_bits) - mmu_pid_bits = 20; -#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE + if (!cpu_has_feature(CPU_FTR_HVMODE) && + cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) { /* - * When KVM is possible, we only use the top half of the - * PID space to avoid collisions between host and guest PIDs - * which can cause problems due to prefetch when exiting the - * guest with AIL=3 + * Older versions of KVM on these machines perfer if the + * guest only uses the low 19 PID bits. */ - mmu_base_pid = 1 << (mmu_pid_bits - 1); -#else - mmu_base_pid = 1; -#endif - } else { - /* The guest uses the bottom half of the PID space */ if (!mmu_pid_bits) mmu_pid_bits = 19; - mmu_base_pid = 1; + } else { + if (!mmu_pid_bits) + mmu_pid_bits = 20; } + mmu_base_pid = 1; /* * Allocate Partition table and process table for the @@ -486,6 +475,7 @@ static int __init radix_dt_scan_page_sizes(unsigned long node, def = &mmu_psize_defs[idx]; def->shift = shift; def->ap = ap; + def->h_rpt_pgsize = psize_to_rpti_pgsize(idx); } /* needed ? */ @@ -560,9 +550,13 @@ void __init radix__early_init_devtree(void) */ mmu_psize_defs[MMU_PAGE_4K].shift = 12; mmu_psize_defs[MMU_PAGE_4K].ap = 0x0; + mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize = + psize_to_rpti_pgsize(MMU_PAGE_4K); mmu_psize_defs[MMU_PAGE_64K].shift = 16; mmu_psize_defs[MMU_PAGE_64K].ap = 0x5; + mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize = + psize_to_rpti_pgsize(MMU_PAGE_64K); } /* diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c index 409e61210789..318ec4f33661 100644 --- a/arch/powerpc/mm/book3s64/radix_tlb.c +++ b/arch/powerpc/mm/book3s64/radix_tlb.c @@ -20,10 +20,6 @@ #include "internal.h" -#define RIC_FLUSH_TLB 0 -#define RIC_FLUSH_PWC 1 -#define RIC_FLUSH_ALL 2 - /* * tlbiel instruction for radix, set invalidation * i.e., r=1 and is=01 or is=10 or is=11 @@ -130,6 +126,21 @@ static __always_inline void __tlbie_pid(unsigned long pid, unsigned long ric) trace_tlbie(0, 0, rb, rs, ric, prs, r); } +static __always_inline void __tlbie_pid_lpid(unsigned long pid, + unsigned long lpid, + unsigned long ric) +{ + unsigned long rb, rs, prs, r; + + rb = PPC_BIT(53); /* IS = 1 */ + rs = (pid << PPC_BITLSHIFT(31)) | (lpid & ~(PPC_BITMASK(0, 31))); + prs = 1; /* process scoped */ + r = 1; /* radix format */ + + asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) + : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); + trace_tlbie(0, 0, rb, rs, ric, prs, r); +} static __always_inline void __tlbie_lpid(unsigned long lpid, unsigned long ric) { unsigned long rb,rs,prs,r; @@ -190,6 +201,23 @@ static __always_inline void __tlbie_va(unsigned long va, unsigned long pid, trace_tlbie(0, 0, rb, rs, ric, prs, r); } +static __always_inline void __tlbie_va_lpid(unsigned long va, unsigned long pid, + unsigned long lpid, + unsigned long ap, unsigned long ric) +{ + unsigned long rb, rs, prs, r; + + rb = va & ~(PPC_BITMASK(52, 63)); + rb |= ap << PPC_BITLSHIFT(58); + rs = (pid << PPC_BITLSHIFT(31)) | (lpid & ~(PPC_BITMASK(0, 31))); + prs = 1; /* process scoped */ + r = 1; /* radix format */ + + asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) + : : "r"(rb), "i"(r), "i"(prs), "i"(ric), "r"(rs) : "memory"); + trace_tlbie(0, 0, rb, rs, ric, prs, r); +} + static __always_inline void __tlbie_lpid_va(unsigned long va, unsigned long lpid, unsigned long ap, unsigned long ric) { @@ -235,6 +263,22 @@ static inline void fixup_tlbie_va_range(unsigned long va, unsigned long pid, } } +static inline void fixup_tlbie_va_range_lpid(unsigned long va, + unsigned long pid, + unsigned long lpid, + unsigned long ap) +{ + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { + asm volatile("ptesync" : : : "memory"); + __tlbie_pid_lpid(0, lpid, RIC_FLUSH_TLB); + } + + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { + asm volatile("ptesync" : : : "memory"); + __tlbie_va_lpid(va, pid, lpid, ap, RIC_FLUSH_TLB); + } +} + static inline void fixup_tlbie_pid(unsigned long pid) { /* @@ -254,6 +298,25 @@ static inline void fixup_tlbie_pid(unsigned long pid) } } +static inline void fixup_tlbie_pid_lpid(unsigned long pid, unsigned long lpid) +{ + /* + * We can use any address for the invalidation, pick one which is + * probably unused as an optimisation. + */ + unsigned long va = ((1UL << 52) - 1); + + if (cpu_has_feature(CPU_FTR_P9_TLBIE_ERAT_BUG)) { + asm volatile("ptesync" : : : "memory"); + __tlbie_pid_lpid(0, lpid, RIC_FLUSH_TLB); + } + + if (cpu_has_feature(CPU_FTR_P9_TLBIE_STQ_BUG)) { + asm volatile("ptesync" : : : "memory"); + __tlbie_va_lpid(va, pid, lpid, mmu_get_ap(MMU_PAGE_64K), + RIC_FLUSH_TLB); + } +} static inline void fixup_tlbie_lpid_va(unsigned long va, unsigned long lpid, unsigned long ap) @@ -344,6 +407,31 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric) asm volatile("eieio; tlbsync; ptesync": : :"memory"); } +static inline void _tlbie_pid_lpid(unsigned long pid, unsigned long lpid, + unsigned long ric) +{ + asm volatile("ptesync" : : : "memory"); + + /* + * Workaround the fact that the "ric" argument to __tlbie_pid + * must be a compile-time contraint to match the "i" constraint + * in the asm statement. + */ + switch (ric) { + case RIC_FLUSH_TLB: + __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB); + fixup_tlbie_pid_lpid(pid, lpid); + break; + case RIC_FLUSH_PWC: + __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC); + break; + case RIC_FLUSH_ALL: + default: + __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_ALL); + fixup_tlbie_pid_lpid(pid, lpid); + } + asm volatile("eieio; tlbsync; ptesync" : : : "memory"); +} struct tlbiel_pid { unsigned long pid; unsigned long ric; @@ -469,6 +557,20 @@ static inline void __tlbie_va_range(unsigned long start, unsigned long end, fixup_tlbie_va_range(addr - page_size, pid, ap); } +static inline void __tlbie_va_range_lpid(unsigned long start, unsigned long end, + unsigned long pid, unsigned long lpid, + unsigned long page_size, + unsigned long psize) +{ + unsigned long addr; + unsigned long ap = mmu_get_ap(psize); + + for (addr = start; addr < end; addr += page_size) + __tlbie_va_lpid(addr, pid, lpid, ap, RIC_FLUSH_TLB); + + fixup_tlbie_va_range_lpid(addr - page_size, pid, lpid, ap); +} + static __always_inline void _tlbie_va(unsigned long va, unsigned long pid, unsigned long psize, unsigned long ric) { @@ -549,6 +651,18 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end, asm volatile("eieio; tlbsync; ptesync": : :"memory"); } +static inline void _tlbie_va_range_lpid(unsigned long start, unsigned long end, + unsigned long pid, unsigned long lpid, + unsigned long page_size, + unsigned long psize, bool also_pwc) +{ + asm volatile("ptesync" : : : "memory"); + if (also_pwc) + __tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC); + __tlbie_va_range_lpid(start, end, pid, lpid, page_size, psize); + asm volatile("eieio; tlbsync; ptesync" : : : "memory"); +} + static inline void _tlbiel_va_range_multicast(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned long pid, unsigned long page_size, @@ -1338,47 +1452,57 @@ void radix__flush_tlb_all(void) } #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE -extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) +/* + * Performs process-scoped invalidations for a given LPID + * as part of H_RPT_INVALIDATE hcall. + */ +void do_h_rpt_invalidate_prt(unsigned long pid, unsigned long lpid, + unsigned long type, unsigned long pg_sizes, + unsigned long start, unsigned long end) { - unsigned long pid = mm->context.id; - - if (unlikely(pid == MMU_NO_CONTEXT)) - return; - - if (!cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) - return; + unsigned long psize, nr_pages; + struct mmu_psize_def *def; + bool flush_pid; /* - * If this context hasn't run on that CPU before and KVM is - * around, there's a slim chance that the guest on another - * CPU just brought in obsolete translation into the TLB of - * this CPU due to a bad prefetch using the guest PID on - * the way into the hypervisor. - * - * We work around this here. If KVM is possible, we check if - * any sibling thread is in KVM. If it is, the window may exist - * and thus we flush that PID from the core. - * - * A potential future improvement would be to mark which PIDs - * have never been used on the system and avoid it if the PID - * is new and the process has no other cpumask bit set. + * A H_RPTI_TYPE_ALL request implies RIC=3, hence + * do a single IS=1 based flush. */ - if (cpu_has_feature(CPU_FTR_HVMODE) && radix_enabled()) { - int cpu = smp_processor_id(); - int sib = cpu_first_thread_sibling(cpu); - bool flush = false; + if ((type & H_RPTI_TYPE_ALL) == H_RPTI_TYPE_ALL) { + _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_ALL); + return; + } - for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) { - if (sib == cpu) - continue; - if (!cpu_possible(sib)) - continue; - if (paca_ptrs[sib]->kvm_hstate.kvm_vcpu) - flush = true; + if (type & H_RPTI_TYPE_PWC) + _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_PWC); + + /* Full PID flush */ + if (start == 0 && end == -1) + return _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB); + + /* Do range invalidation for all the valid page sizes */ + for (psize = 0; psize < MMU_PAGE_COUNT; psize++) { + def = &mmu_psize_defs[psize]; + if (!(pg_sizes & def->h_rpt_pgsize)) + continue; + + nr_pages = (end - start) >> def->shift; + flush_pid = nr_pages > tlb_single_page_flush_ceiling; + + /* + * If the number of pages spanning the range is above + * the ceiling, convert the request into a full PID flush. + * And since PID flush takes out all the page sizes, there + * is no need to consider remaining page sizes. + */ + if (flush_pid) { + _tlbie_pid_lpid(pid, lpid, RIC_FLUSH_TLB); + return; } - if (flush) - _tlbiel_pid(pid, RIC_FLUSH_ALL); + _tlbie_va_range_lpid(start, end, pid, lpid, + (1UL << def->shift), psize, false); } } -EXPORT_SYMBOL_GPL(radix_kvm_prefetch_workaround); +EXPORT_SYMBOL_GPL(do_h_rpt_invalidate_prt); + #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */ diff --git a/arch/powerpc/mm/mmu_context.c b/arch/powerpc/mm/mmu_context.c index a857af401738..74246536b832 100644 --- a/arch/powerpc/mm/mmu_context.c +++ b/arch/powerpc/mm/mmu_context.c @@ -83,9 +83,7 @@ void switch_mm_irqs_off(struct mm_struct *prev, struct mm_struct *next, if (cpu_has_feature(CPU_FTR_ALTIVEC)) asm volatile ("dssall"); - if (new_on_cpu) - radix_kvm_prefetch_workaround(next); - else + if (!new_on_cpu) membarrier_arch_switch_mm(prev, next, tsk); /* diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c index 999997d9e9a9..528a7e0cf83a 100644 --- a/arch/powerpc/platforms/powernv/idle.c +++ b/arch/powerpc/platforms/powernv/idle.c @@ -604,7 +604,7 @@ struct p9_sprs { u64 uamor; }; -static unsigned long power9_idle_stop(unsigned long psscr, bool mmu_on) +static unsigned long power9_idle_stop(unsigned long psscr) { int cpu = raw_smp_processor_id(); int first = cpu_first_thread_sibling(cpu); @@ -620,8 +620,6 @@ static unsigned long power9_idle_stop(unsigned long psscr, bool mmu_on) if (!(psscr & (PSSCR_EC|PSSCR_ESL))) { /* EC=ESL=0 case */ - BUG_ON(!mmu_on); - /* * Wake synchronously. SRESET via xscom may still cause * a 0x100 powersave wakeup with SRR1 reason! @@ -803,8 +801,7 @@ static unsigned long power9_idle_stop(unsigned long psscr, bool mmu_on) __slb_restore_bolted_realmode(); out: - if (mmu_on) - mtmsr(MSR_KERNEL); + mtmsr(MSR_KERNEL); return srr1; } @@ -895,7 +892,7 @@ struct p10_sprs { */ }; -static unsigned long power10_idle_stop(unsigned long psscr, bool mmu_on) +static unsigned long power10_idle_stop(unsigned long psscr) { int cpu = raw_smp_processor_id(); int first = cpu_first_thread_sibling(cpu); @@ -909,8 +906,6 @@ static unsigned long power10_idle_stop(unsigned long psscr, bool mmu_on) if (!(psscr & (PSSCR_EC|PSSCR_ESL))) { /* EC=ESL=0 case */ - BUG_ON(!mmu_on); - /* * Wake synchronously. SRESET via xscom may still cause * a 0x100 powersave wakeup with SRR1 reason! @@ -991,8 +986,7 @@ static unsigned long power10_idle_stop(unsigned long psscr, bool mmu_on) __slb_restore_bolted_realmode(); out: - if (mmu_on) - mtmsr(MSR_KERNEL); + mtmsr(MSR_KERNEL); return srr1; } @@ -1002,40 +996,10 @@ static unsigned long arch300_offline_stop(unsigned long psscr) { unsigned long srr1; -#ifndef CONFIG_KVM_BOOK3S_HV_POSSIBLE - __ppc64_runlatch_off(); if (cpu_has_feature(CPU_FTR_ARCH_31)) - srr1 = power10_idle_stop(psscr, true); + srr1 = power10_idle_stop(psscr); else - srr1 = power9_idle_stop(psscr, true); - __ppc64_runlatch_on(); -#else - /* - * Tell KVM we're entering idle. - * This does not have to be done in real mode because the P9 MMU - * is independent per-thread. Some steppings share radix/hash mode - * between threads, but in that case KVM has a barrier sync in real - * mode before and after switching between radix and hash. - * - * kvm_start_guest must still be called in real mode though, hence - * the false argument. - */ - local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_IDLE; - - __ppc64_runlatch_off(); - if (cpu_has_feature(CPU_FTR_ARCH_31)) - srr1 = power10_idle_stop(psscr, false); - else - srr1 = power9_idle_stop(psscr, false); - __ppc64_runlatch_on(); - - local_paca->kvm_hstate.hwthread_state = KVM_HWTHREAD_IN_KERNEL; - /* Order setting hwthread_state vs. testing hwthread_req */ - smp_mb(); - if (local_paca->kvm_hstate.hwthread_req) - srr1 = idle_kvm_start_guest(srr1); - mtmsr(MSR_KERNEL); -#endif + srr1 = power9_idle_stop(psscr); return srr1; } @@ -1055,9 +1019,9 @@ void arch300_idle_type(unsigned long stop_psscr_val, __ppc64_runlatch_off(); if (cpu_has_feature(CPU_FTR_ARCH_31)) - srr1 = power10_idle_stop(psscr, true); + srr1 = power10_idle_stop(psscr); else - srr1 = power9_idle_stop(psscr, true); + srr1 = power9_idle_stop(psscr); __ppc64_runlatch_on(); fini_irq_for_idle_irqsoff(); diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 8925f3969478..9b4473f76e56 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -361,6 +361,7 @@ struct sie_page { }; struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; u64 exit_userspace; u64 exit_null; u64 exit_external_request; @@ -370,13 +371,7 @@ struct kvm_vcpu_stat { u64 exit_validity; u64 exit_instruction; u64 exit_pei; - u64 halt_successful_poll; - u64 halt_attempted_poll; - u64 halt_poll_invalid; u64 halt_no_poll_steal; - u64 halt_wakeup; - u64 halt_poll_success_ns; - u64 halt_poll_fail_ns; u64 instruction_lctl; u64 instruction_lctlg; u64 instruction_stctl; @@ -755,12 +750,12 @@ struct kvm_vcpu_arch { }; struct kvm_vm_stat { + struct kvm_vm_stat_generic generic; u64 inject_io; u64 inject_float_mchk; u64 inject_pfault_done; u64 inject_service_signal; u64 inject_virtio; - u64 remote_tlb_flush; }; struct kvm_arch_memory_slot { diff --git a/arch/s390/include/asm/page.h b/arch/s390/include/asm/page.h index cc98f9b78fd4..479dc76e0eca 100644 --- a/arch/s390/include/asm/page.h +++ b/arch/s390/include/asm/page.h @@ -68,9 +68,9 @@ static inline void copy_page(void *to, void *from) #define clear_user_page(page, vaddr, pg) clear_page(page) #define copy_user_page(to, from, vaddr, pg) copy_page(to, from) -#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ - alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) -#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE /* * These are used to make use of C type-checking.. diff --git a/arch/s390/kvm/Makefile b/arch/s390/kvm/Makefile index 12decca22e7c..b3aaadc60ead 100644 --- a/arch/s390/kvm/Makefile +++ b/arch/s390/kvm/Makefile @@ -4,7 +4,8 @@ # Copyright IBM Corp. 2008 KVM := ../../../virt/kvm -common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/async_pf.o $(KVM)/irqchip.o $(KVM)/vfio.o +common-objs = $(KVM)/kvm_main.o $(KVM)/eventfd.o $(KVM)/async_pf.o \ + $(KVM)/irqchip.o $(KVM)/vfio.o $(KVM)/binary_stats.o ccflags-y := -Ivirt/kvm -Iarch/s390/kvm diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index 1296fc10f80c..f9fb1e1d960d 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -58,112 +58,132 @@ #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ (KVM_MAX_VCPUS + LOCAL_IRQS)) -struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT("userspace_handled", exit_userspace), - VCPU_STAT("exit_null", exit_null), - VCPU_STAT("pfault_sync", pfault_sync), - VCPU_STAT("exit_validity", exit_validity), - VCPU_STAT("exit_stop_request", exit_stop_request), - VCPU_STAT("exit_external_request", exit_external_request), - VCPU_STAT("exit_io_request", exit_io_request), - VCPU_STAT("exit_external_interrupt", exit_external_interrupt), - VCPU_STAT("exit_instruction", exit_instruction), - VCPU_STAT("exit_pei", exit_pei), - VCPU_STAT("exit_program_interruption", exit_program_interruption), - VCPU_STAT("exit_instr_and_program_int", exit_instr_and_program), - VCPU_STAT("exit_operation_exception", exit_operation_exception), - VCPU_STAT("halt_successful_poll", halt_successful_poll), - VCPU_STAT("halt_attempted_poll", halt_attempted_poll), - VCPU_STAT("halt_poll_invalid", halt_poll_invalid), - VCPU_STAT("halt_no_poll_steal", halt_no_poll_steal), - VCPU_STAT("halt_wakeup", halt_wakeup), - VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), - VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), - VCPU_STAT("instruction_lctlg", instruction_lctlg), - VCPU_STAT("instruction_lctl", instruction_lctl), - VCPU_STAT("instruction_stctl", instruction_stctl), - VCPU_STAT("instruction_stctg", instruction_stctg), - VCPU_STAT("deliver_ckc", deliver_ckc), - VCPU_STAT("deliver_cputm", deliver_cputm), - VCPU_STAT("deliver_emergency_signal", deliver_emergency_signal), - VCPU_STAT("deliver_external_call", deliver_external_call), - VCPU_STAT("deliver_service_signal", deliver_service_signal), - VCPU_STAT("deliver_virtio", deliver_virtio), - VCPU_STAT("deliver_stop_signal", deliver_stop_signal), - VCPU_STAT("deliver_prefix_signal", deliver_prefix_signal), - VCPU_STAT("deliver_restart_signal", deliver_restart_signal), - VCPU_STAT("deliver_program", deliver_program), - VCPU_STAT("deliver_io", deliver_io), - VCPU_STAT("deliver_machine_check", deliver_machine_check), - VCPU_STAT("exit_wait_state", exit_wait_state), - VCPU_STAT("inject_ckc", inject_ckc), - VCPU_STAT("inject_cputm", inject_cputm), - VCPU_STAT("inject_external_call", inject_external_call), - VM_STAT("inject_float_mchk", inject_float_mchk), - VCPU_STAT("inject_emergency_signal", inject_emergency_signal), - VM_STAT("inject_io", inject_io), - VCPU_STAT("inject_mchk", inject_mchk), - VM_STAT("inject_pfault_done", inject_pfault_done), - VCPU_STAT("inject_program", inject_program), - VCPU_STAT("inject_restart", inject_restart), - VM_STAT("inject_service_signal", inject_service_signal), - VCPU_STAT("inject_set_prefix", inject_set_prefix), - VCPU_STAT("inject_stop_signal", inject_stop_signal), - VCPU_STAT("inject_pfault_init", inject_pfault_init), - VM_STAT("inject_virtio", inject_virtio), - VCPU_STAT("instruction_epsw", instruction_epsw), - VCPU_STAT("instruction_gs", instruction_gs), - VCPU_STAT("instruction_io_other", instruction_io_other), - VCPU_STAT("instruction_lpsw", instruction_lpsw), - VCPU_STAT("instruction_lpswe", instruction_lpswe), - VCPU_STAT("instruction_pfmf", instruction_pfmf), - VCPU_STAT("instruction_ptff", instruction_ptff), - VCPU_STAT("instruction_stidp", instruction_stidp), - VCPU_STAT("instruction_sck", instruction_sck), - VCPU_STAT("instruction_sckpf", instruction_sckpf), - VCPU_STAT("instruction_spx", instruction_spx), - VCPU_STAT("instruction_stpx", instruction_stpx), - VCPU_STAT("instruction_stap", instruction_stap), - VCPU_STAT("instruction_iske", instruction_iske), - VCPU_STAT("instruction_ri", instruction_ri), - VCPU_STAT("instruction_rrbe", instruction_rrbe), - VCPU_STAT("instruction_sske", instruction_sske), - VCPU_STAT("instruction_ipte_interlock", instruction_ipte_interlock), - VCPU_STAT("instruction_essa", instruction_essa), - VCPU_STAT("instruction_stsi", instruction_stsi), - VCPU_STAT("instruction_stfl", instruction_stfl), - VCPU_STAT("instruction_tb", instruction_tb), - VCPU_STAT("instruction_tpi", instruction_tpi), - VCPU_STAT("instruction_tprot", instruction_tprot), - VCPU_STAT("instruction_tsch", instruction_tsch), - VCPU_STAT("instruction_sthyi", instruction_sthyi), - VCPU_STAT("instruction_sie", instruction_sie), - VCPU_STAT("instruction_sigp_sense", instruction_sigp_sense), - VCPU_STAT("instruction_sigp_sense_running", instruction_sigp_sense_running), - VCPU_STAT("instruction_sigp_external_call", instruction_sigp_external_call), - VCPU_STAT("instruction_sigp_emergency", instruction_sigp_emergency), - VCPU_STAT("instruction_sigp_cond_emergency", instruction_sigp_cond_emergency), - VCPU_STAT("instruction_sigp_start", instruction_sigp_start), - VCPU_STAT("instruction_sigp_stop", instruction_sigp_stop), - VCPU_STAT("instruction_sigp_stop_store_status", instruction_sigp_stop_store_status), - VCPU_STAT("instruction_sigp_store_status", instruction_sigp_store_status), - VCPU_STAT("instruction_sigp_store_adtl_status", instruction_sigp_store_adtl_status), - VCPU_STAT("instruction_sigp_set_arch", instruction_sigp_arch), - VCPU_STAT("instruction_sigp_set_prefix", instruction_sigp_prefix), - VCPU_STAT("instruction_sigp_restart", instruction_sigp_restart), - VCPU_STAT("instruction_sigp_cpu_reset", instruction_sigp_cpu_reset), - VCPU_STAT("instruction_sigp_init_cpu_reset", instruction_sigp_init_cpu_reset), - VCPU_STAT("instruction_sigp_unknown", instruction_sigp_unknown), - VCPU_STAT("instruction_diag_10", diagnose_10), - VCPU_STAT("instruction_diag_44", diagnose_44), - VCPU_STAT("instruction_diag_9c", diagnose_9c), - VCPU_STAT("diag_9c_ignored", diagnose_9c_ignored), - VCPU_STAT("diag_9c_forward", diagnose_9c_forward), - VCPU_STAT("instruction_diag_258", diagnose_258), - VCPU_STAT("instruction_diag_308", diagnose_308), - VCPU_STAT("instruction_diag_500", diagnose_500), - VCPU_STAT("instruction_diag_other", diagnose_other), - { NULL } +const struct _kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS(), + STATS_DESC_COUNTER(VM, inject_io), + STATS_DESC_COUNTER(VM, inject_float_mchk), + STATS_DESC_COUNTER(VM, inject_pfault_done), + STATS_DESC_COUNTER(VM, inject_service_signal), + STATS_DESC_COUNTER(VM, inject_virtio) +}; +static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == + sizeof(struct kvm_vm_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + STATS_DESC_COUNTER(VCPU, exit_userspace), + STATS_DESC_COUNTER(VCPU, exit_null), + STATS_DESC_COUNTER(VCPU, exit_external_request), + STATS_DESC_COUNTER(VCPU, exit_io_request), + STATS_DESC_COUNTER(VCPU, exit_external_interrupt), + STATS_DESC_COUNTER(VCPU, exit_stop_request), + STATS_DESC_COUNTER(VCPU, exit_validity), + STATS_DESC_COUNTER(VCPU, exit_instruction), + STATS_DESC_COUNTER(VCPU, exit_pei), + STATS_DESC_COUNTER(VCPU, halt_no_poll_steal), + STATS_DESC_COUNTER(VCPU, instruction_lctl), + STATS_DESC_COUNTER(VCPU, instruction_lctlg), + STATS_DESC_COUNTER(VCPU, instruction_stctl), + STATS_DESC_COUNTER(VCPU, instruction_stctg), + STATS_DESC_COUNTER(VCPU, exit_program_interruption), + STATS_DESC_COUNTER(VCPU, exit_instr_and_program), + STATS_DESC_COUNTER(VCPU, exit_operation_exception), + STATS_DESC_COUNTER(VCPU, deliver_ckc), + STATS_DESC_COUNTER(VCPU, deliver_cputm), + STATS_DESC_COUNTER(VCPU, deliver_external_call), + STATS_DESC_COUNTER(VCPU, deliver_emergency_signal), + STATS_DESC_COUNTER(VCPU, deliver_service_signal), + STATS_DESC_COUNTER(VCPU, deliver_virtio), + STATS_DESC_COUNTER(VCPU, deliver_stop_signal), + STATS_DESC_COUNTER(VCPU, deliver_prefix_signal), + STATS_DESC_COUNTER(VCPU, deliver_restart_signal), + STATS_DESC_COUNTER(VCPU, deliver_program), + STATS_DESC_COUNTER(VCPU, deliver_io), + STATS_DESC_COUNTER(VCPU, deliver_machine_check), + STATS_DESC_COUNTER(VCPU, exit_wait_state), + STATS_DESC_COUNTER(VCPU, inject_ckc), + STATS_DESC_COUNTER(VCPU, inject_cputm), + STATS_DESC_COUNTER(VCPU, inject_external_call), + STATS_DESC_COUNTER(VCPU, inject_emergency_signal), + STATS_DESC_COUNTER(VCPU, inject_mchk), + STATS_DESC_COUNTER(VCPU, inject_pfault_init), + STATS_DESC_COUNTER(VCPU, inject_program), + STATS_DESC_COUNTER(VCPU, inject_restart), + STATS_DESC_COUNTER(VCPU, inject_set_prefix), + STATS_DESC_COUNTER(VCPU, inject_stop_signal), + STATS_DESC_COUNTER(VCPU, instruction_epsw), + STATS_DESC_COUNTER(VCPU, instruction_gs), + STATS_DESC_COUNTER(VCPU, instruction_io_other), + STATS_DESC_COUNTER(VCPU, instruction_lpsw), + STATS_DESC_COUNTER(VCPU, instruction_lpswe), + STATS_DESC_COUNTER(VCPU, instruction_pfmf), + STATS_DESC_COUNTER(VCPU, instruction_ptff), + STATS_DESC_COUNTER(VCPU, instruction_sck), + STATS_DESC_COUNTER(VCPU, instruction_sckpf), + STATS_DESC_COUNTER(VCPU, instruction_stidp), + STATS_DESC_COUNTER(VCPU, instruction_spx), + STATS_DESC_COUNTER(VCPU, instruction_stpx), + STATS_DESC_COUNTER(VCPU, instruction_stap), + STATS_DESC_COUNTER(VCPU, instruction_iske), + STATS_DESC_COUNTER(VCPU, instruction_ri), + STATS_DESC_COUNTER(VCPU, instruction_rrbe), + STATS_DESC_COUNTER(VCPU, instruction_sske), + STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock), + STATS_DESC_COUNTER(VCPU, instruction_stsi), + STATS_DESC_COUNTER(VCPU, instruction_stfl), + STATS_DESC_COUNTER(VCPU, instruction_tb), + STATS_DESC_COUNTER(VCPU, instruction_tpi), + STATS_DESC_COUNTER(VCPU, instruction_tprot), + STATS_DESC_COUNTER(VCPU, instruction_tsch), + STATS_DESC_COUNTER(VCPU, instruction_sie), + STATS_DESC_COUNTER(VCPU, instruction_essa), + STATS_DESC_COUNTER(VCPU, instruction_sthyi), + STATS_DESC_COUNTER(VCPU, instruction_sigp_sense), + STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running), + STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call), + STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency), + STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency), + STATS_DESC_COUNTER(VCPU, instruction_sigp_start), + STATS_DESC_COUNTER(VCPU, instruction_sigp_stop), + STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status), + STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status), + STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status), + STATS_DESC_COUNTER(VCPU, instruction_sigp_arch), + STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix), + STATS_DESC_COUNTER(VCPU, instruction_sigp_restart), + STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset), + STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset), + STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown), + STATS_DESC_COUNTER(VCPU, diagnose_10), + STATS_DESC_COUNTER(VCPU, diagnose_44), + STATS_DESC_COUNTER(VCPU, diagnose_9c), + STATS_DESC_COUNTER(VCPU, diagnose_9c_ignored), + STATS_DESC_COUNTER(VCPU, diagnose_9c_forward), + STATS_DESC_COUNTER(VCPU, diagnose_258), + STATS_DESC_COUNTER(VCPU, diagnose_308), + STATS_DESC_COUNTER(VCPU, diagnose_500), + STATS_DESC_COUNTER(VCPU, diagnose_other), + STATS_DESC_COUNTER(VCPU, pfault_sync) +}; +static_assert(ARRAY_SIZE(kvm_vcpu_stats_desc) == + sizeof(struct kvm_vcpu_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), }; /* allow nested virtualization in KVM (if enabled by user space) */ @@ -329,31 +349,31 @@ static void allow_cpu_feat(unsigned long nr) static inline int plo_test_bit(unsigned char nr) { - register unsigned long r0 asm("0") = (unsigned long) nr | 0x100; + unsigned long function = (unsigned long)nr | 0x100; int cc; asm volatile( + " lgr 0,%[function]\n" /* Parameter registers are ignored for "test bit" */ " plo 0,0,0,0(0)\n" " ipm %0\n" " srl %0,28\n" : "=d" (cc) - : "d" (r0) - : "cc"); + : [function] "d" (function) + : "cc", "0"); return cc == 0; } static __always_inline void __insn32_query(unsigned int opcode, u8 *query) { - register unsigned long r0 asm("0") = 0; /* query function */ - register unsigned long r1 asm("1") = (unsigned long) query; - asm volatile( - /* Parameter regs are ignored */ + " lghi 0,0\n" + " lgr 1,%[query]\n" + /* Parameter registers are ignored */ " .insn rrf,%[opc] << 16,2,4,6,0\n" : - : "d" (r0), "a" (r1), [opc] "i" (opcode) - : "cc", "memory"); + : [query] "d" ((unsigned long)query), [opc] "i" (opcode) + : "cc", "memory", "0", "1"); } #define INSN_SORTL 0xb938 @@ -713,6 +733,10 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) set_kvm_facility(kvm->arch.model.fac_mask, 152); set_kvm_facility(kvm->arch.model.fac_list, 152); } + if (test_facility(192)) { + set_kvm_facility(kvm->arch.model.fac_mask, 192); + set_kvm_facility(kvm->arch.model.fac_list, 192); + } r = 0; } else r = -EINVAL; diff --git a/arch/s390/tools/gen_facilities.c b/arch/s390/tools/gen_facilities.c index 61ce5b59b828..606324e56e4e 100644 --- a/arch/s390/tools/gen_facilities.c +++ b/arch/s390/tools/gen_facilities.c @@ -115,6 +115,10 @@ static struct facility_def facility_defs[] = { 12, /* AP Query Configuration Information */ 15, /* AP Facilities Test */ 156, /* etoken facility */ + 165, /* nnpa facility */ + 193, /* bear enhancement facility */ + 194, /* rdp enhancement facility */ + 196, /* processor activity instrumentation facility */ -1 /* END */ } }, diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 0045e1b44190..86dae426798b 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -1693,35 +1693,6 @@ config X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK Set whether the default state of memory_corruption_check is on or off. -config X86_RESERVE_LOW - int "Amount of low memory, in kilobytes, to reserve for the BIOS" - default 64 - range 4 640 - help - Specify the amount of low memory to reserve for the BIOS. - - The first page contains BIOS data structures that the kernel - must not use, so that page must always be reserved. - - By default we reserve the first 64K of physical RAM, as a - number of BIOSes are known to corrupt that memory range - during events such as suspend/resume or monitor cable - insertion, so it must not be used by the kernel. - - You can set this to 4 if you are absolutely sure that you - trust the BIOS to get all its memory reservations and usages - right. If you know your BIOS have problems beyond the - default 64K area, you can set this to 640 to avoid using the - entire low memory range. - - If you have doubts about the BIOS (e.g. suspend/resume does - not work or there's kernel crashes after certain hardware - hotplug events) then you might want to enable - X86_CHECK_BIOS_CORRUPTION=y to allow the kernel to check - typical corruption patterns. - - Leave this to the default value of 64 if you are unsure. - config MATH_EMULATION bool depends on MODIFY_LDT_SYSCALL diff --git a/arch/x86/Makefile b/arch/x86/Makefile index 3bdee26c0146..23171b15f329 100644 --- a/arch/x86/Makefile +++ b/arch/x86/Makefile @@ -257,7 +257,7 @@ drivers-$(CONFIG_FB) += arch/x86/video/ boot := arch/x86/boot -BOOT_TARGETS = bzdisk fdimage fdimage144 fdimage288 isoimage +BOOT_TARGETS = bzdisk fdimage fdimage144 fdimage288 hdimage isoimage PHONY += bzImage $(BOOT_TARGETS) @@ -318,8 +318,9 @@ define archhelp echo ' fdimage - Create 1.4MB boot floppy image (arch/x86/boot/fdimage)' echo ' fdimage144 - Create 1.4MB boot floppy image (arch/x86/boot/fdimage)' echo ' fdimage288 - Create 2.8MB boot floppy image (arch/x86/boot/fdimage)' + echo ' hdimage - Create a BIOS/EFI hard disk image (arch/x86/boot/hdimage)' echo ' isoimage - Create a boot CD-ROM image (arch/x86/boot/image.iso)' - echo ' bzdisk/fdimage*/isoimage also accept:' + echo ' bzdisk/fdimage*/hdimage/isoimage also accept:' echo ' FDARGS="..." arguments for the booted kernel' echo ' FDINITRD=file initrd for the booted kernel' echo '' diff --git a/arch/x86/boot/.gitignore b/arch/x86/boot/.gitignore index 9cc7f1357b9b..1189be057ebd 100644 --- a/arch/x86/boot/.gitignore +++ b/arch/x86/boot/.gitignore @@ -11,3 +11,4 @@ setup.elf fdimage mtools.conf image.iso +hdimage diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index fe605205b4ce..dfbc26a8e924 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -29,7 +29,7 @@ KCOV_INSTRUMENT := n SVGA_MODE := -DSVGA_MODE=NORMAL_VGA targets := vmlinux.bin setup.bin setup.elf bzImage -targets += fdimage fdimage144 fdimage288 image.iso mtools.conf +targets += fdimage fdimage144 fdimage288 image.iso hdimage subdir- := compressed setup-y += a20.o bioscall.o cmdline.o copy.o cpu.o cpuflags.o cpucheck.o @@ -115,47 +115,49 @@ $(obj)/compressed/vmlinux: FORCE $(Q)$(MAKE) $(build)=$(obj)/compressed $@ # Set this if you want to pass append arguments to the -# bzdisk/fdimage/isoimage kernel +# bzdisk/fdimage/hdimage/isoimage kernel FDARGS = -# Set this if you want an initrd included with the -# bzdisk/fdimage/isoimage kernel +# Set this if you want one or more initrds included in the image FDINITRD = -image_cmdline = default linux $(FDARGS) $(if $(FDINITRD),initrd=initrd.img,) +imgdeps = $(obj)/bzImage $(obj)/mtools.conf $(src)/genimage.sh $(obj)/mtools.conf: $(src)/mtools.conf.in sed -e 's|@OBJ@|$(obj)|g' < $< > $@ -quiet_cmd_genimage = GENIMAGE $3 -cmd_genimage = sh $(srctree)/$(src)/genimage.sh $2 $3 $(obj)/bzImage \ - $(obj)/mtools.conf '$(image_cmdline)' $(FDINITRD) +targets += mtools.conf -PHONY += bzdisk fdimage fdimage144 fdimage288 isoimage bzlilo install +# genimage.sh requires bash, but it also has a bunch of other +# external dependencies. +quiet_cmd_genimage = GENIMAGE $3 +cmd_genimage = $(BASH) $(srctree)/$(src)/genimage.sh $2 $3 $(obj)/bzImage \ + $(obj)/mtools.conf '$(FDARGS)' $(FDINITRD) + +PHONY += bzdisk fdimage fdimage144 fdimage288 hdimage isoimage install # This requires write access to /dev/fd0 -bzdisk: $(obj)/bzImage $(obj)/mtools.conf +# All images require syslinux to be installed; hdimage also requires +# EDK2/OVMF if the kernel is compiled with the EFI stub. +bzdisk: $(imgdeps) $(call cmd,genimage,bzdisk,/dev/fd0) -# These require being root or having syslinux 2.02 or higher installed -fdimage fdimage144: $(obj)/bzImage $(obj)/mtools.conf +fdimage fdimage144: $(imgdeps) $(call cmd,genimage,fdimage144,$(obj)/fdimage) @$(kecho) 'Kernel: $(obj)/fdimage is ready' -fdimage288: $(obj)/bzImage $(obj)/mtools.conf +fdimage288: $(imgdeps) $(call cmd,genimage,fdimage288,$(obj)/fdimage) @$(kecho) 'Kernel: $(obj)/fdimage is ready' -isoimage: $(obj)/bzImage +hdimage: $(imgdeps) + $(call cmd,genimage,hdimage,$(obj)/hdimage) + @$(kecho) 'Kernel: $(obj)/hdimage is ready' + +isoimage: $(imgdeps) $(call cmd,genimage,isoimage,$(obj)/image.iso) @$(kecho) 'Kernel: $(obj)/image.iso is ready' -bzlilo: - if [ -f $(INSTALL_PATH)/vmlinuz ]; then mv $(INSTALL_PATH)/vmlinuz $(INSTALL_PATH)/vmlinuz.old; fi - if [ -f $(INSTALL_PATH)/System.map ]; then mv $(INSTALL_PATH)/System.map $(INSTALL_PATH)/System.old; fi - cat $(obj)/bzImage > $(INSTALL_PATH)/vmlinuz - cp System.map $(INSTALL_PATH)/ - if [ -x /sbin/lilo ]; then /sbin/lilo; else /etc/lilo/install; fi - install: - sh $(srctree)/$(src)/install.sh $(KERNELRELEASE) $(obj)/bzImage \ + $(CONFIG_SHELL) $(srctree)/$(src)/install.sh \ + $(KERNELRELEASE) $(obj)/bzImage \ System.map "$(INSTALL_PATH)" diff --git a/arch/x86/boot/genimage.sh b/arch/x86/boot/genimage.sh index 6a10d52a4145..0673fdfc1a11 100644 --- a/arch/x86/boot/genimage.sh +++ b/arch/x86/boot/genimage.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # This file is subject to the terms and conditions of the GNU General Public # License. See the file "COPYING" in the main directory of this archive @@ -8,15 +8,24 @@ # # Adapted from code in arch/x86/boot/Makefile by H. Peter Anvin and others # -# "make fdimage/fdimage144/fdimage288/isoimage" script for x86 architecture +# "make fdimage/fdimage144/fdimage288/hdimage/isoimage" +# script for x86 architecture # # Arguments: -# $1 - fdimage format -# $2 - target image file -# $3 - kernel bzImage file -# $4 - mtool configuration file -# $5 - kernel cmdline -# $6 - inird image file +# $1 - fdimage format +# $2 - target image file +# $3 - kernel bzImage file +# $4 - mtools configuration file +# $5 - kernel cmdline +# $6+ - initrd image file(s) +# +# This script requires: +# bash +# syslinux +# mtools (for fdimage* and hdimage) +# edk2/OVMF (for hdimage) +# +# Otherwise try to stick to POSIX shell commands... # # Use "make V=1" to debug this script @@ -26,105 +35,237 @@ case "${KBUILD_VERBOSE}" in ;; esac -verify () { - if [ ! -f "$1" ]; then - echo "" 1>&2 - echo " *** Missing file: $1" 1>&2 - echo "" 1>&2 - exit 1 +# Exit the top-level shell with an error +topshell=$$ +trap 'exit 1' USR1 +die() { + echo "" 1>&2 + echo " *** $*" 1>&2 + echo "" 1>&2 + kill -USR1 $topshell +} + +# Verify the existence and readability of a file +verify() { + if [ ! -f "$1" -o ! -r "$1" ]; then + die "Missing file: $1" fi } +diskfmt="$1" +FIMAGE="$2" +FBZIMAGE="$3" +MTOOLSRC="$4" +KCMDLINE="$5" +shift 5 # Remaining arguments = initrd files -export MTOOLSRC=$4 -FIMAGE=$2 -FBZIMAGE=$3 -KCMDLINE=$5 -FDINITRD=$6 +export MTOOLSRC + +# common options for dd +dd='dd iflag=fullblock' # Make sure the files actually exist verify "$FBZIMAGE" +declare -a FDINITRDS +irdpfx=' initrd=' +initrdopts_syslinux='' +initrdopts_efi='' +for f in "$@"; do + if [ -f "$f" -a -r "$f" ]; then + FDINITRDS=("${FDINITRDS[@]}" "$f") + fname="$(basename "$f")" + initrdopts_syslinux="${initrdopts_syslinux}${irdpfx}${fname}" + irdpfx=, + initrdopts_efi="${initrdopts_efi} initrd=${fname}" + fi +done + +# Read a $3-byte littleendian unsigned value at offset $2 from file $1 +le() { + local n=0 + local m=1 + for b in $(od -A n -v -j $2 -N $3 -t u1 "$1"); do + n=$((n + b*m)) + m=$((m * 256)) + done + echo $n +} + +# Get the EFI architecture name such that boot{name}.efi is the default +# boot file name. Returns false with no output if the file is not an +# EFI image or otherwise unknown. +efiarch() { + [ -f "$1" ] || return + [ $(le "$1" 0 2) -eq 23117 ] || return # MZ magic + peoffs=$(le "$1" 60 4) # PE header offset + [ $peoffs -ge 64 ] || return + [ $(le "$1" $peoffs 4) -eq 17744 ] || return # PE magic + case $(le "$1" $((peoffs+4+20)) 2) in # PE type + 267) ;; # PE32 + 523) ;; # PE32+ + *) return 1 ;; # Invalid + esac + [ $(le "$1" $((peoffs+4+20+68)) 2) -eq 10 ] || return # EFI app + case $(le "$1" $((peoffs+4)) 2) in # Machine type + 332) echo i386 ;; + 450) echo arm ;; + 512) echo ia64 ;; + 20530) echo riscv32 ;; + 20580) echo riscv64 ;; + 20776) echo riscv128 ;; + 34404) echo x64 ;; + 43620) echo aa64 ;; + esac +} + +# Get the combined sizes in bytes of the files given, counting sparse +# files as full length, and padding each file to a 4K block size +filesizes() { + local t=0 + local s + for s in $(ls -lnL "$@" 2>/dev/null | awk '/^-/{ print $5; }'); do + t=$((t + ((s+4095)/4096)*4096)) + done + echo $t +} + +# Expand directory names which should be in /usr/share into a list +# of possible alternatives +sharedirs() { + local dir file + for dir in /usr/share /usr/lib64 /usr/lib; do + for file; do + echo "$dir/$file" + echo "$dir/${file^^}" + done + done +} +efidirs() { + local dir file + for dir in /usr/share /boot /usr/lib64 /usr/lib; do + for file; do + echo "$dir/$file" + echo "$dir/${file^^}" + done + done +} + +findsyslinux() { + local f="$(find -L $(sharedirs syslinux isolinux) \ + -name "$1" -readable -type f -print -quit 2>/dev/null)" + if [ ! -f "$f" ]; then + die "Need a $1 file, please install syslinux/isolinux." + fi + echo "$f" + return 0 +} + +findovmf() { + local arch="$1" + shift + local -a names=(-false) + local name f + for name; do + names=("${names[@]}" -or -iname "$name") + done + for f in $(find -L $(efidirs edk2 ovmf) \ + \( "${names[@]}" \) -readable -type f \ + -print 2>/dev/null); do + if [ "$(efiarch "$f")" = "$arch" ]; then + echo "$f" + return 0 + fi + done + die "Need a $1 file for $arch, please install EDK2/OVMF." +} + +do_mcopy() { + if [ ${#FDINITRDS[@]} -gt 0 ]; then + mcopy "${FDINITRDS[@]}" "$1" + fi + if [ -n "$efishell" ]; then + mmd "$1"EFI "$1"EFI/Boot + mcopy "$efishell" "$1"EFI/Boot/boot${kefiarch}.efi + fi + if [ -n "$kefiarch" ]; then + echo linux "$KCMDLINE$initrdopts_efi" | \ + mcopy - "$1"startup.nsh + fi + echo default linux "$KCMDLINE$initrdopts_syslinux" | \ + mcopy - "$1"syslinux.cfg + mcopy "$FBZIMAGE" "$1"linux +} + genbzdisk() { verify "$MTOOLSRC" - mformat a: - syslinux $FIMAGE - echo "$KCMDLINE" | mcopy - a:syslinux.cfg - if [ -f "$FDINITRD" ] ; then - mcopy "$FDINITRD" a:initrd.img - fi - mcopy $FBZIMAGE a:linux + mformat -v 'LINUX_BOOT' a: + syslinux "$FIMAGE" + do_mcopy a: } genfdimage144() { verify "$MTOOLSRC" - dd if=/dev/zero of=$FIMAGE bs=1024 count=1440 2> /dev/null - mformat v: - syslinux $FIMAGE - echo "$KCMDLINE" | mcopy - v:syslinux.cfg - if [ -f "$FDINITRD" ] ; then - mcopy "$FDINITRD" v:initrd.img - fi - mcopy $FBZIMAGE v:linux + $dd if=/dev/zero of="$FIMAGE" bs=1024 count=1440 2>/dev/null + mformat -v 'LINUX_BOOT' v: + syslinux "$FIMAGE" + do_mcopy v: } genfdimage288() { verify "$MTOOLSRC" - dd if=/dev/zero of=$FIMAGE bs=1024 count=2880 2> /dev/null - mformat w: - syslinux $FIMAGE - echo "$KCMDLINE" | mcopy - W:syslinux.cfg - if [ -f "$FDINITRD" ] ; then - mcopy "$FDINITRD" w:initrd.img + $dd if=/dev/zero of="$FIMAGE" bs=1024 count=2880 2>/dev/null + mformat -v 'LINUX_BOOT' w: + syslinux "$FIMAGE" + do_mcopy w: +} + +genhdimage() { + verify "$MTOOLSRC" + mbr="$(findsyslinux mbr.bin)" + kefiarch="$(efiarch "$FBZIMAGE")" + if [ -n "$kefiarch" ]; then + # The efishell provides command line handling + efishell="$(findovmf $kefiarch shell.efi shell${kefiarch}.efi)" + ptype='-T 0xef' # EFI system partition, no GPT fi - mcopy $FBZIMAGE w:linux + sizes=$(filesizes "$FBZIMAGE" "${FDINITRDS[@]}" "$efishell") + # Allow 1% + 1 MiB for filesystem and partition table overhead, + # syslinux, and config files + megs=$(((sizes + sizes/100 + 2*1024*1024 - 1)/(1024*1024))) + $dd if=/dev/zero of="$FIMAGE" bs=$((1024*1024)) count=$megs 2>/dev/null + mpartition -I -c -s 32 -h 64 -t $megs $ptype -b 512 -a h: + $dd if="$mbr" of="$FIMAGE" bs=440 count=1 conv=notrunc 2>/dev/null + mformat -v 'LINUX_BOOT' -s 32 -h 64 -t $megs h: + syslinux --offset $((512*512)) "$FIMAGE" + do_mcopy h: } geniso() { - tmp_dir=`dirname $FIMAGE`/isoimage - rm -rf $tmp_dir - mkdir $tmp_dir - for i in lib lib64 share ; do - for j in syslinux ISOLINUX ; do - if [ -f /usr/$i/$j/isolinux.bin ] ; then - isolinux=/usr/$i/$j/isolinux.bin - fi - done - for j in syslinux syslinux/modules/bios ; do - if [ -f /usr/$i/$j/ldlinux.c32 ]; then - ldlinux=/usr/$i/$j/ldlinux.c32 - fi - done - if [ -n "$isolinux" -a -n "$ldlinux" ] ; then - break - fi - done - if [ -z "$isolinux" ] ; then - echo 'Need an isolinux.bin file, please install syslinux/isolinux.' - exit 1 - fi - if [ -z "$ldlinux" ] ; then - echo 'Need an ldlinux.c32 file, please install syslinux/isolinux.' - exit 1 - fi - cp $isolinux $tmp_dir - cp $ldlinux $tmp_dir - cp $FBZIMAGE $tmp_dir/linux - echo "$KCMDLINE" > $tmp_dir/isolinux.cfg - if [ -f "$FDINITRD" ] ; then - cp "$FDINITRD" $tmp_dir/initrd.img - fi - genisoimage -J -r -input-charset=utf-8 -quiet -o $FIMAGE \ - -b isolinux.bin -c boot.cat -no-emul-boot -boot-load-size 4 \ - -boot-info-table $tmp_dir - isohybrid $FIMAGE 2>/dev/null || true - rm -rf $tmp_dir + tmp_dir="$(dirname "$FIMAGE")/isoimage" + rm -rf "$tmp_dir" + mkdir "$tmp_dir" + isolinux=$(findsyslinux isolinux.bin) + ldlinux=$(findsyslinux ldlinux.c32) + cp "$isolinux" "$ldlinux" "$tmp_dir" + cp "$FBZIMAGE" "$tmp_dir"/linux + echo default linux "$KCMDLINE" > "$tmp_dir"/isolinux.cfg + cp "${FDINITRDS[@]}" "$tmp_dir"/ + genisoimage -J -r -appid 'LINUX_BOOT' -input-charset=utf-8 \ + -quiet -o "$FIMAGE" -b isolinux.bin \ + -c boot.cat -no-emul-boot -boot-load-size 4 \ + -boot-info-table "$tmp_dir" + isohybrid "$FIMAGE" 2>/dev/null || true + rm -rf "$tmp_dir" } -case $1 in +rm -f "$FIMAGE" + +case "$diskfmt" in bzdisk) genbzdisk;; fdimage144) genfdimage144;; fdimage288) genfdimage288;; + hdimage) genhdimage;; isoimage) geniso;; - *) echo 'Unknown image format'; exit 1; + *) die "Unknown image format: $diskfmt";; esac diff --git a/arch/x86/boot/mtools.conf.in b/arch/x86/boot/mtools.conf.in index efd6d2490c1d..9e2662d01364 100644 --- a/arch/x86/boot/mtools.conf.in +++ b/arch/x86/boot/mtools.conf.in @@ -14,4 +14,7 @@ drive v: drive w: file="@OBJ@/fdimage" cylinders=80 heads=2 sectors=36 filter +# Hard disk +drive h: + file="@OBJ@/hdimage" partition=1 mformat_only diff --git a/arch/x86/crypto/curve25519-x86_64.c b/arch/x86/crypto/curve25519-x86_64.c index 6706b6cb1d0f..38caf61cd5b7 100644 --- a/arch/x86/crypto/curve25519-x86_64.c +++ b/arch/x86/crypto/curve25519-x86_64.c @@ -1500,7 +1500,7 @@ static int __init curve25519_mod_init(void) static void __exit curve25519_mod_exit(void) { if (IS_REACHABLE(CONFIG_CRYPTO_KPP) && - (boot_cpu_has(X86_FEATURE_BMI2) || boot_cpu_has(X86_FEATURE_ADX))) + static_branch_likely(&curve25519_use_bmi2_adx)) crypto_unregister_kpp(&curve25519_alg); } diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 07a9331d55e7..a4c061fb7c6e 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -6,6 +6,7 @@ #include #include #include +#include /* @@ -62,42 +63,7 @@ For 32-bit we have the following conventions - kernel is built with * for assembly code: */ -/* The layout forms the "struct pt_regs" on the stack: */ -/* - * C ABI says these regs are callee-preserved. They aren't saved on kernel entry - * unless syscall needs a complete, fully filled "struct pt_regs". - */ -#define R15 0*8 -#define R14 1*8 -#define R13 2*8 -#define R12 3*8 -#define RBP 4*8 -#define RBX 5*8 -/* These regs are callee-clobbered. Always saved on kernel entry. */ -#define R11 6*8 -#define R10 7*8 -#define R9 8*8 -#define R8 9*8 -#define RAX 10*8 -#define RCX 11*8 -#define RDX 12*8 -#define RSI 13*8 -#define RDI 14*8 -/* - * On syscall entry, this is syscall#. On CPU exception, this is error code. - * On hw interrupt, it's IRQ number: - */ -#define ORIG_RAX 15*8 -/* Return frame for iretq */ -#define RIP 16*8 -#define CS 17*8 -#define EFLAGS 18*8 -#define RSP 19*8 -#define SS 20*8 - -#define SIZEOF_PTREGS 21*8 - -.macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax save_ret=0 +.macro PUSH_REGS rdx=%rdx rax=%rax save_ret=0 .if \save_ret pushq %rsi /* pt_regs->si */ movq 8(%rsp), %rsi /* temporarily store the return address in %rsi */ @@ -124,7 +90,9 @@ For 32-bit we have the following conventions - kernel is built with .if \save_ret pushq %rsi /* return address on top of stack */ .endif +.endm +.macro CLEAR_REGS /* * Sanitize registers of values that a speculation attack might * otherwise want to exploit. The lower registers are likely clobbered @@ -146,6 +114,11 @@ For 32-bit we have the following conventions - kernel is built with .endm +.macro PUSH_AND_CLEAR_REGS rdx=%rdx rax=%rax save_ret=0 + PUSH_REGS rdx=\rdx, rax=\rax, save_ret=\save_ret + CLEAR_REGS +.endm + .macro POP_REGS pop_rdi=1 skip_r11rcx=0 popq %r15 popq %r14 diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 04bce95bc7e3..f79688648db1 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -36,7 +36,7 @@ #include #ifdef CONFIG_X86_64 -__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) +__visible noinstr void do_syscall_64(struct pt_regs *regs, unsigned long nr) { add_random_kstack_offset(); nr = syscall_enter_from_user_mode(regs, nr); diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index 1886aaf19914..237da6ced61b 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -107,8 +107,8 @@ SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) PUSH_AND_CLEAR_REGS rax=$-ENOSYS /* IRQs are off. */ - movq %rax, %rdi - movq %rsp, %rsi + movq %rsp, %rdi + movq %rax, %rsi call do_syscall_64 /* returns with IRQs disabled */ /* diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index bb0ae4b5c00f..256ad0e34dd2 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -623,7 +623,7 @@ bool hv_query_ext_cap(u64 cap_query) * output parameter to the hypercall below and so it should be * compatible with 'virt_to_phys'. Which means, it's address should be * directly mapped. Use 'static' to keep it compatible; stack variables - * can be virtually mapped, making them imcompatible with + * can be virtually mapped, making them incompatible with * 'virt_to_phys'. * Hypercall input/output addresses should also be 8-byte aligned. */ diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index 0603c7423aca..3ad3da9a7d97 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -3,25 +3,26 @@ #define _ASM_X86_ASM_H #ifdef __ASSEMBLY__ -# define __ASM_FORM(x) x -# define __ASM_FORM_RAW(x) x -# define __ASM_FORM_COMMA(x) x, +# define __ASM_FORM(x, ...) x,## __VA_ARGS__ +# define __ASM_FORM_RAW(x, ...) x,## __VA_ARGS__ +# define __ASM_FORM_COMMA(x, ...) x,## __VA_ARGS__, #else #include - -# define __ASM_FORM(x) " " __stringify(x) " " -# define __ASM_FORM_RAW(x) __stringify(x) -# define __ASM_FORM_COMMA(x) " " __stringify(x) "," +# define __ASM_FORM(x, ...) " " __stringify(x,##__VA_ARGS__) " " +# define __ASM_FORM_RAW(x, ...) __stringify(x,##__VA_ARGS__) +# define __ASM_FORM_COMMA(x, ...) " " __stringify(x,##__VA_ARGS__) "," #endif +#define _ASM_BYTES(x, ...) __ASM_FORM(.byte x,##__VA_ARGS__ ;) + #ifndef __x86_64__ /* 32 bit */ -# define __ASM_SEL(a,b) __ASM_FORM(a) -# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(a) +# define __ASM_SEL(a,b) __ASM_FORM(a) +# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(a) #else /* 64 bit */ -# define __ASM_SEL(a,b) __ASM_FORM(b) -# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(b) +# define __ASM_SEL(a,b) __ASM_FORM(b) +# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(b) #endif #define __ASM_SIZE(inst, ...) __ASM_SEL(inst##l##__VA_ARGS__, \ @@ -119,6 +120,8 @@ # define CC_OUT(c) [_cc_ ## c] "=qm" #endif +#ifdef __KERNEL__ + /* Exception table entry */ #ifdef __ASSEMBLY__ # define _ASM_EXTABLE_HANDLE(from, to, handler) \ @@ -185,4 +188,6 @@ register unsigned long current_stack_pointer asm(_ASM_SP); #define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer) #endif /* __ASSEMBLY__ */ +#endif /* __KERNEL__ */ + #endif /* _ASM_X86_ASM_H */ diff --git a/arch/x86/include/asm/barrier.h b/arch/x86/include/asm/barrier.h index 4819d5e5a335..3ba772a69cc8 100644 --- a/arch/x86/include/asm/barrier.h +++ b/arch/x86/include/asm/barrier.h @@ -54,11 +54,8 @@ static inline unsigned long array_index_mask_nospec(unsigned long index, #define dma_rmb() barrier() #define dma_wmb() barrier() -#ifdef CONFIG_X86_32 -#define __smp_mb() asm volatile("lock; addl $0,-4(%%esp)" ::: "memory", "cc") -#else -#define __smp_mb() asm volatile("lock; addl $0,-4(%%rsp)" ::: "memory", "cc") -#endif +#define __smp_mb() asm volatile("lock; addl $0,-4(%%" _ASM_SP ")" ::: "memory", "cc") + #define __smp_rmb() dma_rmb() #define __smp_wmb() barrier() #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0) diff --git a/arch/x86/include/asm/crash.h b/arch/x86/include/asm/crash.h index f58de66091e5..8b6bd63530dc 100644 --- a/arch/x86/include/asm/crash.h +++ b/arch/x86/include/asm/crash.h @@ -9,10 +9,4 @@ int crash_setup_memmap_entries(struct kimage *image, struct boot_params *params); void crash_smp_send_stop(void); -#ifdef CONFIG_KEXEC_CORE -void __init crash_reserve_low_1M(void); -#else -static inline void __init crash_reserve_low_1M(void) { } -#endif - #endif /* _ASM_X86_CRASH_H */ diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index 476082a83d1c..96021e9bd202 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -421,10 +421,8 @@ extern bool idt_is_f00f_address(unsigned long address); #ifdef CONFIG_X86_64 extern void idt_setup_early_pf(void); -extern void idt_setup_ist_traps(void); #else static inline void idt_setup_early_pf(void) { } -static inline void idt_setup_ist_traps(void) { } #endif extern void idt_invalidate(void *addr); diff --git a/arch/x86/include/asm/hyperv-tlfs.h b/arch/x86/include/asm/hyperv-tlfs.h index 606f5cc579b2..f1366ce609e3 100644 --- a/arch/x86/include/asm/hyperv-tlfs.h +++ b/arch/x86/include/asm/hyperv-tlfs.h @@ -52,7 +52,7 @@ * Support for passing hypercall input parameter block via XMM * registers is available */ -#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE BIT(4) +#define HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE BIT(4) /* Support for a virtual guest idle state is available */ #define HV_X64_GUEST_IDLE_STATE_AVAILABLE BIT(5) /* Frequency MSRs available */ @@ -61,6 +61,11 @@ #define HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE BIT(10) /* Support for debug MSRs available */ #define HV_FEATURE_DEBUG_MSRS_AVAILABLE BIT(11) +/* + * Support for returning hypercall output block via XMM + * registers is available + */ +#define HV_X64_HYPERCALL_XMM_OUTPUT_AVAILABLE BIT(15) /* stimer Direct Mode is available */ #define HV_STIMER_DIRECT_MODE_AVAILABLE BIT(19) @@ -133,6 +138,15 @@ #define HV_X64_NESTED_GUEST_MAPPING_FLUSH BIT(18) #define HV_X64_NESTED_MSR_BITMAP BIT(19) +/* + * This is specific to AMD and specifies that enlightened TLB flush is + * supported. If guest opts in to this feature, ASID invalidations only + * flushes gva -> hpa mapping entries. To flush the TLB entries derived + * from NPT, hypercalls should be used (HvFlushGuestPhysicalAddressSpace + * or HvFlushGuestPhysicalAddressList). + */ +#define HV_X64_NESTED_ENLIGHTENED_TLB BIT(22) + /* HYPERV_CPUID_ISOLATION_CONFIG.EAX bits. */ #define HV_PARAVISOR_PRESENT BIT(0) @@ -314,6 +328,9 @@ struct hv_tsc_emulation_status { #define HV_X64_MSR_TSC_REFERENCE_ENABLE 0x00000001 #define HV_X64_MSR_TSC_REFERENCE_ADDRESS_SHIFT 12 +/* Number of XMM registers used in hypercall input/output */ +#define HV_HYPERCALL_MAX_XMM_REGISTERS 6 + struct hv_nested_enlightenments_control { struct { __u32 directhypercall:1; diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index e7bef91cee04..a12a4987154e 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -87,7 +87,10 @@ KVM_X86_OP(set_identity_map_addr) KVM_X86_OP(get_mt_mask) KVM_X86_OP(load_mmu_pgd) KVM_X86_OP_NULL(has_wbinvd_exit) -KVM_X86_OP(write_l1_tsc_offset) +KVM_X86_OP(get_l2_tsc_offset) +KVM_X86_OP(get_l2_tsc_multiplier) +KVM_X86_OP(write_tsc_offset) +KVM_X86_OP(write_tsc_multiplier) KVM_X86_OP(get_exit_info) KVM_X86_OP(check_intercept) KVM_X86_OP(handle_exit_irqoff) @@ -106,8 +109,8 @@ KVM_X86_OP_NULL(set_hv_timer) KVM_X86_OP_NULL(cancel_hv_timer) KVM_X86_OP(setup_mce) KVM_X86_OP(smi_allowed) -KVM_X86_OP(pre_enter_smm) -KVM_X86_OP(pre_leave_smm) +KVM_X86_OP(enter_smm) +KVM_X86_OP(leave_smm) KVM_X86_OP(enable_smi_window) KVM_X86_OP_NULL(mem_enc_op) KVM_X86_OP_NULL(mem_enc_reg_region) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 9c7ced0e3171..974cbfb1eefe 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -85,7 +85,7 @@ #define KVM_REQ_APICV_UPDATE \ KVM_ARCH_REQ_FLAGS(25, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_TLB_FLUSH_CURRENT KVM_ARCH_REQ(26) -#define KVM_REQ_HV_TLB_FLUSH \ +#define KVM_REQ_TLB_FLUSH_GUEST \ KVM_ARCH_REQ_FLAGS(27, KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_APF_READY KVM_ARCH_REQ(28) #define KVM_REQ_MSR_FILTER_CHANGED KVM_ARCH_REQ(29) @@ -269,12 +269,36 @@ enum x86_intercept_stage; struct kvm_kernel_irq_routing_entry; /* - * the pages used as guest page table on soft mmu are tracked by - * kvm_memory_slot.arch.gfn_track which is 16 bits, so the role bits used - * by indirect shadow page can not be more than 15 bits. + * kvm_mmu_page_role tracks the properties of a shadow page (where shadow page + * also includes TDP pages) to determine whether or not a page can be used in + * the given MMU context. This is a subset of the overall kvm_mmu_role to + * minimize the size of kvm_memory_slot.arch.gfn_track, i.e. allows allocating + * 2 bytes per gfn instead of 4 bytes per gfn. * - * Currently, we used 14 bits that are @level, @gpte_is_8_bytes, @quadrant, @access, - * @nxe, @cr0_wp, @smep_andnot_wp and @smap_andnot_wp. + * Indirect upper-level shadow pages are tracked for write-protection via + * gfn_track. As above, gfn_track is a 16 bit counter, so KVM must not create + * more than 2^16-1 upper-level shadow pages at a single gfn, otherwise + * gfn_track will overflow and explosions will ensure. + * + * A unique shadow page (SP) for a gfn is created if and only if an existing SP + * cannot be reused. The ability to reuse a SP is tracked by its role, which + * incorporates various mode bits and properties of the SP. Roughly speaking, + * the number of unique SPs that can theoretically be created is 2^n, where n + * is the number of bits that are used to compute the role. + * + * But, even though there are 18 bits in the mask below, not all combinations + * of modes and flags are possible. The maximum number of possible upper-level + * shadow pages for a single gfn is in the neighborhood of 2^13. + * + * - invalid shadow pages are not accounted. + * - level is effectively limited to four combinations, not 16 as the number + * bits would imply, as 4k SPs are not tracked (allowed to go unsync). + * - level is effectively unused for non-PAE paging because there is exactly + * one upper level (see 4k SP exception above). + * - quadrant is used only for non-PAE paging and is exclusive with + * gpte_is_8_bytes. + * - execonly and ad_disabled are used only for nested EPT, which makes it + * exclusive with quadrant. */ union kvm_mmu_page_role { u32 word; @@ -285,7 +309,7 @@ union kvm_mmu_page_role { unsigned direct:1; unsigned access:3; unsigned invalid:1; - unsigned nxe:1; + unsigned efer_nx:1; unsigned cr0_wp:1; unsigned smep_andnot_wp:1; unsigned smap_andnot_wp:1; @@ -303,13 +327,26 @@ union kvm_mmu_page_role { }; }; -union kvm_mmu_extended_role { /* - * This structure complements kvm_mmu_page_role caching everything needed for - * MMU configuration. If nothing in both these structures changed, MMU - * re-configuration can be skipped. @valid bit is set on first usage so we don't - * treat all-zero structure as valid data. + * kvm_mmu_extended_role complements kvm_mmu_page_role, tracking properties + * relevant to the current MMU configuration. When loading CR0, CR4, or EFER, + * including on nested transitions, if nothing in the full role changes then + * MMU re-configuration can be skipped. @valid bit is set on first usage so we + * don't treat all-zero structure as valid data. + * + * The properties that are tracked in the extended role but not the page role + * are for things that either (a) do not affect the validity of the shadow page + * or (b) are indirectly reflected in the shadow page's role. For example, + * CR4.PKE only affects permission checks for software walks of the guest page + * tables (because KVM doesn't support Protection Keys with shadow paging), and + * CR0.PG, CR4.PAE, and CR4.PSE are indirectly reflected in role.level. + * + * Note, SMEP and SMAP are not redundant with sm*p_andnot_wp in the page role. + * If CR0.WP=1, KVM can reuse shadow pages for the guest regardless of SMEP and + * SMAP, but the MMU's permission checks for software walks need to be SMEP and + * SMAP aware regardless of CR0.WP. */ +union kvm_mmu_extended_role { u32 word; struct { unsigned int valid:1; @@ -320,7 +357,7 @@ union kvm_mmu_extended_role { unsigned int cr4_pke:1; unsigned int cr4_smap:1; unsigned int cr4_smep:1; - unsigned int maxphyaddr:6; + unsigned int cr4_la57:1; }; }; @@ -420,11 +457,6 @@ struct kvm_mmu { struct rsvd_bits_validate guest_rsvd_check; - /* Can have large pages at levels 2..last_nonleaf_level-1. */ - u8 last_nonleaf_level; - - bool nx; - u64 pdptrs[4]; /* pae */ }; @@ -543,6 +575,15 @@ struct kvm_vcpu_hv { struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); cpumask_t tlb_flush; + bool enforce_cpuid; + struct { + u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */ + u32 features_ebx; /* HYPERV_CPUID_FEATURES.EBX */ + u32 features_edx; /* HYPERV_CPUID_FEATURES.EDX */ + u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */ + u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */ + u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */ + } cpuid_cache; }; /* Xen HVM per vcpu emulation context */ @@ -707,7 +748,7 @@ struct kvm_vcpu_arch { } st; u64 l1_tsc_offset; - u64 tsc_offset; + u64 tsc_offset; /* current tsc offset */ u64 last_guest_tsc; u64 last_host_tsc; u64 tsc_offset_adjustment; @@ -721,7 +762,8 @@ struct kvm_vcpu_arch { u32 virtual_tsc_khz; s64 ia32_tsc_adjust_msr; u64 msr_ia32_power_ctl; - u64 tsc_scaling_ratio; + u64 l1_tsc_scaling_ratio; + u64 tsc_scaling_ratio; /* current scaling ratio */ atomic_t nmi_queued; /* unprocessed asynchronous NMIs */ unsigned nmi_pending; /* NMI queued after currently running handler */ @@ -829,7 +871,7 @@ struct kvm_vcpu_arch { bool l1tf_flush_l1d; /* Host CPU on which VM-entry was most recently attempted */ - unsigned int last_vmentry_cpu; + int last_vmentry_cpu; /* AMD MSRC001_0015 Hardware Configuration */ u64 msr_hwcr; @@ -851,6 +893,16 @@ struct kvm_vcpu_arch { /* Protected Guests */ bool guest_state_protected; + + /* + * Set when PDPTS were loaded directly by the userspace without + * reading the guest memory + */ + bool pdptrs_from_userspace; + +#if IS_ENABLED(CONFIG_HYPERV) + hpa_t hv_root_tdp; +#endif }; struct kvm_lpage_info { @@ -1002,7 +1054,7 @@ struct kvm_arch { struct kvm_apic_map __rcu *apic_map; atomic_t apic_map_dirty; - bool apic_access_page_done; + bool apic_access_memslot_enabled; unsigned long apicv_inhibit_reasons; gpa_t wall_clock; @@ -1062,11 +1114,19 @@ struct kvm_arch { bool exception_payload_enabled; bool bus_lock_detection_enabled; + /* + * If exit_on_emulation_error is set, and the in-kernel instruction + * emulator fails to emulate an instruction, allow userspace + * the opportunity to look at it. + */ + bool exit_on_emulation_error; /* Deflect RDMSR and WRMSR to user space when they trigger a #GP */ u32 user_space_msr_mask; struct kvm_x86_msr_filter __rcu *msr_filter; + u32 hypercall_exit_enabled; + /* Guest can access the SGX PROVISIONKEY. */ bool sgx_provisioning_allowed; @@ -1124,23 +1184,35 @@ struct kvm_arch { */ spinlock_t tdp_mmu_pages_lock; #endif /* CONFIG_X86_64 */ + + /* + * If set, rmaps have been allocated for all memslots and should be + * allocated for any newly created or modified memslots. + */ + bool memslots_have_rmaps; + +#if IS_ENABLED(CONFIG_HYPERV) + hpa_t hv_root_tdp; + spinlock_t hv_root_tdp_lock; +#endif }; struct kvm_vm_stat { - ulong mmu_shadow_zapped; - ulong mmu_pte_write; - ulong mmu_pde_zapped; - ulong mmu_flooded; - ulong mmu_recycled; - ulong mmu_cache_miss; - ulong mmu_unsync; - ulong remote_tlb_flush; - ulong lpages; - ulong nx_lpage_splits; - ulong max_mmu_page_hash_collisions; + struct kvm_vm_stat_generic generic; + u64 mmu_shadow_zapped; + u64 mmu_pte_write; + u64 mmu_pde_zapped; + u64 mmu_flooded; + u64 mmu_recycled; + u64 mmu_cache_miss; + u64 mmu_unsync; + u64 lpages; + u64 nx_lpage_splits; + u64 max_mmu_page_hash_collisions; }; struct kvm_vcpu_stat { + struct kvm_vcpu_stat_generic generic; u64 pf_fixed; u64 pf_guest; u64 tlb_flush; @@ -1154,10 +1226,6 @@ struct kvm_vcpu_stat { u64 nmi_window_exits; u64 l1d_flush; u64 halt_exits; - u64 halt_successful_poll; - u64 halt_attempted_poll; - u64 halt_poll_invalid; - u64 halt_wakeup; u64 request_irq_exits; u64 irq_exits; u64 host_state_reload; @@ -1168,11 +1236,10 @@ struct kvm_vcpu_stat { u64 irq_injections; u64 nmi_injections; u64 req_event; - u64 halt_poll_success_ns; - u64 halt_poll_fail_ns; u64 nested_run; u64 directed_yield_attempted; u64 directed_yield_successful; + u64 guest_mode; }; struct x86_instruction_info; @@ -1304,8 +1371,10 @@ struct kvm_x86_ops { bool (*has_wbinvd_exit)(void); - /* Returns actual tsc_offset set in active VMCS */ - u64 (*write_l1_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); + u64 (*get_l2_tsc_offset)(struct kvm_vcpu *vcpu); + u64 (*get_l2_tsc_multiplier)(struct kvm_vcpu *vcpu); + void (*write_tsc_offset)(struct kvm_vcpu *vcpu, u64 offset); + void (*write_tsc_multiplier)(struct kvm_vcpu *vcpu, u64 multiplier); /* * Retrieve somewhat arbitrary exit information. Intended to be used @@ -1363,8 +1432,8 @@ struct kvm_x86_ops { void (*setup_mce)(struct kvm_vcpu *vcpu); int (*smi_allowed)(struct kvm_vcpu *vcpu, bool for_injection); - int (*pre_enter_smm)(struct kvm_vcpu *vcpu, char *smstate); - int (*pre_leave_smm)(struct kvm_vcpu *vcpu, const char *smstate); + int (*enter_smm)(struct kvm_vcpu *vcpu, char *smstate); + int (*leave_smm)(struct kvm_vcpu *vcpu, const char *smstate); void (*enable_smi_window)(struct kvm_vcpu *vcpu); int (*mem_enc_op)(struct kvm *kvm, void __user *argp); @@ -1423,6 +1492,7 @@ struct kvm_arch_async_pf { extern u32 __read_mostly kvm_nr_uret_msrs; extern u64 __read_mostly host_efer; extern bool __read_mostly allow_smaller_maxphyaddr; +extern bool __read_mostly enable_apicv; extern struct kvm_x86_ops kvm_x86_ops; #define KVM_X86_OP(func) \ @@ -1463,6 +1533,7 @@ int kvm_mmu_create(struct kvm_vcpu *vcpu); void kvm_mmu_init_vm(struct kvm *kvm); void kvm_mmu_uninit_vm(struct kvm *kvm); +void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu); void kvm_mmu_reset_context(struct kvm_vcpu *vcpu); void kvm_mmu_slot_remove_write_access(struct kvm *kvm, struct kvm_memory_slot *memslot, @@ -1477,7 +1548,6 @@ unsigned long kvm_mmu_calculate_default_mmu_pages(struct kvm *kvm); void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages); int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3); -bool pdptrs_changed(struct kvm_vcpu *vcpu); int emulator_write_phys(struct kvm_vcpu *vcpu, gpa_t gpa, const void *val, int bytes); @@ -1650,6 +1720,7 @@ int kvm_mmu_unprotect_page(struct kvm *kvm, gfn_t gfn); void __kvm_mmu_free_some_pages(struct kvm_vcpu *vcpu); void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, ulong roots_to_free); +void kvm_mmu_free_guest_mode_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu); gpa_t translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, u32 access, struct x86_exception *exception); gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva, @@ -1662,7 +1733,6 @@ gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva, struct x86_exception *exception); bool kvm_apicv_activated(struct kvm *kvm); -void kvm_apicv_init(struct kvm *kvm, bool enable); void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu); void kvm_request_apicv_update(struct kvm *kvm, bool activate, unsigned long bit); @@ -1675,8 +1745,7 @@ void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva); void kvm_mmu_invalidate_gva(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, gva_t gva, hpa_t root_hpa); void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid); -void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, - bool skip_mmu_sync); +void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd); void kvm_configure_mmu(bool enable_tdp, int tdp_max_root_level, int tdp_huge_page_level); @@ -1788,8 +1857,10 @@ static inline bool kvm_is_supported_user_return_msr(u32 msr) return kvm_find_user_return_msr(msr) >= 0; } -u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc); +u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc, u64 ratio); u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc); +u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier); +u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier); unsigned long kvm_get_linear_rip(struct kvm_vcpu *vcpu); bool kvm_is_linear_rip(struct kvm_vcpu *vcpu, unsigned long linear_rip); @@ -1863,4 +1934,6 @@ static inline int kvm_cpu_get_apicid(int mps_cpu) int kvm_cpu_dirty_log_size(void); +int alloc_all_memslots_rmaps(struct kvm *kvm); + #endif /* _ASM_X86_KVM_HOST_H */ diff --git a/arch/x86/include/asm/nops.h b/arch/x86/include/asm/nops.h index c1e5e818ba16..c5573eaa5bb9 100644 --- a/arch/x86/include/asm/nops.h +++ b/arch/x86/include/asm/nops.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_NOPS_H #define _ASM_X86_NOPS_H +#include + /* * Define nops for use with alternative() and for tracing. */ @@ -57,20 +59,14 @@ #endif /* CONFIG_64BIT */ -#ifdef __ASSEMBLY__ -#define _ASM_MK_NOP(x) .byte x -#else -#define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n" -#endif - -#define ASM_NOP1 _ASM_MK_NOP(BYTES_NOP1) -#define ASM_NOP2 _ASM_MK_NOP(BYTES_NOP2) -#define ASM_NOP3 _ASM_MK_NOP(BYTES_NOP3) -#define ASM_NOP4 _ASM_MK_NOP(BYTES_NOP4) -#define ASM_NOP5 _ASM_MK_NOP(BYTES_NOP5) -#define ASM_NOP6 _ASM_MK_NOP(BYTES_NOP6) -#define ASM_NOP7 _ASM_MK_NOP(BYTES_NOP7) -#define ASM_NOP8 _ASM_MK_NOP(BYTES_NOP8) +#define ASM_NOP1 _ASM_BYTES(BYTES_NOP1) +#define ASM_NOP2 _ASM_BYTES(BYTES_NOP2) +#define ASM_NOP3 _ASM_BYTES(BYTES_NOP3) +#define ASM_NOP4 _ASM_BYTES(BYTES_NOP4) +#define ASM_NOP5 _ASM_BYTES(BYTES_NOP5) +#define ASM_NOP6 _ASM_BYTES(BYTES_NOP6) +#define ASM_NOP7 _ASM_BYTES(BYTES_NOP7) +#define ASM_NOP8 _ASM_BYTES(BYTES_NOP8) #define ASM_NOP_MAX 8 diff --git a/arch/x86/include/asm/page.h b/arch/x86/include/asm/page.h index 7555b48803a8..4d5810c8fab7 100644 --- a/arch/x86/include/asm/page.h +++ b/arch/x86/include/asm/page.h @@ -34,9 +34,9 @@ static inline void copy_user_page(void *to, void *from, unsigned long vaddr, copy_page(to, from); } -#define __alloc_zeroed_user_highpage(movableflags, vma, vaddr) \ - alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO | movableflags, vma, vaddr) -#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +#define alloc_zeroed_user_highpage_movable(vma, vaddr) \ + alloc_page_vma(GFP_HIGHUSER_MOVABLE | __GFP_ZERO, vma, vaddr) +#define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE #ifndef __pa #define __pa(x) __phys_addr((unsigned long)(x)) diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 556b2b17c3e2..364d0e42e280 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -663,6 +663,7 @@ extern void load_direct_gdt(int); extern void load_fixmap_gdt(int); extern void load_percpu_segment(int); extern void cpu_init(void); +extern void cpu_init_secondary(void); extern void cpu_init_exception_handling(void); extern void cr4_init(void); diff --git a/arch/x86/include/asm/sgx.h b/arch/x86/include/asm/sgx.h index 9c31e0ebc55b..05f3e21f01a7 100644 --- a/arch/x86/include/asm/sgx.h +++ b/arch/x86/include/asm/sgx.h @@ -13,7 +13,7 @@ /* * This file contains both data structures defined by SGX architecture and Linux * defined software data structures and functions. The two should not be mixed - * together for better readibility. The architectural definitions come first. + * together for better readability. The architectural definitions come first. */ /* The SGX specific CPUID function. */ diff --git a/arch/x86/include/asm/stackprotector.h b/arch/x86/include/asm/stackprotector.h index b6ffe58c70fa..24a8d6c4fb18 100644 --- a/arch/x86/include/asm/stackprotector.h +++ b/arch/x86/include/asm/stackprotector.h @@ -11,7 +11,7 @@ * The same segment is shared by percpu area and stack canary. On * x86_64, percpu symbols are zero based and %gs (64-bit) points to the * base of percpu area. The first occupant of the percpu area is always - * fixed_percpu_data which contains stack_canary at the approproate + * fixed_percpu_data which contains stack_canary at the appropriate * offset. On x86_32, the stack canary is just a regular percpu * variable. * diff --git a/arch/x86/include/asm/svm.h b/arch/x86/include/asm/svm.h index 772e60efe243..e322676039f4 100644 --- a/arch/x86/include/asm/svm.h +++ b/arch/x86/include/asm/svm.h @@ -156,6 +156,12 @@ struct __attribute__ ((__packed__)) vmcb_control_area { u64 avic_physical_id; /* Offset 0xf8 */ u8 reserved_7[8]; u64 vmsa_pa; /* Used for an SEV-ES guest */ + u8 reserved_8[720]; + /* + * Offset 0x3e0, 32 bytes reserved + * for use by hypervisor/software. + */ + u8 reserved_sw[32]; }; @@ -314,7 +320,7 @@ struct ghcb { #define EXPECTED_VMCB_SAVE_AREA_SIZE 1032 -#define EXPECTED_VMCB_CONTROL_AREA_SIZE 272 +#define EXPECTED_VMCB_CONTROL_AREA_SIZE 1024 #define EXPECTED_GHCB_SIZE PAGE_SIZE static inline void __unused_size_checks(void) @@ -326,7 +332,6 @@ static inline void __unused_size_checks(void) struct vmcb { struct vmcb_control_area control; - u8 reserved_control[1024 - sizeof(struct vmcb_control_area)]; struct vmcb_save_area save; } __packed; diff --git a/arch/x86/include/asm/syscall.h b/arch/x86/include/asm/syscall.h index 7cbf733d11af..f6593cafdbd9 100644 --- a/arch/x86/include/asm/syscall.h +++ b/arch/x86/include/asm/syscall.h @@ -21,13 +21,12 @@ extern const sys_call_ptr_t sys_call_table[]; #if defined(CONFIG_X86_32) #define ia32_sys_call_table sys_call_table -#endif - -#if defined(CONFIG_IA32_EMULATION) +#else +/* + * These may not exist, but still put the prototypes in so we + * can use IS_ENABLED(). + */ extern const sys_call_ptr_t ia32_sys_call_table[]; -#endif - -#ifdef CONFIG_X86_X32_ABI extern const sys_call_ptr_t x32_sys_call_table[]; #endif @@ -160,7 +159,7 @@ static inline int syscall_get_arch(struct task_struct *task) ? AUDIT_ARCH_I386 : AUDIT_ARCH_X86_64; } -void do_syscall_64(unsigned long nr, struct pt_regs *regs); +void do_syscall_64(struct pt_regs *regs, unsigned long nr); void do_int80_syscall_32(struct pt_regs *regs); long do_fast_syscall_32(struct pt_regs *regs); diff --git a/arch/x86/include/uapi/asm/hwcap2.h b/arch/x86/include/uapi/asm/hwcap2.h index 5fdfcb47000f..054604aba9f0 100644 --- a/arch/x86/include/uapi/asm/hwcap2.h +++ b/arch/x86/include/uapi/asm/hwcap2.h @@ -2,10 +2,12 @@ #ifndef _ASM_X86_HWCAP2_H #define _ASM_X86_HWCAP2_H +#include + /* MONITOR/MWAIT enabled in Ring 3 */ -#define HWCAP2_RING3MWAIT (1 << 0) +#define HWCAP2_RING3MWAIT _BITUL(0) /* Kernel allows FSGSBASE instructions available in Ring 3 */ -#define HWCAP2_FSGSBASE BIT(1) +#define HWCAP2_FSGSBASE _BITUL(1) #endif diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h index 0662f644aad9..a6c327f8ad9e 100644 --- a/arch/x86/include/uapi/asm/kvm.h +++ b/arch/x86/include/uapi/asm/kvm.h @@ -159,6 +159,19 @@ struct kvm_sregs { __u64 interrupt_bitmap[(KVM_NR_INTERRUPTS + 63) / 64]; }; +struct kvm_sregs2 { + /* out (KVM_GET_SREGS2) / in (KVM_SET_SREGS2) */ + struct kvm_segment cs, ds, es, fs, gs, ss; + struct kvm_segment tr, ldt; + struct kvm_dtable gdt, idt; + __u64 cr0, cr2, cr3, cr4, cr8; + __u64 efer; + __u64 apic_base; + __u64 flags; + __u64 pdptrs[4]; +}; +#define KVM_SREGS2_FLAGS_PDPTRS_VALID 1 + /* for KVM_GET_FPU and KVM_SET_FPU */ struct kvm_fpu { __u8 fpr[8][16]; diff --git a/arch/x86/include/uapi/asm/kvm_para.h b/arch/x86/include/uapi/asm/kvm_para.h index 950afebfba88..5146bbab84d4 100644 --- a/arch/x86/include/uapi/asm/kvm_para.h +++ b/arch/x86/include/uapi/asm/kvm_para.h @@ -33,6 +33,8 @@ #define KVM_FEATURE_PV_SCHED_YIELD 13 #define KVM_FEATURE_ASYNC_PF_INT 14 #define KVM_FEATURE_MSI_EXT_DEST_ID 15 +#define KVM_FEATURE_HC_MAP_GPA_RANGE 16 +#define KVM_FEATURE_MIGRATION_CONTROL 17 #define KVM_HINTS_REALTIME 0 @@ -54,6 +56,7 @@ #define MSR_KVM_POLL_CONTROL 0x4b564d05 #define MSR_KVM_ASYNC_PF_INT 0x4b564d06 #define MSR_KVM_ASYNC_PF_ACK 0x4b564d07 +#define MSR_KVM_MIGRATION_CONTROL 0x4b564d08 struct kvm_steal_time { __u64 steal; @@ -90,6 +93,16 @@ struct kvm_clock_pairing { /* MSR_KVM_ASYNC_PF_INT */ #define KVM_ASYNC_PF_VEC_MASK GENMASK(7, 0) +/* MSR_KVM_MIGRATION_CONTROL */ +#define KVM_MIGRATION_READY (1 << 0) + +/* KVM_HC_MAP_GPA_RANGE */ +#define KVM_MAP_GPA_RANGE_PAGE_SZ_4K 0 +#define KVM_MAP_GPA_RANGE_PAGE_SZ_2M (1 << 0) +#define KVM_MAP_GPA_RANGE_PAGE_SZ_1G (1 << 1) +#define KVM_MAP_GPA_RANGE_ENC_STAT(n) (n << 4) +#define KVM_MAP_GPA_RANGE_ENCRYPTED KVM_MAP_GPA_RANGE_ENC_STAT(1) +#define KVM_MAP_GPA_RANGE_DECRYPTED KVM_MAP_GPA_RANGE_ENC_STAT(0) /* Operations for KVM_HC_MMU_OP */ #define KVM_MMU_OP_WRITE_PTE 1 diff --git a/arch/x86/include/uapi/asm/svm.h b/arch/x86/include/uapi/asm/svm.h index 554f75fe013c..efa969325ede 100644 --- a/arch/x86/include/uapi/asm/svm.h +++ b/arch/x86/include/uapi/asm/svm.h @@ -110,6 +110,9 @@ #define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1 #define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff +/* Exit code reserved for hypervisor/software use */ +#define SVM_EXIT_SW 0xf0000000 + #define SVM_EXIT_ERR -1 #define SVM_EXIT_REASONS \ diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 6fe5b44fcbc9..e9da3dc71254 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -75,7 +75,7 @@ do { \ } \ } while (0) -const unsigned char x86nops[] = +static const unsigned char x86nops[] = { BYTES_NOP1, BYTES_NOP2, @@ -301,8 +301,8 @@ void __init_or_module noinline apply_alternatives(struct alt_instr *start, instr, instr, a->instrlen, replacement, a->replacementlen); - DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr); - DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement); + DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr); + DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement); memcpy(insn_buff, replacement, a->replacementlen); insn_buff_sz = a->replacementlen; diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index a1b756c49a93..a99d00393206 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1773,10 +1773,16 @@ void syscall_init(void) wrmsrl_safe(MSR_IA32_SYSENTER_EIP, 0ULL); #endif - /* Flags to clear on syscall */ + /* + * Flags to clear on syscall; clear as much as possible + * to minimize user space-kernel interference. + */ wrmsrl(MSR_SYSCALL_MASK, - X86_EFLAGS_TF|X86_EFLAGS_DF|X86_EFLAGS_IF| - X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT); + X86_EFLAGS_CF|X86_EFLAGS_PF|X86_EFLAGS_AF| + X86_EFLAGS_ZF|X86_EFLAGS_SF|X86_EFLAGS_TF| + X86_EFLAGS_IF|X86_EFLAGS_DF|X86_EFLAGS_OF| + X86_EFLAGS_IOPL|X86_EFLAGS_NT|X86_EFLAGS_RF| + X86_EFLAGS_AC|X86_EFLAGS_ID); } #else /* CONFIG_X86_64 */ @@ -1938,13 +1944,12 @@ void cpu_init_exception_handling(void) /* * cpu_init() initializes state that is per-CPU. Some data is already - * initialized (naturally) in the bootstrap process, such as the GDT - * and IDT. We reload them nevertheless, this function acts as a - * 'CPU state barrier', nothing should get across. + * initialized (naturally) in the bootstrap process, such as the GDT. We + * reload it nevertheless, this function acts as a 'CPU state barrier', + * nothing should get across. */ void cpu_init(void) { - struct tss_struct *tss = this_cpu_ptr(&cpu_tss_rw); struct task_struct *cur = current; int cpu = raw_smp_processor_id(); @@ -1957,8 +1962,6 @@ void cpu_init(void) early_cpu_to_node(cpu) != NUMA_NO_NODE) set_numa_node(early_cpu_to_node(cpu)); #endif - setup_getcpu(cpu); - pr_debug("Initializing CPU#%d\n", cpu); if (IS_ENABLED(CONFIG_X86_64) || cpu_feature_enabled(X86_FEATURE_VME) || @@ -1970,7 +1973,6 @@ void cpu_init(void) * and set up the GDT descriptor: */ switch_to_new_gdt(cpu); - load_current_idt(); if (IS_ENABLED(CONFIG_X86_64)) { loadsegment(fs, 0); @@ -1990,12 +1992,6 @@ void cpu_init(void) initialize_tlbstate_and_flush(); enter_lazy_tlb(&init_mm, cur); - /* Initialize the TSS. */ - tss_setup_ist(tss); - tss_setup_io_bitmap(tss); - set_tss_desc(cpu, &get_cpu_entry_area(cpu)->tss.x86_tss); - - load_TR_desc(); /* * sp0 points to the entry trampoline stack regardless of what task * is running. @@ -2017,6 +2013,18 @@ void cpu_init(void) load_fixmap_gdt(cpu); } +#ifdef CONFIG_SMP +void cpu_init_secondary(void) +{ + /* + * Relies on the BP having set-up the IDT tables, which are loaded + * on this CPU in cpu_init_exception_handling(). + */ + cpu_init_exception_handling(); + cpu_init(); +} +#endif + /* * The microcode loader calls this upon late microcode load to recheck features, * only when microcode has been updated. Caller holds microcode_mutex and CPU diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 861e919eba9a..8321c43554a1 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -41,6 +42,7 @@ enum split_lock_detect_state { sld_off = 0, sld_warn, sld_fatal, + sld_ratelimit, }; /* @@ -999,13 +1001,30 @@ static const struct { { "off", sld_off }, { "warn", sld_warn }, { "fatal", sld_fatal }, + { "ratelimit:", sld_ratelimit }, }; +static struct ratelimit_state bld_ratelimit; + static inline bool match_option(const char *arg, int arglen, const char *opt) { - int len = strlen(opt); + int len = strlen(opt), ratelimit; - return len == arglen && !strncmp(arg, opt, len); + if (strncmp(arg, opt, len)) + return false; + + /* + * Min ratelimit is 1 bus lock/sec. + * Max ratelimit is 1000 bus locks/sec. + */ + if (sscanf(arg, "ratelimit:%d", &ratelimit) == 1 && + ratelimit > 0 && ratelimit <= 1000) { + ratelimit_state_init(&bld_ratelimit, HZ, ratelimit); + ratelimit_set_flags(&bld_ratelimit, RATELIMIT_MSG_ON_RELEASE); + return true; + } + + return len == arglen; } static bool split_lock_verify_msr(bool on) @@ -1084,6 +1103,15 @@ static void sld_update_msr(bool on) static void split_lock_init(void) { + /* + * #DB for bus lock handles ratelimit and #AC for split lock is + * disabled. + */ + if (sld_state == sld_ratelimit) { + split_lock_verify_msr(false); + return; + } + if (cpu_model_supports_sld) split_lock_verify_msr(sld_state != sld_off); } @@ -1156,6 +1184,12 @@ void handle_bus_lock(struct pt_regs *regs) switch (sld_state) { case sld_off: break; + case sld_ratelimit: + /* Enforce no more than bld_ratelimit bus locks/sec. */ + while (!__ratelimit(&bld_ratelimit)) + msleep(20); + /* Warn on the bus lock. */ + fallthrough; case sld_warn: pr_warn_ratelimited("#DB: %s/%d took a bus_lock trap at address: 0x%lx\n", current->comm, current->pid, regs->ip); @@ -1261,6 +1295,10 @@ static void sld_state_show(void) " from non-WB" : ""); } break; + case sld_ratelimit: + if (boot_cpu_has(X86_FEATURE_BUS_LOCK_DETECT)) + pr_info("#DB: setting system wide bus lock rate limit to %u/sec\n", bld_ratelimit.burst); + break; } } diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index 22f13343b5da..c268c2730048 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -252,6 +252,7 @@ static void __init hv_smp_prepare_cpus(unsigned int max_cpus) static void __init ms_hyperv_init_platform(void) { + int hv_max_functions_eax; int hv_host_info_eax; int hv_host_info_ebx; int hv_host_info_ecx; @@ -269,6 +270,8 @@ static void __init ms_hyperv_init_platform(void) ms_hyperv.misc_features = cpuid_edx(HYPERV_CPUID_FEATURES); ms_hyperv.hints = cpuid_eax(HYPERV_CPUID_ENLIGHTMENT_INFO); + hv_max_functions_eax = cpuid_eax(HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS); + pr_info("Hyper-V: privilege flags low 0x%x, high 0x%x, hints 0x%x, misc 0x%x\n", ms_hyperv.features, ms_hyperv.priv_high, ms_hyperv.hints, ms_hyperv.misc_features); @@ -298,8 +301,7 @@ static void __init ms_hyperv_init_platform(void) /* * Extract host information. */ - if (cpuid_eax(HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS) >= - HYPERV_CPUID_VERSION) { + if (hv_max_functions_eax >= HYPERV_CPUID_VERSION) { hv_host_info_eax = cpuid_eax(HYPERV_CPUID_VERSION); hv_host_info_ebx = cpuid_ebx(HYPERV_CPUID_VERSION); hv_host_info_ecx = cpuid_ecx(HYPERV_CPUID_VERSION); @@ -325,9 +327,11 @@ static void __init ms_hyperv_init_platform(void) ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b); } - if (ms_hyperv.hints & HV_X64_ENLIGHTENED_VMCS_RECOMMENDED) { + if (hv_max_functions_eax >= HYPERV_CPUID_NESTED_FEATURES) { ms_hyperv.nested_features = cpuid_eax(HYPERV_CPUID_NESTED_FEATURES); + pr_info("Hyper-V: Nested features: 0x%x\n", + ms_hyperv.nested_features); } /* diff --git a/arch/x86/kernel/cpu/resctrl/internal.h b/arch/x86/kernel/cpu/resctrl/internal.h index c4d320d02fd5..6a5f60a37219 100644 --- a/arch/x86/kernel/cpu/resctrl/internal.h +++ b/arch/x86/kernel/cpu/resctrl/internal.h @@ -70,6 +70,7 @@ DECLARE_STATIC_KEY_FALSE(rdt_mon_enable_key); * struct mon_evt - Entry in the event list of a resource * @evtid: event id * @name: name of the event + * @list: entry in &rdt_resource->evt_list */ struct mon_evt { u32 evtid; @@ -78,10 +79,13 @@ struct mon_evt { }; /** - * struct mon_data_bits - Monitoring details for each event file - * @rid: Resource id associated with the event file. + * union mon_data_bits - Monitoring details for each event file + * @priv: Used to store monitoring event data in @u + * as kernfs private data + * @rid: Resource id associated with the event file * @evtid: Event id associated with the event file * @domid: The domain to which the event file belongs + * @u: Name of the bit fields struct */ union mon_data_bits { void *priv; @@ -119,6 +123,7 @@ enum rdt_group_type { * @RDT_MODE_PSEUDO_LOCKSETUP: Resource group will be used for Pseudo-Locking * @RDT_MODE_PSEUDO_LOCKED: No sharing of this resource group's allocations * allowed AND the allocations are Cache Pseudo-Locked + * @RDT_NUM_MODES: Total number of modes * * The mode of a resource group enables control over the allowed overlap * between allocations associated with different resource groups (classes @@ -142,7 +147,7 @@ enum rdtgrp_mode { /** * struct mongroup - store mon group's data in resctrl fs. - * @mon_data_kn kernlfs node for the mon_data directory + * @mon_data_kn: kernfs node for the mon_data directory * @parent: parent rdtgrp * @crdtgrp_list: child rdtgroup node list * @rmid: rmid for this rdtgroup @@ -282,11 +287,11 @@ struct rftype { /** * struct mbm_state - status for each MBM counter in each domain * @chunks: Total data moved (multiply by rdt_group.mon_scale to get bytes) - * @prev_msr Value of IA32_QM_CTR for this RMID last time we read it + * @prev_msr: Value of IA32_QM_CTR for this RMID last time we read it * @prev_bw_msr:Value of previous IA32_QM_CTR for bandwidth counting - * @prev_bw The most recent bandwidth in MBps - * @delta_bw Difference between the current and previous bandwidth - * @delta_comp Indicates whether to compute the delta_bw + * @prev_bw: The most recent bandwidth in MBps + * @delta_bw: Difference between the current and previous bandwidth + * @delta_comp: Indicates whether to compute the delta_bw */ struct mbm_state { u64 chunks; @@ -456,11 +461,13 @@ struct rdt_parse_data { * @data_width: Character width of data when displaying * @domains: All domains for this resource * @cache: Cache allocation related data + * @membw: If the component has bandwidth controls, their properties. * @format_str: Per resource format string to show domain value * @parse_ctrlval: Per resource function pointer to parse control values * @evt_list: List of monitoring events * @num_rmid: Number of RMIDs available * @mon_scale: cqm counter * mon_scale = occupancy in bytes + * @mbm_width: Monitor width, to detect and correct for overflow. * @fflags: flags to choose base and info files */ struct rdt_resource { diff --git a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c index 05a89e33fde2..2207916cae65 100644 --- a/arch/x86/kernel/cpu/resctrl/pseudo_lock.c +++ b/arch/x86/kernel/cpu/resctrl/pseudo_lock.c @@ -49,6 +49,7 @@ static struct class *pseudo_lock_class; /** * get_prefetch_disable_bits - prefetch disable bits of supported platforms + * @void: It takes no parameters. * * Capture the list of platforms that have been validated to support * pseudo-locking. This includes testing to ensure pseudo-locked regions @@ -162,7 +163,7 @@ static struct rdtgroup *region_find_by_minor(unsigned int minor) } /** - * pseudo_lock_pm_req - A power management QoS request list entry + * struct pseudo_lock_pm_req - A power management QoS request list entry * @list: Entry within the @pm_reqs list for a pseudo-locked region * @req: PM QoS request */ @@ -184,6 +185,7 @@ static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr) /** * pseudo_lock_cstates_constrain - Restrict cores from entering C6 + * @plr: Pseudo-locked region * * To prevent the cache from being affected by power management entering * C6 has to be avoided. This is accomplished by requesting a latency @@ -196,6 +198,8 @@ static void pseudo_lock_cstates_relax(struct pseudo_lock_region *plr) * the ACPI latencies need to be considered while keeping in mind that C2 * may be set to map to deeper sleep states. In this case the latency * requirement needs to prevent entering C2 also. + * + * Return: 0 on success, <0 on failure */ static int pseudo_lock_cstates_constrain(struct pseudo_lock_region *plr) { @@ -520,7 +524,7 @@ static int pseudo_lock_fn(void *_rdtgrp) /** * rdtgroup_monitor_in_progress - Test if monitoring in progress - * @r: resource group being queried + * @rdtgrp: resource group being queried * * Return: 1 if monitor groups have been created for this resource * group, 0 otherwise. @@ -1140,6 +1144,8 @@ static int measure_l3_residency(void *_plr) /** * pseudo_lock_measure_cycles - Trigger latency measure to pseudo-locked region + * @rdtgrp: Resource group to which the pseudo-locked region belongs. + * @sel: Selector of which measurement to perform on a pseudo-locked region. * * The measurement of latency to access a pseudo-locked region should be * done from a cpu that is associated with that pseudo-locked region. diff --git a/arch/x86/kernel/cpu/sgx/encl.c b/arch/x86/kernel/cpu/sgx/encl.c index 3be203297988..001808e3901c 100644 --- a/arch/x86/kernel/cpu/sgx/encl.c +++ b/arch/x86/kernel/cpu/sgx/encl.c @@ -383,7 +383,7 @@ const struct vm_operations_struct sgx_vm_ops = { /** * sgx_encl_release - Destroy an enclave instance - * @kref: address of a kref inside &sgx_encl + * @ref: address of a kref inside &sgx_encl * * Used together with kref_put(). Frees all the resources associated with the * enclave and the instance itself. diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c index 54ce999ed321..e8326a8d1c5d 100644 --- a/arch/x86/kernel/crash.c +++ b/arch/x86/kernel/crash.c @@ -70,19 +70,6 @@ static inline void cpu_crash_vmclear_loaded_vmcss(void) rcu_read_unlock(); } -/* - * When the crashkernel option is specified, only use the low - * 1M for the real mode trampoline. - */ -void __init crash_reserve_low_1M(void) -{ - if (cmdline_find_option(boot_command_line, "crashkernel", NULL, 0) < 0) - return; - - memblock_reserve(0, 1<<20); - pr_info("Reserving the low 1M of memory for crashkernel\n"); -} - #if defined(CONFIG_SMP) && defined(CONFIG_X86_LOCAL_APIC) static void kdump_nmi_callback(int cpu, struct pt_regs *regs) diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 04bddaaba8e2..d8b3ebd2bb85 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -62,7 +62,7 @@ SYM_CODE_START_NOALIGN(startup_64) */ /* Set up the stack for verify_cpu(), similar to initial_stack below */ - leaq (__end_init_task - SIZEOF_PTREGS)(%rip), %rsp + leaq (__end_init_task - FRAME_SIZE)(%rip), %rsp leaq _text(%rip), %rdi pushq %rsi @@ -343,10 +343,10 @@ SYM_DATA(initial_vc_handler, .quad handle_vc_boot_ghcb) #endif /* - * The SIZEOF_PTREGS gap is a convention which helps the in-kernel unwinder + * The FRAME_SIZE gap is a convention which helps the in-kernel unwinder * reliably detect the end of the stack. */ -SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - SIZEOF_PTREGS) +SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - FRAME_SIZE) __FINITDATA __INIT diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index d552f177eca0..6cce6047fa12 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -35,12 +35,16 @@ #define SYSG(_vector, _addr) \ G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS) +#ifdef CONFIG_X86_64 /* * Interrupt gate with interrupt stack. The _ist index is the index in * the tss.ist[] array, but for the descriptor it needs to start at 1. */ #define ISTG(_vector, _addr, _ist) \ G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS) +#else +#define ISTG(_vector, _addr, _ist) INTG(_vector, _addr) +#endif /* Task gate */ #define TSKG(_vector, _gdt) \ @@ -74,7 +78,7 @@ static const __initconst struct idt_data early_idts[] = { */ static const __initconst struct idt_data def_idts[] = { INTG(X86_TRAP_DE, asm_exc_divide_error), - INTG(X86_TRAP_NMI, asm_exc_nmi), + ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI), INTG(X86_TRAP_BR, asm_exc_bounds), INTG(X86_TRAP_UD, asm_exc_invalid_op), INTG(X86_TRAP_NM, asm_exc_device_not_available), @@ -91,12 +95,16 @@ static const __initconst struct idt_data def_idts[] = { #ifdef CONFIG_X86_32 TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS), #else - INTG(X86_TRAP_DF, asm_exc_double_fault), + ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF), #endif - INTG(X86_TRAP_DB, asm_exc_debug), + ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB), #ifdef CONFIG_X86_MCE - INTG(X86_TRAP_MC, asm_exc_machine_check), + ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE), +#endif + +#ifdef CONFIG_AMD_MEM_ENCRYPT + ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC), #endif SYSG(X86_TRAP_OF, asm_exc_overflow), @@ -221,22 +229,6 @@ static const __initconst struct idt_data early_pf_idts[] = { INTG(X86_TRAP_PF, asm_exc_page_fault), }; -/* - * The exceptions which use Interrupt stacks. They are setup after - * cpu_init() when the TSS has been initialized. - */ -static const __initconst struct idt_data ist_idts[] = { - ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB), - ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI), - ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF), -#ifdef CONFIG_X86_MCE - ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE), -#endif -#ifdef CONFIG_AMD_MEM_ENCRYPT - ISTG(X86_TRAP_VC, asm_exc_vmm_communication, IST_INDEX_VC), -#endif -}; - /** * idt_setup_early_pf - Initialize the idt table with early pagefault handler * @@ -254,14 +246,6 @@ void __init idt_setup_early_pf(void) idt_setup_from_table(idt_table, early_pf_idts, ARRAY_SIZE(early_pf_idts), true); } - -/** - * idt_setup_ist_traps - Initialize the idt table with traps using IST - */ -void __init idt_setup_ist_traps(void) -{ - idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true); -} #endif static void __init idt_map_in_cea(void) diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 1b3fe0edd329..c492ad3001ca 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -674,7 +674,7 @@ static int prepare_emulation(struct kprobe *p, struct insn *insn) break; if (insn->addr_bytes != sizeof(unsigned long)) - return -EOPNOTSUPP; /* Don't support differnt size */ + return -EOPNOTSUPP; /* Don't support different size */ if (X86_MODRM_MOD(opcode) != 3) return -EOPNOTSUPP; /* TODO: support memory addressing */ diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c index 87a4143aa7d7..4c208ea3bd9f 100644 --- a/arch/x86/kernel/ptrace.c +++ b/arch/x86/kernel/ptrace.c @@ -911,7 +911,7 @@ static int putreg32(struct task_struct *child, unsigned regno, u32 value) * syscall with TS_COMPAT still set. */ regs->orig_ax = value; - if (syscall_get_nr(child, regs) >= 0) + if (syscall_get_nr(child, regs) != -1) child->thread_info.status |= TS_I386_REGS_POKED; break; diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index 1e720626069a..85acd22f8022 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -695,30 +695,6 @@ static void __init e820_add_kernel_range(void) e820__range_add(start, size, E820_TYPE_RAM); } -static unsigned reserve_low = CONFIG_X86_RESERVE_LOW << 10; - -static int __init parse_reservelow(char *p) -{ - unsigned long long size; - - if (!p) - return -EINVAL; - - size = memparse(p, &p); - - if (size < 4096) - size = 4096; - - if (size > 640*1024) - size = 640*1024; - - reserve_low = size; - - return 0; -} - -early_param("reservelow", parse_reservelow); - static void __init early_reserve_memory(void) { /* @@ -1084,17 +1060,18 @@ void __init setup_arch(char **cmdline_p) #endif /* - * Find free memory for the real mode trampoline and place it - * there. - * If there is not enough free memory under 1M, on EFI-enabled - * systems there will be additional attempt to reclaim the memory - * for the real mode trampoline at efi_free_boot_services(). + * Find free memory for the real mode trampoline and place it there. If + * there is not enough free memory under 1M, on EFI-enabled systems + * there will be additional attempt to reclaim the memory for the real + * mode trampoline at efi_free_boot_services(). * - * Unconditionally reserve the entire first 1M of RAM because - * BIOSes are know to corrupt low memory and several - * hundred kilobytes are not worth complex detection what memory gets - * clobbered. Moreover, on machines with SandyBridge graphics or in - * setups that use crashkernel the entire 1M is reserved anyway. + * Unconditionally reserve the entire first 1M of RAM because BIOSes + * are known to corrupt low memory and several hundred kilobytes are not + * worth complex detection what memory gets clobbered. Windows does the + * same thing for very similar reasons. + * + * Moreover, on machines with SandyBridge graphics or in setups that use + * crashkernel the entire 1M is reserved anyway. */ reserve_real_mode(); diff --git a/arch/x86/kernel/signal.c b/arch/x86/kernel/signal.c index a06cb107c0e8..e12779a2714d 100644 --- a/arch/x86/kernel/signal.c +++ b/arch/x86/kernel/signal.c @@ -713,7 +713,7 @@ handle_signal(struct ksignal *ksig, struct pt_regs *regs) save_v86_state((struct kernel_vm86_regs *) regs, VM86_SIGNAL); /* Are we from a system call? */ - if (syscall_get_nr(current, regs) >= 0) { + if (syscall_get_nr(current, regs) != -1) { /* If so, check system call restarting.. */ switch (syscall_get_error(current, regs)) { case -ERESTART_RESTARTBLOCK: @@ -793,7 +793,7 @@ void arch_do_signal_or_restart(struct pt_regs *regs, bool has_signal) } /* Did we come from a system call? */ - if (syscall_get_nr(current, regs) >= 0) { + if (syscall_get_nr(current, regs) != -1) { /* Restart the system call - no handlers present */ switch (syscall_get_error(current, regs)) { case -ERESTARTNOHAND: diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index ec2d64aa2163..9320285a5e29 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -232,8 +232,7 @@ static void notrace start_secondary(void *unused) load_cr3(swapper_pg_dir); __flush_tlb_all(); #endif - cpu_init_exception_handling(); - cpu_init(); + cpu_init_secondary(); rcu_cpu_starting(raw_smp_processor_id()); x86_cpuinit.early_percpu_clock_init(); smp_callin(); diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 853ea7a80806..ed540e09a399 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -1160,12 +1160,9 @@ void __init trap_init(void) /* Init GHCB memory pages when running as an SEV-ES guest */ sev_es_init_vc_handling(); + /* Initialize TSS before setting up traps so ISTs work */ + cpu_init_exception_handling(); + /* Setup traps as cpu_init() might #GP */ idt_setup_traps(); - - /* - * Should be a barrier for any external CPU state: - */ cpu_init(); - - idt_setup_ist_traps(); } diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index fb8efb387aff..ac69894eab88 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -43,6 +43,7 @@ config KVM select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_VFIO select SRCU + select HAVE_KVM_PM_NOTIFIER if PM help Support hosting fully virtualized guest machines using hardware virtualization extensions. You will need a fairly recent diff --git a/arch/x86/kvm/Makefile b/arch/x86/kvm/Makefile index c589db5d91b3..75dfd27b6e8a 100644 --- a/arch/x86/kvm/Makefile +++ b/arch/x86/kvm/Makefile @@ -11,13 +11,18 @@ KVM := ../../../virt/kvm kvm-y += $(KVM)/kvm_main.o $(KVM)/coalesced_mmio.o \ $(KVM)/eventfd.o $(KVM)/irqchip.o $(KVM)/vfio.o \ - $(KVM)/dirty_ring.o + $(KVM)/dirty_ring.o $(KVM)/binary_stats.o kvm-$(CONFIG_KVM_ASYNC_PF) += $(KVM)/async_pf.o kvm-y += x86.o emulate.o i8259.o irq.o lapic.o \ i8254.o ioapic.o irq_comm.o cpuid.o pmu.o mtrr.o \ hyperv.o debugfs.o mmu/mmu.o mmu/page_track.o \ mmu/spte.o + +ifdef CONFIG_HYPERV +kvm-y += kvm_onhyperv.o +endif + kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o kvm-$(CONFIG_KVM_XEN) += xen.o @@ -27,6 +32,10 @@ kvm-intel-$(CONFIG_X86_SGX_KVM) += vmx/sgx.o kvm-amd-y += svm/svm.o svm/vmenter.o svm/pmu.o svm/nested.o svm/avic.o svm/sev.o +ifdef CONFIG_HYPERV +kvm-amd-y += svm/svm_onhyperv.o +endif + obj-$(CONFIG_KVM) += kvm.o obj-$(CONFIG_KVM_INTEL) += kvm-intel.o obj-$(CONFIG_KVM_AMD) += kvm-amd.o diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index b4da665bb892..c42613cfb5ba 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -202,10 +202,10 @@ static void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) static_call(kvm_x86_vcpu_after_set_cpuid)(vcpu); /* - * Except for the MMU, which needs to be reset after any vendor - * specific adjustments to the reserved GPA bits. + * Except for the MMU, which needs to do its thing any vendor specific + * adjustments to the reserved GPA bits. */ - kvm_mmu_reset_context(vcpu); + kvm_mmu_after_set_cpuid(vcpu); } static int is_efer_nx(void) diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c index 7e818d64bb4d..95a98413dc32 100644 --- a/arch/x86/kvm/debugfs.c +++ b/arch/x86/kvm/debugfs.c @@ -17,6 +17,15 @@ static int vcpu_get_timer_advance_ns(void *data, u64 *val) DEFINE_SIMPLE_ATTRIBUTE(vcpu_timer_advance_ns_fops, vcpu_get_timer_advance_ns, NULL, "%llu\n"); +static int vcpu_get_guest_mode(void *data, u64 *val) +{ + struct kvm_vcpu *vcpu = (struct kvm_vcpu *) data; + *val = vcpu->stat.guest_mode; + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(vcpu_guest_mode_fops, vcpu_get_guest_mode, NULL, "%lld\n"); + static int vcpu_get_tsc_offset(void *data, u64 *val) { struct kvm_vcpu *vcpu = (struct kvm_vcpu *) data; @@ -45,6 +54,8 @@ DEFINE_SIMPLE_ATTRIBUTE(vcpu_tsc_scaling_frac_fops, vcpu_get_tsc_scaling_frac_bi void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry) { + debugfs_create_file("guest_mode", 0444, debugfs_dentry, vcpu, + &vcpu_guest_mode_fops); debugfs_create_file("tsc-offset", 0444, debugfs_dentry, vcpu, &vcpu_tsc_offset_fops); diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 5e5de05a8fbf..2837110e66ed 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -22,7 +22,6 @@ #include "kvm_cache_regs.h" #include "kvm_emulate.h" #include -#include #include #include @@ -1081,116 +1080,14 @@ static void fetch_register_operand(struct operand *op) } } -static void emulator_get_fpu(void) -{ - fpregs_lock(); - - fpregs_assert_state_consistent(); - if (test_thread_flag(TIF_NEED_FPU_LOAD)) - switch_fpu_return(); -} - -static void emulator_put_fpu(void) -{ - fpregs_unlock(); -} - -static void read_sse_reg(sse128_t *data, int reg) -{ - emulator_get_fpu(); - switch (reg) { - case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break; - case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break; - case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break; - case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break; - case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break; - case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break; - case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break; - case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break; -#ifdef CONFIG_X86_64 - case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break; - case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break; - case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break; - case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break; - case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break; - case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break; - case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break; - case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break; -#endif - default: BUG(); - } - emulator_put_fpu(); -} - -static void write_sse_reg(sse128_t *data, int reg) -{ - emulator_get_fpu(); - switch (reg) { - case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break; - case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break; - case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break; - case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break; - case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break; - case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break; - case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break; - case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break; -#ifdef CONFIG_X86_64 - case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break; - case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break; - case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break; - case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break; - case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break; - case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break; - case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break; - case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break; -#endif - default: BUG(); - } - emulator_put_fpu(); -} - -static void read_mmx_reg(u64 *data, int reg) -{ - emulator_get_fpu(); - switch (reg) { - case 0: asm("movq %%mm0, %0" : "=m"(*data)); break; - case 1: asm("movq %%mm1, %0" : "=m"(*data)); break; - case 2: asm("movq %%mm2, %0" : "=m"(*data)); break; - case 3: asm("movq %%mm3, %0" : "=m"(*data)); break; - case 4: asm("movq %%mm4, %0" : "=m"(*data)); break; - case 5: asm("movq %%mm5, %0" : "=m"(*data)); break; - case 6: asm("movq %%mm6, %0" : "=m"(*data)); break; - case 7: asm("movq %%mm7, %0" : "=m"(*data)); break; - default: BUG(); - } - emulator_put_fpu(); -} - -static void write_mmx_reg(u64 *data, int reg) -{ - emulator_get_fpu(); - switch (reg) { - case 0: asm("movq %0, %%mm0" : : "m"(*data)); break; - case 1: asm("movq %0, %%mm1" : : "m"(*data)); break; - case 2: asm("movq %0, %%mm2" : : "m"(*data)); break; - case 3: asm("movq %0, %%mm3" : : "m"(*data)); break; - case 4: asm("movq %0, %%mm4" : : "m"(*data)); break; - case 5: asm("movq %0, %%mm5" : : "m"(*data)); break; - case 6: asm("movq %0, %%mm6" : : "m"(*data)); break; - case 7: asm("movq %0, %%mm7" : : "m"(*data)); break; - default: BUG(); - } - emulator_put_fpu(); -} - static int em_fninit(struct x86_emulate_ctxt *ctxt) { if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) return emulate_nm(ctxt); - emulator_get_fpu(); + kvm_fpu_get(); asm volatile("fninit"); - emulator_put_fpu(); + kvm_fpu_put(); return X86EMUL_CONTINUE; } @@ -1201,9 +1098,9 @@ static int em_fnstcw(struct x86_emulate_ctxt *ctxt) if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) return emulate_nm(ctxt); - emulator_get_fpu(); + kvm_fpu_get(); asm volatile("fnstcw %0": "+m"(fcw)); - emulator_put_fpu(); + kvm_fpu_put(); ctxt->dst.val = fcw; @@ -1217,9 +1114,9 @@ static int em_fnstsw(struct x86_emulate_ctxt *ctxt) if (ctxt->ops->get_cr(ctxt, 0) & (X86_CR0_TS | X86_CR0_EM)) return emulate_nm(ctxt); - emulator_get_fpu(); + kvm_fpu_get(); asm volatile("fnstsw %0": "+m"(fsw)); - emulator_put_fpu(); + kvm_fpu_put(); ctxt->dst.val = fsw; @@ -1238,7 +1135,7 @@ static void decode_register_operand(struct x86_emulate_ctxt *ctxt, op->type = OP_XMM; op->bytes = 16; op->addr.xmm = reg; - read_sse_reg(&op->vec_val, reg); + kvm_read_sse_reg(reg, &op->vec_val); return; } if (ctxt->d & Mmx) { @@ -1289,7 +1186,7 @@ static int decode_modrm(struct x86_emulate_ctxt *ctxt, op->type = OP_XMM; op->bytes = 16; op->addr.xmm = ctxt->modrm_rm; - read_sse_reg(&op->vec_val, ctxt->modrm_rm); + kvm_read_sse_reg(ctxt->modrm_rm, &op->vec_val); return rc; } if (ctxt->d & Mmx) { @@ -1866,10 +1763,10 @@ static int writeback(struct x86_emulate_ctxt *ctxt, struct operand *op) op->bytes * op->count); break; case OP_XMM: - write_sse_reg(&op->vec_val, op->addr.xmm); + kvm_write_sse_reg(op->addr.xmm, &op->vec_val); break; case OP_MM: - write_mmx_reg(&op->mm_val, op->addr.mm); + kvm_write_mmx_reg(op->addr.mm, &op->mm_val); break; case OP_NONE: /* no writeback */ @@ -2638,8 +2535,7 @@ static int em_rsm(struct x86_emulate_ctxt *ctxt) if ((ctxt->ops->get_hflags(ctxt) & X86EMUL_SMM_INSIDE_NMI_MASK) == 0) ctxt->ops->set_nmi_mask(ctxt, false); - ctxt->ops->set_hflags(ctxt, ctxt->ops->get_hflags(ctxt) & - ~(X86EMUL_SMM_INSIDE_NMI_MASK | X86EMUL_SMM_MASK)); + ctxt->ops->exiting_smm(ctxt); /* * Get back to real mode, to prepare a safe state in which to load @@ -2678,12 +2574,12 @@ static int em_rsm(struct x86_emulate_ctxt *ctxt) } /* - * Give pre_leave_smm() a chance to make ISA-specific changes to the - * vCPU state (e.g. enter guest mode) before loading state from the SMM + * Give leave_smm() a chance to make ISA-specific changes to the vCPU + * state (e.g. enter guest mode) before loading state from the SMM * state-save area. */ - if (ctxt->ops->pre_leave_smm(ctxt, buf)) - return X86EMUL_UNHANDLEABLE; + if (ctxt->ops->leave_smm(ctxt, buf)) + goto emulate_shutdown; #ifdef CONFIG_X86_64 if (emulator_has_longmode(ctxt)) @@ -2692,13 +2588,21 @@ static int em_rsm(struct x86_emulate_ctxt *ctxt) #endif ret = rsm_load_state_32(ctxt, buf); - if (ret != X86EMUL_CONTINUE) { - /* FIXME: should triple fault */ - return X86EMUL_UNHANDLEABLE; - } + if (ret != X86EMUL_CONTINUE) + goto emulate_shutdown; - ctxt->ops->post_leave_smm(ctxt); + /* + * Note, the ctxt->ops callbacks are responsible for handling side + * effects when writing MSRs and CRs, e.g. MMU context resets, CPUID + * runtime updates, etc... If that changes, e.g. this flow is moved + * out of the emulator to make it look more like enter_smm(), then + * those side effects need to be explicitly handled for both success + * and shutdown. + */ + return X86EMUL_CONTINUE; +emulate_shutdown: + ctxt->ops->triple_fault(ctxt); return X86EMUL_CONTINUE; } @@ -4124,11 +4028,11 @@ static int em_fxsave(struct x86_emulate_ctxt *ctxt) if (rc != X86EMUL_CONTINUE) return rc; - emulator_get_fpu(); + kvm_fpu_get(); rc = asm_safe("fxsave %[fx]", , [fx] "+m"(fx_state)); - emulator_put_fpu(); + kvm_fpu_put(); if (rc != X86EMUL_CONTINUE) return rc; @@ -4172,7 +4076,7 @@ static int em_fxrstor(struct x86_emulate_ctxt *ctxt) if (rc != X86EMUL_CONTINUE) return rc; - emulator_get_fpu(); + kvm_fpu_get(); if (size < __fxstate_size(16)) { rc = fxregs_fixup(&fx_state, size); @@ -4189,7 +4093,7 @@ static int em_fxrstor(struct x86_emulate_ctxt *ctxt) rc = asm_safe("fxrstor %[fx]", : [fx] "m"(fx_state)); out: - emulator_put_fpu(); + kvm_fpu_put(); return rc; } @@ -5437,9 +5341,9 @@ static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt) { int rc; - emulator_get_fpu(); + kvm_fpu_get(); rc = asm_safe("fwait"); - emulator_put_fpu(); + kvm_fpu_put(); if (unlikely(rc != X86EMUL_CONTINUE)) return emulate_exception(ctxt, MF_VECTOR, 0, false); @@ -5450,7 +5354,7 @@ static int flush_pending_x87_faults(struct x86_emulate_ctxt *ctxt) static void fetch_possible_mmx_operand(struct operand *op) { if (op->type == OP_MM) - read_mmx_reg(&op->mm_val, op->addr.mm); + kvm_read_mmx_reg(op->addr.mm, &op->mm_val); } static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop) diff --git a/arch/x86/kvm/fpu.h b/arch/x86/kvm/fpu.h new file mode 100644 index 000000000000..3ba12888bf66 --- /dev/null +++ b/arch/x86/kvm/fpu.h @@ -0,0 +1,140 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __KVM_FPU_H_ +#define __KVM_FPU_H_ + +#include + +typedef u32 __attribute__((vector_size(16))) sse128_t; +#define __sse128_u union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; } +#define sse128_lo(x) ({ __sse128_u t; t.vec = x; t.as_u64[0]; }) +#define sse128_hi(x) ({ __sse128_u t; t.vec = x; t.as_u64[1]; }) +#define sse128_l0(x) ({ __sse128_u t; t.vec = x; t.as_u32[0]; }) +#define sse128_l1(x) ({ __sse128_u t; t.vec = x; t.as_u32[1]; }) +#define sse128_l2(x) ({ __sse128_u t; t.vec = x; t.as_u32[2]; }) +#define sse128_l3(x) ({ __sse128_u t; t.vec = x; t.as_u32[3]; }) +#define sse128(lo, hi) ({ __sse128_u t; t.as_u64[0] = lo; t.as_u64[1] = hi; t.vec; }) + +static inline void _kvm_read_sse_reg(int reg, sse128_t *data) +{ + switch (reg) { + case 0: asm("movdqa %%xmm0, %0" : "=m"(*data)); break; + case 1: asm("movdqa %%xmm1, %0" : "=m"(*data)); break; + case 2: asm("movdqa %%xmm2, %0" : "=m"(*data)); break; + case 3: asm("movdqa %%xmm3, %0" : "=m"(*data)); break; + case 4: asm("movdqa %%xmm4, %0" : "=m"(*data)); break; + case 5: asm("movdqa %%xmm5, %0" : "=m"(*data)); break; + case 6: asm("movdqa %%xmm6, %0" : "=m"(*data)); break; + case 7: asm("movdqa %%xmm7, %0" : "=m"(*data)); break; +#ifdef CONFIG_X86_64 + case 8: asm("movdqa %%xmm8, %0" : "=m"(*data)); break; + case 9: asm("movdqa %%xmm9, %0" : "=m"(*data)); break; + case 10: asm("movdqa %%xmm10, %0" : "=m"(*data)); break; + case 11: asm("movdqa %%xmm11, %0" : "=m"(*data)); break; + case 12: asm("movdqa %%xmm12, %0" : "=m"(*data)); break; + case 13: asm("movdqa %%xmm13, %0" : "=m"(*data)); break; + case 14: asm("movdqa %%xmm14, %0" : "=m"(*data)); break; + case 15: asm("movdqa %%xmm15, %0" : "=m"(*data)); break; +#endif + default: BUG(); + } +} + +static inline void _kvm_write_sse_reg(int reg, const sse128_t *data) +{ + switch (reg) { + case 0: asm("movdqa %0, %%xmm0" : : "m"(*data)); break; + case 1: asm("movdqa %0, %%xmm1" : : "m"(*data)); break; + case 2: asm("movdqa %0, %%xmm2" : : "m"(*data)); break; + case 3: asm("movdqa %0, %%xmm3" : : "m"(*data)); break; + case 4: asm("movdqa %0, %%xmm4" : : "m"(*data)); break; + case 5: asm("movdqa %0, %%xmm5" : : "m"(*data)); break; + case 6: asm("movdqa %0, %%xmm6" : : "m"(*data)); break; + case 7: asm("movdqa %0, %%xmm7" : : "m"(*data)); break; +#ifdef CONFIG_X86_64 + case 8: asm("movdqa %0, %%xmm8" : : "m"(*data)); break; + case 9: asm("movdqa %0, %%xmm9" : : "m"(*data)); break; + case 10: asm("movdqa %0, %%xmm10" : : "m"(*data)); break; + case 11: asm("movdqa %0, %%xmm11" : : "m"(*data)); break; + case 12: asm("movdqa %0, %%xmm12" : : "m"(*data)); break; + case 13: asm("movdqa %0, %%xmm13" : : "m"(*data)); break; + case 14: asm("movdqa %0, %%xmm14" : : "m"(*data)); break; + case 15: asm("movdqa %0, %%xmm15" : : "m"(*data)); break; +#endif + default: BUG(); + } +} + +static inline void _kvm_read_mmx_reg(int reg, u64 *data) +{ + switch (reg) { + case 0: asm("movq %%mm0, %0" : "=m"(*data)); break; + case 1: asm("movq %%mm1, %0" : "=m"(*data)); break; + case 2: asm("movq %%mm2, %0" : "=m"(*data)); break; + case 3: asm("movq %%mm3, %0" : "=m"(*data)); break; + case 4: asm("movq %%mm4, %0" : "=m"(*data)); break; + case 5: asm("movq %%mm5, %0" : "=m"(*data)); break; + case 6: asm("movq %%mm6, %0" : "=m"(*data)); break; + case 7: asm("movq %%mm7, %0" : "=m"(*data)); break; + default: BUG(); + } +} + +static inline void _kvm_write_mmx_reg(int reg, const u64 *data) +{ + switch (reg) { + case 0: asm("movq %0, %%mm0" : : "m"(*data)); break; + case 1: asm("movq %0, %%mm1" : : "m"(*data)); break; + case 2: asm("movq %0, %%mm2" : : "m"(*data)); break; + case 3: asm("movq %0, %%mm3" : : "m"(*data)); break; + case 4: asm("movq %0, %%mm4" : : "m"(*data)); break; + case 5: asm("movq %0, %%mm5" : : "m"(*data)); break; + case 6: asm("movq %0, %%mm6" : : "m"(*data)); break; + case 7: asm("movq %0, %%mm7" : : "m"(*data)); break; + default: BUG(); + } +} + +static inline void kvm_fpu_get(void) +{ + fpregs_lock(); + + fpregs_assert_state_consistent(); + if (test_thread_flag(TIF_NEED_FPU_LOAD)) + switch_fpu_return(); +} + +static inline void kvm_fpu_put(void) +{ + fpregs_unlock(); +} + +static inline void kvm_read_sse_reg(int reg, sse128_t *data) +{ + kvm_fpu_get(); + _kvm_read_sse_reg(reg, data); + kvm_fpu_put(); +} + +static inline void kvm_write_sse_reg(int reg, const sse128_t *data) +{ + kvm_fpu_get(); + _kvm_write_sse_reg(reg, data); + kvm_fpu_put(); +} + +static inline void kvm_read_mmx_reg(int reg, u64 *data) +{ + kvm_fpu_get(); + _kvm_read_mmx_reg(reg, data); + kvm_fpu_put(); +} + +static inline void kvm_write_mmx_reg(int reg, const u64 *data) +{ + kvm_fpu_get(); + _kvm_write_mmx_reg(reg, data); + kvm_fpu_put(); +} + +#endif diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index f00830e5202f..b07592ca92f0 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -36,6 +36,7 @@ #include "trace.h" #include "irq.h" +#include "fpu.h" /* "Hv#1" signature */ #define HYPERV_CPUID_SIGNATURE_EAX 0x31237648 @@ -273,15 +274,10 @@ static int synic_set_msr(struct kvm_vcpu_hv_synic *synic, static bool kvm_hv_is_syndbg_enabled(struct kvm_vcpu *vcpu) { - struct kvm_cpuid_entry2 *entry; + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); - entry = kvm_find_cpuid_entry(vcpu, - HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES, - 0); - if (!entry) - return false; - - return entry->eax & HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; + return hv_vcpu->cpuid_cache.syndbg_cap_eax & + HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; } static int kvm_hv_syndbg_complete_userspace(struct kvm_vcpu *vcpu) @@ -635,11 +631,17 @@ static int stimer_set_config(struct kvm_vcpu_hv_stimer *stimer, u64 config, union hv_stimer_config new_config = {.as_uint64 = config}, old_config = {.as_uint64 = stimer->config.as_uint64}; struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); struct kvm_vcpu_hv_synic *synic = to_hv_synic(vcpu); if (!synic->active && !host) return 1; + if (unlikely(!host && hv_vcpu->enforce_cpuid && new_config.direct_mode && + !(hv_vcpu->cpuid_cache.features_edx & + HV_STIMER_DIRECT_MODE_AVAILABLE))) + return 1; + trace_kvm_hv_stimer_set_config(hv_stimer_to_vcpu(stimer)->vcpu_id, stimer->index, config, host); @@ -1206,12 +1208,90 @@ void kvm_hv_invalidate_tsc_page(struct kvm *kvm) mutex_unlock(&hv->hv_lock); } + +static bool hv_check_msr_access(struct kvm_vcpu_hv *hv_vcpu, u32 msr) +{ + if (!hv_vcpu->enforce_cpuid) + return true; + + switch (msr) { + case HV_X64_MSR_GUEST_OS_ID: + case HV_X64_MSR_HYPERCALL: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_HYPERCALL_AVAILABLE; + case HV_X64_MSR_VP_RUNTIME: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_VP_RUNTIME_AVAILABLE; + case HV_X64_MSR_TIME_REF_COUNT: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_TIME_REF_COUNT_AVAILABLE; + case HV_X64_MSR_VP_INDEX: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_VP_INDEX_AVAILABLE; + case HV_X64_MSR_RESET: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_RESET_AVAILABLE; + case HV_X64_MSR_REFERENCE_TSC: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_REFERENCE_TSC_AVAILABLE; + case HV_X64_MSR_SCONTROL: + case HV_X64_MSR_SVERSION: + case HV_X64_MSR_SIEFP: + case HV_X64_MSR_SIMP: + case HV_X64_MSR_EOM: + case HV_X64_MSR_SINT0 ... HV_X64_MSR_SINT15: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_SYNIC_AVAILABLE; + case HV_X64_MSR_STIMER0_CONFIG: + case HV_X64_MSR_STIMER1_CONFIG: + case HV_X64_MSR_STIMER2_CONFIG: + case HV_X64_MSR_STIMER3_CONFIG: + case HV_X64_MSR_STIMER0_COUNT: + case HV_X64_MSR_STIMER1_COUNT: + case HV_X64_MSR_STIMER2_COUNT: + case HV_X64_MSR_STIMER3_COUNT: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_SYNTIMER_AVAILABLE; + case HV_X64_MSR_EOI: + case HV_X64_MSR_ICR: + case HV_X64_MSR_TPR: + case HV_X64_MSR_VP_ASSIST_PAGE: + return hv_vcpu->cpuid_cache.features_eax & + HV_MSR_APIC_ACCESS_AVAILABLE; + break; + case HV_X64_MSR_TSC_FREQUENCY: + case HV_X64_MSR_APIC_FREQUENCY: + return hv_vcpu->cpuid_cache.features_eax & + HV_ACCESS_FREQUENCY_MSRS; + case HV_X64_MSR_REENLIGHTENMENT_CONTROL: + case HV_X64_MSR_TSC_EMULATION_CONTROL: + case HV_X64_MSR_TSC_EMULATION_STATUS: + return hv_vcpu->cpuid_cache.features_eax & + HV_ACCESS_REENLIGHTENMENT; + case HV_X64_MSR_CRASH_P0 ... HV_X64_MSR_CRASH_P4: + case HV_X64_MSR_CRASH_CTL: + return hv_vcpu->cpuid_cache.features_edx & + HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; + case HV_X64_MSR_SYNDBG_OPTIONS: + case HV_X64_MSR_SYNDBG_CONTROL ... HV_X64_MSR_SYNDBG_PENDING_BUFFER: + return hv_vcpu->cpuid_cache.features_edx & + HV_FEATURE_DEBUG_MSRS_AVAILABLE; + default: + break; + } + + return false; +} + static int kvm_hv_set_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) { struct kvm *kvm = vcpu->kvm; struct kvm_hv *hv = to_kvm_hv(kvm); + if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) + return 1; + switch (msr) { case HV_X64_MSR_GUEST_OS_ID: hv->hv_guest_os_id = data; @@ -1340,6 +1420,9 @@ static int kvm_hv_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data, bool host) { struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); + if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) + return 1; + switch (msr) { case HV_X64_MSR_VP_INDEX: { struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); @@ -1454,6 +1537,9 @@ static int kvm_hv_get_msr_pw(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, struct kvm *kvm = vcpu->kvm; struct kvm_hv *hv = to_kvm_hv(kvm); + if (unlikely(!host && !hv_check_msr_access(to_hv_vcpu(vcpu), msr))) + return 1; + switch (msr) { case HV_X64_MSR_GUEST_OS_ID: data = hv->hv_guest_os_id; @@ -1503,6 +1589,9 @@ static int kvm_hv_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata, u64 data = 0; struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); + if (unlikely(!host && !hv_check_msr_access(hv_vcpu, msr))) + return 1; + switch (msr) { case HV_X64_MSR_VP_INDEX: data = hv_vcpu->vp_index; @@ -1631,8 +1720,22 @@ static __always_inline unsigned long *sparse_set_to_vcpu_mask( return vcpu_bitmap; } -static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, u64 ingpa, u16 rep_cnt, bool ex) +struct kvm_hv_hcall { + u64 param; + u64 ingpa; + u64 outgpa; + u16 code; + u16 rep_cnt; + u16 rep_idx; + bool fast; + bool rep; + sse128_t xmm[HV_HYPERCALL_MAX_XMM_REGISTERS]; +}; + +static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool ex) { + int i; + gpa_t gpa; struct kvm *kvm = vcpu->kvm; struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); struct hv_tlb_flush_ex flush_ex; @@ -1646,8 +1749,15 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, u64 ingpa, u16 rep_cnt, bool bool all_cpus; if (!ex) { - if (unlikely(kvm_read_guest(kvm, ingpa, &flush, sizeof(flush)))) - return HV_STATUS_INVALID_HYPERCALL_INPUT; + if (hc->fast) { + flush.address_space = hc->ingpa; + flush.flags = hc->outgpa; + flush.processor_mask = sse128_lo(hc->xmm[0]); + } else { + if (unlikely(kvm_read_guest(kvm, hc->ingpa, + &flush, sizeof(flush)))) + return HV_STATUS_INVALID_HYPERCALL_INPUT; + } trace_kvm_hv_flush_tlb(flush.processor_mask, flush.address_space, flush.flags); @@ -1665,9 +1775,16 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, u64 ingpa, u16 rep_cnt, bool all_cpus = (flush.flags & HV_FLUSH_ALL_PROCESSORS) || flush.processor_mask == 0; } else { - if (unlikely(kvm_read_guest(kvm, ingpa, &flush_ex, - sizeof(flush_ex)))) - return HV_STATUS_INVALID_HYPERCALL_INPUT; + if (hc->fast) { + flush_ex.address_space = hc->ingpa; + flush_ex.flags = hc->outgpa; + memcpy(&flush_ex.hv_vp_set, + &hc->xmm[0], sizeof(hc->xmm[0])); + } else { + if (unlikely(kvm_read_guest(kvm, hc->ingpa, &flush_ex, + sizeof(flush_ex)))) + return HV_STATUS_INVALID_HYPERCALL_INPUT; + } trace_kvm_hv_flush_tlb_ex(flush_ex.hv_vp_set.valid_bank_mask, flush_ex.hv_vp_set.format, @@ -1678,20 +1795,28 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, u64 ingpa, u16 rep_cnt, bool all_cpus = flush_ex.hv_vp_set.format != HV_GENERIC_SET_SPARSE_4K; - sparse_banks_len = - bitmap_weight((unsigned long *)&valid_bank_mask, 64) * - sizeof(sparse_banks[0]); + sparse_banks_len = bitmap_weight((unsigned long *)&valid_bank_mask, 64); if (!sparse_banks_len && !all_cpus) goto ret_success; - if (!all_cpus && - kvm_read_guest(kvm, - ingpa + offsetof(struct hv_tlb_flush_ex, - hv_vp_set.bank_contents), - sparse_banks, - sparse_banks_len)) - return HV_STATUS_INVALID_HYPERCALL_INPUT; + if (!all_cpus) { + if (hc->fast) { + if (sparse_banks_len > HV_HYPERCALL_MAX_XMM_REGISTERS - 1) + return HV_STATUS_INVALID_HYPERCALL_INPUT; + for (i = 0; i < sparse_banks_len; i += 2) { + sparse_banks[i] = sse128_lo(hc->xmm[i / 2 + 1]); + sparse_banks[i + 1] = sse128_hi(hc->xmm[i / 2 + 1]); + } + } else { + gpa = hc->ingpa + offsetof(struct hv_tlb_flush_ex, + hv_vp_set.bank_contents); + if (unlikely(kvm_read_guest(kvm, gpa, sparse_banks, + sparse_banks_len * + sizeof(sparse_banks[0])))) + return HV_STATUS_INVALID_HYPERCALL_INPUT; + } + } } cpumask_clear(&hv_vcpu->tlb_flush); @@ -1704,13 +1829,13 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, u64 ingpa, u16 rep_cnt, bool * vcpu->arch.cr3 may not be up-to-date for running vCPUs so we can't * analyze it here, flush TLB regardless of the specified address space. */ - kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, + kvm_make_vcpus_request_mask(kvm, KVM_REQ_TLB_FLUSH_GUEST, NULL, vcpu_mask, &hv_vcpu->tlb_flush); ret_success: - /* We always do full TLB flush, set rep_done = rep_cnt. */ + /* We always do full TLB flush, set 'Reps completed' = 'Rep Count' */ return (u64)HV_STATUS_SUCCESS | - ((u64)rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET); + ((u64)hc->rep_cnt << HV_HYPERCALL_REP_COMP_OFFSET); } static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector, @@ -1732,8 +1857,7 @@ static void kvm_send_ipi_to_many(struct kvm *kvm, u32 vector, } } -static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, u64 ingpa, u64 outgpa, - bool ex, bool fast) +static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc, bool ex) { struct kvm *kvm = vcpu->kvm; struct hv_send_ipi_ex send_ipi_ex; @@ -1748,25 +1872,25 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, u64 ingpa, u64 outgpa, bool all_cpus; if (!ex) { - if (!fast) { - if (unlikely(kvm_read_guest(kvm, ingpa, &send_ipi, + if (!hc->fast) { + if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi, sizeof(send_ipi)))) return HV_STATUS_INVALID_HYPERCALL_INPUT; sparse_banks[0] = send_ipi.cpu_mask; vector = send_ipi.vector; } else { /* 'reserved' part of hv_send_ipi should be 0 */ - if (unlikely(ingpa >> 32 != 0)) + if (unlikely(hc->ingpa >> 32 != 0)) return HV_STATUS_INVALID_HYPERCALL_INPUT; - sparse_banks[0] = outgpa; - vector = (u32)ingpa; + sparse_banks[0] = hc->outgpa; + vector = (u32)hc->ingpa; } all_cpus = false; valid_bank_mask = BIT_ULL(0); trace_kvm_hv_send_ipi(vector, sparse_banks[0]); } else { - if (unlikely(kvm_read_guest(kvm, ingpa, &send_ipi_ex, + if (unlikely(kvm_read_guest(kvm, hc->ingpa, &send_ipi_ex, sizeof(send_ipi_ex)))) return HV_STATUS_INVALID_HYPERCALL_INPUT; @@ -1786,8 +1910,8 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, u64 ingpa, u64 outgpa, if (!all_cpus && kvm_read_guest(kvm, - ingpa + offsetof(struct hv_send_ipi_ex, - vp_set.bank_contents), + hc->ingpa + offsetof(struct hv_send_ipi_ex, + vp_set.bank_contents), sparse_banks, sparse_banks_len)) return HV_STATUS_INVALID_HYPERCALL_INPUT; @@ -1809,12 +1933,67 @@ static u64 kvm_hv_send_ipi(struct kvm_vcpu *vcpu, u64 ingpa, u64 outgpa, void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu) { struct kvm_cpuid_entry2 *entry; + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_INTERFACE, 0); - if (entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX) + if (entry && entry->eax == HYPERV_CPUID_SIGNATURE_EAX) { vcpu->arch.hyperv_enabled = true; - else + } else { vcpu->arch.hyperv_enabled = false; + return; + } + + if (!to_hv_vcpu(vcpu) && kvm_hv_vcpu_init(vcpu)) + return; + + hv_vcpu = to_hv_vcpu(vcpu); + + entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_FEATURES, 0); + if (entry) { + hv_vcpu->cpuid_cache.features_eax = entry->eax; + hv_vcpu->cpuid_cache.features_ebx = entry->ebx; + hv_vcpu->cpuid_cache.features_edx = entry->edx; + } else { + hv_vcpu->cpuid_cache.features_eax = 0; + hv_vcpu->cpuid_cache.features_ebx = 0; + hv_vcpu->cpuid_cache.features_edx = 0; + } + + entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_ENLIGHTMENT_INFO, 0); + if (entry) { + hv_vcpu->cpuid_cache.enlightenments_eax = entry->eax; + hv_vcpu->cpuid_cache.enlightenments_ebx = entry->ebx; + } else { + hv_vcpu->cpuid_cache.enlightenments_eax = 0; + hv_vcpu->cpuid_cache.enlightenments_ebx = 0; + } + + entry = kvm_find_cpuid_entry(vcpu, HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES, 0); + if (entry) + hv_vcpu->cpuid_cache.syndbg_cap_eax = entry->eax; + else + hv_vcpu->cpuid_cache.syndbg_cap_eax = 0; +} + +int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce) +{ + struct kvm_vcpu_hv *hv_vcpu; + int ret = 0; + + if (!to_hv_vcpu(vcpu)) { + if (enforce) { + ret = kvm_hv_vcpu_init(vcpu); + if (ret) + return ret; + } else { + return 0; + } + } + + hv_vcpu = to_hv_vcpu(vcpu); + hv_vcpu->enforce_cpuid = enforce; + + return ret; } bool kvm_hv_hypercall_enabled(struct kvm_vcpu *vcpu) @@ -1847,20 +2026,21 @@ static int kvm_hv_hypercall_complete_userspace(struct kvm_vcpu *vcpu) return kvm_hv_hypercall_complete(vcpu, vcpu->run->hyperv.u.hcall.result); } -static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param) +static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) { struct kvm_hv *hv = to_kvm_hv(vcpu->kvm); struct eventfd_ctx *eventfd; - if (unlikely(!fast)) { + if (unlikely(!hc->fast)) { int ret; - gpa_t gpa = param; + gpa_t gpa = hc->ingpa; - if ((gpa & (__alignof__(param) - 1)) || - offset_in_page(gpa) + sizeof(param) > PAGE_SIZE) + if ((gpa & (__alignof__(hc->ingpa) - 1)) || + offset_in_page(gpa) + sizeof(hc->ingpa) > PAGE_SIZE) return HV_STATUS_INVALID_ALIGNMENT; - ret = kvm_vcpu_read_guest(vcpu, gpa, ¶m, sizeof(param)); + ret = kvm_vcpu_read_guest(vcpu, gpa, + &hc->ingpa, sizeof(hc->ingpa)); if (ret < 0) return HV_STATUS_INVALID_ALIGNMENT; } @@ -1870,15 +2050,15 @@ static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param) * have no use for it, and in all known usecases it is zero, so just * report lookup failure if it isn't. */ - if (param & 0xffff00000000ULL) + if (hc->ingpa & 0xffff00000000ULL) return HV_STATUS_INVALID_PORT_ID; /* remaining bits are reserved-zero */ - if (param & ~KVM_HYPERV_CONN_ID_MASK) + if (hc->ingpa & ~KVM_HYPERV_CONN_ID_MASK) return HV_STATUS_INVALID_HYPERCALL_INPUT; /* the eventfd is protected by vcpu->kvm->srcu, but conn_to_evt isn't */ rcu_read_lock(); - eventfd = idr_find(&hv->conn_to_evt, param); + eventfd = idr_find(&hv->conn_to_evt, hc->ingpa); rcu_read_unlock(); if (!eventfd) return HV_STATUS_INVALID_PORT_ID; @@ -1887,11 +2067,80 @@ static u16 kvm_hvcall_signal_event(struct kvm_vcpu *vcpu, bool fast, u64 param) return HV_STATUS_SUCCESS; } +static bool is_xmm_fast_hypercall(struct kvm_hv_hcall *hc) +{ + switch (hc->code) { + case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: + case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: + case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: + case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: + return true; + } + + return false; +} + +static void kvm_hv_hypercall_read_xmm(struct kvm_hv_hcall *hc) +{ + int reg; + + kvm_fpu_get(); + for (reg = 0; reg < HV_HYPERCALL_MAX_XMM_REGISTERS; reg++) + _kvm_read_sse_reg(reg, &hc->xmm[reg]); + kvm_fpu_put(); +} + +static bool hv_check_hypercall_access(struct kvm_vcpu_hv *hv_vcpu, u16 code) +{ + if (!hv_vcpu->enforce_cpuid) + return true; + + switch (code) { + case HVCALL_NOTIFY_LONG_SPIN_WAIT: + return hv_vcpu->cpuid_cache.enlightenments_ebx && + hv_vcpu->cpuid_cache.enlightenments_ebx != U32_MAX; + case HVCALL_POST_MESSAGE: + return hv_vcpu->cpuid_cache.features_ebx & HV_POST_MESSAGES; + case HVCALL_SIGNAL_EVENT: + return hv_vcpu->cpuid_cache.features_ebx & HV_SIGNAL_EVENTS; + case HVCALL_POST_DEBUG_DATA: + case HVCALL_RETRIEVE_DEBUG_DATA: + case HVCALL_RESET_DEBUG_SESSION: + /* + * Return 'true' when SynDBG is disabled so the resulting code + * will be HV_STATUS_INVALID_HYPERCALL_CODE. + */ + return !kvm_hv_is_syndbg_enabled(hv_vcpu->vcpu) || + hv_vcpu->cpuid_cache.features_ebx & HV_DEBUGGING; + case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: + case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: + if (!(hv_vcpu->cpuid_cache.enlightenments_eax & + HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) + return false; + fallthrough; + case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: + case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: + return hv_vcpu->cpuid_cache.enlightenments_eax & + HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; + case HVCALL_SEND_IPI_EX: + if (!(hv_vcpu->cpuid_cache.enlightenments_eax & + HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED)) + return false; + fallthrough; + case HVCALL_SEND_IPI: + return hv_vcpu->cpuid_cache.enlightenments_eax & + HV_X64_CLUSTER_IPI_RECOMMENDED; + default: + break; + } + + return true; +} + int kvm_hv_hypercall(struct kvm_vcpu *vcpu) { - u64 param, ingpa, outgpa, ret = HV_STATUS_SUCCESS; - uint16_t code, rep_idx, rep_cnt; - bool fast, rep; + struct kvm_hv_hcall hc; + u64 ret = HV_STATUS_SUCCESS; /* * hypercall generates UD from non zero cpl and real mode @@ -1904,104 +2153,113 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) #ifdef CONFIG_X86_64 if (is_64_bit_mode(vcpu)) { - param = kvm_rcx_read(vcpu); - ingpa = kvm_rdx_read(vcpu); - outgpa = kvm_r8_read(vcpu); + hc.param = kvm_rcx_read(vcpu); + hc.ingpa = kvm_rdx_read(vcpu); + hc.outgpa = kvm_r8_read(vcpu); } else #endif { - param = ((u64)kvm_rdx_read(vcpu) << 32) | - (kvm_rax_read(vcpu) & 0xffffffff); - ingpa = ((u64)kvm_rbx_read(vcpu) << 32) | - (kvm_rcx_read(vcpu) & 0xffffffff); - outgpa = ((u64)kvm_rdi_read(vcpu) << 32) | - (kvm_rsi_read(vcpu) & 0xffffffff); + hc.param = ((u64)kvm_rdx_read(vcpu) << 32) | + (kvm_rax_read(vcpu) & 0xffffffff); + hc.ingpa = ((u64)kvm_rbx_read(vcpu) << 32) | + (kvm_rcx_read(vcpu) & 0xffffffff); + hc.outgpa = ((u64)kvm_rdi_read(vcpu) << 32) | + (kvm_rsi_read(vcpu) & 0xffffffff); } - code = param & 0xffff; - fast = !!(param & HV_HYPERCALL_FAST_BIT); - rep_cnt = (param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff; - rep_idx = (param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff; - rep = !!(rep_cnt || rep_idx); + hc.code = hc.param & 0xffff; + hc.fast = !!(hc.param & HV_HYPERCALL_FAST_BIT); + hc.rep_cnt = (hc.param >> HV_HYPERCALL_REP_COMP_OFFSET) & 0xfff; + hc.rep_idx = (hc.param >> HV_HYPERCALL_REP_START_OFFSET) & 0xfff; + hc.rep = !!(hc.rep_cnt || hc.rep_idx); - trace_kvm_hv_hypercall(code, fast, rep_cnt, rep_idx, ingpa, outgpa); + if (hc.fast && is_xmm_fast_hypercall(&hc)) + kvm_hv_hypercall_read_xmm(&hc); - switch (code) { + trace_kvm_hv_hypercall(hc.code, hc.fast, hc.rep_cnt, hc.rep_idx, + hc.ingpa, hc.outgpa); + + if (unlikely(!hv_check_hypercall_access(to_hv_vcpu(vcpu), hc.code))) { + ret = HV_STATUS_ACCESS_DENIED; + goto hypercall_complete; + } + + switch (hc.code) { case HVCALL_NOTIFY_LONG_SPIN_WAIT: - if (unlikely(rep)) { + if (unlikely(hc.rep)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } kvm_vcpu_on_spin(vcpu, true); break; case HVCALL_SIGNAL_EVENT: - if (unlikely(rep)) { + if (unlikely(hc.rep)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } - ret = kvm_hvcall_signal_event(vcpu, fast, ingpa); + ret = kvm_hvcall_signal_event(vcpu, &hc); if (ret != HV_STATUS_INVALID_PORT_ID) break; fallthrough; /* maybe userspace knows this conn_id */ case HVCALL_POST_MESSAGE: /* don't bother userspace if it has no way to handle it */ - if (unlikely(rep || !to_hv_synic(vcpu)->active)) { + if (unlikely(hc.rep || !to_hv_synic(vcpu)->active)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } vcpu->run->exit_reason = KVM_EXIT_HYPERV; vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL; - vcpu->run->hyperv.u.hcall.input = param; - vcpu->run->hyperv.u.hcall.params[0] = ingpa; - vcpu->run->hyperv.u.hcall.params[1] = outgpa; + vcpu->run->hyperv.u.hcall.input = hc.param; + vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa; + vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa; vcpu->arch.complete_userspace_io = kvm_hv_hypercall_complete_userspace; return 0; case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST: - if (unlikely(fast || !rep_cnt || rep_idx)) { + if (unlikely(!hc.rep_cnt || hc.rep_idx)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } - ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, false); + ret = kvm_hv_flush_tlb(vcpu, &hc, false); break; case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE: - if (unlikely(fast || rep)) { + if (unlikely(hc.rep)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } - ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, false); + ret = kvm_hv_flush_tlb(vcpu, &hc, false); break; case HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX: - if (unlikely(fast || !rep_cnt || rep_idx)) { + if (unlikely(!hc.rep_cnt || hc.rep_idx)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } - ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, true); + ret = kvm_hv_flush_tlb(vcpu, &hc, true); break; case HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX: - if (unlikely(fast || rep)) { + if (unlikely(hc.rep)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } - ret = kvm_hv_flush_tlb(vcpu, ingpa, rep_cnt, true); + ret = kvm_hv_flush_tlb(vcpu, &hc, true); break; case HVCALL_SEND_IPI: - if (unlikely(rep)) { + if (unlikely(hc.rep)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } - ret = kvm_hv_send_ipi(vcpu, ingpa, outgpa, false, fast); + ret = kvm_hv_send_ipi(vcpu, &hc, false); break; case HVCALL_SEND_IPI_EX: - if (unlikely(fast || rep)) { + if (unlikely(hc.fast || hc.rep)) { ret = HV_STATUS_INVALID_HYPERCALL_INPUT; break; } - ret = kvm_hv_send_ipi(vcpu, ingpa, outgpa, true, false); + ret = kvm_hv_send_ipi(vcpu, &hc, true); break; case HVCALL_POST_DEBUG_DATA: case HVCALL_RETRIEVE_DEBUG_DATA: - if (unlikely(fast)) { + if (unlikely(hc.fast)) { ret = HV_STATUS_INVALID_PARAMETER; break; } @@ -2020,9 +2278,9 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) } vcpu->run->exit_reason = KVM_EXIT_HYPERV; vcpu->run->hyperv.type = KVM_EXIT_HYPERV_HCALL; - vcpu->run->hyperv.u.hcall.input = param; - vcpu->run->hyperv.u.hcall.params[0] = ingpa; - vcpu->run->hyperv.u.hcall.params[1] = outgpa; + vcpu->run->hyperv.u.hcall.input = hc.param; + vcpu->run->hyperv.u.hcall.params[0] = hc.ingpa; + vcpu->run->hyperv.u.hcall.params[1] = hc.outgpa; vcpu->arch.complete_userspace_io = kvm_hv_hypercall_complete_userspace; return 0; @@ -2032,6 +2290,7 @@ int kvm_hv_hypercall(struct kvm_vcpu *vcpu) break; } +hypercall_complete: return kvm_hv_hypercall_complete(vcpu, ret); } @@ -2180,6 +2439,7 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, ent->ebx |= HV_POST_MESSAGES; ent->ebx |= HV_SIGNAL_EVENTS; + ent->edx |= HV_X64_HYPERCALL_XMM_INPUT_AVAILABLE; ent->edx |= HV_FEATURE_FREQUENCY_MSRS_AVAILABLE; ent->edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 60547d5cb6d7..730da8537d05 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -138,6 +138,7 @@ void kvm_hv_invalidate_tsc_page(struct kvm *kvm); void kvm_hv_init_vm(struct kvm *kvm); void kvm_hv_destroy_vm(struct kvm *kvm); void kvm_hv_set_cpuid(struct kvm_vcpu *vcpu); +int kvm_hv_set_enforce_cpuid(struct kvm_vcpu *vcpu, bool enforce); int kvm_vm_ioctl_hv_eventfd(struct kvm *kvm, struct kvm_hyperv_eventfd *args); int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, struct kvm_cpuid_entry2 __user *entries); diff --git a/arch/x86/kvm/kvm_cache_regs.h b/arch/x86/kvm/kvm_cache_regs.h index 3db5c42c9ecd..90e1ffdc05b7 100644 --- a/arch/x86/kvm/kvm_cache_regs.h +++ b/arch/x86/kvm/kvm_cache_regs.h @@ -55,6 +55,13 @@ static inline void kvm_register_mark_available(struct kvm_vcpu *vcpu, __set_bit(reg, (unsigned long *)&vcpu->arch.regs_avail); } +static inline void kvm_register_clear_available(struct kvm_vcpu *vcpu, + enum kvm_reg reg) +{ + __clear_bit(reg, (unsigned long *)&vcpu->arch.regs_avail); + __clear_bit(reg, (unsigned long *)&vcpu->arch.regs_dirty); +} + static inline void kvm_register_mark_dirty(struct kvm_vcpu *vcpu, enum kvm_reg reg) { @@ -118,6 +125,11 @@ static inline u64 kvm_pdptr_read(struct kvm_vcpu *vcpu, int index) return vcpu->arch.walk_mmu->pdptrs[index]; } +static inline void kvm_pdptr_write(struct kvm_vcpu *vcpu, int index, u64 value) +{ + vcpu->arch.walk_mmu->pdptrs[index] = value; +} + static inline ulong kvm_read_cr0_bits(struct kvm_vcpu *vcpu, ulong mask) { ulong tmask = mask & KVM_POSSIBLE_CR0_GUEST_BITS; @@ -162,6 +174,7 @@ static inline u64 kvm_read_edx_eax(struct kvm_vcpu *vcpu) static inline void enter_guest_mode(struct kvm_vcpu *vcpu) { vcpu->arch.hflags |= HF_GUEST_MASK; + vcpu->stat.guest_mode = 1; } static inline void leave_guest_mode(struct kvm_vcpu *vcpu) @@ -172,6 +185,8 @@ static inline void leave_guest_mode(struct kvm_vcpu *vcpu) vcpu->arch.load_eoi_exitmap_pending = false; kvm_make_request(KVM_REQ_LOAD_EOI_EXITMAP, vcpu); } + + vcpu->stat.guest_mode = 0; } static inline bool is_guest_mode(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h index 3e870bf9ca4d..68b420289d7e 100644 --- a/arch/x86/kvm/kvm_emulate.h +++ b/arch/x86/kvm/kvm_emulate.h @@ -13,6 +13,7 @@ #define _ASM_X86_KVM_X86_EMULATE_H #include +#include "fpu.h" struct x86_emulate_ctxt; enum x86_intercept; @@ -229,15 +230,12 @@ struct x86_emulate_ops { void (*set_nmi_mask)(struct x86_emulate_ctxt *ctxt, bool masked); unsigned (*get_hflags)(struct x86_emulate_ctxt *ctxt); - void (*set_hflags)(struct x86_emulate_ctxt *ctxt, unsigned hflags); - int (*pre_leave_smm)(struct x86_emulate_ctxt *ctxt, - const char *smstate); - void (*post_leave_smm)(struct x86_emulate_ctxt *ctxt); + void (*exiting_smm)(struct x86_emulate_ctxt *ctxt); + int (*leave_smm)(struct x86_emulate_ctxt *ctxt, const char *smstate); + void (*triple_fault)(struct x86_emulate_ctxt *ctxt); int (*set_xcr)(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr); }; -typedef u32 __attribute__((vector_size(16))) sse128_t; - /* Type, address-of, and value of an instruction's operand. */ struct operand { enum { OP_REG, OP_MEM, OP_MEM_STR, OP_IMM, OP_XMM, OP_MM, OP_NONE } type; diff --git a/arch/x86/kvm/kvm_onhyperv.c b/arch/x86/kvm/kvm_onhyperv.c new file mode 100644 index 000000000000..c7db2df50a7a --- /dev/null +++ b/arch/x86/kvm/kvm_onhyperv.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM L1 hypervisor optimizations on Hyper-V. + */ + +#include +#include + +#include "hyperv.h" +#include "kvm_onhyperv.h" + +static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush, + void *data) +{ + struct kvm_tlb_range *range = data; + + return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn, + range->pages); +} + +static inline int hv_remote_flush_root_tdp(hpa_t root_tdp, + struct kvm_tlb_range *range) +{ + if (range) + return hyperv_flush_guest_mapping_range(root_tdp, + kvm_fill_hv_flush_list_func, (void *)range); + else + return hyperv_flush_guest_mapping(root_tdp); +} + +int hv_remote_flush_tlb_with_range(struct kvm *kvm, + struct kvm_tlb_range *range) +{ + struct kvm_arch *kvm_arch = &kvm->arch; + struct kvm_vcpu *vcpu; + int ret = 0, i, nr_unique_valid_roots; + hpa_t root; + + spin_lock(&kvm_arch->hv_root_tdp_lock); + + if (!VALID_PAGE(kvm_arch->hv_root_tdp)) { + nr_unique_valid_roots = 0; + + /* + * Flush all valid roots, and see if all vCPUs have converged + * on a common root, in which case future flushes can skip the + * loop and flush the common root. + */ + kvm_for_each_vcpu(i, vcpu, kvm) { + root = vcpu->arch.hv_root_tdp; + if (!VALID_PAGE(root) || root == kvm_arch->hv_root_tdp) + continue; + + /* + * Set the tracked root to the first valid root. Keep + * this root for the entirety of the loop even if more + * roots are encountered as a low effort optimization + * to avoid flushing the same (first) root again. + */ + if (++nr_unique_valid_roots == 1) + kvm_arch->hv_root_tdp = root; + + if (!ret) + ret = hv_remote_flush_root_tdp(root, range); + + /* + * Stop processing roots if a failure occurred and + * multiple valid roots have already been detected. + */ + if (ret && nr_unique_valid_roots > 1) + break; + } + + /* + * The optimized flush of a single root can't be used if there + * are multiple valid roots (obviously). + */ + if (nr_unique_valid_roots > 1) + kvm_arch->hv_root_tdp = INVALID_PAGE; + } else { + ret = hv_remote_flush_root_tdp(kvm_arch->hv_root_tdp, range); + } + + spin_unlock(&kvm_arch->hv_root_tdp_lock); + return ret; +} +EXPORT_SYMBOL_GPL(hv_remote_flush_tlb_with_range); + +int hv_remote_flush_tlb(struct kvm *kvm) +{ + return hv_remote_flush_tlb_with_range(kvm, NULL); +} +EXPORT_SYMBOL_GPL(hv_remote_flush_tlb); diff --git a/arch/x86/kvm/kvm_onhyperv.h b/arch/x86/kvm/kvm_onhyperv.h new file mode 100644 index 000000000000..1c67abf2eba9 --- /dev/null +++ b/arch/x86/kvm/kvm_onhyperv.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * KVM L1 hypervisor optimizations on Hyper-V. + */ + +#ifndef __ARCH_X86_KVM_KVM_ONHYPERV_H__ +#define __ARCH_X86_KVM_KVM_ONHYPERV_H__ + +#if IS_ENABLED(CONFIG_HYPERV) +int hv_remote_flush_tlb_with_range(struct kvm *kvm, + struct kvm_tlb_range *range); +int hv_remote_flush_tlb(struct kvm *kvm); + +static inline void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp) +{ + struct kvm_arch *kvm_arch = &vcpu->kvm->arch; + + if (kvm_x86_ops.tlb_remote_flush == hv_remote_flush_tlb) { + spin_lock(&kvm_arch->hv_root_tdp_lock); + vcpu->arch.hv_root_tdp = root_tdp; + if (root_tdp != kvm_arch->hv_root_tdp) + kvm_arch->hv_root_tdp = INVALID_PAGE; + spin_unlock(&kvm_arch->hv_root_tdp_lock); + } +} +#else /* !CONFIG_HYPERV */ +static inline void hv_track_root_tdp(struct kvm_vcpu *vcpu, hpa_t root_tdp) +{ +} +#endif /* !CONFIG_HYPERV */ + +#endif diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 17fa4ab1b834..ba5a27879f1d 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -2631,6 +2631,7 @@ int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s) apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0)); update_divide_count(apic); __start_apic_timer(apic, APIC_TMCCT); + kvm_lapic_set_reg(apic, APIC_TMCCT, 0); kvm_apic_update_apicv(vcpu); apic->highest_isr_cache = -1; if (vcpu->arch.apicv_active) { @@ -2872,7 +2873,7 @@ int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len) return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len); } -void kvm_apic_accept_events(struct kvm_vcpu *vcpu) +int kvm_apic_accept_events(struct kvm_vcpu *vcpu) { struct kvm_lapic *apic = vcpu->arch.apic; u8 sipi_vector; @@ -2880,7 +2881,7 @@ void kvm_apic_accept_events(struct kvm_vcpu *vcpu) unsigned long pe; if (!lapic_in_kernel(vcpu)) - return; + return 0; /* * Read pending events before calling the check_events @@ -2888,12 +2889,12 @@ void kvm_apic_accept_events(struct kvm_vcpu *vcpu) */ pe = smp_load_acquire(&apic->pending_events); if (!pe) - return; + return 0; if (is_guest_mode(vcpu)) { r = kvm_check_nested_events(vcpu); if (r < 0) - return; + return r == -EBUSY ? 0 : r; /* * If an event has happened and caused a vmexit, * we know INITs are latched and therefore @@ -2914,7 +2915,7 @@ void kvm_apic_accept_events(struct kvm_vcpu *vcpu) WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED); if (test_bit(KVM_APIC_SIPI, &pe)) clear_bit(KVM_APIC_SIPI, &apic->pending_events); - return; + return 0; } if (test_bit(KVM_APIC_INIT, &pe)) { @@ -2935,6 +2936,7 @@ void kvm_apic_accept_events(struct kvm_vcpu *vcpu) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; } } + return 0; } void kvm_lapic_exit(void) diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 997c45a5963a..d7c25d0c1354 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -76,7 +76,7 @@ void kvm_free_lapic(struct kvm_vcpu *vcpu); int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu); int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu); int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu); -void kvm_apic_accept_events(struct kvm_vcpu *vcpu); +int kvm_apic_accept_events(struct kvm_vcpu *vcpu); void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event); u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu); void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8); diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index 88d0ed5225a4..83e6c6965f1e 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -44,6 +44,12 @@ #define PT32_ROOT_LEVEL 2 #define PT32E_ROOT_LEVEL 3 +#define KVM_MMU_CR4_ROLE_BITS (X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | \ + X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE | \ + X86_CR4_LA57) + +#define KVM_MMU_CR0_ROLE_BITS (X86_CR0_PG | X86_CR0_WP) + static __always_inline u64 rsvd_bits(int s, int e) { BUILD_BUG_ON(__builtin_constant_p(e) && __builtin_constant_p(s) && e < s); @@ -62,12 +68,9 @@ static __always_inline u64 rsvd_bits(int s, int e) void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask); void kvm_mmu_set_ept_masks(bool has_ad_bits, bool has_exec_only); -void -reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context); - -void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots); -void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer, - gpa_t nested_cr3); +void kvm_init_mmu(struct kvm_vcpu *vcpu); +void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, + unsigned long cr4, u64 efer, gpa_t nested_cr3); void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, bool accessed_dirty, gpa_t new_eptp); bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu); @@ -162,11 +165,6 @@ static inline bool is_writable_pte(unsigned long pte) return pte & PT_WRITABLE_MASK; } -static inline bool is_write_protection(struct kvm_vcpu *vcpu) -{ - return kvm_read_cr0_bits(vcpu, X86_CR0_WP); -} - /* * Check if a given access (described through the I/D, W/R and U/S bits of a * page fault error code pfec) causes a permission fault with the given PTE @@ -232,4 +230,14 @@ int kvm_arch_write_log_dirty(struct kvm_vcpu *vcpu); int kvm_mmu_post_init_vm(struct kvm *kvm); void kvm_mmu_pre_destroy_vm(struct kvm *kvm); +static inline bool kvm_memslots_have_rmaps(struct kvm *kvm) +{ + /* + * Read memslot_have_rmaps before rmap pointers. Hence, threads reading + * memslots_have_rmaps in any lock context are guaranteed to see the + * pointers. Pairs with smp_store_release in alloc_all_memslots_rmaps. + */ + return smp_load_acquire(&kvm->arch.memslots_have_rmaps); +} + #endif diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 8d5876dfc6b7..845d114ae075 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -55,7 +55,7 @@ extern bool itlb_multihit_kvm_mitigation; -static int __read_mostly nx_huge_pages = -1; +int __read_mostly nx_huge_pages = -1; #ifdef CONFIG_PREEMPT_RT /* Recovery can cause latency spikes, disable it for PREEMPT_RT. */ static uint __read_mostly nx_huge_pages_recovery_ratio = 0; @@ -176,9 +176,80 @@ static void mmu_spte_set(u64 *sptep, u64 spte); static union kvm_mmu_page_role kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu); +struct kvm_mmu_role_regs { + const unsigned long cr0; + const unsigned long cr4; + const u64 efer; +}; + #define CREATE_TRACE_POINTS #include "mmutrace.h" +/* + * Yes, lot's of underscores. They're a hint that you probably shouldn't be + * reading from the role_regs. Once the mmu_role is constructed, it becomes + * the single source of truth for the MMU's state. + */ +#define BUILD_MMU_ROLE_REGS_ACCESSOR(reg, name, flag) \ +static inline bool ____is_##reg##_##name(struct kvm_mmu_role_regs *regs)\ +{ \ + return !!(regs->reg & flag); \ +} +BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, pg, X86_CR0_PG); +BUILD_MMU_ROLE_REGS_ACCESSOR(cr0, wp, X86_CR0_WP); +BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pse, X86_CR4_PSE); +BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pae, X86_CR4_PAE); +BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smep, X86_CR4_SMEP); +BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, smap, X86_CR4_SMAP); +BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, pke, X86_CR4_PKE); +BUILD_MMU_ROLE_REGS_ACCESSOR(cr4, la57, X86_CR4_LA57); +BUILD_MMU_ROLE_REGS_ACCESSOR(efer, nx, EFER_NX); +BUILD_MMU_ROLE_REGS_ACCESSOR(efer, lma, EFER_LMA); + +/* + * The MMU itself (with a valid role) is the single source of truth for the + * MMU. Do not use the regs used to build the MMU/role, nor the vCPU. The + * regs don't account for dependencies, e.g. clearing CR4 bits if CR0.PG=1, + * and the vCPU may be incorrect/irrelevant. + */ +#define BUILD_MMU_ROLE_ACCESSOR(base_or_ext, reg, name) \ +static inline bool is_##reg##_##name(struct kvm_mmu *mmu) \ +{ \ + return !!(mmu->mmu_role. base_or_ext . reg##_##name); \ +} +BUILD_MMU_ROLE_ACCESSOR(ext, cr0, pg); +BUILD_MMU_ROLE_ACCESSOR(base, cr0, wp); +BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pse); +BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pae); +BUILD_MMU_ROLE_ACCESSOR(ext, cr4, smep); +BUILD_MMU_ROLE_ACCESSOR(ext, cr4, smap); +BUILD_MMU_ROLE_ACCESSOR(ext, cr4, pke); +BUILD_MMU_ROLE_ACCESSOR(ext, cr4, la57); +BUILD_MMU_ROLE_ACCESSOR(base, efer, nx); + +static struct kvm_mmu_role_regs vcpu_to_role_regs(struct kvm_vcpu *vcpu) +{ + struct kvm_mmu_role_regs regs = { + .cr0 = kvm_read_cr0_bits(vcpu, KVM_MMU_CR0_ROLE_BITS), + .cr4 = kvm_read_cr4_bits(vcpu, KVM_MMU_CR4_ROLE_BITS), + .efer = vcpu->arch.efer, + }; + + return regs; +} + +static int role_regs_to_root_level(struct kvm_mmu_role_regs *regs) +{ + if (!____is_cr0_pg(regs)) + return 0; + else if (____is_efer_lma(regs)) + return ____is_cr4_la57(regs) ? PT64_ROOT_5LEVEL : + PT64_ROOT_4LEVEL; + else if (____is_cr4_pae(regs)) + return PT32E_ROOT_LEVEL; + else + return PT32_ROOT_LEVEL; +} static inline bool kvm_available_flush_tlb_with_range(void) { @@ -208,11 +279,6 @@ void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, kvm_flush_remote_tlbs_with_range(kvm, &range); } -bool is_nx_huge_page_enabled(void) -{ - return READ_ONCE(nx_huge_pages); -} - static void mark_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, u64 gfn, unsigned int access) { @@ -269,11 +335,6 @@ static int is_cpuid_PSE36(void) return 1; } -static int is_nx(struct kvm_vcpu *vcpu) -{ - return vcpu->arch.efer & EFER_NX; -} - static gfn_t pse36_gfn_delta(u32 gpte) { int shift = 32 - PT32_DIR_PSE36_SHIFT - PAGE_SHIFT; @@ -1177,8 +1238,7 @@ static bool __rmap_clear_dirty(struct kvm *kvm, struct kvm_rmap_head *rmap_head, * @gfn_offset: start of the BITS_PER_LONG pages we care about * @mask: indicates which pages we should protect * - * Used when we do not need to care about huge page mappings: e.g. during dirty - * logging we do not have any such mappings. + * Used when we do not need to care about huge page mappings. */ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, @@ -1189,6 +1249,10 @@ static void kvm_mmu_write_protect_pt_masked(struct kvm *kvm, if (is_tdp_mmu_enabled(kvm)) kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot, slot->base_gfn + gfn_offset, mask, true); + + if (!kvm_memslots_have_rmaps(kvm)) + return; + while (mask) { rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), PG_LEVEL_4K, slot); @@ -1218,6 +1282,10 @@ static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, if (is_tdp_mmu_enabled(kvm)) kvm_tdp_mmu_clear_dirty_pt_masked(kvm, slot, slot->base_gfn + gfn_offset, mask, false); + + if (!kvm_memslots_have_rmaps(kvm)) + return; + while (mask) { rmap_head = __gfn_to_rmap(slot->base_gfn + gfn_offset + __ffs(mask), PG_LEVEL_4K, slot); @@ -1235,13 +1303,36 @@ static void kvm_mmu_clear_dirty_pt_masked(struct kvm *kvm, * It calls kvm_mmu_write_protect_pt_masked to write protect selected pages to * enable dirty logging for them. * - * Used when we do not need to care about huge page mappings: e.g. during dirty - * logging we do not have any such mappings. + * We need to care about huge page mappings: e.g. during dirty logging we may + * have such mappings. */ void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm, struct kvm_memory_slot *slot, gfn_t gfn_offset, unsigned long mask) { + /* + * Huge pages are NOT write protected when we start dirty logging in + * initially-all-set mode; must write protect them here so that they + * are split to 4K on the first write. + * + * The gfn_offset is guaranteed to be aligned to 64, but the base_gfn + * of memslot has no such restriction, so the range can cross two large + * pages. + */ + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) { + gfn_t start = slot->base_gfn + gfn_offset + __ffs(mask); + gfn_t end = slot->base_gfn + gfn_offset + __fls(mask); + + kvm_mmu_slot_gfn_write_protect(kvm, slot, start, PG_LEVEL_2M); + + /* Cross two large pages? */ + if (ALIGN(start << PAGE_SHIFT, PMD_SIZE) != + ALIGN(end << PAGE_SHIFT, PMD_SIZE)) + kvm_mmu_slot_gfn_write_protect(kvm, slot, end, + PG_LEVEL_2M); + } + + /* Now handle 4K PTEs. */ if (kvm_x86_ops.cpu_dirty_log_size) kvm_mmu_clear_dirty_pt_masked(kvm, slot, gfn_offset, mask); else @@ -1254,20 +1345,23 @@ int kvm_cpu_dirty_log_size(void) } bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, - struct kvm_memory_slot *slot, u64 gfn) + struct kvm_memory_slot *slot, u64 gfn, + int min_level) { struct kvm_rmap_head *rmap_head; int i; bool write_protected = false; - for (i = PG_LEVEL_4K; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { - rmap_head = __gfn_to_rmap(gfn, i, slot); - write_protected |= __rmap_write_protect(kvm, rmap_head, true); + if (kvm_memslots_have_rmaps(kvm)) { + for (i = min_level; i <= KVM_MAX_HUGEPAGE_LEVEL; ++i) { + rmap_head = __gfn_to_rmap(gfn, i, slot); + write_protected |= __rmap_write_protect(kvm, rmap_head, true); + } } if (is_tdp_mmu_enabled(kvm)) write_protected |= - kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn); + kvm_tdp_mmu_write_protect_gfn(kvm, slot, gfn, min_level); return write_protected; } @@ -1277,7 +1371,7 @@ static bool rmap_write_protect(struct kvm_vcpu *vcpu, u64 gfn) struct kvm_memory_slot *slot; slot = kvm_vcpu_gfn_to_memslot(vcpu, gfn); - return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn); + return kvm_mmu_slot_gfn_write_protect(vcpu->kvm, slot, gfn, PG_LEVEL_4K); } static bool kvm_zap_rmapp(struct kvm *kvm, struct kvm_rmap_head *rmap_head, @@ -1433,9 +1527,10 @@ static __always_inline bool kvm_handle_gfn_range(struct kvm *kvm, bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) { - bool flush; + bool flush = false; - flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp); + if (kvm_memslots_have_rmaps(kvm)) + flush = kvm_handle_gfn_range(kvm, range, kvm_unmap_rmapp); if (is_tdp_mmu_enabled(kvm)) flush |= kvm_tdp_mmu_unmap_gfn_range(kvm, range, flush); @@ -1445,9 +1540,10 @@ bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range) bool kvm_set_spte_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - bool flush; + bool flush = false; - flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmapp); + if (kvm_memslots_have_rmaps(kvm)) + flush = kvm_handle_gfn_range(kvm, range, kvm_set_pte_rmapp); if (is_tdp_mmu_enabled(kvm)) flush |= kvm_tdp_mmu_set_spte_gfn(kvm, range); @@ -1500,9 +1596,10 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn) bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - bool young; + bool young = false; - young = kvm_handle_gfn_range(kvm, range, kvm_age_rmapp); + if (kvm_memslots_have_rmaps(kvm)) + young = kvm_handle_gfn_range(kvm, range, kvm_age_rmapp); if (is_tdp_mmu_enabled(kvm)) young |= kvm_tdp_mmu_age_gfn_range(kvm, range); @@ -1512,9 +1609,10 @@ bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range) { - bool young; + bool young = false; - young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmapp); + if (kvm_memslots_have_rmaps(kvm)) + young = kvm_handle_gfn_range(kvm, range, kvm_test_age_rmapp); if (is_tdp_mmu_enabled(kvm)) young |= kvm_tdp_mmu_test_age_gfn(kvm, range); @@ -1748,17 +1846,10 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm, &(_kvm)->arch.mmu_page_hash[kvm_page_table_hashfn(_gfn)]) \ if ((_sp)->gfn != (_gfn) || (_sp)->role.direct) {} else -static inline bool is_ept_sp(struct kvm_mmu_page *sp) +static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, + struct list_head *invalid_list) { - return sp->role.cr0_wp && sp->role.smap_andnot_wp; -} - -/* @sp->gfn should be write-protected at the call site */ -static bool __kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, - struct list_head *invalid_list) -{ - if ((!is_ept_sp(sp) && sp->role.gpte_is_8_bytes != !!is_pae(vcpu)) || - vcpu->arch.mmu->sync_page(vcpu, sp) == 0) { + if (vcpu->arch.mmu->sync_page(vcpu, sp) == 0) { kvm_mmu_prepare_zap_page(vcpu->kvm, sp, invalid_list); return false; } @@ -1804,31 +1895,6 @@ static bool is_obsolete_sp(struct kvm *kvm, struct kvm_mmu_page *sp) unlikely(sp->mmu_valid_gen != kvm->arch.mmu_valid_gen); } -static bool kvm_sync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, - struct list_head *invalid_list) -{ - kvm_unlink_unsync_page(vcpu->kvm, sp); - return __kvm_sync_page(vcpu, sp, invalid_list); -} - -/* @gfn should be write-protected at the call site */ -static bool kvm_sync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, - struct list_head *invalid_list) -{ - struct kvm_mmu_page *s; - bool ret = false; - - for_each_gfn_indirect_valid_sp(vcpu->kvm, s, gfn) { - if (!s->unsync) - continue; - - WARN_ON(s->role.level != PG_LEVEL_4K); - ret |= kvm_sync_page(vcpu, s, invalid_list); - } - - return ret; -} - struct mmu_page_path { struct kvm_mmu_page *parent[PT64_ROOT_MAX_LEVEL]; unsigned int idx[PT64_ROOT_MAX_LEVEL]; @@ -1923,6 +1989,7 @@ static void mmu_sync_children(struct kvm_vcpu *vcpu, } for_each_sp(pages, sp, parents, i) { + kvm_unlink_unsync_page(vcpu->kvm, sp); flush |= kvm_sync_page(vcpu, sp, &invalid_list); mmu_pages_clear_parents(&parents); } @@ -1958,8 +2025,6 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, struct hlist_head *sp_list; unsigned quadrant; struct kvm_mmu_page *sp; - bool need_sync = false; - bool flush = false; int collisions = 0; LIST_HEAD(invalid_list); @@ -1982,20 +2047,39 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, continue; } - if (!need_sync && sp->unsync) - need_sync = true; - - if (sp->role.word != role.word) + if (sp->role.word != role.word) { + /* + * If the guest is creating an upper-level page, zap + * unsync pages for the same gfn. While it's possible + * the guest is using recursive page tables, in all + * likelihood the guest has stopped using the unsync + * page and is installing a completely unrelated page. + * Unsync pages must not be left as is, because the new + * upper-level page will be write-protected. + */ + if (level > PG_LEVEL_4K && sp->unsync) + kvm_mmu_prepare_zap_page(vcpu->kvm, sp, + &invalid_list); continue; + } if (direct_mmu) goto trace_get_page; if (sp->unsync) { - /* The page is good, but __kvm_sync_page might still end - * up zapping it. If so, break in order to rebuild it. + /* + * The page is good, but is stale. kvm_sync_page does + * get the latest guest state, but (unlike mmu_unsync_children) + * it doesn't write-protect the page or mark it synchronized! + * This way the validity of the mapping is ensured, but the + * overhead of write protection is not incurred until the + * guest invalidates the TLB mapping. This allows multiple + * SPs for a single gfn to be unsync. + * + * If the sync fails, the page is zapped. If so, break + * in order to rebuild it. */ - if (!__kvm_sync_page(vcpu, sp, &invalid_list)) + if (!kvm_sync_page(vcpu, sp, &invalid_list)) break; WARN_ON(!list_empty(&invalid_list)); @@ -2020,22 +2104,14 @@ static struct kvm_mmu_page *kvm_mmu_get_page(struct kvm_vcpu *vcpu, sp->role = role; hlist_add_head(&sp->hash_link, sp_list); if (!direct) { - /* - * we should do write protection before syncing pages - * otherwise the content of the synced shadow page may - * be inconsistent with guest page table. - */ account_shadowed(vcpu->kvm, sp); if (level == PG_LEVEL_4K && rmap_write_protect(vcpu, gfn)) kvm_flush_remote_tlbs_with_address(vcpu->kvm, gfn, 1); - - if (level > PG_LEVEL_4K && need_sync) - flush |= kvm_sync_pages(vcpu, gfn, &invalid_list); } trace_kvm_mmu_get_page(sp, true); - - kvm_mmu_flush_or_zap(vcpu, &invalid_list, false, flush); out: + kvm_mmu_commit_zap_page(vcpu->kvm, &invalid_list); + if (collisions > vcpu->kvm->stat.max_mmu_page_hash_collisions) vcpu->kvm->stat.max_mmu_page_hash_collisions = collisions; return sp; @@ -2374,7 +2450,7 @@ static int make_mmu_pages_available(struct kvm_vcpu *vcpu) * page is available, while the caller may end up allocating as many as * four pages, e.g. for PAE roots or for 5-level paging. Temporarily * exceeding the (arbitrary by default) limit will not harm the host, - * being too agressive may unnecessarily kill the guest, and getting an + * being too aggressive may unnecessarily kill the guest, and getting an * exact count is far more trouble than it's worth, especially in the * page fault paths. */ @@ -2448,17 +2524,33 @@ static void kvm_unsync_page(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) kvm_mmu_mark_parents_unsync(sp); } -bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, - bool can_unsync) +/* + * Attempt to unsync any shadow pages that can be reached by the specified gfn, + * KVM is creating a writable mapping for said gfn. Returns 0 if all pages + * were marked unsync (or if there is no shadow page), -EPERM if the SPTE must + * be write-protected. + */ +int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync) { struct kvm_mmu_page *sp; + /* + * Force write-protection if the page is being tracked. Note, the page + * track machinery is used to write-protect upper-level shadow pages, + * i.e. this guards the role.level == 4K assertion below! + */ if (kvm_page_track_is_active(vcpu, gfn, KVM_PAGE_TRACK_WRITE)) - return true; + return -EPERM; + /* + * The page is not write-tracked, mark existing shadow pages unsync + * unless KVM is synchronizing an unsync SP (can_unsync = false). In + * that case, KVM must complete emulation of the guest TLB flush before + * allowing shadow pages to become unsync (writable by the guest). + */ for_each_gfn_indirect_valid_sp(vcpu->kvm, sp, gfn) { if (!can_unsync) - return true; + return -EPERM; if (sp->unsync) continue; @@ -2489,8 +2581,8 @@ bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, * 2.2 Guest issues TLB flush. * That causes a VM Exit. * - * 2.3 kvm_mmu_sync_pages() reads sp->unsync. - * Since it is false, so it just returns. + * 2.3 Walking of unsync pages sees sp->unsync is + * false and skips the page. * * 2.4 Guest accesses GVA X. * Since the mapping in the SP was not updated, @@ -2506,7 +2598,7 @@ bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, */ smp_wmb(); - return false; + return 0; } static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep, @@ -2827,9 +2919,6 @@ static int __direct_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, gfn_t gfn = gpa >> PAGE_SHIFT; gfn_t base_gfn = gfn; - if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) - return RET_PF_RETRY; - level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn, huge_page_disallowed, &req_level); @@ -3180,6 +3269,33 @@ void kvm_mmu_free_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, } EXPORT_SYMBOL_GPL(kvm_mmu_free_roots); +void kvm_mmu_free_guest_mode_roots(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) +{ + unsigned long roots_to_free = 0; + hpa_t root_hpa; + int i; + + /* + * This should not be called while L2 is active, L2 can't invalidate + * _only_ its own roots, e.g. INVVPID unconditionally exits. + */ + WARN_ON_ONCE(mmu->mmu_role.base.guest_mode); + + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) { + root_hpa = mmu->prev_roots[i].hpa; + if (!VALID_PAGE(root_hpa)) + continue; + + if (!to_shadow_page(root_hpa) || + to_shadow_page(root_hpa)->role.guest_mode) + roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); + } + + kvm_mmu_free_roots(vcpu, mmu, roots_to_free); +} +EXPORT_SYMBOL_GPL(kvm_mmu_free_guest_mode_roots); + + static int mmu_check_root(struct kvm_vcpu *vcpu, gfn_t root_gfn) { int ret = 0; @@ -3280,6 +3396,10 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu) } } + r = alloc_all_memslots_rmaps(vcpu->kvm); + if (r) + return r; + write_lock(&vcpu->kvm->mmu_lock); r = make_mmu_pages_available(vcpu); if (r < 0) @@ -3423,8 +3543,8 @@ void kvm_mmu_sync_roots(struct kvm_vcpu *vcpu) * flush strictly after those changes are made. We only need to * ensure that the other CPU sets these flags before any actual * changes to the page tables are made. The comments in - * mmu_need_write_protect() describe what could go wrong if this - * requirement isn't satisfied. + * mmu_try_to_unsync_pages() describe what could go wrong if + * this requirement isn't satisfied. */ if (!smp_load_acquire(&sp->unsync) && !smp_load_acquire(&sp->unsync_children)) @@ -3474,19 +3594,6 @@ static gpa_t nonpaging_gva_to_gpa_nested(struct kvm_vcpu *vcpu, gpa_t vaddr, return vcpu->arch.nested_mmu.translate_gpa(vcpu, vaddr, access, exception); } -static bool -__is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, u64 pte, int level) -{ - int bit7 = (pte >> 7) & 1; - - return pte & rsvd_check->rsvd_bits_mask[bit7][level-1]; -} - -static bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check, u64 pte) -{ - return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f); -} - static bool mmio_info_in_cache(struct kvm_vcpu *vcpu, u64 addr, bool direct) { /* @@ -3540,12 +3647,7 @@ static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) int root, leaf, level; bool reserved = false; - if (!VALID_PAGE(vcpu->arch.mmu->root_hpa)) { - *sptep = 0ull; - return reserved; - } - - if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) + if (is_tdp_mmu(vcpu->arch.mmu)) leaf = kvm_tdp_mmu_get_walk(vcpu, addr, sptes, &root); else leaf = get_walk(vcpu, addr, sptes, &root); @@ -3569,13 +3671,7 @@ static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) rsvd_check = &vcpu->arch.mmu->shadow_zero_check; for (level = root; level >= leaf; level--) - /* - * Use a bitwise-OR instead of a logical-OR to aggregate the - * reserved bit and EPT's invalid memtype/XWR checks to avoid - * adding a Jcc in the loop. - */ - reserved |= __is_bad_mt_xwr(rsvd_check, sptes[level]) | - __is_rsvd_bits_set(rsvd_check, sptes[level], level); + reserved |= is_rsvd_spte(rsvd_check, sptes[level], level); if (reserved) { pr_err("%s: reserved bits set on MMU-present spte, addr 0x%llx, hierarchy:\n", @@ -3583,7 +3679,7 @@ static bool get_mmio_spte(struct kvm_vcpu *vcpu, u64 addr, u64 *sptep) for (level = root; level >= leaf; level--) pr_err("------ spte = 0x%llx level = %d, rsvd bits = 0x%llx", sptes[level], level, - rsvd_check->rsvd_bits_mask[(sptes[level] >> 7) & 1][level-1]); + get_rsvd_bits(rsvd_check, sptes[level], level)); } return reserved; @@ -3717,6 +3813,7 @@ static bool try_async_pf(struct kvm_vcpu *vcpu, bool prefault, gfn_t gfn, static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, bool prefault, int max_level, bool is_tdp) { + bool is_tdp_mmu_fault = is_tdp_mmu(vcpu->arch.mmu); bool write = error_code & PFERR_WRITE_MASK; bool map_writable; @@ -3729,7 +3826,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, if (page_fault_handle_page_track(vcpu, error_code, gfn)) return RET_PF_EMULATE; - if (!is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) { + if (!is_tdp_mmu_fault) { r = fast_page_fault(vcpu, gpa, error_code); if (r != RET_PF_INVALID) return r; @@ -3751,7 +3848,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, r = RET_PF_RETRY; - if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) + if (is_tdp_mmu_fault) read_lock(&vcpu->kvm->mmu_lock); else write_lock(&vcpu->kvm->mmu_lock); @@ -3762,7 +3859,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, if (r) goto out_unlock; - if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) + if (is_tdp_mmu_fault) r = kvm_tdp_mmu_map(vcpu, gpa, error_code, map_writable, max_level, pfn, prefault); else @@ -3770,7 +3867,7 @@ static int direct_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, prefault, is_tdp); out_unlock: - if (is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa)) + if (is_tdp_mmu_fault) read_unlock(&vcpu->kvm->mmu_lock); else write_unlock(&vcpu->kvm->mmu_lock); @@ -3840,17 +3937,13 @@ int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, max_level, true); } -static void nonpaging_init_context(struct kvm_vcpu *vcpu, - struct kvm_mmu *context) +static void nonpaging_init_context(struct kvm_mmu *context) { context->page_fault = nonpaging_page_fault; context->gva_to_gpa = nonpaging_gva_to_gpa; context->sync_page = nonpaging_sync_page; context->invlpg = NULL; - context->root_level = 0; - context->shadow_root_level = PT32E_ROOT_LEVEL; context->direct_map = true; - context->nx = false; } static inline bool is_root_usable(struct kvm_mmu_root_info *root, gpa_t pgd, @@ -3913,8 +4006,7 @@ static bool fast_pgd_switch(struct kvm_vcpu *vcpu, gpa_t new_pgd, } static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, - union kvm_mmu_page_role new_role, - bool skip_tlb_flush, bool skip_mmu_sync) + union kvm_mmu_page_role new_role) { if (!fast_pgd_switch(vcpu, new_pgd, new_role)) { kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, KVM_MMU_ROOT_CURRENT); @@ -3929,10 +4021,10 @@ static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, */ kvm_make_request(KVM_REQ_LOAD_MMU_PGD, vcpu); - if (!skip_mmu_sync || force_flush_and_sync_on_reuse) + if (force_flush_and_sync_on_reuse) { kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); - if (!skip_tlb_flush || force_flush_and_sync_on_reuse) kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + } /* * The last MMIO access's GVA and GPA are cached in the VCPU. When @@ -3951,11 +4043,9 @@ static void __kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, to_shadow_page(vcpu->arch.mmu->root_hpa)); } -void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd, bool skip_tlb_flush, - bool skip_mmu_sync) +void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd) { - __kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu), - skip_tlb_flush, skip_mmu_sync); + __kvm_mmu_new_pgd(vcpu, new_pgd, kvm_mmu_calc_root_page_role(vcpu)); } EXPORT_SYMBOL_GPL(kvm_mmu_new_pgd); @@ -3981,26 +4071,6 @@ static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64 *sptep, gfn_t gfn, return false; } -static inline bool is_last_gpte(struct kvm_mmu *mmu, - unsigned level, unsigned gpte) -{ - /* - * The RHS has bit 7 set iff level < mmu->last_nonleaf_level. - * If it is clear, there are no large pages at this level, so clear - * PT_PAGE_SIZE_MASK in gpte if that is the case. - */ - gpte &= level - mmu->last_nonleaf_level; - - /* - * PG_LEVEL_4K always terminates. The RHS has bit 7 set - * iff level <= PG_LEVEL_4K, which for our purpose means - * level == PG_LEVEL_4K; set PT_PAGE_SIZE_MASK in gpte then. - */ - gpte |= level - PG_LEVEL_4K - 1; - - return gpte & PT_PAGE_SIZE_MASK; -} - #define PTTYPE_EPT 18 /* arbitrary */ #define PTTYPE PTTYPE_EPT #include "paging_tmpl.h" @@ -4015,8 +4085,7 @@ static inline bool is_last_gpte(struct kvm_mmu *mmu, #undef PTTYPE static void -__reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, - struct rsvd_bits_validate *rsvd_check, +__reset_rsvds_bits_mask(struct rsvd_bits_validate *rsvd_check, u64 pa_bits_rsvd, int level, bool nx, bool gbpages, bool pse, bool amd) { @@ -4105,14 +4174,29 @@ __reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, } } +static bool guest_can_use_gbpages(struct kvm_vcpu *vcpu) +{ + /* + * If TDP is enabled, let the guest use GBPAGES if they're supported in + * hardware. The hardware page walker doesn't let KVM disable GBPAGES, + * i.e. won't treat them as reserved, and KVM doesn't redo the GVA->GPA + * walk for performance and complexity reasons. Not to mention KVM + * _can't_ solve the problem because GVA->GPA walks aren't visible to + * KVM once a TDP translation is installed. Mimic hardware behavior so + * that KVM's is at least consistent, i.e. doesn't randomly inject #PF. + */ + return tdp_enabled ? boot_cpu_has(X86_FEATURE_GBPAGES) : + guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES); +} + static void reset_rsvds_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context) { - __reset_rsvds_bits_mask(vcpu, &context->guest_rsvd_check, + __reset_rsvds_bits_mask(&context->guest_rsvd_check, vcpu->arch.reserved_gpa_bits, - context->root_level, context->nx, - guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), - is_pse(vcpu), + context->root_level, is_efer_nx(context), + guest_can_use_gbpages(vcpu), + is_cr4_pse(context), guest_cpuid_is_amd_or_hygon(vcpu)); } @@ -4165,24 +4249,32 @@ static inline u64 reserved_hpa_bits(void) * table in guest or amd nested guest, its mmu features completely * follow the features in guest. */ -void -reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context) +static void reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, + struct kvm_mmu *context) { - bool uses_nx = context->nx || - context->mmu_role.base.smep_andnot_wp; + /* + * KVM uses NX when TDP is disabled to handle a variety of scenarios, + * notably for huge SPTEs if iTLB multi-hit mitigation is enabled and + * to generate correct permissions for CR0.WP=0/CR4.SMEP=1/EFER.NX=0. + * The iTLB multi-hit workaround can be toggled at any time, so assume + * NX can be used by any non-nested shadow MMU to avoid having to reset + * MMU contexts. Note, KVM forces EFER.NX=1 when TDP is disabled. + */ + bool uses_nx = is_efer_nx(context) || !tdp_enabled; + + /* @amd adds a check on bit of SPTEs, which KVM shouldn't use anyways. */ + bool is_amd = true; + /* KVM doesn't use 2-level page tables for the shadow MMU. */ + bool is_pse = false; struct rsvd_bits_validate *shadow_zero_check; int i; - /* - * Passing "true" to the last argument is okay; it adds a check - * on bit 8 of the SPTEs which KVM doesn't use anyway. - */ + WARN_ON_ONCE(context->shadow_root_level < PT32E_ROOT_LEVEL); + shadow_zero_check = &context->shadow_zero_check; - __reset_rsvds_bits_mask(vcpu, shadow_zero_check, - reserved_hpa_bits(), + __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(), context->shadow_root_level, uses_nx, - guest_cpuid_has(vcpu, X86_FEATURE_GBPAGES), - is_pse(vcpu), true); + guest_can_use_gbpages(vcpu), is_pse, is_amd); if (!shadow_me_mask) return; @@ -4193,7 +4285,6 @@ reset_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, struct kvm_mmu *context) } } -EXPORT_SYMBOL_GPL(reset_shadow_zero_bits_mask); static inline bool boot_cpu_is_amd(void) { @@ -4215,11 +4306,10 @@ reset_tdp_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, shadow_zero_check = &context->shadow_zero_check; if (boot_cpu_is_amd()) - __reset_rsvds_bits_mask(vcpu, shadow_zero_check, - reserved_hpa_bits(), + __reset_rsvds_bits_mask(shadow_zero_check, reserved_hpa_bits(), context->shadow_root_level, false, boot_cpu_has(X86_FEATURE_GBPAGES), - true, true); + false, true); else __reset_rsvds_bits_mask_ept(shadow_zero_check, reserved_hpa_bits(), false); @@ -4255,8 +4345,7 @@ reset_ept_shadow_zero_bits_mask(struct kvm_vcpu *vcpu, (7 & (access) ? 128 : 0)) -static void update_permission_bitmask(struct kvm_vcpu *vcpu, - struct kvm_mmu *mmu, bool ept) +static void update_permission_bitmask(struct kvm_mmu *mmu, bool ept) { unsigned byte; @@ -4264,9 +4353,10 @@ static void update_permission_bitmask(struct kvm_vcpu *vcpu, const u8 w = BYTE_MASK(ACC_WRITE_MASK); const u8 u = BYTE_MASK(ACC_USER_MASK); - bool cr4_smep = kvm_read_cr4_bits(vcpu, X86_CR4_SMEP) != 0; - bool cr4_smap = kvm_read_cr4_bits(vcpu, X86_CR4_SMAP) != 0; - bool cr0_wp = is_write_protection(vcpu); + bool cr4_smep = is_cr4_smep(mmu); + bool cr4_smap = is_cr4_smap(mmu); + bool cr0_wp = is_cr0_wp(mmu); + bool efer_nx = is_efer_nx(mmu); for (byte = 0; byte < ARRAY_SIZE(mmu->permissions); ++byte) { unsigned pfec = byte << 1; @@ -4292,7 +4382,7 @@ static void update_permission_bitmask(struct kvm_vcpu *vcpu, u8 kf = (pfec & PFERR_USER_MASK) ? 0 : u; /* Not really needed: !nx will cause pte.nx to fault */ - if (!mmu->nx) + if (!efer_nx) ff = 0; /* Allow supervisor writes if !cr0.wp */ @@ -4351,24 +4441,17 @@ static void update_permission_bitmask(struct kvm_vcpu *vcpu, * away both AD and WD. For all reads or if the last condition holds, WD * only will be masked away. */ -static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, - bool ept) +static void update_pkru_bitmask(struct kvm_mmu *mmu) { unsigned bit; bool wp; - if (ept) { + if (!is_cr4_pke(mmu)) { mmu->pkru_mask = 0; return; } - /* PKEY is enabled only if CR4.PKE and EFER.LMA are both set. */ - if (!kvm_read_cr4_bits(vcpu, X86_CR4_PKE) || !is_long_mode(vcpu)) { - mmu->pkru_mask = 0; - return; - } - - wp = is_write_protection(vcpu); + wp = is_cr0_wp(mmu); for (bit = 0; bit < ARRAY_SIZE(mmu->permissions); ++bit) { unsigned pfec, pkey_bits; @@ -4402,81 +4485,51 @@ static void update_pkru_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, } } -static void update_last_nonleaf_level(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu) +static void reset_guest_paging_metadata(struct kvm_vcpu *vcpu, + struct kvm_mmu *mmu) { - unsigned root_level = mmu->root_level; + if (!is_cr0_pg(mmu)) + return; - mmu->last_nonleaf_level = root_level; - if (root_level == PT32_ROOT_LEVEL && is_pse(vcpu)) - mmu->last_nonleaf_level++; + reset_rsvds_bits_mask(vcpu, mmu); + update_permission_bitmask(mmu, false); + update_pkru_bitmask(mmu); } -static void paging64_init_context_common(struct kvm_vcpu *vcpu, - struct kvm_mmu *context, - int level) +static void paging64_init_context(struct kvm_mmu *context) { - context->nx = is_nx(vcpu); - context->root_level = level; - - reset_rsvds_bits_mask(vcpu, context); - update_permission_bitmask(vcpu, context, false); - update_pkru_bitmask(vcpu, context, false); - update_last_nonleaf_level(vcpu, context); - - MMU_WARN_ON(!is_pae(vcpu)); context->page_fault = paging64_page_fault; context->gva_to_gpa = paging64_gva_to_gpa; context->sync_page = paging64_sync_page; context->invlpg = paging64_invlpg; - context->shadow_root_level = level; context->direct_map = false; } -static void paging64_init_context(struct kvm_vcpu *vcpu, - struct kvm_mmu *context) +static void paging32_init_context(struct kvm_mmu *context) { - int root_level = is_la57_mode(vcpu) ? - PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; - - paging64_init_context_common(vcpu, context, root_level); -} - -static void paging32_init_context(struct kvm_vcpu *vcpu, - struct kvm_mmu *context) -{ - context->nx = false; - context->root_level = PT32_ROOT_LEVEL; - - reset_rsvds_bits_mask(vcpu, context); - update_permission_bitmask(vcpu, context, false); - update_pkru_bitmask(vcpu, context, false); - update_last_nonleaf_level(vcpu, context); - context->page_fault = paging32_page_fault; context->gva_to_gpa = paging32_gva_to_gpa; context->sync_page = paging32_sync_page; context->invlpg = paging32_invlpg; - context->shadow_root_level = PT32E_ROOT_LEVEL; context->direct_map = false; } -static void paging32E_init_context(struct kvm_vcpu *vcpu, - struct kvm_mmu *context) -{ - paging64_init_context_common(vcpu, context, PT32E_ROOT_LEVEL); -} - -static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu) +static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu, + struct kvm_mmu_role_regs *regs) { union kvm_mmu_extended_role ext = {0}; - ext.cr0_pg = !!is_paging(vcpu); - ext.cr4_pae = !!is_pae(vcpu); - ext.cr4_smep = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMEP); - ext.cr4_smap = !!kvm_read_cr4_bits(vcpu, X86_CR4_SMAP); - ext.cr4_pse = !!is_pse(vcpu); - ext.cr4_pke = !!kvm_read_cr4_bits(vcpu, X86_CR4_PKE); - ext.maxphyaddr = cpuid_maxphyaddr(vcpu); + if (____is_cr0_pg(regs)) { + ext.cr0_pg = 1; + ext.cr4_pae = ____is_cr4_pae(regs); + ext.cr4_smep = ____is_cr4_smep(regs); + ext.cr4_smap = ____is_cr4_smap(regs); + ext.cr4_pse = ____is_cr4_pse(regs); + + /* PKEY and LA57 are active iff long mode is active. */ + ext.cr4_pke = ____is_efer_lma(regs) && ____is_cr4_pke(regs); + ext.cr4_la57 = ____is_efer_lma(regs) && ____is_cr4_la57(regs); + } ext.valid = 1; @@ -4484,20 +4537,23 @@ static union kvm_mmu_extended_role kvm_calc_mmu_role_ext(struct kvm_vcpu *vcpu) } static union kvm_mmu_role kvm_calc_mmu_role_common(struct kvm_vcpu *vcpu, + struct kvm_mmu_role_regs *regs, bool base_only) { union kvm_mmu_role role = {0}; role.base.access = ACC_ALL; - role.base.nxe = !!is_nx(vcpu); - role.base.cr0_wp = is_write_protection(vcpu); + if (____is_cr0_pg(regs)) { + role.base.efer_nx = ____is_efer_nx(regs); + role.base.cr0_wp = ____is_cr0_wp(regs); + } role.base.smm = is_smm(vcpu); role.base.guest_mode = is_guest_mode(vcpu); if (base_only) return role; - role.ext = kvm_calc_mmu_role_ext(vcpu); + role.ext = kvm_calc_mmu_role_ext(vcpu, regs); return role; } @@ -4512,9 +4568,10 @@ static inline int kvm_mmu_get_tdp_level(struct kvm_vcpu *vcpu) } static union kvm_mmu_role -kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) +kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, + struct kvm_mmu_role_regs *regs, bool base_only) { - union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); + union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only); role.base.ad_disabled = (shadow_accessed_mask == 0); role.base.level = kvm_mmu_get_tdp_level(vcpu); @@ -4527,8 +4584,9 @@ kvm_calc_tdp_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) { struct kvm_mmu *context = &vcpu->arch.root_mmu; + struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); union kvm_mmu_role new_role = - kvm_calc_tdp_mmu_root_page_role(vcpu, false); + kvm_calc_tdp_mmu_root_page_role(vcpu, ®s, false); if (new_role.as_u64 == context->mmu_role.as_u64) return; @@ -4542,60 +4600,44 @@ static void init_kvm_tdp_mmu(struct kvm_vcpu *vcpu) context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; + context->root_level = role_regs_to_root_level(®s); - if (!is_paging(vcpu)) { - context->nx = false; + if (!is_cr0_pg(context)) context->gva_to_gpa = nonpaging_gva_to_gpa; - context->root_level = 0; - } else if (is_long_mode(vcpu)) { - context->nx = is_nx(vcpu); - context->root_level = is_la57_mode(vcpu) ? - PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; - reset_rsvds_bits_mask(vcpu, context); + else if (is_cr4_pae(context)) context->gva_to_gpa = paging64_gva_to_gpa; - } else if (is_pae(vcpu)) { - context->nx = is_nx(vcpu); - context->root_level = PT32E_ROOT_LEVEL; - reset_rsvds_bits_mask(vcpu, context); - context->gva_to_gpa = paging64_gva_to_gpa; - } else { - context->nx = false; - context->root_level = PT32_ROOT_LEVEL; - reset_rsvds_bits_mask(vcpu, context); + else context->gva_to_gpa = paging32_gva_to_gpa; - } - update_permission_bitmask(vcpu, context, false); - update_pkru_bitmask(vcpu, context, false); - update_last_nonleaf_level(vcpu, context); + reset_guest_paging_metadata(vcpu, context); reset_tdp_shadow_zero_bits_mask(vcpu, context); } static union kvm_mmu_role -kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, bool base_only) +kvm_calc_shadow_root_page_role_common(struct kvm_vcpu *vcpu, + struct kvm_mmu_role_regs *regs, bool base_only) { - union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, base_only); + union kvm_mmu_role role = kvm_calc_mmu_role_common(vcpu, regs, base_only); - role.base.smep_andnot_wp = role.ext.cr4_smep && - !is_write_protection(vcpu); - role.base.smap_andnot_wp = role.ext.cr4_smap && - !is_write_protection(vcpu); - role.base.gpte_is_8_bytes = !!is_pae(vcpu); + role.base.smep_andnot_wp = role.ext.cr4_smep && !____is_cr0_wp(regs); + role.base.smap_andnot_wp = role.ext.cr4_smap && !____is_cr0_wp(regs); + role.base.gpte_is_8_bytes = ____is_cr0_pg(regs) && ____is_cr4_pae(regs); return role; } static union kvm_mmu_role -kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) +kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, + struct kvm_mmu_role_regs *regs, bool base_only) { union kvm_mmu_role role = - kvm_calc_shadow_root_page_role_common(vcpu, base_only); + kvm_calc_shadow_root_page_role_common(vcpu, regs, base_only); - role.base.direct = !is_paging(vcpu); + role.base.direct = !____is_cr0_pg(regs); - if (!is_long_mode(vcpu)) + if (!____is_efer_lma(regs)) role.base.level = PT32E_ROOT_LEVEL; - else if (is_la57_mode(vcpu)) + else if (____is_cr4_la57(regs)) role.base.level = PT64_ROOT_5LEVEL; else role.base.level = PT64_ROOT_4LEVEL; @@ -4604,37 +4646,44 @@ kvm_calc_shadow_mmu_root_page_role(struct kvm_vcpu *vcpu, bool base_only) } static void shadow_mmu_init_context(struct kvm_vcpu *vcpu, struct kvm_mmu *context, - u32 cr0, u32 cr4, u32 efer, + struct kvm_mmu_role_regs *regs, union kvm_mmu_role new_role) { - if (!(cr0 & X86_CR0_PG)) - nonpaging_init_context(vcpu, context); - else if (efer & EFER_LMA) - paging64_init_context(vcpu, context); - else if (cr4 & X86_CR4_PAE) - paging32E_init_context(vcpu, context); - else - paging32_init_context(vcpu, context); + if (new_role.as_u64 == context->mmu_role.as_u64) + return; context->mmu_role.as_u64 = new_role.as_u64; + + if (!is_cr0_pg(context)) + nonpaging_init_context(context); + else if (is_cr4_pae(context)) + paging64_init_context(context); + else + paging32_init_context(context); + context->root_level = role_regs_to_root_level(regs); + + reset_guest_paging_metadata(vcpu, context); + context->shadow_root_level = new_role.base.level; + reset_shadow_zero_bits_mask(vcpu, context); } -static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer) +static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu, + struct kvm_mmu_role_regs *regs) { struct kvm_mmu *context = &vcpu->arch.root_mmu; union kvm_mmu_role new_role = - kvm_calc_shadow_mmu_root_page_role(vcpu, false); + kvm_calc_shadow_mmu_root_page_role(vcpu, regs, false); - if (new_role.as_u64 != context->mmu_role.as_u64) - shadow_mmu_init_context(vcpu, context, cr0, cr4, efer, new_role); + shadow_mmu_init_context(vcpu, context, regs, new_role); } static union kvm_mmu_role -kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu) +kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu, + struct kvm_mmu_role_regs *regs) { union kvm_mmu_role role = - kvm_calc_shadow_root_page_role_common(vcpu, false); + kvm_calc_shadow_root_page_role_common(vcpu, regs, false); role.base.direct = false; role.base.level = kvm_mmu_get_tdp_level(vcpu); @@ -4642,23 +4691,22 @@ kvm_calc_shadow_npt_root_page_role(struct kvm_vcpu *vcpu) return role; } -void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, u32 cr0, u32 cr4, u32 efer, - gpa_t nested_cr3) +void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr0, + unsigned long cr4, u64 efer, gpa_t nested_cr3) { struct kvm_mmu *context = &vcpu->arch.guest_mmu; - union kvm_mmu_role new_role = kvm_calc_shadow_npt_root_page_role(vcpu); + struct kvm_mmu_role_regs regs = { + .cr0 = cr0, + .cr4 = cr4, + .efer = efer, + }; + union kvm_mmu_role new_role; - __kvm_mmu_new_pgd(vcpu, nested_cr3, new_role.base, false, false); + new_role = kvm_calc_shadow_npt_root_page_role(vcpu, ®s); - if (new_role.as_u64 != context->mmu_role.as_u64) { - shadow_mmu_init_context(vcpu, context, cr0, cr4, efer, new_role); + __kvm_mmu_new_pgd(vcpu, nested_cr3, new_role.base); - /* - * Override the level set by the common init helper, nested TDP - * always uses the host's TDP configuration. - */ - context->shadow_root_level = new_role.base.level; - } + shadow_mmu_init_context(vcpu, context, ®s, new_role); } EXPORT_SYMBOL_GPL(kvm_init_shadow_npt_mmu); @@ -4678,15 +4726,10 @@ kvm_calc_shadow_ept_root_page_role(struct kvm_vcpu *vcpu, bool accessed_dirty, role.base.guest_mode = true; role.base.access = ACC_ALL; - /* - * WP=1 and NOT_WP=1 is an impossible combination, use WP and the - * SMAP variation to denote shadow EPT entries. - */ - role.base.cr0_wp = true; - role.base.smap_andnot_wp = true; - - role.ext = kvm_calc_mmu_role_ext(vcpu); + /* EPT, and thus nested EPT, does not consume CR0, CR4, nor EFER. */ + role.ext.word = 0; role.ext.execonly = execonly; + role.ext.valid = 1; return role; } @@ -4700,14 +4743,15 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, kvm_calc_shadow_ept_root_page_role(vcpu, accessed_dirty, execonly, level); - __kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base, true, true); + __kvm_mmu_new_pgd(vcpu, new_eptp, new_role.base); if (new_role.as_u64 == context->mmu_role.as_u64) return; + context->mmu_role.as_u64 = new_role.as_u64; + context->shadow_root_level = level; - context->nx = true; context->ept_ad = accessed_dirty; context->page_fault = ept_page_fault; context->gva_to_gpa = ept_gva_to_gpa; @@ -4715,11 +4759,9 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly, context->invlpg = ept_invlpg; context->root_level = level; context->direct_map = false; - context->mmu_role.as_u64 = new_role.as_u64; - update_permission_bitmask(vcpu, context, true); - update_pkru_bitmask(vcpu, context, true); - update_last_nonleaf_level(vcpu, context); + update_permission_bitmask(context, true); + update_pkru_bitmask(context); reset_rsvds_bits_mask_ept(vcpu, context, execonly); reset_ept_shadow_zero_bits_mask(vcpu, context, execonly); } @@ -4728,20 +4770,21 @@ EXPORT_SYMBOL_GPL(kvm_init_shadow_ept_mmu); static void init_kvm_softmmu(struct kvm_vcpu *vcpu) { struct kvm_mmu *context = &vcpu->arch.root_mmu; + struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); - kvm_init_shadow_mmu(vcpu, - kvm_read_cr0_bits(vcpu, X86_CR0_PG), - kvm_read_cr4_bits(vcpu, X86_CR4_PAE), - vcpu->arch.efer); + kvm_init_shadow_mmu(vcpu, ®s); context->get_guest_pgd = get_cr3; context->get_pdptr = kvm_pdptr_read; context->inject_page_fault = kvm_inject_page_fault; } -static union kvm_mmu_role kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu) +static union kvm_mmu_role +kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu, struct kvm_mmu_role_regs *regs) { - union kvm_mmu_role role = kvm_calc_shadow_root_page_role_common(vcpu, false); + union kvm_mmu_role role; + + role = kvm_calc_shadow_root_page_role_common(vcpu, regs, false); /* * Nested MMUs are used only for walking L2's gva->gpa, they never have @@ -4749,23 +4792,14 @@ static union kvm_mmu_role kvm_calc_nested_mmu_role(struct kvm_vcpu *vcpu) * to "true" to try to detect bogus usage of the nested MMU. */ role.base.direct = true; - - if (!is_paging(vcpu)) - role.base.level = 0; - else if (is_long_mode(vcpu)) - role.base.level = is_la57_mode(vcpu) ? PT64_ROOT_5LEVEL : - PT64_ROOT_4LEVEL; - else if (is_pae(vcpu)) - role.base.level = PT32E_ROOT_LEVEL; - else - role.base.level = PT32_ROOT_LEVEL; - + role.base.level = role_regs_to_root_level(regs); return role; } static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) { - union kvm_mmu_role new_role = kvm_calc_nested_mmu_role(vcpu); + struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); + union kvm_mmu_role new_role = kvm_calc_nested_mmu_role(vcpu, ®s); struct kvm_mmu *g_context = &vcpu->arch.nested_mmu; if (new_role.as_u64 == g_context->mmu_role.as_u64) @@ -4775,6 +4809,7 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) g_context->get_guest_pgd = get_cr3; g_context->get_pdptr = kvm_pdptr_read; g_context->inject_page_fault = kvm_inject_page_fault; + g_context->root_level = new_role.base.level; /* * L2 page tables are never shadowed, so there is no need to sync @@ -4790,44 +4825,20 @@ static void init_kvm_nested_mmu(struct kvm_vcpu *vcpu) * nested page tables as the second level of translation. Basically * the gva_to_gpa functions between mmu and nested_mmu are swapped. */ - if (!is_paging(vcpu)) { - g_context->nx = false; - g_context->root_level = 0; + if (!is_paging(vcpu)) g_context->gva_to_gpa = nonpaging_gva_to_gpa_nested; - } else if (is_long_mode(vcpu)) { - g_context->nx = is_nx(vcpu); - g_context->root_level = is_la57_mode(vcpu) ? - PT64_ROOT_5LEVEL : PT64_ROOT_4LEVEL; - reset_rsvds_bits_mask(vcpu, g_context); + else if (is_long_mode(vcpu)) g_context->gva_to_gpa = paging64_gva_to_gpa_nested; - } else if (is_pae(vcpu)) { - g_context->nx = is_nx(vcpu); - g_context->root_level = PT32E_ROOT_LEVEL; - reset_rsvds_bits_mask(vcpu, g_context); + else if (is_pae(vcpu)) g_context->gva_to_gpa = paging64_gva_to_gpa_nested; - } else { - g_context->nx = false; - g_context->root_level = PT32_ROOT_LEVEL; - reset_rsvds_bits_mask(vcpu, g_context); + else g_context->gva_to_gpa = paging32_gva_to_gpa_nested; - } - update_permission_bitmask(vcpu, g_context, false); - update_pkru_bitmask(vcpu, g_context, false); - update_last_nonleaf_level(vcpu, g_context); + reset_guest_paging_metadata(vcpu, g_context); } -void kvm_init_mmu(struct kvm_vcpu *vcpu, bool reset_roots) +void kvm_init_mmu(struct kvm_vcpu *vcpu) { - if (reset_roots) { - uint i; - - vcpu->arch.mmu->root_hpa = INVALID_PAGE; - - for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) - vcpu->arch.mmu->prev_roots[i] = KVM_MMU_ROOT_INFO_INVALID; - } - if (mmu_is_nested(vcpu)) init_kvm_nested_mmu(vcpu); else if (tdp_enabled) @@ -4840,20 +4851,53 @@ EXPORT_SYMBOL_GPL(kvm_init_mmu); static union kvm_mmu_page_role kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu) { + struct kvm_mmu_role_regs regs = vcpu_to_role_regs(vcpu); union kvm_mmu_role role; if (tdp_enabled) - role = kvm_calc_tdp_mmu_root_page_role(vcpu, true); + role = kvm_calc_tdp_mmu_root_page_role(vcpu, ®s, true); else - role = kvm_calc_shadow_mmu_root_page_role(vcpu, true); + role = kvm_calc_shadow_mmu_root_page_role(vcpu, ®s, true); return role.base; } +void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu) +{ + /* + * Invalidate all MMU roles to force them to reinitialize as CPUID + * information is factored into reserved bit calculations. + */ + vcpu->arch.root_mmu.mmu_role.ext.valid = 0; + vcpu->arch.guest_mmu.mmu_role.ext.valid = 0; + vcpu->arch.nested_mmu.mmu_role.ext.valid = 0; + kvm_mmu_reset_context(vcpu); + + /* + * KVM does not correctly handle changing guest CPUID after KVM_RUN, as + * MAXPHYADDR, GBPAGES support, AMD reserved bit behavior, etc.. aren't + * tracked in kvm_mmu_page_role. As a result, KVM may miss guest page + * faults due to reusing SPs/SPTEs. Alert userspace, but otherwise + * sweep the problem under the rug. + * + * KVM's horrific CPUID ABI makes the problem all but impossible to + * solve, as correctly handling multiple vCPU models (with respect to + * paging and physical address properties) in a single VM would require + * tracking all relevant CPUID information in kvm_mmu_page_role. That + * is very undesirable as it would double the memory requirements for + * gfn_track (see struct kvm_mmu_page_role comments), and in practice + * no sane VMM mucks with the core vCPU model on the fly. + */ + if (vcpu->arch.last_vmentry_cpu != -1) { + pr_warn_ratelimited("KVM: KVM_SET_CPUID{,2} after KVM_RUN may cause guest instability\n"); + pr_warn_ratelimited("KVM: KVM_SET_CPUID{,2} will fail after KVM_RUN starting with Linux 5.16\n"); + } +} + void kvm_mmu_reset_context(struct kvm_vcpu *vcpu) { kvm_mmu_unload(vcpu); - kvm_init_mmu(vcpu, true); + kvm_init_mmu(vcpu); } EXPORT_SYMBOL_GPL(kvm_mmu_reset_context); @@ -5491,7 +5535,13 @@ void kvm_mmu_init_vm(struct kvm *kvm) { struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker; - kvm_mmu_init_tdp_mmu(kvm); + if (!kvm_mmu_init_tdp_mmu(kvm)) + /* + * No smp_load/store wrappers needed here as we are in + * VM init and there cannot be any memslots / other threads + * accessing this struct kvm yet. + */ + kvm->arch.memslots_have_rmaps = true; node->track_write = kvm_mmu_pte_write; node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot; @@ -5514,29 +5564,29 @@ void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end) int i; bool flush = false; - write_lock(&kvm->mmu_lock); - for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { - slots = __kvm_memslots(kvm, i); - kvm_for_each_memslot(memslot, slots) { - gfn_t start, end; + if (kvm_memslots_have_rmaps(kvm)) { + write_lock(&kvm->mmu_lock); + for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + slots = __kvm_memslots(kvm, i); + kvm_for_each_memslot(memslot, slots) { + gfn_t start, end; - start = max(gfn_start, memslot->base_gfn); - end = min(gfn_end, memslot->base_gfn + memslot->npages); - if (start >= end) - continue; + start = max(gfn_start, memslot->base_gfn); + end = min(gfn_end, memslot->base_gfn + memslot->npages); + if (start >= end) + continue; - flush = slot_handle_level_range(kvm, memslot, kvm_zap_rmapp, - PG_LEVEL_4K, - KVM_MAX_HUGEPAGE_LEVEL, - start, end - 1, true, flush); + flush = slot_handle_level_range(kvm, memslot, + kvm_zap_rmapp, PG_LEVEL_4K, + KVM_MAX_HUGEPAGE_LEVEL, start, + end - 1, true, flush); + } } + if (flush) + kvm_flush_remote_tlbs_with_address(kvm, gfn_start, gfn_end); + write_unlock(&kvm->mmu_lock); } - if (flush) - kvm_flush_remote_tlbs_with_address(kvm, gfn_start, gfn_end); - - write_unlock(&kvm->mmu_lock); - if (is_tdp_mmu_enabled(kvm)) { flush = false; @@ -5563,12 +5613,15 @@ void kvm_mmu_slot_remove_write_access(struct kvm *kvm, struct kvm_memory_slot *memslot, int start_level) { - bool flush; + bool flush = false; - write_lock(&kvm->mmu_lock); - flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, - start_level, KVM_MAX_HUGEPAGE_LEVEL, false); - write_unlock(&kvm->mmu_lock); + if (kvm_memslots_have_rmaps(kvm)) { + write_lock(&kvm->mmu_lock); + flush = slot_handle_level(kvm, memslot, slot_rmap_write_protect, + start_level, KVM_MAX_HUGEPAGE_LEVEL, + false); + write_unlock(&kvm->mmu_lock); + } if (is_tdp_mmu_enabled(kvm)) { read_lock(&kvm->mmu_lock); @@ -5636,18 +5689,17 @@ void kvm_mmu_zap_collapsible_sptes(struct kvm *kvm, { /* FIXME: const-ify all uses of struct kvm_memory_slot. */ struct kvm_memory_slot *slot = (struct kvm_memory_slot *)memslot; - bool flush; + bool flush = false; - write_lock(&kvm->mmu_lock); - flush = slot_handle_leaf(kvm, slot, kvm_mmu_zap_collapsible_spte, true); - - if (flush) - kvm_arch_flush_remote_tlbs_memslot(kvm, slot); - write_unlock(&kvm->mmu_lock); + if (kvm_memslots_have_rmaps(kvm)) { + write_lock(&kvm->mmu_lock); + flush = slot_handle_leaf(kvm, slot, kvm_mmu_zap_collapsible_spte, true); + if (flush) + kvm_arch_flush_remote_tlbs_memslot(kvm, slot); + write_unlock(&kvm->mmu_lock); + } if (is_tdp_mmu_enabled(kvm)) { - flush = false; - read_lock(&kvm->mmu_lock); flush = kvm_tdp_mmu_zap_collapsible_sptes(kvm, slot, flush); if (flush) @@ -5674,11 +5726,14 @@ void kvm_arch_flush_remote_tlbs_memslot(struct kvm *kvm, void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm, struct kvm_memory_slot *memslot) { - bool flush; + bool flush = false; - write_lock(&kvm->mmu_lock); - flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, false); - write_unlock(&kvm->mmu_lock); + if (kvm_memslots_have_rmaps(kvm)) { + write_lock(&kvm->mmu_lock); + flush = slot_handle_leaf(kvm, memslot, __rmap_clear_dirty, + false); + write_unlock(&kvm->mmu_lock); + } if (is_tdp_mmu_enabled(kvm)) { read_lock(&kvm->mmu_lock); @@ -5981,6 +6036,7 @@ static int set_nx_huge_pages_recovery_ratio(const char *val, const struct kernel static void kvm_recover_nx_lpages(struct kvm *kvm) { + unsigned long nx_lpage_splits = kvm->stat.nx_lpage_splits; int rcu_idx; struct kvm_mmu_page *sp; unsigned int ratio; @@ -5992,7 +6048,7 @@ static void kvm_recover_nx_lpages(struct kvm *kvm) write_lock(&kvm->mmu_lock); ratio = READ_ONCE(nx_huge_pages_recovery_ratio); - to_zap = ratio ? DIV_ROUND_UP(kvm->stat.nx_lpage_splits, ratio) : 0; + to_zap = ratio ? DIV_ROUND_UP(nx_lpage_splits, ratio) : 0; for ( ; to_zap; --to_zap) { if (list_empty(&kvm->arch.lpage_disallowed_mmu_pages)) break; diff --git a/arch/x86/kvm/mmu/mmu_internal.h b/arch/x86/kvm/mmu/mmu_internal.h index d64ccb417c60..35567293c1fd 100644 --- a/arch/x86/kvm/mmu/mmu_internal.h +++ b/arch/x86/kvm/mmu/mmu_internal.h @@ -116,14 +116,19 @@ static inline bool kvm_vcpu_ad_need_write_protect(struct kvm_vcpu *vcpu) kvm_x86_ops.cpu_dirty_log_size; } -bool is_nx_huge_page_enabled(void); -bool mmu_need_write_protect(struct kvm_vcpu *vcpu, gfn_t gfn, - bool can_unsync); +extern int nx_huge_pages; +static inline bool is_nx_huge_page_enabled(void) +{ + return READ_ONCE(nx_huge_pages); +} + +int mmu_try_to_unsync_pages(struct kvm_vcpu *vcpu, gfn_t gfn, bool can_unsync); void kvm_mmu_gfn_disallow_lpage(struct kvm_memory_slot *slot, gfn_t gfn); void kvm_mmu_gfn_allow_lpage(struct kvm_memory_slot *slot, gfn_t gfn); bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm, - struct kvm_memory_slot *slot, u64 gfn); + struct kvm_memory_slot *slot, u64 gfn, + int min_level); void kvm_flush_remote_tlbs_with_address(struct kvm *kvm, u64 start_gfn, u64 pages); @@ -158,8 +163,6 @@ int kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, gfn_t gfn, void disallowed_hugepage_adjust(u64 spte, gfn_t gfn, int cur_level, kvm_pfn_t *pfnp, int *goal_levelp); -bool is_nx_huge_page_enabled(void); - void *mmu_memory_cache_alloc(struct kvm_mmu_memory_cache *mc); void account_huge_nx_page(struct kvm *kvm, struct kvm_mmu_page *sp); diff --git a/arch/x86/kvm/mmu/mmutrace.h b/arch/x86/kvm/mmu/mmutrace.h index e798489b56b5..efbad33a0645 100644 --- a/arch/x86/kvm/mmu/mmutrace.h +++ b/arch/x86/kvm/mmu/mmutrace.h @@ -40,7 +40,7 @@ role.direct ? " direct" : "", \ access_str[role.access], \ role.invalid ? " invalid" : "", \ - role.nxe ? "" : "!", \ + role.efer_nx ? "" : "!", \ role.ad_disabled ? "!" : "", \ __entry->root_count, \ __entry->unsync ? "unsync" : "sync", 0); \ diff --git a/arch/x86/kvm/mmu/page_track.c b/arch/x86/kvm/mmu/page_track.c index 34bb0ec69bd8..91a9f7e0fd91 100644 --- a/arch/x86/kvm/mmu/page_track.c +++ b/arch/x86/kvm/mmu/page_track.c @@ -100,7 +100,7 @@ void kvm_slot_page_track_add_page(struct kvm *kvm, kvm_mmu_gfn_disallow_lpage(slot, gfn); if (mode == KVM_PAGE_TRACK_WRITE) - if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn)) + if (kvm_mmu_slot_gfn_write_protect(kvm, slot, gfn, PG_LEVEL_4K)) kvm_flush_remote_tlbs(kvm); } EXPORT_SYMBOL_GPL(kvm_slot_page_track_add_page); diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index 823a5919f9fa..490a028ddabe 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -305,6 +305,35 @@ static inline unsigned FNAME(gpte_pkeys)(struct kvm_vcpu *vcpu, u64 gpte) return pkeys; } +static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu, + unsigned int level, unsigned int gpte) +{ + /* + * For EPT and PAE paging (both variants), bit 7 is either reserved at + * all level or indicates a huge page (ignoring CR3/EPTP). In either + * case, bit 7 being set terminates the walk. + */ +#if PTTYPE == 32 + /* + * 32-bit paging requires special handling because bit 7 is ignored if + * CR4.PSE=0, not reserved. Clear bit 7 in the gpte if the level is + * greater than the last level for which bit 7 is the PAGE_SIZE bit. + * + * The RHS has bit 7 set iff level < (2 + PSE). If it is clear, bit 7 + * is not reserved and does not indicate a large page at this level, + * so clear PT_PAGE_SIZE_MASK in gpte if that is the case. + */ + gpte &= level - (PT32_ROOT_LEVEL + mmu->mmu_role.ext.cr4_pse); +#endif + /* + * PG_LEVEL_4K always terminates. The RHS has bit 7 set + * iff level <= PG_LEVEL_4K, which for our purpose means + * level == PG_LEVEL_4K; set PT_PAGE_SIZE_MASK in gpte then. + */ + gpte |= level - PG_LEVEL_4K - 1; + + return gpte & PT_PAGE_SIZE_MASK; +} /* * Fetch a guest pte for a guest virtual address, or for an L2's GPA. */ @@ -421,7 +450,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, /* Convert to ACC_*_MASK flags for struct guest_walker. */ walker->pt_access[walker->level - 1] = FNAME(gpte_access)(pt_access ^ walk_nx_mask); - } while (!is_last_gpte(mmu, walker->level, pte)); + } while (!FNAME(is_last_gpte)(mmu, walker->level, pte)); pte_pkey = FNAME(gpte_pkeys)(vcpu, pte); accessed_dirty = have_ad ? pte_access & PT_GUEST_ACCESSED_MASK : 0; @@ -471,8 +500,7 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker, error: errcode |= write_fault | user_fault; - if (fetch_fault && (mmu->nx || - kvm_read_cr4_bits(vcpu, X86_CR4_SMEP))) + if (fetch_fault && (is_efer_nx(mmu) || is_cr4_smep(mmu))) errcode |= PFERR_FETCH_MASK; walker->fault.vector = PF_VECTOR; @@ -767,7 +795,7 @@ FNAME(is_self_change_mapping)(struct kvm_vcpu *vcpu, bool self_changed = false; if (!(walker->pte_access & ACC_WRITE_MASK || - (!is_write_protection(vcpu) && !user_fault))) + (!is_cr0_wp(vcpu->arch.mmu) && !user_fault))) return false; for (level = walker->level; level <= walker->max_level; level++) { @@ -865,8 +893,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, * we will cache the incorrect access into mmio spte. */ if (write_fault && !(walker.pte_access & ACC_WRITE_MASK) && - !is_write_protection(vcpu) && !user_fault && - !is_noslot_pfn(pfn)) { + !is_cr0_wp(vcpu->arch.mmu) && !user_fault && !is_noslot_pfn(pfn)) { walker.pte_access |= ACC_WRITE_MASK; walker.pte_access &= ~ACC_USER_MASK; @@ -876,7 +903,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, gpa_t addr, u32 error_code, * then we should prevent the kernel from executing it * if SMEP is enabled. */ - if (kvm_read_cr4_bits(vcpu, X86_CR4_SMEP)) + if (is_cr4_smep(vcpu->arch.mmu)) walker.pte_access &= ~ACC_EXEC_MASK; } @@ -1031,13 +1058,36 @@ static gpa_t FNAME(gva_to_gpa_nested)(struct kvm_vcpu *vcpu, gpa_t vaddr, */ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp) { + union kvm_mmu_page_role mmu_role = vcpu->arch.mmu->mmu_role.base; int i, nr_present = 0; bool host_writable; gpa_t first_pte_gpa; int set_spte_ret = 0; - /* direct kvm_mmu_page can not be unsync. */ - BUG_ON(sp->role.direct); + /* + * Ignore various flags when verifying that it's safe to sync a shadow + * page using the current MMU context. + * + * - level: not part of the overall MMU role and will never match as the MMU's + * level tracks the root level + * - access: updated based on the new guest PTE + * - quadrant: not part of the overall MMU role (similar to level) + */ + const union kvm_mmu_page_role sync_role_ign = { + .level = 0xf, + .access = 0x7, + .quadrant = 0x3, + }; + + /* + * Direct pages can never be unsync, and KVM should never attempt to + * sync a shadow page for a different MMU context, e.g. if the role + * differs then the memslot lookup (SMM vs. non-SMM) will be bogus, the + * reserved bits checks will be wrong, etc... + */ + if (WARN_ON_ONCE(sp->role.direct || + (sp->role.word ^ mmu_role.word) & ~sync_role_ign.word)) + return 0; first_pte_gpa = FNAME(get_level1_sp_gpa)(sp); diff --git a/arch/x86/kvm/mmu/spte.c b/arch/x86/kvm/mmu/spte.c index 66d43cec0c31..3e97cdb13eb7 100644 --- a/arch/x86/kvm/mmu/spte.c +++ b/arch/x86/kvm/mmu/spte.c @@ -102,13 +102,6 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, else if (kvm_vcpu_ad_need_write_protect(vcpu)) spte |= SPTE_TDP_AD_WRPROT_ONLY_MASK; - /* - * Bits 62:52 of PAE SPTEs are reserved. WARN if said bits are set - * if PAE paging may be employed (shadow paging or any 32-bit KVM). - */ - WARN_ON_ONCE((!tdp_enabled || !IS_ENABLED(CONFIG_X86_64)) && - (spte & SPTE_TDP_AD_MASK)); - /* * For the EPT case, shadow_present_mask is 0 if hardware * supports exec-only page table entries. In that case, @@ -154,13 +147,19 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, /* * Optimization: for pte sync, if spte was writable the hash * lookup is unnecessary (and expensive). Write protection - * is responsibility of mmu_get_page / kvm_sync_page. + * is responsibility of kvm_mmu_get_page / kvm_mmu_sync_roots. * Same reasoning can be applied to dirty page accounting. */ if (!can_unsync && is_writable_pte(old_spte)) goto out; - if (mmu_need_write_protect(vcpu, gfn, can_unsync)) { + /* + * Unsync shadow pages that are reachable by the new, writable + * SPTE. Write-protect the SPTE if the page can't be unsync'd, + * e.g. it's write-tracked (upper-level SPs) or has one or more + * shadow pages and unsync'ing pages is not allowed. + */ + if (mmu_try_to_unsync_pages(vcpu, gfn, can_unsync)) { pgprintk("%s: found shadow page for %llx, marking ro\n", __func__, gfn); ret |= SET_SPTE_WRITE_PROTECTED_PT; @@ -176,7 +175,10 @@ int make_spte(struct kvm_vcpu *vcpu, unsigned int pte_access, int level, spte = mark_spte_for_access_track(spte); out: - WARN_ON(is_mmio_spte(spte)); + WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->shadow_zero_check, spte, level), + "spte = 0x%llx, level = %d, rsvd bits = 0x%llx", spte, level, + get_rsvd_bits(&vcpu->arch.mmu->shadow_zero_check, spte, level)); + *new_spte = spte; return ret; } diff --git a/arch/x86/kvm/mmu/spte.h b/arch/x86/kvm/mmu/spte.h index bca0ba11cccf..7a5ce9314107 100644 --- a/arch/x86/kvm/mmu/spte.h +++ b/arch/x86/kvm/mmu/spte.h @@ -293,6 +293,38 @@ static inline bool is_dirty_spte(u64 spte) return dirty_mask ? spte & dirty_mask : spte & PT_WRITABLE_MASK; } +static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte, + int level) +{ + int bit7 = (pte >> 7) & 1; + + return rsvd_check->rsvd_bits_mask[bit7][level-1]; +} + +static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check, + u64 pte, int level) +{ + return pte & get_rsvd_bits(rsvd_check, pte, level); +} + +static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check, + u64 pte) +{ + return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f); +} + +static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check, + u64 spte, int level) +{ + /* + * Use a bitwise-OR instead of a logical-OR to aggregate the reserved + * bits and EPT's invalid memtype/XWR checks to avoid an extra Jcc + * (this is extremely unlikely to be short-circuited as true). + */ + return __is_bad_mt_xwr(rsvd_check, spte) | + __is_rsvd_bits_set(rsvd_check, spte, level); +} + static inline bool spte_can_locklessly_be_made_writable(u64 spte) { return (spte & shadow_host_writable_mask) && diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index 237317b1eddd..0853370bd811 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -14,10 +14,10 @@ static bool __read_mostly tdp_mmu_enabled = false; module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644); /* Initializes the TDP MMU for the VM, if enabled. */ -void kvm_mmu_init_tdp_mmu(struct kvm *kvm) +bool kvm_mmu_init_tdp_mmu(struct kvm *kvm) { if (!tdp_enabled || !READ_ONCE(tdp_mmu_enabled)) - return; + return false; /* This should not be changed for the lifetime of the VM. */ kvm->arch.tdp_mmu_enabled = true; @@ -25,6 +25,8 @@ void kvm_mmu_init_tdp_mmu(struct kvm *kvm) INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots); spin_lock_init(&kvm->arch.tdp_mmu_pages_lock); INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages); + + return true; } static __always_inline void kvm_lockdep_assert_mmu_lock_held(struct kvm *kvm, @@ -335,7 +337,7 @@ static void handle_removed_tdp_mmu_page(struct kvm *kvm, tdp_ptep_t pt, for (i = 0; i < PT64_ENT_PER_PAGE; i++) { sptep = rcu_dereference(pt) + i; - gfn = base_gfn + (i * KVM_PAGES_PER_HPAGE(level - 1)); + gfn = base_gfn + i * KVM_PAGES_PER_HPAGE(level); if (shared) { /* @@ -377,12 +379,12 @@ static void handle_removed_tdp_mmu_page(struct kvm *kvm, tdp_ptep_t pt, WRITE_ONCE(*sptep, REMOVED_SPTE); } handle_changed_spte(kvm, kvm_mmu_page_as_id(sp), gfn, - old_child_spte, REMOVED_SPTE, level - 1, + old_child_spte, REMOVED_SPTE, level, shared); } kvm_flush_remote_tlbs_with_address(kvm, gfn, - KVM_PAGES_PER_HPAGE(level)); + KVM_PAGES_PER_HPAGE(level + 1)); call_rcu(&sp->rcu_head, tdp_mmu_free_sp_rcu_callback); } @@ -912,7 +914,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, kvm_pfn_t pfn, bool prefault) { u64 new_spte; - int ret = 0; + int ret = RET_PF_FIXED; int make_spte_ret = 0; if (unlikely(is_noslot_pfn(pfn))) @@ -949,7 +951,11 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu, int write, rcu_dereference(iter->sptep)); } - if (!prefault) + /* + * Increase pf_fixed in both RET_PF_EMULATE and RET_PF_FIXED to be + * consistent with legacy MMU behavior. + */ + if (ret != RET_PF_SPURIOUS) vcpu->stat.pf_fixed++; return ret; @@ -977,11 +983,6 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, int level; int req_level; - if (WARN_ON(!VALID_PAGE(vcpu->arch.mmu->root_hpa))) - return RET_PF_RETRY; - if (WARN_ON(!is_tdp_mmu_root(vcpu->kvm, vcpu->arch.mmu->root_hpa))) - return RET_PF_RETRY; - level = kvm_mmu_hugepage_adjust(vcpu, gfn, max_level, &pfn, huge_page_disallowed, &req_level); @@ -1017,14 +1018,14 @@ int kvm_tdp_mmu_map(struct kvm_vcpu *vcpu, gpa_t gpa, u32 error_code, if (!is_shadow_present_pte(iter.old_spte)) { /* - * If SPTE has been forzen by another thread, just + * If SPTE has been frozen by another thread, just * give up and retry, avoiding unnecessary page table * allocation and free. */ if (is_removed_spte(iter.old_spte)) break; - sp = alloc_tdp_mmu_page(vcpu, iter.gfn, iter.level); + sp = alloc_tdp_mmu_page(vcpu, iter.gfn, iter.level - 1); child_pt = sp->spt; new_spte = make_nonleaf_spte(child_pt, @@ -1462,15 +1463,22 @@ bool kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, * Returns true if an SPTE was set and a TLB flush is needed. */ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, - gfn_t gfn) + gfn_t gfn, int min_level) { struct tdp_iter iter; u64 new_spte; bool spte_set = false; + BUG_ON(min_level > KVM_MAX_HUGEPAGE_LEVEL); + rcu_read_lock(); - tdp_root_for_each_leaf_pte(iter, root, gfn, gfn + 1) { + for_each_tdp_pte_min_level(iter, root->spt, root->role.level, + min_level, gfn, gfn + 1) { + if (!is_shadow_present_pte(iter.old_spte) || + !is_last_spte(iter.old_spte, iter.level)) + continue; + if (!is_writable_pte(iter.old_spte)) break; @@ -1492,14 +1500,15 @@ static bool write_protect_gfn(struct kvm *kvm, struct kvm_mmu_page *root, * Returns true if an SPTE was set and a TLB flush is needed. */ bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, - struct kvm_memory_slot *slot, gfn_t gfn) + struct kvm_memory_slot *slot, gfn_t gfn, + int min_level) { struct kvm_mmu_page *root; bool spte_set = false; lockdep_assert_held_write(&kvm->mmu_lock); for_each_tdp_mmu_root(kvm, root, slot->as_id) - spte_set |= write_protect_gfn(kvm, root, gfn); + spte_set |= write_protect_gfn(kvm, root, gfn, min_level); return spte_set; } diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h index 5fdf63090451..1cae4485b3bc 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.h +++ b/arch/x86/kvm/mmu/tdp_mmu.h @@ -31,7 +31,7 @@ static inline bool kvm_tdp_mmu_zap_gfn_range(struct kvm *kvm, int as_id, } static inline bool kvm_tdp_mmu_zap_sp(struct kvm *kvm, struct kvm_mmu_page *sp) { - gfn_t end = sp->gfn + KVM_PAGES_PER_HPAGE(sp->role.level); + gfn_t end = sp->gfn + KVM_PAGES_PER_HPAGE(sp->role.level + 1); /* * Don't allow yielding, as the caller may have a flush pending. Note, @@ -74,37 +74,40 @@ bool kvm_tdp_mmu_zap_collapsible_sptes(struct kvm *kvm, bool flush); bool kvm_tdp_mmu_write_protect_gfn(struct kvm *kvm, - struct kvm_memory_slot *slot, gfn_t gfn); + struct kvm_memory_slot *slot, gfn_t gfn, + int min_level); int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes, int *root_level); #ifdef CONFIG_X86_64 -void kvm_mmu_init_tdp_mmu(struct kvm *kvm); +bool kvm_mmu_init_tdp_mmu(struct kvm *kvm); void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm); static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return kvm->arch.tdp_mmu_enabled; } static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; } -#else -static inline void kvm_mmu_init_tdp_mmu(struct kvm *kvm) {} -static inline void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) {} -static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; } -static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; } -#endif -static inline bool is_tdp_mmu_root(struct kvm *kvm, hpa_t hpa) +static inline bool is_tdp_mmu(struct kvm_mmu *mmu) { struct kvm_mmu_page *sp; + hpa_t hpa = mmu->root_hpa; - if (!is_tdp_mmu_enabled(kvm)) - return false; if (WARN_ON(!VALID_PAGE(hpa))) return false; + /* + * A NULL shadow page is legal when shadowing a non-paging guest with + * PAE paging, as the MMU will be direct with root_hpa pointing at the + * pae_root page, not a shadow page. + */ sp = to_shadow_page(hpa); - if (WARN_ON(!sp)) - return false; - - return is_tdp_mmu_page(sp) && sp->root_count; + return sp && is_tdp_mmu_page(sp) && sp->root_count; } +#else +static inline bool kvm_mmu_init_tdp_mmu(struct kvm *kvm) { return false; } +static inline void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) {} +static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; } +static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; } +static inline bool is_tdp_mmu(struct kvm_mmu *mmu) { return false; } +#endif #endif /* __KVM_X86_MMU_TDP_MMU_H */ diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 5e7e920113f3..1d01da64c333 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -27,10 +27,6 @@ #include "irq.h" #include "svm.h" -/* enable / disable AVIC */ -bool avic; -module_param(avic, bool, S_IRUGO); - #define SVM_AVIC_DOORBELL 0xc001011b #define AVIC_HPA_MASK ~((0xFFFULL << 52) | 0xFFF) @@ -124,7 +120,7 @@ void avic_vm_destroy(struct kvm *kvm) unsigned long flags; struct kvm_svm *kvm_svm = to_kvm_svm(kvm); - if (!avic) + if (!enable_apicv) return; if (kvm_svm->avic_logical_id_table_page) @@ -147,7 +143,7 @@ int avic_vm_init(struct kvm *kvm) struct page *l_page; u32 vm_id; - if (!avic) + if (!enable_apicv) return 0; /* Allocating physical APIC ID table (4KB) */ @@ -240,7 +236,7 @@ static int avic_update_access_page(struct kvm *kvm, bool activate) * APICv mode change, which update APIC_ACCESS_PAGE_PRIVATE_MEMSLOT * memory region. So, we need to ensure that kvm->mm == current->mm. */ - if ((kvm->arch.apic_access_page_done == activate) || + if ((kvm->arch.apic_access_memslot_enabled == activate) || (kvm->mm != current->mm)) goto out; @@ -253,7 +249,7 @@ static int avic_update_access_page(struct kvm *kvm, bool activate) goto out; } - kvm->arch.apic_access_page_done = activate; + kvm->arch.apic_access_memslot_enabled = activate; out: mutex_unlock(&kvm->slots_lock); return r; @@ -569,7 +565,7 @@ int avic_init_vcpu(struct vcpu_svm *svm) int ret; struct kvm_vcpu *vcpu = &svm->vcpu; - if (!avic || !irqchip_in_kernel(vcpu->kvm)) + if (!enable_apicv || !irqchip_in_kernel(vcpu->kvm)) return 0; ret = avic_init_backing_page(vcpu); @@ -593,7 +589,7 @@ void avic_post_state_restore(struct kvm_vcpu *vcpu) void svm_toggle_avic_for_irq_window(struct kvm_vcpu *vcpu, bool activate) { - if (!avic || !lapic_in_kernel(vcpu)) + if (!enable_apicv || !lapic_in_kernel(vcpu)) return; srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); @@ -653,7 +649,7 @@ void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu) struct vmcb *vmcb = svm->vmcb; bool activated = kvm_vcpu_apicv_active(vcpu); - if (!avic) + if (!enable_apicv) return; if (activated) { diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 5e8d8443154e..21d03e3a5dfd 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -98,13 +98,18 @@ static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu) WARN_ON(mmu_is_nested(vcpu)); vcpu->arch.mmu = &vcpu->arch.guest_mmu; + + /* + * The NPT format depends on L1's CR4 and EFER, which is in vmcb01. Note, + * when called via KVM_SET_NESTED_STATE, that state may _not_ match current + * vCPU state. CR0.WP is explicitly ignored, while CR0.PG is required. + */ kvm_init_shadow_npt_mmu(vcpu, X86_CR0_PG, svm->vmcb01.ptr->save.cr4, svm->vmcb01.ptr->save.efer, svm->nested.ctl.nested_cr3); vcpu->arch.mmu->get_guest_pgd = nested_svm_get_tdp_cr3; vcpu->arch.mmu->get_pdptr = nested_svm_get_tdp_pdptr; vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit; - reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu); vcpu->arch.walk_mmu = &vcpu->arch.nested_mmu; } @@ -380,33 +385,47 @@ static inline bool nested_npt_enabled(struct vcpu_svm *svm) return svm->nested.ctl.nested_ctl & SVM_NESTED_CTL_NP_ENABLE; } +static void nested_svm_transition_tlb_flush(struct kvm_vcpu *vcpu) +{ + /* + * TODO: optimize unconditional TLB flush/MMU sync. A partial list of + * things to fix before this can be conditional: + * + * - Flush TLBs for both L1 and L2 remote TLB flush + * - Honor L1's request to flush an ASID on nested VMRUN + * - Sync nested NPT MMU on VMRUN that flushes L2's ASID[*] + * - Don't crush a pending TLB flush in vmcb02 on nested VMRUN + * - Flush L1's ASID on KVM_REQ_TLB_FLUSH_GUEST + * + * [*] Unlike nested EPT, SVM's ASID management can invalidate nested + * NPT guest-physical mappings on VMRUN. + */ + kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); +} + /* * Load guest's/host's cr3 on nested vmentry or vmexit. @nested_npt is true * if we are emulating VM-Entry into a guest with NPT enabled. */ static int nested_svm_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, - bool nested_npt) + bool nested_npt, bool reload_pdptrs) { if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) return -EINVAL; - if (!nested_npt && is_pae_paging(vcpu) && - (cr3 != kvm_read_cr3(vcpu) || pdptrs_changed(vcpu))) { - if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) - return -EINVAL; - } + if (reload_pdptrs && !nested_npt && is_pae_paging(vcpu) && + CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) + return -EINVAL; - /* - * TODO: optimize unconditional TLB flush/MMU sync here and in - * kvm_init_shadow_npt_mmu(). - */ if (!nested_npt) - kvm_mmu_new_pgd(vcpu, cr3, false, false); + kvm_mmu_new_pgd(vcpu, cr3); vcpu->arch.cr3 = cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); - kvm_init_mmu(vcpu, false); + /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ + kvm_init_mmu(vcpu); return 0; } @@ -481,6 +500,7 @@ static void nested_vmcb02_prepare_save(struct vcpu_svm *svm, struct vmcb *vmcb12 static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) { const u32 mask = V_INTR_MASKING_MASK | V_GIF_ENABLE_MASK | V_GIF_MASK; + struct kvm_vcpu *vcpu = &svm->vcpu; /* * Filled at exit: exit_code, exit_code_hi, exit_info_1, exit_info_2, @@ -505,10 +525,10 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) /* nested_cr3. */ if (nested_npt_enabled(svm)) - nested_svm_init_mmu_context(&svm->vcpu); + nested_svm_init_mmu_context(vcpu); - svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset = - svm->vcpu.arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; + svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset = + vcpu->arch.l1_tsc_offset + svm->nested.ctl.tsc_offset; svm->vmcb->control.int_ctl = (svm->nested.ctl.int_ctl & ~mask) | @@ -523,8 +543,10 @@ static void nested_vmcb02_prepare_control(struct vcpu_svm *svm) svm->vmcb->control.pause_filter_count = svm->nested.ctl.pause_filter_count; svm->vmcb->control.pause_filter_thresh = svm->nested.ctl.pause_filter_thresh; + nested_svm_transition_tlb_flush(vcpu); + /* Enter Guest-Mode */ - enter_guest_mode(&svm->vcpu); + enter_guest_mode(vcpu); /* * Merge guest and host intercepts - must be called with vcpu in @@ -576,7 +598,7 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, nested_vmcb02_prepare_save(svm, vmcb12); ret = nested_svm_load_cr3(&svm->vcpu, vmcb12->save.cr3, - nested_npt_enabled(svm)); + nested_npt_enabled(svm), true); if (ret) return ret; @@ -596,8 +618,6 @@ int nested_svm_vmrun(struct kvm_vcpu *vcpu) struct kvm_host_map map; u64 vmcb12_gpa; - ++vcpu->stat.nested_run; - if (is_smm(vcpu)) { kvm_queue_exception(vcpu, UD_VECTOR); return 1; @@ -803,9 +823,11 @@ int nested_svm_vmexit(struct vcpu_svm *svm) kvm_vcpu_unmap(vcpu, &map, true); + nested_svm_transition_tlb_flush(vcpu); + nested_svm_uninit_mmu_context(vcpu); - rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false); + rc = nested_svm_load_cr3(vcpu, svm->vmcb->save.cr3, false, true); if (rc) return 1; @@ -1228,8 +1250,8 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, &user_kvm_nested_state->data.svm[0]; struct vmcb_control_area *ctl; struct vmcb_save_area *save; + unsigned long cr0; int ret; - u32 cr0; BUILD_BUG_ON(sizeof(struct vmcb_control_area) + sizeof(struct vmcb_save_area) > KVM_STATE_NESTED_SVM_VMCB_SIZE); @@ -1301,6 +1323,19 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, !nested_vmcb_valid_sregs(vcpu, save)) goto out_free; + /* + * While the nested guest CR3 is already checked and set by + * KVM_SET_SREGS, it was set when nested state was yet loaded, + * thus MMU might not be initialized correctly. + * Set it again to fix this. + */ + + ret = nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, + nested_npt_enabled(svm), false); + if (WARN_ON_ONCE(ret)) + goto out_free; + + /* * All checks done, we can enter guest mode. Userspace provides * vmcb12.control, which will be combined with L1 and stored into @@ -1358,9 +1393,15 @@ static bool svm_get_nested_state_pages(struct kvm_vcpu *vcpu) if (WARN_ON(!is_guest_mode(vcpu))) return true; - if (nested_svm_load_cr3(&svm->vcpu, vcpu->arch.cr3, - nested_npt_enabled(svm))) - return false; + if (!vcpu->arch.pdptrs_from_userspace && + !nested_npt_enabled(svm) && is_pae_paging(vcpu)) + /* + * Reload the guest's PDPTRs since after a migration + * the guest CR3 might be restored prior to setting the nested + * state which can lead to a load of wrong PDPTRs. + */ + if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))) + return false; if (!nested_svm_vmrun_msrpm(svm)) { vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index e088086f3de6..8834822c00cd 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -43,6 +43,9 @@ #include "svm.h" #include "svm_ops.h" +#include "kvm_onhyperv.h" +#include "svm_onhyperv.h" + #define __ex(x) __kvm_handle_fault_on_reboot(x) MODULE_AUTHOR("Qumranet"); @@ -185,6 +188,13 @@ module_param(vls, int, 0444); static int vgif = true; module_param(vgif, int, 0444); +/* + * enable / disable AVIC. Because the defaults differ for APICv + * support between VMX and SVM we cannot use module_param_named. + */ +static bool avic; +module_param(avic, bool, 0444); + bool __read_mostly dump_invalid_vmcb; module_param(dump_invalid_vmcb, bool, 0644); @@ -673,6 +683,9 @@ static void set_msr_interception_bitmap(struct kvm_vcpu *vcpu, u32 *msrpm, write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp); msrpm[offset] = tmp; + + svm_hv_vmcb_dirty_nested_enlightenments(vcpu); + } void set_msr_interception(struct kvm_vcpu *vcpu, u32 *msrpm, u32 msr, @@ -939,6 +952,16 @@ static __init int svm_hardware_setup(void) int r; unsigned int order = get_order(IOPM_SIZE); + /* + * NX is required for shadow paging and for NPT if the NX huge pages + * mitigation is enabled. + */ + if (!boot_cpu_has(X86_FEATURE_NX)) { + pr_err_ratelimited("NX (Execute Disable) not supported\n"); + return -EOPNOTSUPP; + } + kvm_enable_efer_bits(EFER_NX); + iopm_pages = alloc_pages(GFP_KERNEL, order); if (!iopm_pages) @@ -952,9 +975,6 @@ static __init int svm_hardware_setup(void) supported_xcr0 &= ~(XFEATURE_MASK_BNDREGS | XFEATURE_MASK_BNDCSR); - if (boot_cpu_has(X86_FEATURE_NX)) - kvm_enable_efer_bits(EFER_NX); - if (boot_cpu_has(X86_FEATURE_FXSR_OPT)) kvm_enable_efer_bits(EFER_FFXSR); @@ -996,6 +1016,8 @@ static __init int svm_hardware_setup(void) /* Note, SEV setup consumes npt_enabled. */ sev_hardware_setup(); + svm_hv_hardware_setup(); + svm_adjust_mmio_mask(); for_each_possible_cpu(cpu) { @@ -1009,14 +1031,12 @@ static __init int svm_hardware_setup(void) nrips = false; } - if (avic) { - if (!npt_enabled || !boot_cpu_has(X86_FEATURE_AVIC)) { - avic = false; - } else { - pr_info("AVIC enabled\n"); + enable_apicv = avic = avic && npt_enabled && boot_cpu_has(X86_FEATURE_AVIC); - amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); - } + if (enable_apicv) { + pr_info("AVIC enabled\n"); + + amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier); } if (vls) { @@ -1080,26 +1100,30 @@ static void init_sys_seg(struct vmcb_seg *seg, uint32_t type) seg->base = 0; } -static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) +static u64 svm_get_l2_tsc_offset(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); - u64 g_tsc_offset = 0; - if (is_guest_mode(vcpu)) { - /* Write L1's TSC offset. */ - g_tsc_offset = svm->vmcb->control.tsc_offset - - svm->vmcb01.ptr->control.tsc_offset; - svm->vmcb01.ptr->control.tsc_offset = offset; - } + return svm->nested.ctl.tsc_offset; +} - trace_kvm_write_tsc_offset(vcpu->vcpu_id, - svm->vmcb->control.tsc_offset - g_tsc_offset, - offset); +static u64 svm_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) +{ + return kvm_default_tsc_scaling_ratio; +} - svm->vmcb->control.tsc_offset = offset + g_tsc_offset; +static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) +{ + struct vcpu_svm *svm = to_svm(vcpu); + svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; + svm->vmcb->control.tsc_offset = offset; vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); - return svm->vmcb->control.tsc_offset; +} + +static void svm_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) +{ + wrmsrl(MSR_AMD64_TSC_RATIO, multiplier); } /* Evaluate instruction intercepts that depend on guest CPUID features. */ @@ -1287,6 +1311,8 @@ static void init_vmcb(struct kvm_vcpu *vcpu) } } + svm_hv_init_vmcb(svm->vmcb); + vmcb_mark_all_dirty(svm->vmcb); enable_gif(svm); @@ -3106,6 +3132,8 @@ static void dump_vmcb(struct kvm_vcpu *vcpu) return; } + pr_err("VMCB %p, last attempted VMRUN on CPU %d\n", + svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu); pr_err("VMCB Control Area:\n"); pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); @@ -3762,6 +3790,8 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) } svm->vmcb->save.cr2 = vcpu->arch.cr2; + svm_hv_update_vp_id(svm->vmcb, vcpu); + /* * Run with all-zero DR6 unless needed, so that we can get the exact cause * of a #DB. @@ -3835,6 +3865,12 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) svm->next_rip = 0; if (is_guest_mode(vcpu)) { nested_sync_control_from_vmcb02(svm); + + /* Track VMRUNs that have made past consistency checking */ + if (svm->nested.nested_run_pending && + svm->vmcb->control.exit_code != SVM_EXIT_ERR) + ++vcpu->stat.nested_run; + svm->nested.nested_run_pending = 0; } @@ -3846,10 +3882,8 @@ static __no_kcsan fastpath_t svm_vcpu_run(struct kvm_vcpu *vcpu) vcpu->arch.apf.host_apf_flags = kvm_read_and_reset_apf_flags(); - if (npt_enabled) { - vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR); - vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR); - } + if (npt_enabled) + kvm_register_clear_available(vcpu, VCPU_EXREG_PDPTR); /* * We need to handle MC intercepts here before the vcpu has a chance to @@ -3877,6 +3911,8 @@ static void svm_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); vmcb_mark_dirty(svm->vmcb, VMCB_NPT); + hv_track_root_tdp(vcpu, root_hpa); + /* Loading L2's CR3 is handled by enter_svm_guest_mode. */ if (!test_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail)) return; @@ -4249,7 +4285,7 @@ static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) return !svm_smi_blocked(vcpu); } -static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) +static int svm_enter_smm(struct kvm_vcpu *vcpu, char *smstate) { struct vcpu_svm *svm = to_svm(vcpu); int ret; @@ -4271,7 +4307,7 @@ static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) return 0; } -static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) +static int svm_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_host_map map; @@ -4427,13 +4463,12 @@ static int svm_vm_init(struct kvm *kvm) if (!pause_filter_count || !pause_filter_thresh) kvm->arch.pause_in_guest = true; - if (avic) { + if (enable_apicv) { int ret = avic_vm_init(kvm); if (ret) return ret; } - kvm_apicv_init(kvm, avic); return 0; } @@ -4524,7 +4559,10 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .has_wbinvd_exit = svm_has_wbinvd_exit, - .write_l1_tsc_offset = svm_write_l1_tsc_offset, + .get_l2_tsc_offset = svm_get_l2_tsc_offset, + .get_l2_tsc_multiplier = svm_get_l2_tsc_multiplier, + .write_tsc_offset = svm_write_tsc_offset, + .write_tsc_multiplier = svm_write_tsc_multiplier, .load_mmu_pgd = svm_load_mmu_pgd, @@ -4544,8 +4582,8 @@ static struct kvm_x86_ops svm_x86_ops __initdata = { .setup_mce = svm_setup_mce, .smi_allowed = svm_smi_allowed, - .pre_enter_smm = svm_pre_enter_smm, - .pre_leave_smm = svm_pre_leave_smm, + .enter_smm = svm_enter_smm, + .leave_smm = svm_leave_smm, .enable_smi_window = svm_enable_smi_window, .mem_enc_op = svm_mem_enc_op, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 2908c6ab5bb4..f89b623bb591 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -32,6 +32,11 @@ extern u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly; extern bool npt_enabled; +/* + * Clean bits in VMCB. + * VMCB_ALL_CLEAN_MASK might also need to + * be updated if this enum is modified. + */ enum { VMCB_INTERCEPTS, /* Intercept vectors, TSC offset, pause filter count */ @@ -49,9 +54,17 @@ enum { * AVIC PHYSICAL_TABLE pointer, * AVIC LOGICAL_TABLE pointer */ - VMCB_DIRTY_MAX, + VMCB_SW = 31, /* Reserved for hypervisor/software use */ }; +#define VMCB_ALL_CLEAN_MASK ( \ + (1U << VMCB_INTERCEPTS) | (1U << VMCB_PERM_MAP) | \ + (1U << VMCB_ASID) | (1U << VMCB_INTR) | \ + (1U << VMCB_NPT) | (1U << VMCB_CR) | (1U << VMCB_DR) | \ + (1U << VMCB_DT) | (1U << VMCB_SEG) | (1U << VMCB_CR2) | \ + (1U << VMCB_LBR) | (1U << VMCB_AVIC) | \ + (1U << VMCB_SW)) + /* TPR and CR2 are always written before VMRUN */ #define VMCB_ALWAYS_DIRTY_MASK ((1U << VMCB_INTR) | (1U << VMCB_CR2)) @@ -238,10 +251,15 @@ static inline void vmcb_mark_all_dirty(struct vmcb *vmcb) static inline void vmcb_mark_all_clean(struct vmcb *vmcb) { - vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1) + vmcb->control.clean = VMCB_ALL_CLEAN_MASK & ~VMCB_ALWAYS_DIRTY_MASK; } +static inline bool vmcb_is_clean(struct vmcb *vmcb, int bit) +{ + return (vmcb->control.clean & (1 << bit)); +} + static inline void vmcb_mark_dirty(struct vmcb *vmcb, int bit) { vmcb->control.clean &= ~(1 << bit); @@ -480,8 +498,6 @@ extern struct kvm_x86_nested_ops svm_nested_ops; #define VMCB_AVIC_APIC_BAR_MASK 0xFFFFFFFFFF000ULL -extern bool avic; - static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data) { svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK; diff --git a/arch/x86/kvm/svm/svm_onhyperv.c b/arch/x86/kvm/svm/svm_onhyperv.c new file mode 100644 index 000000000000..98aa981c04ec --- /dev/null +++ b/arch/x86/kvm/svm/svm_onhyperv.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM L1 hypervisor optimizations on Hyper-V for SVM. + */ + +#include +#include "kvm_cache_regs.h" + +#include + +#include "svm.h" +#include "svm_ops.h" + +#include "hyperv.h" +#include "kvm_onhyperv.h" +#include "svm_onhyperv.h" + +int svm_hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) +{ + struct hv_enlightenments *hve; + struct hv_partition_assist_pg **p_hv_pa_pg = + &to_kvm_hv(vcpu->kvm)->hv_pa_pg; + + if (!*p_hv_pa_pg) + *p_hv_pa_pg = kzalloc(PAGE_SIZE, GFP_KERNEL); + + if (!*p_hv_pa_pg) + return -ENOMEM; + + hve = (struct hv_enlightenments *)to_svm(vcpu)->vmcb->control.reserved_sw; + + hve->partition_assist_page = __pa(*p_hv_pa_pg); + hve->hv_vm_id = (unsigned long)vcpu->kvm; + if (!hve->hv_enlightenments_control.nested_flush_hypercall) { + hve->hv_enlightenments_control.nested_flush_hypercall = 1; + vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS); + } + + return 0; +} + diff --git a/arch/x86/kvm/svm/svm_onhyperv.h b/arch/x86/kvm/svm/svm_onhyperv.h new file mode 100644 index 000000000000..9b9a55abc29f --- /dev/null +++ b/arch/x86/kvm/svm/svm_onhyperv.h @@ -0,0 +1,130 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * KVM L1 hypervisor optimizations on Hyper-V for SVM. + */ + +#ifndef __ARCH_X86_KVM_SVM_ONHYPERV_H__ +#define __ARCH_X86_KVM_SVM_ONHYPERV_H__ + +#if IS_ENABLED(CONFIG_HYPERV) +#include + +#include "hyperv.h" +#include "kvm_onhyperv.h" + +static struct kvm_x86_ops svm_x86_ops; + +/* + * Hyper-V uses the software reserved 32 bytes in VMCB + * control area to expose SVM enlightenments to guests. + */ +struct hv_enlightenments { + struct __packed hv_enlightenments_control { + u32 nested_flush_hypercall:1; + u32 msr_bitmap:1; + u32 enlightened_npt_tlb: 1; + u32 reserved:29; + } __packed hv_enlightenments_control; + u32 hv_vp_id; + u64 hv_vm_id; + u64 partition_assist_page; + u64 reserved; +} __packed; + +/* + * Hyper-V uses the software reserved clean bit in VMCB + */ +#define VMCB_HV_NESTED_ENLIGHTENMENTS VMCB_SW + +int svm_hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu); + +static inline void svm_hv_init_vmcb(struct vmcb *vmcb) +{ + struct hv_enlightenments *hve = + (struct hv_enlightenments *)vmcb->control.reserved_sw; + + if (npt_enabled && + ms_hyperv.nested_features & HV_X64_NESTED_ENLIGHTENED_TLB) + hve->hv_enlightenments_control.enlightened_npt_tlb = 1; +} + +static inline void svm_hv_hardware_setup(void) +{ + if (npt_enabled && + ms_hyperv.nested_features & HV_X64_NESTED_ENLIGHTENED_TLB) { + pr_info("kvm: Hyper-V enlightened NPT TLB flush enabled\n"); + svm_x86_ops.tlb_remote_flush = hv_remote_flush_tlb; + svm_x86_ops.tlb_remote_flush_with_range = + hv_remote_flush_tlb_with_range; + } + + if (ms_hyperv.nested_features & HV_X64_NESTED_DIRECT_FLUSH) { + int cpu; + + pr_info("kvm: Hyper-V Direct TLB Flush enabled\n"); + for_each_online_cpu(cpu) { + struct hv_vp_assist_page *vp_ap = + hv_get_vp_assist_page(cpu); + + if (!vp_ap) + continue; + + vp_ap->nested_control.features.directhypercall = 1; + } + svm_x86_ops.enable_direct_tlbflush = + svm_hv_enable_direct_tlbflush; + } +} + +static inline void svm_hv_vmcb_dirty_nested_enlightenments( + struct kvm_vcpu *vcpu) +{ + struct vmcb *vmcb = to_svm(vcpu)->vmcb; + struct hv_enlightenments *hve = + (struct hv_enlightenments *)vmcb->control.reserved_sw; + + /* + * vmcb can be NULL if called during early vcpu init. + * And its okay not to mark vmcb dirty during vcpu init + * as we mark it dirty unconditionally towards end of vcpu + * init phase. + */ + if (vmcb && vmcb_is_clean(vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS) && + hve->hv_enlightenments_control.msr_bitmap) + vmcb_mark_dirty(vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS); +} + +static inline void svm_hv_update_vp_id(struct vmcb *vmcb, + struct kvm_vcpu *vcpu) +{ + struct hv_enlightenments *hve = + (struct hv_enlightenments *)vmcb->control.reserved_sw; + u32 vp_index = kvm_hv_get_vpindex(vcpu); + + if (hve->hv_vp_id != vp_index) { + hve->hv_vp_id = vp_index; + vmcb_mark_dirty(vmcb, VMCB_HV_NESTED_ENLIGHTENMENTS); + } +} +#else + +static inline void svm_hv_init_vmcb(struct vmcb *vmcb) +{ +} + +static inline void svm_hv_hardware_setup(void) +{ +} + +static inline void svm_hv_vmcb_dirty_nested_enlightenments( + struct kvm_vcpu *vcpu) +{ +} + +static inline void svm_hv_update_vp_id(struct vmcb *vmcb, + struct kvm_vcpu *vcpu) +{ +} +#endif /* CONFIG_HYPERV */ + +#endif /* __ARCH_X86_KVM_SVM_ONHYPERV_H__ */ diff --git a/arch/x86/kvm/trace.h b/arch/x86/kvm/trace.h index 4f839148948b..b484141ea15b 100644 --- a/arch/x86/kvm/trace.h +++ b/arch/x86/kvm/trace.h @@ -997,7 +997,7 @@ TRACE_EVENT(kvm_wait_lapic_expire, __entry->delta < 0 ? "early" : "late") ); -TRACE_EVENT(kvm_enter_smm, +TRACE_EVENT(kvm_smm_transition, TP_PROTO(unsigned int vcpu_id, u64 smbase, bool entering), TP_ARGS(vcpu_id, smbase, entering), diff --git a/arch/x86/kvm/vmx/capabilities.h b/arch/x86/kvm/vmx/capabilities.h index aa0e7872fcc9..4705ad55abb5 100644 --- a/arch/x86/kvm/vmx/capabilities.h +++ b/arch/x86/kvm/vmx/capabilities.h @@ -12,7 +12,6 @@ extern bool __read_mostly enable_ept; extern bool __read_mostly enable_unrestricted_guest; extern bool __read_mostly enable_ept_ad_bits; extern bool __read_mostly enable_pml; -extern bool __read_mostly enable_apicv; extern int __read_mostly pt_mode; #define PT_MODE_SYSTEM 0 diff --git a/arch/x86/kvm/vmx/evmcs.c b/arch/x86/kvm/vmx/evmcs.c index 41f24661af04..896b2a50b4aa 100644 --- a/arch/x86/kvm/vmx/evmcs.c +++ b/arch/x86/kvm/vmx/evmcs.c @@ -319,6 +319,9 @@ bool nested_enlightened_vmentry(struct kvm_vcpu *vcpu, u64 *evmcs_gpa) if (unlikely(!assist_page.enlighten_vmentry)) return false; + if (unlikely(!evmptr_is_valid(assist_page.current_nested_vmcs))) + return false; + *evmcs_gpa = assist_page.current_nested_vmcs; return true; diff --git a/arch/x86/kvm/vmx/evmcs.h b/arch/x86/kvm/vmx/evmcs.h index bd41d9462355..2ec9b46f0d0c 100644 --- a/arch/x86/kvm/vmx/evmcs.h +++ b/arch/x86/kvm/vmx/evmcs.h @@ -197,6 +197,14 @@ static inline void evmcs_load(u64 phys_addr) {} static inline void evmcs_touch_msr_bitmap(void) {} #endif /* IS_ENABLED(CONFIG_HYPERV) */ +#define EVMPTR_INVALID (-1ULL) +#define EVMPTR_MAP_PENDING (-2ULL) + +static inline bool evmptr_is_valid(u64 evmptr) +{ + return evmptr != EVMPTR_INVALID && evmptr != EVMPTR_MAP_PENDING; +} + enum nested_evmptrld_status { EVMPTRLD_DISABLED, EVMPTRLD_SUCCEEDED, diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6058a65a6ede..1a52134b0c42 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -173,9 +173,13 @@ static int nested_vmx_failValid(struct kvm_vcpu *vcpu, | X86_EFLAGS_ZF); get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error; /* - * We don't need to force a shadow sync because - * VM_INSTRUCTION_ERROR is not shadowed + * We don't need to force sync to shadow VMCS because + * VM_INSTRUCTION_ERROR is not shadowed. Enlightened VMCS 'shadows' all + * fields and thus must be synced. */ + if (to_vmx(vcpu)->nested.hv_evmcs_vmptr != EVMPTR_INVALID) + to_vmx(vcpu)->nested.need_vmcs12_to_shadow_sync = true; + return kvm_skip_emulated_instruction(vcpu); } @@ -187,7 +191,8 @@ static int nested_vmx_fail(struct kvm_vcpu *vcpu, u32 vm_instruction_error) * failValid writes the error number to the current VMCS, which * can't be done if there isn't a current VMCS. */ - if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs) + if (vmx->nested.current_vmptr == -1ull && + !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) return nested_vmx_failInvalid(vcpu); return nested_vmx_failValid(vcpu, vm_instruction_error); @@ -221,12 +226,12 @@ static inline void nested_release_evmcs(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (!vmx->nested.hv_evmcs) - return; + if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { + kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); + vmx->nested.hv_evmcs = NULL; + } - kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true); - vmx->nested.hv_evmcs_vmptr = 0; - vmx->nested.hv_evmcs = NULL; + vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; } static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx, @@ -346,16 +351,21 @@ static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu, vmcs12->guest_physical_address = fault->address; } +static void nested_ept_new_eptp(struct kvm_vcpu *vcpu) +{ + kvm_init_shadow_ept_mmu(vcpu, + to_vmx(vcpu)->nested.msrs.ept_caps & + VMX_EPT_EXECUTE_ONLY_BIT, + nested_ept_ad_enabled(vcpu), + nested_ept_get_eptp(vcpu)); +} + static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu) { WARN_ON(mmu_is_nested(vcpu)); vcpu->arch.mmu = &vcpu->arch.guest_mmu; - kvm_init_shadow_ept_mmu(vcpu, - to_vmx(vcpu)->nested.msrs.ept_caps & - VMX_EPT_EXECUTE_ONLY_BIT, - nested_ept_ad_enabled(vcpu), - nested_ept_get_eptp(vcpu)); + nested_ept_new_eptp(vcpu); vcpu->arch.mmu->get_guest_pgd = nested_ept_get_eptp; vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault; vcpu->arch.mmu->get_pdptr = kvm_pdptr_read; @@ -1057,55 +1067,14 @@ static void prepare_vmx_msr_autostore_list(struct kvm_vcpu *vcpu, } } -/* - * Returns true if the MMU needs to be sync'd on nested VM-Enter/VM-Exit. - * tl;dr: the MMU needs a sync if L0 is using shadow paging and L1 didn't - * enable VPID for L2 (implying it expects a TLB flush on VMX transitions). - * Here's why. - * - * If EPT is enabled by L0 a sync is never needed: - * - if it is disabled by L1, then L0 is not shadowing L1 or L2 PTEs, there - * cannot be unsync'd SPTEs for either L1 or L2. - * - * - if it is also enabled by L1, then L0 doesn't need to sync on VM-Enter - * VM-Enter as VM-Enter isn't required to invalidate guest-physical mappings - * (irrespective of VPID), i.e. L1 can't rely on the (virtual) CPU to flush - * stale guest-physical mappings for L2 from the TLB. And as above, L0 isn't - * shadowing L1 PTEs so there are no unsync'd SPTEs to sync on VM-Exit. - * - * If EPT is disabled by L0: - * - if VPID is enabled by L1 (for L2), the situation is similar to when L1 - * enables EPT: L0 doesn't need to sync as VM-Enter and VM-Exit aren't - * required to invalidate linear mappings (EPT is disabled so there are - * no combined or guest-physical mappings), i.e. L1 can't rely on the - * (virtual) CPU to flush stale linear mappings for either L2 or itself (L1). - * - * - however if VPID is disabled by L1, then a sync is needed as L1 expects all - * linear mappings (EPT is disabled so there are no combined or guest-physical - * mappings) to be invalidated on both VM-Enter and VM-Exit. - * - * Note, this logic is subtly different than nested_has_guest_tlb_tag(), which - * additionally checks that L2 has been assigned a VPID (when EPT is disabled). - * Whether or not L2 has been assigned a VPID by L0 is irrelevant with respect - * to L1's expectations, e.g. L0 needs to invalidate hardware TLB entries if L2 - * doesn't have a unique VPID to prevent reusing L1's entries (assuming L1 has - * been assigned a VPID), but L0 doesn't need to do a MMU sync because L1 - * doesn't expect stale (virtual) TLB entries to be flushed, i.e. L1 doesn't - * know that L0 will flush the TLB and so L1 will do INVVPID as needed to flush - * stale TLB entries, at which point L0 will sync L2's MMU. - */ -static bool nested_vmx_transition_mmu_sync(struct kvm_vcpu *vcpu) -{ - return !enable_ept && !nested_cpu_has_vpid(get_vmcs12(vcpu)); -} - /* * Load guest's/host's cr3 at nested entry/exit. @nested_ept is true if we are * emulating VM-Entry into a guest with EPT enabled. On failure, the expected * Exit Qualification (for a VM-Entry consistency check VM-Exit) is assigned to * @entry_failure_code. */ -static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept, +static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, + bool nested_ept, bool reload_pdptrs, enum vm_entry_failure_code *entry_failure_code) { if (CC(kvm_vcpu_is_illegal_gpa(vcpu, cr3))) { @@ -1117,27 +1086,20 @@ static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool ne * If PAE paging and EPT are both on, CR3 is not used by the CPU and * must not be dereferenced. */ - if (!nested_ept && is_pae_paging(vcpu) && - (cr3 != kvm_read_cr3(vcpu) || pdptrs_changed(vcpu))) { - if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) { - *entry_failure_code = ENTRY_FAIL_PDPTE; - return -EINVAL; - } + if (reload_pdptrs && !nested_ept && is_pae_paging(vcpu) && + CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) { + *entry_failure_code = ENTRY_FAIL_PDPTE; + return -EINVAL; } - /* - * Unconditionally skip the TLB flush on fast CR3 switch, all TLB - * flushes are handled by nested_vmx_transition_tlb_flush(). See - * nested_vmx_transition_mmu_sync for details on skipping the MMU sync. - */ if (!nested_ept) - kvm_mmu_new_pgd(vcpu, cr3, true, - !nested_vmx_transition_mmu_sync(vcpu)); + kvm_mmu_new_pgd(vcpu, cr3); vcpu->arch.cr3 = cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); - kvm_init_mmu(vcpu, false); + /* Re-initialize the MMU, e.g. to pick up CR4 MMU role changes. */ + kvm_init_mmu(vcpu); return 0; } @@ -1170,17 +1132,28 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, struct vcpu_vmx *vmx = to_vmx(vcpu); /* - * If VPID is disabled, linear and combined mappings are flushed on - * VM-Enter/VM-Exit, and guest-physical mappings are valid only for - * their associated EPTP. + * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings + * for *all* contexts to be flushed on VM-Enter/VM-Exit, i.e. it's a + * full TLB flush from the guest's perspective. This is required even + * if VPID is disabled in the host as KVM may need to synchronize the + * MMU in response to the guest TLB flush. + * + * Note, using TLB_FLUSH_GUEST is correct even if nested EPT is in use. + * EPT is a special snowflake, as guest-physical mappings aren't + * flushed on VPID invalidations, including VM-Enter or VM-Exit with + * VPID disabled. As a result, KVM _never_ needs to sync nEPT + * entries on VM-Enter because L1 can't rely on VM-Enter to flush + * those mappings. */ - if (!enable_vpid) + if (!nested_cpu_has_vpid(vmcs12)) { + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); return; + } + + /* L2 should never have a VPID if VPID is disabled. */ + WARN_ON(!enable_vpid); /* - * If vmcs12 doesn't use VPID, L1 expects linear and combined mappings - * for *all* contexts to be flushed on VM-Enter/VM-Exit. - * * If VPID is enabled and used by vmc12, but L2 does not have a unique * TLB tag (ASID), i.e. EPT is disabled and KVM was unable to allocate * a VPID for L2, flush the current context as the effective ASID is @@ -1192,13 +1165,12 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, * * If a TLB flush isn't required due to any of the above, and vpid12 is * changing then the new "virtual" VPID (vpid12) will reuse the same - * "real" VPID (vpid02), and so needs to be sync'd. There is no direct + * "real" VPID (vpid02), and so needs to be flushed. There's no direct * mapping between vpid02 and vpid12, vpid02 is per-vCPU and reused for - * all nested vCPUs. + * all nested vCPUs. Remember, a flush on VM-Enter does not invalidate + * guest-physical mappings, so there is no need to sync the nEPT MMU. */ - if (!nested_cpu_has_vpid(vmcs12)) { - kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); - } else if (!nested_has_guest_tlb_tag(vcpu)) { + if (!nested_has_guest_tlb_tag(vcpu)) { kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); } else if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { @@ -1586,7 +1558,7 @@ static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx) vmcs_load(vmx->loaded_vmcs->vmcs); } -static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) +static void copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx, u32 hv_clean_fields) { struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; @@ -1595,7 +1567,7 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) vmcs12->tpr_threshold = evmcs->tpr_threshold; vmcs12->guest_rip = evmcs->guest_rip; - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) { vmcs12->guest_rsp = evmcs->guest_rsp; vmcs12->guest_rflags = evmcs->guest_rflags; @@ -1603,23 +1575,23 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) evmcs->guest_interruptibility_info; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) { vmcs12->cpu_based_vm_exec_control = evmcs->cpu_based_vm_exec_control; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) { vmcs12->exception_bitmap = evmcs->exception_bitmap; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) { vmcs12->vm_entry_controls = evmcs->vm_entry_controls; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) { vmcs12->vm_entry_intr_info_field = evmcs->vm_entry_intr_info_field; @@ -1629,7 +1601,7 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) evmcs->vm_entry_instruction_len; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) { vmcs12->host_ia32_pat = evmcs->host_ia32_pat; vmcs12->host_ia32_efer = evmcs->host_ia32_efer; @@ -1649,7 +1621,7 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) vmcs12->host_tr_selector = evmcs->host_tr_selector; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) { vmcs12->pin_based_vm_exec_control = evmcs->pin_based_vm_exec_control; @@ -1658,18 +1630,18 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) evmcs->secondary_vm_exec_control; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) { vmcs12->io_bitmap_a = evmcs->io_bitmap_a; vmcs12->io_bitmap_b = evmcs->io_bitmap_b; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) { vmcs12->msr_bitmap = evmcs->msr_bitmap; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) { vmcs12->guest_es_base = evmcs->guest_es_base; vmcs12->guest_cs_base = evmcs->guest_cs_base; @@ -1709,14 +1681,14 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) vmcs12->guest_tr_selector = evmcs->guest_tr_selector; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) { vmcs12->tsc_offset = evmcs->tsc_offset; vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr; vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) { vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask; vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask; @@ -1728,7 +1700,7 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) vmcs12->guest_dr7 = evmcs->guest_dr7; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) { vmcs12->host_fs_base = evmcs->host_fs_base; vmcs12->host_gs_base = evmcs->host_gs_base; @@ -1738,13 +1710,13 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) vmcs12->host_rsp = evmcs->host_rsp; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) { vmcs12->ept_pointer = evmcs->ept_pointer; vmcs12->virtual_processor_id = evmcs->virtual_processor_id; } - if (unlikely(!(evmcs->hv_clean_fields & + if (unlikely(!(hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) { vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer; vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl; @@ -1799,10 +1771,10 @@ static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx) * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip; */ - return 0; + return; } -static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) +static void copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) { struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12; struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs; @@ -1962,7 +1934,7 @@ static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx) evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs; - return 0; + return; } /* @@ -1979,13 +1951,13 @@ static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( if (likely(!vmx->nested.enlightened_vmcs_enabled)) return EVMPTRLD_DISABLED; - if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa)) + if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa)) { + nested_release_evmcs(vcpu); return EVMPTRLD_DISABLED; + } - if (unlikely(!vmx->nested.hv_evmcs || - evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { - if (!vmx->nested.hv_evmcs) - vmx->nested.current_vmptr = -1ull; + if (unlikely(evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) { + vmx->nested.current_vmptr = -1ull; nested_release_evmcs(vcpu); @@ -2023,7 +1995,6 @@ static enum nested_evmptrld_status nested_vmx_handle_enlightened_vmptrld( return EVMPTRLD_VMFAIL; } - vmx->nested.dirty_vmcs12 = true; vmx->nested.hv_evmcs_vmptr = evmcs_gpa; evmcs_gpa_changed = true; @@ -2056,14 +2027,10 @@ void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (vmx->nested.hv_evmcs) { + if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) copy_vmcs12_to_enlightened(vmx); - /* All fields are clean */ - vmx->nested.hv_evmcs->hv_clean_fields |= - HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; - } else { + else copy_vmcs12_to_shadow(vmx); - } vmx->nested.need_vmcs12_to_shadow_sync = false; } @@ -2208,7 +2175,7 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) u32 exec_control; u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12); - if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs) + if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) prepare_vmcs02_early_rare(vmx, vmcs12); /* @@ -2277,7 +2244,8 @@ static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE | SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY | SECONDARY_EXEC_APIC_REGISTER_VIRT | - SECONDARY_EXEC_ENABLE_VMFUNC); + SECONDARY_EXEC_ENABLE_VMFUNC | + SECONDARY_EXEC_TSC_SCALING); if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) exec_control |= vmcs12->secondary_vm_exec_control; @@ -2488,18 +2456,18 @@ static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12) * is assigned to entry_failure_code on failure. */ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, + bool from_vmentry, enum vm_entry_failure_code *entry_failure_code) { struct vcpu_vmx *vmx = to_vmx(vcpu); - struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs; bool load_guest_pdptrs_vmcs12 = false; - if (vmx->nested.dirty_vmcs12 || hv_evmcs) { + if (vmx->nested.dirty_vmcs12 || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { prepare_vmcs02_rare(vmx, vmcs12); vmx->nested.dirty_vmcs12 = false; - load_guest_pdptrs_vmcs12 = !hv_evmcs || - !(hv_evmcs->hv_clean_fields & + load_guest_pdptrs_vmcs12 = !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) || + !(vmx->nested.hv_evmcs->hv_clean_fields & HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1); } @@ -2532,10 +2500,18 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat); } - vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); + vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( + vcpu->arch.l1_tsc_offset, + vmx_get_l2_tsc_offset(vcpu), + vmx_get_l2_tsc_multiplier(vcpu)); + vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( + vcpu->arch.l1_tsc_scaling_ratio, + vmx_get_l2_tsc_multiplier(vcpu)); + + vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); if (kvm_has_tsc_control) - decache_tsc_multiplier(vmx); + vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); nested_vmx_transition_tlb_flush(vcpu, vmcs12, true); @@ -2572,7 +2548,7 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, /* Shadow page tables on either EPT or shadow page tables. */ if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12), - entry_failure_code)) + from_vmentry, entry_failure_code)) return -EINVAL; /* @@ -2604,6 +2580,17 @@ static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12, kvm_rsp_write(vcpu, vmcs12->guest_rsp); kvm_rip_write(vcpu, vmcs12->guest_rip); + + /* + * It was observed that genuine Hyper-V running in L1 doesn't reset + * 'hv_clean_fields' by itself, it only sets the corresponding dirty + * bits when it changes a field in eVMCS. Mark all fields as clean + * here. + */ + if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + vmx->nested.hv_evmcs->hv_clean_fields |= + HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL; + return 0; } @@ -3093,13 +3080,20 @@ static bool nested_get_evmcs_page(struct kvm_vcpu *vcpu) * L2 was running), map it here to make sure vmcs12 changes are * properly reflected. */ - if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs) { + if (vmx->nested.enlightened_vmcs_enabled && + vmx->nested.hv_evmcs_vmptr == EVMPTR_MAP_PENDING) { enum nested_evmptrld_status evmptrld_status = nested_vmx_handle_enlightened_vmptrld(vcpu, false); if (evmptrld_status == EVMPTRLD_VMFAIL || evmptrld_status == EVMPTRLD_ERROR) return false; + + /* + * Post migration VMCS12 always provides the most actual + * information, copy it to eVMCS upon entry. + */ + vmx->nested.need_vmcs12_to_shadow_sync = true; } return true; @@ -3113,6 +3107,18 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) struct page *page; u64 hpa; + if (!vcpu->arch.pdptrs_from_userspace && + !nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) { + /* + * Reload the guest's PDPTRs since after a migration + * the guest CR3 might be restored prior to setting the nested + * state which can lead to a load of wrong PDPTRs. + */ + if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, vcpu->arch.cr3))) + return false; + } + + if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) { /* * Translate L1 physical address to host physical @@ -3175,6 +3181,15 @@ static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu) offset_in_page(vmcs12->posted_intr_desc_addr)); vmcs_write64(POSTED_INTR_DESC_ADDR, pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr)); + } else { + /* + * Defer the KVM_INTERNAL_EXIT until KVM tries to + * access the contents of the VMCS12 posted interrupt + * descriptor. (Note that KVM may do this when it + * should not, per the architectural specification.) + */ + vmx->nested.pi_desc = NULL; + pin_controls_clearbit(vmx, PIN_BASED_POSTED_INTR); } } if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12)) @@ -3354,10 +3369,8 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, } enter_guest_mode(vcpu); - if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) - vcpu->arch.tsc_offset += vmcs12->tsc_offset; - if (prepare_vmcs02(vcpu, vmcs12, &entry_failure_code)) { + if (prepare_vmcs02(vcpu, vmcs12, from_vmentry, &entry_failure_code)) { exit_reason.basic = EXIT_REASON_INVALID_STATE; vmcs12->exit_qualification = entry_failure_code; goto vmentry_fail_vmexit_guest_mode; @@ -3437,7 +3450,7 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, load_vmcs12_host_state(vcpu, vmcs12); vmcs12->vm_exit_reason = exit_reason.full; - if (enable_shadow_vmcs || vmx->nested.hv_evmcs) + if (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) vmx->nested.need_vmcs12_to_shadow_sync = true; return NVMX_VMENTRY_VMEXIT; } @@ -3454,8 +3467,6 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu); enum nested_evmptrld_status evmptrld_status; - ++vcpu->stat.nested_run; - if (!nested_vmx_check_permission(vcpu)) return 1; @@ -3467,7 +3478,8 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) return nested_vmx_failInvalid(vcpu); } - if (CC(!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull)) + if (CC(!evmptr_is_valid(vmx->nested.hv_evmcs_vmptr) && + vmx->nested.current_vmptr == -1ull)) return nested_vmx_failInvalid(vcpu); vmcs12 = get_vmcs12(vcpu); @@ -3481,8 +3493,8 @@ static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch) if (CC(vmcs12->hdr.shadow_vmcs)) return nested_vmx_failInvalid(vcpu); - if (vmx->nested.hv_evmcs) { - copy_enlightened_to_vmcs12(vmx); + if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) { + copy_enlightened_to_vmcs12(vmx, vmx->nested.hv_evmcs->hv_clean_fields); /* Enlightened VMCS doesn't have launch state */ vmcs12->launch_state = !launch; } else if (enable_shadow_vmcs) { @@ -3682,25 +3694,29 @@ void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu) } } -static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) +static int vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); int max_irr; void *vapic_page; u16 status; - if (!vmx->nested.pi_desc || !vmx->nested.pi_pending) - return; + if (!vmx->nested.pi_pending) + return 0; + + if (!vmx->nested.pi_desc) + goto mmio_needed; vmx->nested.pi_pending = false; + if (!pi_test_and_clear_on(vmx->nested.pi_desc)) - return; + return 0; max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256); if (max_irr != 256) { vapic_page = vmx->nested.virtual_apic_map.hva; if (!vapic_page) - return; + goto mmio_needed; __kvm_apic_update_irr(vmx->nested.pi_desc->pir, vapic_page, &max_irr); @@ -3713,6 +3729,11 @@ static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu) } nested_mark_vmcs12_pages_dirty(vcpu); + return 0; + +mmio_needed: + kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); + return -ENXIO; } static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu, @@ -3887,8 +3908,7 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) } no_vmexit: - vmx_complete_nested_posted_interrupt(vcpu); - return 0; + return vmx_complete_nested_posted_interrupt(vcpu); } static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu) @@ -4032,10 +4052,11 @@ static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12) { struct vcpu_vmx *vmx = to_vmx(vcpu); - if (vmx->nested.hv_evmcs) + if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12); - vmx->nested.need_sync_vmcs02_to_vmcs12_rare = !vmx->nested.hv_evmcs; + vmx->nested.need_sync_vmcs02_to_vmcs12_rare = + !evmptr_is_valid(vmx->nested.hv_evmcs_vmptr); vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12); vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12); @@ -4206,7 +4227,7 @@ static void load_vmcs12_host_state(struct kvm_vcpu *vcpu, * Only PDPTE load can fail as the value of cr3 was checked on entry and * couldn't have changed. */ - if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &ignored)) + if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, true, &ignored)) nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL); nested_vmx_transition_tlb_flush(vcpu, vmcs12, false); @@ -4463,8 +4484,11 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, if (nested_cpu_has_preemption_timer(vmcs12)) hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer); - if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) - vcpu->arch.tsc_offset -= vmcs12->tsc_offset; + if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) { + vcpu->arch.tsc_offset = vcpu->arch.l1_tsc_offset; + if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) + vcpu->arch.tsc_scaling_ratio = vcpu->arch.l1_tsc_scaling_ratio; + } if (likely(!vmx->fail)) { sync_vmcs02_to_vmcs12(vcpu, vmcs12); @@ -4501,12 +4525,12 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr); vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr); vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset); + if (kvm_has_tsc_control) + vmcs_write64(TSC_MULTIPLIER, vcpu->arch.tsc_scaling_ratio); + if (vmx->nested.l1_tpr_threshold != -1) vmcs_write32(TPR_THRESHOLD, vmx->nested.l1_tpr_threshold); - if (kvm_has_tsc_control) - decache_tsc_multiplier(vmx); - if (vmx->nested.change_vmcs01_virtual_apic_mode) { vmx->nested.change_vmcs01_virtual_apic_mode = false; vmx_set_virtual_apic_mode(vcpu); @@ -4532,7 +4556,7 @@ void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, } if ((vm_exit_reason != -1) && - (enable_shadow_vmcs || vmx->nested.hv_evmcs)) + (enable_shadow_vmcs || evmptr_is_valid(vmx->nested.hv_evmcs_vmptr))) vmx->nested.need_vmcs12_to_shadow_sync = true; /* in case we halted in L2 */ @@ -4987,6 +5011,8 @@ static int handle_vmclear(struct kvm_vcpu *vcpu) vmptr + offsetof(struct vmcs12, launch_state), &zero, sizeof(zero)); + } else if (vmx->nested.hv_evmcs && vmptr == vmx->nested.hv_evmcs_vmptr) { + nested_release_evmcs(vcpu); } return nested_vmx_succeed(vcpu); @@ -5228,7 +5254,7 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) return nested_vmx_fail(vcpu, VMXERR_VMPTRLD_VMXON_POINTER); /* Forbid normal VMPTRLD if Enlightened version was used */ - if (vmx->nested.hv_evmcs) + if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) return 1; if (vmx->nested.current_vmptr != vmptr) { @@ -5284,7 +5310,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) if (!nested_vmx_check_permission(vcpu)) return 1; - if (unlikely(to_vmx(vcpu)->nested.hv_evmcs)) + if (unlikely(evmptr_is_valid(to_vmx(vcpu)->nested.hv_evmcs_vmptr))) return 1; if (get_vmx_mem_address(vcpu, exit_qual, instr_info, @@ -5461,8 +5487,8 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) /* * Sync the shadow page tables if EPT is disabled, L1 is invalidating - * linear mappings for L2 (tagged with L2's VPID). Free all roots as - * VPIDs are not tracked in the MMU role. + * linear mappings for L2 (tagged with L2's VPID). Free all guest + * roots as VPIDs are not tracked in the MMU role. * * Note, this operates on root_mmu, not guest_mmu, as L1 and L2 share * an MMU when EPT is disabled. @@ -5470,8 +5496,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) * TODO: sync only the affected SPTEs for INVDIVIDUAL_ADDR. */ if (!enable_ept) - kvm_mmu_free_roots(vcpu, &vcpu->arch.root_mmu, - KVM_MMU_ROOTS_ALL); + kvm_mmu_free_guest_mode_roots(vcpu, &vcpu->arch.root_mmu); return nested_vmx_succeed(vcpu); } @@ -5481,23 +5506,16 @@ static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, { u32 index = kvm_rcx_read(vcpu); u64 new_eptp; - bool accessed_dirty; - struct kvm_mmu *mmu = vcpu->arch.walk_mmu; - if (!nested_cpu_has_eptp_switching(vmcs12) || - !nested_cpu_has_ept(vmcs12)) + if (WARN_ON_ONCE(!nested_cpu_has_ept(vmcs12))) return 1; - if (index >= VMFUNC_EPTP_ENTRIES) return 1; - if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT, &new_eptp, index * 8, 8)) return 1; - accessed_dirty = !!(new_eptp & VMX_EPTP_AD_ENABLE_BIT); - /* * If the (L2) guest does a vmfunc to the currently * active ept pointer, we don't have to do anything else @@ -5506,11 +5524,11 @@ static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu, if (!nested_vmx_check_eptp(vcpu, new_eptp)) return 1; - mmu->ept_ad = accessed_dirty; - mmu->mmu_role.base.ad_disabled = !accessed_dirty; vmcs12->ept_pointer = new_eptp; + nested_ept_new_eptp(vcpu); - kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); + if (!nested_cpu_has_vpid(vmcs12)) + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); } return 0; @@ -5533,7 +5551,17 @@ static int handle_vmfunc(struct kvm_vcpu *vcpu) } vmcs12 = get_vmcs12(vcpu); - if ((vmcs12->vm_function_control & (1 << function)) == 0) + + /* + * #UD on out-of-bounds function has priority over VM-Exit, and VMFUNC + * is enabled in vmcs02 if and only if it's enabled in vmcs12. + */ + if (WARN_ON_ONCE((function > 63) || !nested_cpu_has_vmfunc(vmcs12))) { + kvm_queue_exception(vcpu, UD_VECTOR); + return 1; + } + + if (!(vmcs12->vm_function_control & BIT_ULL(function))) goto fail; switch (function) { @@ -5806,6 +5834,9 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, else if (is_breakpoint(intr_info) && vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) return true; + else if (is_alignment_check(intr_info) && + !vmx_guest_inject_ac(vcpu)) + return true; return false; case EXIT_REASON_EXTERNAL_INTERRUPT: return true; @@ -6056,7 +6087,8 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, if (vmx_has_valid_vmcs12(vcpu)) { kvm_state.size += sizeof(user_vmx_nested_state->vmcs12); - if (vmx->nested.hv_evmcs) + /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ + if (vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID) kvm_state.flags |= KVM_STATE_NESTED_EVMCS; if (is_guest_mode(vcpu) && @@ -6112,8 +6144,15 @@ static int vmx_get_nested_state(struct kvm_vcpu *vcpu, } else { copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu)); if (!vmx->nested.need_vmcs12_to_shadow_sync) { - if (vmx->nested.hv_evmcs) - copy_enlightened_to_vmcs12(vmx); + if (evmptr_is_valid(vmx->nested.hv_evmcs_vmptr)) + /* + * L1 hypervisor is not obliged to keep eVMCS + * clean fields data always up-to-date while + * not in guest mode, 'hv_clean_fields' is only + * supposed to be actual upon vmentry so we need + * to ignore it here and do full copy. + */ + copy_enlightened_to_vmcs12(vmx, 0); else if (enable_shadow_vmcs) copy_shadow_to_vmcs12(vmx); } @@ -6255,6 +6294,7 @@ static int vmx_set_nested_state(struct kvm_vcpu *vcpu, * restored yet. EVMCS will be mapped from * nested_get_vmcs12_pages(). */ + vmx->nested.hv_evmcs_vmptr = EVMPTR_MAP_PENDING; kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); } else { return -EINVAL; @@ -6338,6 +6378,40 @@ void nested_vmx_set_vmcs_shadowing_bitmap(void) } } +/* + * Indexing into the vmcs12 uses the VMCS encoding rotated left by 6. Undo + * that madness to get the encoding for comparison. + */ +#define VMCS12_IDX_TO_ENC(idx) ((u16)(((u16)(idx) >> 6) | ((u16)(idx) << 10))) + +static u64 nested_vmx_calc_vmcs_enum_msr(void) +{ + /* + * Note these are the so called "index" of the VMCS field encoding, not + * the index into vmcs12. + */ + unsigned int max_idx, idx; + int i; + + /* + * For better or worse, KVM allows VMREAD/VMWRITE to all fields in + * vmcs12, regardless of whether or not the associated feature is + * exposed to L1. Simply find the field with the highest index. + */ + max_idx = 0; + for (i = 0; i < nr_vmcs12_fields; i++) { + /* The vmcs12 table is very, very sparsely populated. */ + if (!vmcs_field_to_offset_table[i]) + continue; + + idx = vmcs_field_index(VMCS12_IDX_TO_ENC(i)); + if (idx > max_idx) + max_idx = idx; + } + + return (u64)max_idx << VMCS_FIELD_INDEX_SHIFT; +} + /* * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be * returned for the various VMX controls MSRs when nested VMX is enabled. @@ -6474,7 +6548,8 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) SECONDARY_EXEC_RDRAND_EXITING | SECONDARY_EXEC_ENABLE_INVPCID | SECONDARY_EXEC_RDSEED_EXITING | - SECONDARY_EXEC_XSAVES; + SECONDARY_EXEC_XSAVES | + SECONDARY_EXEC_TSC_SCALING; /* * We can emulate "VMCS shadowing," even if the hardware @@ -6582,8 +6657,7 @@ void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps) rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1); rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1); - /* highest index: VMX_PREEMPTION_TIMER_VALUE */ - msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1; + msrs->vmcs_enum = nested_vmx_calc_vmcs_enum_msr(); } void nested_vmx_hardware_unsetup(void) diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 184418baeb3c..b69a80f43b37 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -56,14 +56,9 @@ static inline int vmx_has_valid_vmcs12(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); - /* - * In case we do two consecutive get/set_nested_state()s while L2 was - * running hv_evmcs may end up not being mapped (we map it from - * nested_vmx_run()/vmx_vcpu_run()). Check is_guest_mode() as we always - * have vmcs12 if it is true. - */ - return is_guest_mode(vcpu) || vmx->nested.current_vmptr != -1ull || - vmx->nested.hv_evmcs; + /* 'hv_evmcs_vmptr' can also be EVMPTR_MAP_PENDING here */ + return vmx->nested.current_vmptr != -1ull || + vmx->nested.hv_evmcs_vmptr != EVMPTR_INVALID; } static inline u16 nested_get_vpid02(struct kvm_vcpu *vcpu) diff --git a/arch/x86/kvm/vmx/vmcs.h b/arch/x86/kvm/vmx/vmcs.h index 1472c6c376f7..4b9957e2bf5b 100644 --- a/arch/x86/kvm/vmx/vmcs.h +++ b/arch/x86/kvm/vmx/vmcs.h @@ -117,6 +117,11 @@ static inline bool is_gp_fault(u32 intr_info) return is_exception_n(intr_info, GP_VECTOR); } +static inline bool is_alignment_check(u32 intr_info) +{ + return is_exception_n(intr_info, AC_VECTOR); +} + static inline bool is_machine_check(u32 intr_info) { return is_exception_n(intr_info, MC_VECTOR); @@ -164,4 +169,12 @@ static inline int vmcs_field_readonly(unsigned long field) return (((field >> 10) & 0x3) == 1); } +#define VMCS_FIELD_INDEX_SHIFT (1) +#define VMCS_FIELD_INDEX_MASK GENMASK(9, 1) + +static inline unsigned int vmcs_field_index(unsigned long field) +{ + return (field & VMCS_FIELD_INDEX_MASK) >> VMCS_FIELD_INDEX_SHIFT; +} + #endif /* __KVM_X86_VMX_VMCS_H */ diff --git a/arch/x86/kvm/vmx/vmcs12.c b/arch/x86/kvm/vmx/vmcs12.c index 034adb6404dc..d9f5d7c56ae3 100644 --- a/arch/x86/kvm/vmx/vmcs12.c +++ b/arch/x86/kvm/vmx/vmcs12.c @@ -37,6 +37,7 @@ const unsigned short vmcs_field_to_offset_table[] = { FIELD64(VM_ENTRY_MSR_LOAD_ADDR, vm_entry_msr_load_addr), FIELD64(PML_ADDRESS, pml_address), FIELD64(TSC_OFFSET, tsc_offset), + FIELD64(TSC_MULTIPLIER, tsc_multiplier), FIELD64(VIRTUAL_APIC_PAGE_ADDR, virtual_apic_page_addr), FIELD64(APIC_ACCESS_ADDR, apic_access_addr), FIELD64(POSTED_INTR_DESC_ADDR, posted_intr_desc_addr), diff --git a/arch/x86/kvm/vmx/vmcs12.h b/arch/x86/kvm/vmx/vmcs12.h index 13494956d0e9..5e0e1b39f495 100644 --- a/arch/x86/kvm/vmx/vmcs12.h +++ b/arch/x86/kvm/vmx/vmcs12.h @@ -70,7 +70,8 @@ struct __packed vmcs12 { u64 eptp_list_address; u64 pml_address; u64 encls_exiting_bitmap; - u64 padding64[2]; /* room for future expansion */ + u64 tsc_multiplier; + u64 padding64[1]; /* room for future expansion */ /* * To allow migration of L1 (complete with its L2 guests) between * machines of different natural widths (32 or 64 bit), we cannot have @@ -204,12 +205,6 @@ struct __packed vmcs12 { */ #define VMCS12_SIZE KVM_STATE_NESTED_VMX_VMCS_SIZE -/* - * VMCS12_MAX_FIELD_INDEX is the highest index value used in any - * supported VMCS12 field encoding. - */ -#define VMCS12_MAX_FIELD_INDEX 0x17 - /* * For save/restore compatibility, the vmcs12 field offsets must not change. */ @@ -258,6 +253,7 @@ static inline void vmx_check_vmcs12_offsets(void) CHECK_OFFSET(eptp_list_address, 304); CHECK_OFFSET(pml_address, 312); CHECK_OFFSET(encls_exiting_bitmap, 320); + CHECK_OFFSET(tsc_multiplier, 328); CHECK_OFFSET(cr0_guest_host_mask, 344); CHECK_OFFSET(cr4_guest_host_mask, 352); CHECK_OFFSET(cr0_read_shadow, 360); diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index c2a779b688e6..927a552393b9 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -52,6 +52,7 @@ #include "cpuid.h" #include "evmcs.h" #include "hyperv.h" +#include "kvm_onhyperv.h" #include "irq.h" #include "kvm_cache_regs.h" #include "lapic.h" @@ -101,7 +102,6 @@ module_param(emulate_invalid_guest_state, bool, S_IRUGO); static bool __read_mostly fasteoi = 1; module_param(fasteoi, bool, S_IRUGO); -bool __read_mostly enable_apicv = 1; module_param(enable_apicv, bool, S_IRUGO); /* @@ -459,86 +459,6 @@ static unsigned long host_idt_base; static bool __read_mostly enlightened_vmcs = true; module_param(enlightened_vmcs, bool, 0444); -static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush, - void *data) -{ - struct kvm_tlb_range *range = data; - - return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn, - range->pages); -} - -static inline int hv_remote_flush_root_ept(hpa_t root_ept, - struct kvm_tlb_range *range) -{ - if (range) - return hyperv_flush_guest_mapping_range(root_ept, - kvm_fill_hv_flush_list_func, (void *)range); - else - return hyperv_flush_guest_mapping(root_ept); -} - -static int hv_remote_flush_tlb_with_range(struct kvm *kvm, - struct kvm_tlb_range *range) -{ - struct kvm_vmx *kvm_vmx = to_kvm_vmx(kvm); - struct kvm_vcpu *vcpu; - int ret = 0, i, nr_unique_valid_roots; - hpa_t root; - - spin_lock(&kvm_vmx->hv_root_ept_lock); - - if (!VALID_PAGE(kvm_vmx->hv_root_ept)) { - nr_unique_valid_roots = 0; - - /* - * Flush all valid roots, and see if all vCPUs have converged - * on a common root, in which case future flushes can skip the - * loop and flush the common root. - */ - kvm_for_each_vcpu(i, vcpu, kvm) { - root = to_vmx(vcpu)->hv_root_ept; - if (!VALID_PAGE(root) || root == kvm_vmx->hv_root_ept) - continue; - - /* - * Set the tracked root to the first valid root. Keep - * this root for the entirety of the loop even if more - * roots are encountered as a low effort optimization - * to avoid flushing the same (first) root again. - */ - if (++nr_unique_valid_roots == 1) - kvm_vmx->hv_root_ept = root; - - if (!ret) - ret = hv_remote_flush_root_ept(root, range); - - /* - * Stop processing roots if a failure occurred and - * multiple valid roots have already been detected. - */ - if (ret && nr_unique_valid_roots > 1) - break; - } - - /* - * The optimized flush of a single root can't be used if there - * are multiple valid roots (obviously). - */ - if (nr_unique_valid_roots > 1) - kvm_vmx->hv_root_ept = INVALID_PAGE; - } else { - ret = hv_remote_flush_root_ept(kvm_vmx->hv_root_ept, range); - } - - spin_unlock(&kvm_vmx->hv_root_ept_lock); - return ret; -} -static int hv_remote_flush_tlb(struct kvm *kvm) -{ - return hv_remote_flush_tlb_with_range(kvm, NULL); -} - static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) { struct hv_enlightened_vmcs *evmcs; @@ -566,21 +486,6 @@ static int hv_enable_direct_tlbflush(struct kvm_vcpu *vcpu) #endif /* IS_ENABLED(CONFIG_HYPERV) */ -static void hv_track_root_ept(struct kvm_vcpu *vcpu, hpa_t root_ept) -{ -#if IS_ENABLED(CONFIG_HYPERV) - struct kvm_vmx *kvm_vmx = to_kvm_vmx(vcpu->kvm); - - if (kvm_x86_ops.tlb_remote_flush == hv_remote_flush_tlb) { - spin_lock(&kvm_vmx->hv_root_ept_lock); - to_vmx(vcpu)->hv_root_ept = root_ept; - if (root_ept != kvm_vmx->hv_root_ept) - kvm_vmx->hv_root_ept = INVALID_PAGE; - spin_unlock(&kvm_vmx->hv_root_ept_lock); - } -#endif -} - /* * Comment's format: document - errata name - stepping - processor name. * Refer from @@ -842,16 +747,21 @@ void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu) if (is_guest_mode(vcpu)) eb |= get_vmcs12(vcpu)->exception_bitmap; else { - /* - * If EPT is enabled, #PF is only trapped if MAXPHYADDR is mismatched - * between guest and host. In that case we only care about present - * faults. For vmcs02, however, PFEC_MASK and PFEC_MATCH are set in - * prepare_vmcs02_rare. - */ - bool selective_pf_trap = enable_ept && (eb & (1u << PF_VECTOR)); - int mask = selective_pf_trap ? PFERR_PRESENT_MASK : 0; + int mask = 0, match = 0; + + if (enable_ept && (eb & (1u << PF_VECTOR))) { + /* + * If EPT is enabled, #PF is currently only intercepted + * if MAXPHYADDR is smaller on the guest than on the + * host. In that case we only care about present, + * non-reserved faults. For vmcs02, however, PFEC_MASK + * and PFEC_MATCH are set in prepare_vmcs02_rare. + */ + mask = PFERR_PRESENT_MASK | PFERR_RSVD_MASK; + match = PFERR_PRESENT_MASK; + } vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK, mask); - vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, mask); + vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH, match); } vmcs_write32(EXCEPTION_BITMAP, eb); @@ -1390,11 +1300,6 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu, vmx->loaded_vmcs->cpu = cpu; } - - /* Setup TSC multiplier */ - if (kvm_has_tsc_control && - vmx->current_tsc_ratio != vcpu->arch.tsc_scaling_ratio) - decache_tsc_multiplier(vmx); } /* @@ -1787,26 +1692,35 @@ static void setup_msrs(struct vcpu_vmx *vmx) vmx->guest_uret_msrs_loaded = false; } -static u64 vmx_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) +u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu) { struct vmcs12 *vmcs12 = get_vmcs12(vcpu); - u64 g_tsc_offset = 0; - /* - * We're here if L1 chose not to trap WRMSR to TSC. According - * to the spec, this should set L1's TSC; The offset that L1 - * set for L2 remains unchanged, and still needs to be added - * to the newly set TSC to get L2's TSC. - */ - if (is_guest_mode(vcpu) && - (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)) - g_tsc_offset = vmcs12->tsc_offset; + if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING)) + return vmcs12->tsc_offset; - trace_kvm_write_tsc_offset(vcpu->vcpu_id, - vcpu->arch.tsc_offset - g_tsc_offset, - offset); - vmcs_write64(TSC_OFFSET, offset + g_tsc_offset); - return offset + g_tsc_offset; + return 0; +} + +u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu) +{ + struct vmcs12 *vmcs12 = get_vmcs12(vcpu); + + if (nested_cpu_has(vmcs12, CPU_BASED_USE_TSC_OFFSETTING) && + nested_cpu_has2(vmcs12, SECONDARY_EXEC_TSC_SCALING)) + return vmcs12->tsc_multiplier; + + return kvm_default_tsc_scaling_ratio; +} + +static void vmx_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) +{ + vmcs_write64(TSC_OFFSET, offset); +} + +static void vmx_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 multiplier) +{ + vmcs_write64(TSC_MULTIPLIER, multiplier); } /* @@ -3181,7 +3095,7 @@ static void vmx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, eptp = construct_eptp(vcpu, root_hpa, root_level); vmcs_write64(EPT_POINTER, eptp); - hv_track_root_ept(vcpu, root_hpa); + hv_track_root_tdp(vcpu, root_hpa); if (!enable_unrestricted_guest && !is_paging(vcpu)) guest_cr3 = to_kvm_vmx(kvm)->ept_identity_map_addr; @@ -3707,7 +3621,7 @@ static int alloc_apic_access_page(struct kvm *kvm) int ret = 0; mutex_lock(&kvm->slots_lock); - if (kvm->arch.apic_access_page_done) + if (kvm->arch.apic_access_memslot_enabled) goto out; hva = __x86_set_memory_region(kvm, APIC_ACCESS_PAGE_PRIVATE_MEMSLOT, APIC_DEFAULT_PHYS_BASE, PAGE_SIZE); @@ -3727,7 +3641,7 @@ static int alloc_apic_access_page(struct kvm *kvm) * is able to migrate it. */ put_page(page); - kvm->arch.apic_access_page_done = true; + kvm->arch.apic_access_memslot_enabled = true; out: mutex_unlock(&kvm->slots_lock); return ret; @@ -4829,7 +4743,7 @@ static int handle_machine_check(struct kvm_vcpu *vcpu) * - Guest has #AC detection enabled in CR0 * - Guest EFLAGS has AC bit set */ -static inline bool guest_inject_ac(struct kvm_vcpu *vcpu) +bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu) { if (!boot_cpu_has(X86_FEATURE_SPLIT_LOCK_DETECT)) return true; @@ -4937,7 +4851,7 @@ static int handle_exception_nmi(struct kvm_vcpu *vcpu) kvm_run->debug.arch.exception = ex_no; break; case AC_VECTOR: - if (guest_inject_ac(vcpu)) { + if (vmx_guest_inject_ac(vcpu)) { kvm_queue_exception_e(vcpu, AC_VECTOR, error_code); return 1; } @@ -5810,6 +5724,8 @@ void dump_vmcs(struct kvm_vcpu *vcpu) if (cpu_has_secondary_exec_ctrls()) secondary_exec_control = vmcs_read32(SECONDARY_VM_EXEC_CONTROL); + pr_err("VMCS %p, last attempted VM-entry on CPU %d\n", + vmx->loaded_vmcs->vmcs, vcpu->arch.last_vmentry_cpu); pr_err("*** Guest State ***\n"); pr_err("CR0: actual=0x%016lx, shadow=0x%016lx, gh_mask=%016lx\n", vmcs_readl(GUEST_CR0), vmcs_readl(CR0_READ_SHADOW), @@ -6806,7 +6722,18 @@ static fastpath_t vmx_vcpu_run(struct kvm_vcpu *vcpu) kvm_load_host_xsave_state(vcpu); - vmx->nested.nested_run_pending = 0; + if (is_guest_mode(vcpu)) { + /* + * Track VMLAUNCH/VMRESUME that have made past guest state + * checking. + */ + if (vmx->nested.nested_run_pending && + !vmx->exit_reason.failed_vmentry) + ++vcpu->stat.nested_run; + + vmx->nested.nested_run_pending = 0; + } + vmx->idt_vectoring_info = 0; if (unlikely(vmx->fail)) { @@ -6941,6 +6868,7 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) vmx->nested.posted_intr_nv = -1; vmx->nested.current_vmptr = -1ull; + vmx->nested.hv_evmcs_vmptr = EVMPTR_INVALID; vcpu->arch.microcode_version = 0x100000000ULL; vmx->msr_ia32_feature_control_valid_bits = FEAT_CTL_LOCKED; @@ -6952,9 +6880,6 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) vmx->pi_desc.nv = POSTED_INTR_VECTOR; vmx->pi_desc.sn = 1; -#if IS_ENABLED(CONFIG_HYPERV) - vmx->hv_root_ept = INVALID_PAGE; -#endif return 0; free_vmcs: @@ -6971,10 +6896,6 @@ static int vmx_create_vcpu(struct kvm_vcpu *vcpu) static int vmx_vm_init(struct kvm *kvm) { -#if IS_ENABLED(CONFIG_HYPERV) - spin_lock_init(&to_kvm_vmx(kvm)->hv_root_ept_lock); -#endif - if (!ple_gap) kvm->arch.pause_in_guest = true; @@ -7001,7 +6922,6 @@ static int vmx_vm_init(struct kvm *kvm) break; } } - kvm_apicv_init(kvm, enable_apicv); return 0; } @@ -7453,10 +7373,10 @@ static int vmx_set_hv_timer(struct kvm_vcpu *vcpu, u64 guest_deadline_tsc, delta_tsc = 0; /* Convert to host delta tsc if tsc scaling is enabled */ - if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && + if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio && delta_tsc && u64_shl_div_u64(delta_tsc, kvm_tsc_scaling_ratio_frac_bits, - vcpu->arch.tsc_scaling_ratio, &delta_tsc)) + vcpu->arch.l1_tsc_scaling_ratio, &delta_tsc)) return -ERANGE; /* @@ -7542,7 +7462,7 @@ static int vmx_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) return !is_smm(vcpu); } -static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) +static int vmx_enter_smm(struct kvm_vcpu *vcpu, char *smstate) { struct vcpu_vmx *vmx = to_vmx(vcpu); @@ -7556,7 +7476,7 @@ static int vmx_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate) return 0; } -static int vmx_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) +static int vmx_leave_smm(struct kvm_vcpu *vcpu, const char *smstate) { struct vcpu_vmx *vmx = to_vmx(vcpu); int ret; @@ -7700,7 +7620,10 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .has_wbinvd_exit = cpu_has_vmx_wbinvd_exit, - .write_l1_tsc_offset = vmx_write_l1_tsc_offset, + .get_l2_tsc_offset = vmx_get_l2_tsc_offset, + .get_l2_tsc_multiplier = vmx_get_l2_tsc_multiplier, + .write_tsc_offset = vmx_write_tsc_offset, + .write_tsc_multiplier = vmx_write_tsc_multiplier, .load_mmu_pgd = vmx_load_mmu_pgd, @@ -7731,8 +7654,8 @@ static struct kvm_x86_ops vmx_x86_ops __initdata = { .setup_mce = vmx_setup_mce, .smi_allowed = vmx_smi_allowed, - .pre_enter_smm = vmx_pre_enter_smm, - .pre_leave_smm = vmx_pre_leave_smm, + .enter_smm = vmx_enter_smm, + .leave_smm = vmx_leave_smm, .enable_smi_window = vmx_enable_smi_window, .can_emulate_instruction = vmx_can_emulate_instruction, @@ -7807,6 +7730,12 @@ static __init int hardware_setup(void) !cpu_has_vmx_invept_global()) enable_ept = 0; + /* NX support is required for shadow paging. */ + if (!enable_ept && !boot_cpu_has(X86_FEATURE_NX)) { + pr_err_ratelimited("kvm: NX (Execute Disable) not supported\n"); + return -EOPNOTSUPP; + } + if (!cpu_has_vmx_ept_ad_bits() || !enable_ept) enable_ept_ad_bits = 0; @@ -7996,6 +7925,8 @@ static void vmx_exit(void) } #endif vmx_cleanup_l1d_flush(); + + allow_smaller_maxphyaddr = false; } module_exit(vmx_exit); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 16e4e457ba23..3979a947933a 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -322,8 +322,6 @@ struct vcpu_vmx { /* apic deadline value in host tsc */ u64 hv_deadline_tsc; - u64 current_tsc_ratio; - unsigned long host_debugctlmsr; /* @@ -336,10 +334,6 @@ struct vcpu_vmx { /* SGX Launch Control public key hash */ u64 msr_ia32_sgxlepubkeyhash[4]; -#if IS_ENABLED(CONFIG_HYPERV) - u64 hv_root_ept; -#endif - struct pt_desc pt_desc; struct lbr_desc lbr_desc; @@ -357,11 +351,6 @@ struct kvm_vmx { unsigned int tss_addr; bool ept_identity_pagetable_done; gpa_t ept_identity_map_addr; - -#if IS_ENABLED(CONFIG_HYPERV) - hpa_t hv_root_ept; - spinlock_t hv_root_ept_lock; -#endif }; bool nested_vmx_allowed(struct kvm_vcpu *vcpu); @@ -387,6 +376,7 @@ void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); void vmx_set_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg); u64 construct_eptp(struct kvm_vcpu *vcpu, hpa_t root_hpa, int root_level); +bool vmx_guest_inject_ac(struct kvm_vcpu *vcpu); void vmx_update_exception_bitmap(struct kvm_vcpu *vcpu); void vmx_update_msr_bitmap(struct kvm_vcpu *vcpu); bool vmx_nmi_blocked(struct kvm_vcpu *vcpu); @@ -404,6 +394,9 @@ void vmx_ept_load_pdptrs(struct kvm_vcpu *vcpu); void vmx_disable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type); void vmx_enable_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type); +u64 vmx_get_l2_tsc_offset(struct kvm_vcpu *vcpu); +u64 vmx_get_l2_tsc_multiplier(struct kvm_vcpu *vcpu); + static inline void vmx_set_intercept_for_msr(struct kvm_vcpu *vcpu, u32 msr, int type, bool value) { @@ -529,12 +522,6 @@ static inline struct vmcs *alloc_vmcs(bool shadow) GFP_KERNEL_ACCOUNT); } -static inline void decache_tsc_multiplier(struct vcpu_vmx *vmx) -{ - vmx->current_tsc_ratio = vmx->vcpu.arch.tsc_scaling_ratio; - vmcs_write64(TSC_MULTIPLIER, vmx->current_tsc_ratio); -} - static inline bool vmx_has_waitpkg(struct vcpu_vmx *vmx) { return vmx->secondary_exec_control & diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index e0f4a46649d7..17468d983fbd 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -58,6 +58,7 @@ #include #include #include +#include #include @@ -102,6 +103,8 @@ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); static u64 __read_mostly cr4_reserved_bits = CR4_RESERVED_BITS; +#define KVM_EXIT_HYPERCALL_VALID_MASK (1 << KVM_HC_MAP_GPA_RANGE) + #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK) @@ -113,6 +116,9 @@ static void __kvm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags); static void store_regs(struct kvm_vcpu *vcpu); static int sync_regs(struct kvm_vcpu *vcpu); +static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); +static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2); + struct kvm_x86_ops kvm_x86_ops __read_mostly; EXPORT_SYMBOL_GPL(kvm_x86_ops); @@ -209,55 +215,78 @@ EXPORT_SYMBOL_GPL(host_efer); bool __read_mostly allow_smaller_maxphyaddr = 0; EXPORT_SYMBOL_GPL(allow_smaller_maxphyaddr); +bool __read_mostly enable_apicv = true; +EXPORT_SYMBOL_GPL(enable_apicv); + u64 __read_mostly host_xss; EXPORT_SYMBOL_GPL(host_xss); u64 __read_mostly supported_xss; EXPORT_SYMBOL_GPL(supported_xss); -struct kvm_stats_debugfs_item debugfs_entries[] = { - VCPU_STAT("pf_fixed", pf_fixed), - VCPU_STAT("pf_guest", pf_guest), - VCPU_STAT("tlb_flush", tlb_flush), - VCPU_STAT("invlpg", invlpg), - VCPU_STAT("exits", exits), - VCPU_STAT("io_exits", io_exits), - VCPU_STAT("mmio_exits", mmio_exits), - VCPU_STAT("signal_exits", signal_exits), - VCPU_STAT("irq_window", irq_window_exits), - VCPU_STAT("nmi_window", nmi_window_exits), - VCPU_STAT("halt_exits", halt_exits), - VCPU_STAT("halt_successful_poll", halt_successful_poll), - VCPU_STAT("halt_attempted_poll", halt_attempted_poll), - VCPU_STAT("halt_poll_invalid", halt_poll_invalid), - VCPU_STAT("halt_wakeup", halt_wakeup), - VCPU_STAT("hypercalls", hypercalls), - VCPU_STAT("request_irq", request_irq_exits), - VCPU_STAT("irq_exits", irq_exits), - VCPU_STAT("host_state_reload", host_state_reload), - VCPU_STAT("fpu_reload", fpu_reload), - VCPU_STAT("insn_emulation", insn_emulation), - VCPU_STAT("insn_emulation_fail", insn_emulation_fail), - VCPU_STAT("irq_injections", irq_injections), - VCPU_STAT("nmi_injections", nmi_injections), - VCPU_STAT("req_event", req_event), - VCPU_STAT("l1d_flush", l1d_flush), - VCPU_STAT("halt_poll_success_ns", halt_poll_success_ns), - VCPU_STAT("halt_poll_fail_ns", halt_poll_fail_ns), - VCPU_STAT("nested_run", nested_run), - VCPU_STAT("directed_yield_attempted", directed_yield_attempted), - VCPU_STAT("directed_yield_successful", directed_yield_successful), - VM_STAT("mmu_shadow_zapped", mmu_shadow_zapped), - VM_STAT("mmu_pte_write", mmu_pte_write), - VM_STAT("mmu_pde_zapped", mmu_pde_zapped), - VM_STAT("mmu_flooded", mmu_flooded), - VM_STAT("mmu_recycled", mmu_recycled), - VM_STAT("mmu_cache_miss", mmu_cache_miss), - VM_STAT("mmu_unsync", mmu_unsync), - VM_STAT("remote_tlb_flush", remote_tlb_flush), - VM_STAT("largepages", lpages, .mode = 0444), - VM_STAT("nx_largepages_splitted", nx_lpage_splits, .mode = 0444), - VM_STAT("max_mmu_page_hash_collisions", max_mmu_page_hash_collisions), - { NULL } +const struct _kvm_stats_desc kvm_vm_stats_desc[] = { + KVM_GENERIC_VM_STATS(), + STATS_DESC_COUNTER(VM, mmu_shadow_zapped), + STATS_DESC_COUNTER(VM, mmu_pte_write), + STATS_DESC_COUNTER(VM, mmu_pde_zapped), + STATS_DESC_COUNTER(VM, mmu_flooded), + STATS_DESC_COUNTER(VM, mmu_recycled), + STATS_DESC_COUNTER(VM, mmu_cache_miss), + STATS_DESC_ICOUNTER(VM, mmu_unsync), + STATS_DESC_ICOUNTER(VM, lpages), + STATS_DESC_ICOUNTER(VM, nx_lpage_splits), + STATS_DESC_PCOUNTER(VM, max_mmu_page_hash_collisions) +}; +static_assert(ARRAY_SIZE(kvm_vm_stats_desc) == + sizeof(struct kvm_vm_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vm_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vm_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vm_stats_desc), +}; + +const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { + KVM_GENERIC_VCPU_STATS(), + STATS_DESC_COUNTER(VCPU, pf_fixed), + STATS_DESC_COUNTER(VCPU, pf_guest), + STATS_DESC_COUNTER(VCPU, tlb_flush), + STATS_DESC_COUNTER(VCPU, invlpg), + STATS_DESC_COUNTER(VCPU, exits), + STATS_DESC_COUNTER(VCPU, io_exits), + STATS_DESC_COUNTER(VCPU, mmio_exits), + STATS_DESC_COUNTER(VCPU, signal_exits), + STATS_DESC_COUNTER(VCPU, irq_window_exits), + STATS_DESC_COUNTER(VCPU, nmi_window_exits), + STATS_DESC_COUNTER(VCPU, l1d_flush), + STATS_DESC_COUNTER(VCPU, halt_exits), + STATS_DESC_COUNTER(VCPU, request_irq_exits), + STATS_DESC_COUNTER(VCPU, irq_exits), + STATS_DESC_COUNTER(VCPU, host_state_reload), + STATS_DESC_COUNTER(VCPU, fpu_reload), + STATS_DESC_COUNTER(VCPU, insn_emulation), + STATS_DESC_COUNTER(VCPU, insn_emulation_fail), + STATS_DESC_COUNTER(VCPU, hypercalls), + STATS_DESC_COUNTER(VCPU, irq_injections), + STATS_DESC_COUNTER(VCPU, nmi_injections), + STATS_DESC_COUNTER(VCPU, req_event), + STATS_DESC_COUNTER(VCPU, nested_run), + STATS_DESC_COUNTER(VCPU, directed_yield_attempted), + STATS_DESC_COUNTER(VCPU, directed_yield_successful), + STATS_DESC_ICOUNTER(VCPU, guest_mode) +}; +static_assert(ARRAY_SIZE(kvm_vcpu_stats_desc) == + sizeof(struct kvm_vcpu_stat) / sizeof(u64)); + +const struct kvm_stats_header kvm_vcpu_stats_header = { + .name_size = KVM_STATS_NAME_SIZE, + .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), + .id_offset = sizeof(struct kvm_stats_header), + .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, + .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + + sizeof(kvm_vcpu_stats_desc), }; u64 __read_mostly host_xcr0; @@ -778,13 +807,6 @@ int kvm_read_guest_page_mmu(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, } EXPORT_SYMBOL_GPL(kvm_read_guest_page_mmu); -static int kvm_read_nested_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, - void *data, int offset, int len, u32 access) -{ - return kvm_read_guest_page_mmu(vcpu, vcpu->arch.walk_mmu, gfn, - data, offset, len, access); -} - static inline u64 pdptr_rsvd_bits(struct kvm_vcpu *vcpu) { return vcpu->arch.reserved_gpa_bits | rsvd_bits(5, 8) | rsvd_bits(1, 2); @@ -819,6 +841,7 @@ int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3) memcpy(mmu->pdptrs, pdpte, sizeof(mmu->pdptrs)); kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); + vcpu->arch.pdptrs_from_userspace = false; out: @@ -826,40 +849,14 @@ int load_pdptrs(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu, unsigned long cr3) } EXPORT_SYMBOL_GPL(load_pdptrs); -bool pdptrs_changed(struct kvm_vcpu *vcpu) -{ - u64 pdpte[ARRAY_SIZE(vcpu->arch.walk_mmu->pdptrs)]; - int offset; - gfn_t gfn; - int r; - - if (!is_pae_paging(vcpu)) - return false; - - if (!kvm_register_is_available(vcpu, VCPU_EXREG_PDPTR)) - return true; - - gfn = (kvm_read_cr3(vcpu) & 0xffffffe0ul) >> PAGE_SHIFT; - offset = (kvm_read_cr3(vcpu) & 0xffffffe0ul) & (PAGE_SIZE - 1); - r = kvm_read_nested_guest_page(vcpu, gfn, pdpte, offset, sizeof(pdpte), - PFERR_USER_MASK | PFERR_WRITE_MASK); - if (r < 0) - return true; - - return memcmp(pdpte, vcpu->arch.walk_mmu->pdptrs, sizeof(pdpte)) != 0; -} -EXPORT_SYMBOL_GPL(pdptrs_changed); - void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0) { - unsigned long update_bits = X86_CR0_PG | X86_CR0_WP; - if ((cr0 ^ old_cr0) & X86_CR0_PG) { kvm_clear_async_pf_completion_queue(vcpu); kvm_async_pf_hash_reset(vcpu); } - if ((cr0 ^ old_cr0) & update_bits) + if ((cr0 ^ old_cr0) & KVM_MMU_CR0_ROLE_BITS) kvm_mmu_reset_context(vcpu); if (((cr0 ^ old_cr0) & X86_CR0_CD) && @@ -1038,10 +1035,7 @@ EXPORT_SYMBOL_GPL(kvm_is_valid_cr4); void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4) { - unsigned long mmu_role_bits = X86_CR4_PGE | X86_CR4_PSE | X86_CR4_PAE | - X86_CR4_SMEP | X86_CR4_SMAP | X86_CR4_PKE; - - if (((cr4 ^ old_cr4) & mmu_role_bits) || + if (((cr4 ^ old_cr4) & KVM_MMU_CR4_ROLE_BITS) || (!(cr4 & X86_CR4_PCIDE) && (old_cr4 & X86_CR4_PCIDE))) kvm_mmu_reset_context(vcpu); } @@ -1084,25 +1078,46 @@ int kvm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4) } EXPORT_SYMBOL_GPL(kvm_set_cr4); +static void kvm_invalidate_pcid(struct kvm_vcpu *vcpu, unsigned long pcid) +{ + struct kvm_mmu *mmu = vcpu->arch.mmu; + unsigned long roots_to_free = 0; + int i; + + /* + * If neither the current CR3 nor any of the prev_roots use the given + * PCID, then nothing needs to be done here because a resync will + * happen anyway before switching to any other CR3. + */ + if (kvm_get_active_pcid(vcpu) == pcid) { + kvm_make_request(KVM_REQ_MMU_SYNC, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); + } + + for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) + if (kvm_get_pcid(vcpu, mmu->prev_roots[i].pgd) == pcid) + roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); + + kvm_mmu_free_roots(vcpu, mmu, roots_to_free); +} + int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) { bool skip_tlb_flush = false; + unsigned long pcid = 0; #ifdef CONFIG_X86_64 bool pcid_enabled = kvm_read_cr4_bits(vcpu, X86_CR4_PCIDE); if (pcid_enabled) { skip_tlb_flush = cr3 & X86_CR3_PCID_NOFLUSH; cr3 &= ~X86_CR3_PCID_NOFLUSH; + pcid = cr3 & X86_CR3_PCID_MASK; } #endif - if (cr3 == kvm_read_cr3(vcpu) && !pdptrs_changed(vcpu)) { - if (!skip_tlb_flush) { - kvm_mmu_sync_roots(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); - } - return 0; - } + /* PDPTRs are always reloaded for PAE paging. */ + if (cr3 == kvm_read_cr3(vcpu) && !is_pae_paging(vcpu)) + goto handle_tlb_flush; /* * Do not condition the GPA check on long mode, this helper is used to @@ -1115,10 +1130,23 @@ int kvm_set_cr3(struct kvm_vcpu *vcpu, unsigned long cr3) if (is_pae_paging(vcpu) && !load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3)) return 1; - kvm_mmu_new_pgd(vcpu, cr3, skip_tlb_flush, skip_tlb_flush); + if (cr3 != kvm_read_cr3(vcpu)) + kvm_mmu_new_pgd(vcpu, cr3); + vcpu->arch.cr3 = cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); +handle_tlb_flush: + /* + * A load of CR3 that flushes the TLB flushes only the current PCID, + * even if PCID is disabled, in which case PCID=0 is flushed. It's a + * moot point in the end because _disabling_ PCID will flush all PCIDs, + * and it's impossible to use a non-zero PCID when PCID is disabled, + * i.e. only PCID=0 can be relevant. + */ + if (!skip_tlb_flush) + kvm_invalidate_pcid(vcpu, pcid); + return 0; } EXPORT_SYMBOL_GPL(kvm_set_cr3); @@ -2179,13 +2207,15 @@ static u32 adjust_tsc_khz(u32 khz, s32 ppm) return v; } +static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier); + static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale) { u64 ratio; /* Guest TSC same frequency as host TSC? */ if (!scale) { - vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; + kvm_vcpu_write_tsc_multiplier(vcpu, kvm_default_tsc_scaling_ratio); return 0; } @@ -2211,7 +2241,7 @@ static int set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz, bool scale) return -1; } - vcpu->arch.tsc_scaling_ratio = ratio; + kvm_vcpu_write_tsc_multiplier(vcpu, ratio); return 0; } @@ -2223,7 +2253,7 @@ static int kvm_set_tsc_khz(struct kvm_vcpu *vcpu, u32 user_tsc_khz) /* tsc_khz can be zero if TSC calibration fails */ if (user_tsc_khz == 0) { /* set tsc_scaling_ratio to a safe value */ - vcpu->arch.tsc_scaling_ratio = kvm_default_tsc_scaling_ratio; + kvm_vcpu_write_tsc_multiplier(vcpu, kvm_default_tsc_scaling_ratio); return -1; } @@ -2305,10 +2335,9 @@ static inline u64 __scale_tsc(u64 ratio, u64 tsc) return mul_u64_u64_shr(tsc, ratio, kvm_tsc_scaling_ratio_frac_bits); } -u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc) +u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc, u64 ratio) { u64 _tsc = tsc; - u64 ratio = vcpu->arch.tsc_scaling_ratio; if (ratio != kvm_default_tsc_scaling_ratio) _tsc = __scale_tsc(ratio, tsc); @@ -2317,25 +2346,86 @@ u64 kvm_scale_tsc(struct kvm_vcpu *vcpu, u64 tsc) } EXPORT_SYMBOL_GPL(kvm_scale_tsc); -static u64 kvm_compute_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) +static u64 kvm_compute_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 target_tsc) { u64 tsc; - tsc = kvm_scale_tsc(vcpu, rdtsc()); + tsc = kvm_scale_tsc(vcpu, rdtsc(), vcpu->arch.l1_tsc_scaling_ratio); return target_tsc - tsc; } u64 kvm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc) { - return vcpu->arch.l1_tsc_offset + kvm_scale_tsc(vcpu, host_tsc); + return vcpu->arch.l1_tsc_offset + + kvm_scale_tsc(vcpu, host_tsc, vcpu->arch.l1_tsc_scaling_ratio); } EXPORT_SYMBOL_GPL(kvm_read_l1_tsc); -static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset) +u64 kvm_calc_nested_tsc_offset(u64 l1_offset, u64 l2_offset, u64 l2_multiplier) { - vcpu->arch.l1_tsc_offset = offset; - vcpu->arch.tsc_offset = static_call(kvm_x86_write_l1_tsc_offset)(vcpu, offset); + u64 nested_offset; + + if (l2_multiplier == kvm_default_tsc_scaling_ratio) + nested_offset = l1_offset; + else + nested_offset = mul_s64_u64_shr((s64) l1_offset, l2_multiplier, + kvm_tsc_scaling_ratio_frac_bits); + + nested_offset += l2_offset; + return nested_offset; +} +EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_offset); + +u64 kvm_calc_nested_tsc_multiplier(u64 l1_multiplier, u64 l2_multiplier) +{ + if (l2_multiplier != kvm_default_tsc_scaling_ratio) + return mul_u64_u64_shr(l1_multiplier, l2_multiplier, + kvm_tsc_scaling_ratio_frac_bits); + + return l1_multiplier; +} +EXPORT_SYMBOL_GPL(kvm_calc_nested_tsc_multiplier); + +static void kvm_vcpu_write_tsc_offset(struct kvm_vcpu *vcpu, u64 l1_offset) +{ + trace_kvm_write_tsc_offset(vcpu->vcpu_id, + vcpu->arch.l1_tsc_offset, + l1_offset); + + vcpu->arch.l1_tsc_offset = l1_offset; + + /* + * If we are here because L1 chose not to trap WRMSR to TSC then + * according to the spec this should set L1's TSC (as opposed to + * setting L1's offset for L2). + */ + if (is_guest_mode(vcpu)) + vcpu->arch.tsc_offset = kvm_calc_nested_tsc_offset( + l1_offset, + static_call(kvm_x86_get_l2_tsc_offset)(vcpu), + static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu)); + else + vcpu->arch.tsc_offset = l1_offset; + + static_call(kvm_x86_write_tsc_offset)(vcpu, vcpu->arch.tsc_offset); +} + +static void kvm_vcpu_write_tsc_multiplier(struct kvm_vcpu *vcpu, u64 l1_multiplier) +{ + vcpu->arch.l1_tsc_scaling_ratio = l1_multiplier; + + /* Userspace is changing the multiplier while L2 is active */ + if (is_guest_mode(vcpu)) + vcpu->arch.tsc_scaling_ratio = kvm_calc_nested_tsc_multiplier( + l1_multiplier, + static_call(kvm_x86_get_l2_tsc_multiplier)(vcpu)); + else + vcpu->arch.tsc_scaling_ratio = l1_multiplier; + + if (kvm_has_tsc_control) + static_call(kvm_x86_write_tsc_multiplier)( + vcpu, vcpu->arch.tsc_scaling_ratio); } static inline bool kvm_check_tsc_unstable(void) @@ -2361,7 +2451,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data) bool synchronizing = false; raw_spin_lock_irqsave(&kvm->arch.tsc_write_lock, flags); - offset = kvm_compute_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, data); ns = get_kvmclock_base_ns(); elapsed = ns - kvm->arch.last_tsc_nsec; @@ -2400,7 +2490,7 @@ static void kvm_synchronize_tsc(struct kvm_vcpu *vcpu, u64 data) } else { u64 delta = nsec_to_cycles(vcpu, elapsed); data += delta; - offset = kvm_compute_tsc_offset(vcpu, data); + offset = kvm_compute_l1_tsc_offset(vcpu, data); } matched = true; already_matched = (vcpu->arch.this_tsc_generation == kvm->arch.cur_tsc_generation); @@ -2459,9 +2549,10 @@ static inline void adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, static inline void adjust_tsc_offset_host(struct kvm_vcpu *vcpu, s64 adjustment) { - if (vcpu->arch.tsc_scaling_ratio != kvm_default_tsc_scaling_ratio) + if (vcpu->arch.l1_tsc_scaling_ratio != kvm_default_tsc_scaling_ratio) WARN_ON(adjustment < 0); - adjustment = kvm_scale_tsc(vcpu, (u64) adjustment); + adjustment = kvm_scale_tsc(vcpu, (u64) adjustment, + vcpu->arch.l1_tsc_scaling_ratio); adjust_tsc_offset_guest(vcpu, adjustment); } @@ -2844,7 +2935,8 @@ static int kvm_guest_time_update(struct kvm_vcpu *v) /* With all the info we got, fill in the values */ if (kvm_has_tsc_control) - tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz); + tgt_tsc_khz = kvm_scale_tsc(v, tgt_tsc_khz, + v->arch.l1_tsc_scaling_ratio); if (unlikely(vcpu->hw_tsc_khz != tgt_tsc_khz)) { kvm_get_time_scale(NSEC_PER_SEC, tgt_tsc_khz * 1000LL, @@ -3250,7 +3342,7 @@ int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (msr_info->host_initiated) { kvm_synchronize_tsc(vcpu, data); } else { - u64 adj = kvm_compute_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; + u64 adj = kvm_compute_l1_tsc_offset(vcpu, data) - vcpu->arch.l1_tsc_offset; adjust_tsc_offset_guest(vcpu, adj); vcpu->arch.ia32_tsc_adjust_msr += adj; } @@ -3552,10 +3644,17 @@ int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr_info) * return L1's TSC value to ensure backwards-compatible * behavior for migration. */ - u64 tsc_offset = msr_info->host_initiated ? vcpu->arch.l1_tsc_offset : - vcpu->arch.tsc_offset; + u64 offset, ratio; - msr_info->data = kvm_scale_tsc(vcpu, rdtsc()) + tsc_offset; + if (msr_info->host_initiated) { + offset = vcpu->arch.l1_tsc_offset; + ratio = vcpu->arch.l1_tsc_scaling_ratio; + } else { + offset = vcpu->arch.tsc_offset; + ratio = vcpu->arch.tsc_scaling_ratio; + } + + msr_info->data = kvm_scale_tsc(vcpu, rdtsc(), ratio) + offset; break; } case MSR_MTRRcap: @@ -3879,6 +3978,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_HYPERV_TLBFLUSH: case KVM_CAP_HYPERV_SEND_IPI: case KVM_CAP_HYPERV_CPUID: + case KVM_CAP_HYPERV_ENFORCE_CPUID: case KVM_CAP_SYS_HYPERV_CPUID: case KVM_CAP_PCI_SEGMENT: case KVM_CAP_DEBUGREGS: @@ -3909,8 +4009,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) case KVM_CAP_SGX_ATTRIBUTE: #endif case KVM_CAP_VM_COPY_ENC_CONTEXT_FROM: + case KVM_CAP_SREGS2: + case KVM_CAP_EXIT_ON_EMULATION_FAILURE: r = 1; break; + case KVM_CAP_EXIT_HYPERCALL: + r = KVM_EXIT_HYPERCALL_VALID_MASK; + break; case KVM_CAP_SET_GUEST_DEBUG2: return KVM_GUESTDBG_VALID_MASK; #ifdef CONFIG_KVM_XEN @@ -4138,7 +4243,7 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) mark_tsc_unstable("KVM discovered backwards TSC"); if (kvm_check_tsc_unstable()) { - u64 offset = kvm_compute_tsc_offset(vcpu, + u64 offset = kvm_compute_l1_tsc_offset(vcpu, vcpu->arch.last_guest_tsc); kvm_vcpu_write_tsc_offset(vcpu, offset); vcpu->arch.tsc_catchup = 1; @@ -4457,7 +4562,7 @@ static void kvm_vcpu_ioctl_x86_get_vcpu_events(struct kvm_vcpu *vcpu, memset(&events->reserved, 0, sizeof(events->reserved)); } -static void kvm_smm_changed(struct kvm_vcpu *vcpu); +static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm); static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, struct kvm_vcpu_events *events) @@ -4517,13 +4622,8 @@ static int kvm_vcpu_ioctl_x86_set_vcpu_events(struct kvm_vcpu *vcpu, vcpu->arch.apic->sipi_vector = events->sipi_vector; if (events->flags & KVM_VCPUEVENT_VALID_SMM) { - if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) { - if (events->smi.smm) - vcpu->arch.hflags |= HF_SMM_MASK; - else - vcpu->arch.hflags &= ~HF_SMM_MASK; - kvm_smm_changed(vcpu); - } + if (!!(vcpu->arch.hflags & HF_SMM_MASK) != events->smi.smm) + kvm_smm_changed(vcpu, events->smi.smm); vcpu->arch.smi_pending = events->smi.pending; @@ -4807,6 +4907,9 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, return static_call(kvm_x86_enable_direct_tlbflush)(vcpu); + case KVM_CAP_HYPERV_ENFORCE_CPUID: + return kvm_hv_set_enforce_cpuid(vcpu, cap->args[0]); + case KVM_CAP_ENFORCE_PV_FEATURE_CPUID: vcpu->arch.pv_cpuid.enforce = cap->args[0]; if (vcpu->arch.pv_cpuid.enforce) @@ -4825,6 +4928,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, void __user *argp = (void __user *)arg; int r; union { + struct kvm_sregs2 *sregs2; struct kvm_lapic_state *lapic; struct kvm_xsave *xsave; struct kvm_xcrs *xcrs; @@ -5197,6 +5301,28 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; } #endif + case KVM_GET_SREGS2: { + u.sregs2 = kzalloc(sizeof(struct kvm_sregs2), GFP_KERNEL); + r = -ENOMEM; + if (!u.sregs2) + goto out; + __get_sregs2(vcpu, u.sregs2); + r = -EFAULT; + if (copy_to_user(argp, u.sregs2, sizeof(struct kvm_sregs2))) + goto out; + r = 0; + break; + } + case KVM_SET_SREGS2: { + u.sregs2 = memdup_user(argp, sizeof(struct kvm_sregs2)); + if (IS_ERR(u.sregs2)) { + r = PTR_ERR(u.sregs2); + u.sregs2 = NULL; + goto out; + } + r = __set_sregs2(vcpu, u.sregs2); + break; + } default: r = -EINVAL; } @@ -5516,6 +5642,21 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, if (kvm_x86_ops.vm_copy_enc_context_from) r = kvm_x86_ops.vm_copy_enc_context_from(kvm, cap->args[0]); return r; + case KVM_CAP_EXIT_HYPERCALL: + if (cap->args[0] & ~KVM_EXIT_HYPERCALL_VALID_MASK) { + r = -EINVAL; + break; + } + kvm->arch.hypercall_exit_enabled = cap->args[0]; + r = 0; + break; + case KVM_CAP_EXIT_ON_EMULATION_FAILURE: + r = -EINVAL; + if (cap->args[0] & ~1) + break; + kvm->arch.exit_on_emulation_error = cap->args[0]; + r = 0; + break; default: r = -EINVAL; break; @@ -5630,6 +5771,41 @@ static int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, void __user *argp) return 0; } +#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER +static int kvm_arch_suspend_notifier(struct kvm *kvm) +{ + struct kvm_vcpu *vcpu; + int i, ret = 0; + + mutex_lock(&kvm->lock); + kvm_for_each_vcpu(i, vcpu, kvm) { + if (!vcpu->arch.pv_time_enabled) + continue; + + ret = kvm_set_guest_paused(vcpu); + if (ret) { + kvm_err("Failed to pause guest VCPU%d: %d\n", + vcpu->vcpu_id, ret); + break; + } + } + mutex_unlock(&kvm->lock); + + return ret ? NOTIFY_BAD : NOTIFY_DONE; +} + +int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state) +{ + switch (state) { + case PM_HIBERNATION_PREPARE: + case PM_SUSPEND_PREPARE: + return kvm_arch_suspend_notifier(kvm); + } + + return NOTIFY_DONE; +} +#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ + long kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -7104,23 +7280,22 @@ static unsigned emulator_get_hflags(struct x86_emulate_ctxt *ctxt) return emul_to_vcpu(ctxt)->arch.hflags; } -static void emulator_set_hflags(struct x86_emulate_ctxt *ctxt, unsigned emul_flags) +static void emulator_exiting_smm(struct x86_emulate_ctxt *ctxt) { struct kvm_vcpu *vcpu = emul_to_vcpu(ctxt); - vcpu->arch.hflags = emul_flags; - kvm_mmu_reset_context(vcpu); + kvm_smm_changed(vcpu, false); } -static int emulator_pre_leave_smm(struct x86_emulate_ctxt *ctxt, +static int emulator_leave_smm(struct x86_emulate_ctxt *ctxt, const char *smstate) { - return static_call(kvm_x86_pre_leave_smm)(emul_to_vcpu(ctxt), smstate); + return static_call(kvm_x86_leave_smm)(emul_to_vcpu(ctxt), smstate); } -static void emulator_post_leave_smm(struct x86_emulate_ctxt *ctxt) +static void emulator_triple_fault(struct x86_emulate_ctxt *ctxt) { - kvm_smm_changed(emul_to_vcpu(ctxt)); + kvm_make_request(KVM_REQ_TRIPLE_FAULT, emul_to_vcpu(ctxt)); } static int emulator_set_xcr(struct x86_emulate_ctxt *ctxt, u32 index, u64 xcr) @@ -7169,9 +7344,9 @@ static const struct x86_emulate_ops emulate_ops = { .guest_has_fxsr = emulator_guest_has_fxsr, .set_nmi_mask = emulator_set_nmi_mask, .get_hflags = emulator_get_hflags, - .set_hflags = emulator_set_hflags, - .pre_leave_smm = emulator_pre_leave_smm, - .post_leave_smm = emulator_post_leave_smm, + .exiting_smm = emulator_exiting_smm, + .leave_smm = emulator_leave_smm, + .triple_fault = emulator_triple_fault, .set_xcr = emulator_set_xcr, }; @@ -7277,8 +7452,33 @@ void kvm_inject_realmode_interrupt(struct kvm_vcpu *vcpu, int irq, int inc_eip) } EXPORT_SYMBOL_GPL(kvm_inject_realmode_interrupt); +static void prepare_emulation_failure_exit(struct kvm_vcpu *vcpu) +{ + struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; + u32 insn_size = ctxt->fetch.end - ctxt->fetch.data; + struct kvm_run *run = vcpu->run; + + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + run->emulation_failure.suberror = KVM_INTERNAL_ERROR_EMULATION; + run->emulation_failure.ndata = 0; + run->emulation_failure.flags = 0; + + if (insn_size) { + run->emulation_failure.ndata = 3; + run->emulation_failure.flags |= + KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES; + run->emulation_failure.insn_size = insn_size; + memset(run->emulation_failure.insn_bytes, 0x90, + sizeof(run->emulation_failure.insn_bytes)); + memcpy(run->emulation_failure.insn_bytes, + ctxt->fetch.data, insn_size); + } +} + static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) { + struct kvm *kvm = vcpu->kvm; + ++vcpu->stat.insn_emulation_fail; trace_kvm_emulate_insn_failed(vcpu); @@ -7287,10 +7487,9 @@ static int handle_emulation_failure(struct kvm_vcpu *vcpu, int emulation_type) return 1; } - if (emulation_type & EMULTYPE_SKIP) { - vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; - vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; - vcpu->run->internal.ndata = 0; + if (kvm->arch.exit_on_emulation_error || + (emulation_type & EMULTYPE_SKIP)) { + prepare_emulation_failure_exit(vcpu); return 0; } @@ -7432,11 +7631,14 @@ static bool retry_instruction(struct x86_emulate_ctxt *ctxt, static int complete_emulated_mmio(struct kvm_vcpu *vcpu); static int complete_emulated_pio(struct kvm_vcpu *vcpu); -static void kvm_smm_changed(struct kvm_vcpu *vcpu) +static void kvm_smm_changed(struct kvm_vcpu *vcpu, bool entering_smm) { - if (!(vcpu->arch.hflags & HF_SMM_MASK)) { - /* This is a good place to trace that we are exiting SMM. */ - trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, false); + trace_kvm_smm_transition(vcpu->vcpu_id, vcpu->arch.smbase, entering_smm); + + if (entering_smm) { + vcpu->arch.hflags |= HF_SMM_MASK; + } else { + vcpu->arch.hflags &= ~(HF_SMM_MASK | HF_SMM_INSIDE_NMI_MASK); /* Process a latched INIT or SMI, if any. */ kvm_make_request(KVM_REQ_EVENT, vcpu); @@ -8361,16 +8563,15 @@ bool kvm_apicv_activated(struct kvm *kvm) } EXPORT_SYMBOL_GPL(kvm_apicv_activated); -void kvm_apicv_init(struct kvm *kvm, bool enable) +static void kvm_apicv_init(struct kvm *kvm) { - if (enable) + if (enable_apicv) clear_bit(APICV_INHIBIT_REASON_DISABLE, &kvm->arch.apicv_inhibit_reasons); else set_bit(APICV_INHIBIT_REASON_DISABLE, &kvm->arch.apicv_inhibit_reasons); } -EXPORT_SYMBOL_GPL(kvm_apicv_init); static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) { @@ -8406,6 +8607,17 @@ static void kvm_sched_yield(struct kvm_vcpu *vcpu, unsigned long dest_id) return; } +static int complete_hypercall_exit(struct kvm_vcpu *vcpu) +{ + u64 ret = vcpu->run->hypercall.ret; + + if (!is_64_bit_mode(vcpu)) + ret = (u32)ret; + kvm_rax_write(vcpu, ret); + ++vcpu->stat.hypercalls; + return kvm_skip_emulated_instruction(vcpu); +} + int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) { unsigned long nr, a0, a1, a2, a3, ret; @@ -8471,6 +8683,28 @@ int kvm_emulate_hypercall(struct kvm_vcpu *vcpu) kvm_sched_yield(vcpu, a0); ret = 0; break; + case KVM_HC_MAP_GPA_RANGE: { + u64 gpa = a0, npages = a1, attrs = a2; + + ret = -KVM_ENOSYS; + if (!(vcpu->kvm->arch.hypercall_exit_enabled & (1 << KVM_HC_MAP_GPA_RANGE))) + break; + + if (!PAGE_ALIGNED(gpa) || !npages || + gpa_to_gfn(gpa) + npages <= gpa_to_gfn(gpa)) { + ret = -KVM_EINVAL; + break; + } + + vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; + vcpu->run->hypercall.nr = KVM_HC_MAP_GPA_RANGE; + vcpu->run->hypercall.args[0] = gpa; + vcpu->run->hypercall.args[1] = npages; + vcpu->run->hypercall.args[2] = attrs; + vcpu->run->hypercall.longmode = op_64_bit; + vcpu->arch.complete_userspace_io = complete_hypercall_exit; + return 0; + } default: ret = -KVM_ENOSYS; break; @@ -8554,9 +8788,6 @@ static void update_cr8_intercept(struct kvm_vcpu *vcpu) int kvm_check_nested_events(struct kvm_vcpu *vcpu) { - if (WARN_ON_ONCE(!is_guest_mode(vcpu))) - return -EIO; - if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { kvm_x86_ops.nested_ops->triple_fault(vcpu); return 1; @@ -8572,7 +8803,7 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu) static_call(kvm_x86_queue_exception)(vcpu); } -static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit) +static int inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit) { int r; bool can_inject = true; @@ -8619,7 +8850,7 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit if (is_guest_mode(vcpu)) { r = kvm_check_nested_events(vcpu); if (r < 0) - goto busy; + goto out; } /* try to inject new event if pending */ @@ -8661,7 +8892,7 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit if (vcpu->arch.smi_pending) { r = can_inject ? static_call(kvm_x86_smi_allowed)(vcpu, true) : -EBUSY; if (r < 0) - goto busy; + goto out; if (r) { vcpu->arch.smi_pending = false; ++vcpu->arch.smi_count; @@ -8674,7 +8905,7 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit if (vcpu->arch.nmi_pending) { r = can_inject ? static_call(kvm_x86_nmi_allowed)(vcpu, true) : -EBUSY; if (r < 0) - goto busy; + goto out; if (r) { --vcpu->arch.nmi_pending; vcpu->arch.nmi_injected = true; @@ -8689,7 +8920,7 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit if (kvm_cpu_has_injectable_intr(vcpu)) { r = can_inject ? static_call(kvm_x86_interrupt_allowed)(vcpu, true) : -EBUSY; if (r < 0) - goto busy; + goto out; if (r) { kvm_queue_interrupt(vcpu, kvm_cpu_get_interrupt(vcpu), false); static_call(kvm_x86_set_irq)(vcpu); @@ -8705,11 +8936,14 @@ static void inject_pending_event(struct kvm_vcpu *vcpu, bool *req_immediate_exit *req_immediate_exit = true; WARN_ON(vcpu->arch.exception.pending); - return; + return 0; -busy: - *req_immediate_exit = true; - return; +out: + if (r == -EBUSY) { + *req_immediate_exit = true; + r = 0; + } + return r; } static void process_nmi(struct kvm_vcpu *vcpu) @@ -8888,10 +9122,9 @@ static void enter_smm(struct kvm_vcpu *vcpu) { struct kvm_segment cs, ds; struct desc_ptr dt; + unsigned long cr0; char buf[512]; - u32 cr0; - trace_kvm_enter_smm(vcpu->vcpu_id, vcpu->arch.smbase, true); memset(buf, 0, 512); #ifdef CONFIG_X86_64 if (guest_cpuid_has(vcpu, X86_FEATURE_LM)) @@ -8901,13 +9134,13 @@ static void enter_smm(struct kvm_vcpu *vcpu) enter_smm_save_state_32(vcpu, buf); /* - * Give pre_enter_smm() a chance to make ISA-specific changes to the - * vCPU state (e.g. leave guest mode) after we've saved the state into - * the SMM state-save area. + * Give enter_smm() a chance to make ISA-specific changes to the vCPU + * state (e.g. leave guest mode) after we've saved the state into the + * SMM state-save area. */ - static_call(kvm_x86_pre_enter_smm)(vcpu, buf); + static_call(kvm_x86_enter_smm)(vcpu, buf); - vcpu->arch.hflags |= HF_SMM_MASK; + kvm_smm_changed(vcpu, true); kvm_vcpu_write_guest(vcpu, vcpu->arch.smbase + 0xfe00, buf, sizeof(buf)); if (static_call(kvm_x86_get_nmi_mask)(vcpu)) @@ -8996,6 +9229,15 @@ void kvm_vcpu_update_apicv(struct kvm_vcpu *vcpu) vcpu->arch.apicv_active = kvm_apicv_activated(vcpu->kvm); kvm_apic_update_apicv(vcpu); static_call(kvm_x86_refresh_apicv_exec_ctrl)(vcpu); + + /* + * When APICv gets disabled, we may still have injected interrupts + * pending. At the same time, KVM_REQ_EVENT may not be set as APICv was + * still active when the interrupt got accepted. Make sure + * inject_pending_event() is called to check for that. + */ + if (!vcpu->arch.apicv_active) + kvm_make_request(KVM_REQ_EVENT, vcpu); } EXPORT_SYMBOL_GPL(kvm_vcpu_update_apicv); @@ -9171,7 +9413,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) } if (kvm_check_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu)) kvm_vcpu_flush_tlb_current(vcpu); - if (kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) + if (kvm_check_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu)) kvm_vcpu_flush_tlb_guest(vcpu); if (kvm_check_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu)) { @@ -9264,13 +9506,21 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || kvm_xen_has_interrupt(vcpu)) { ++vcpu->stat.req_event; - kvm_apic_accept_events(vcpu); + r = kvm_apic_accept_events(vcpu); + if (r < 0) { + r = 0; + goto out; + } if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) { r = 1; goto out; } - inject_pending_event(vcpu, &req_immediate_exit); + r = inject_pending_event(vcpu, &req_immediate_exit); + if (r < 0) { + r = 0; + goto out; + } if (req_int_win) static_call(kvm_x86_enable_irq_window)(vcpu); @@ -9472,7 +9722,8 @@ static inline int vcpu_block(struct kvm *kvm, struct kvm_vcpu *vcpu) return 1; } - kvm_apic_accept_events(vcpu); + if (kvm_apic_accept_events(vcpu) < 0) + return 0; switch(vcpu->arch.mp_state) { case KVM_MP_STATE_HALTED: case KVM_MP_STATE_AP_RESET_HOLD: @@ -9696,7 +9947,10 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) goto out; } kvm_vcpu_block(vcpu); - kvm_apic_accept_events(vcpu); + if (kvm_apic_accept_events(vcpu) < 0) { + r = 0; + goto out; + } kvm_clear_request(KVM_REQ_UNHALT, vcpu); r = -EAGAIN; if (signal_pending(current)) { @@ -9845,7 +10099,7 @@ void kvm_get_cs_db_l_bits(struct kvm_vcpu *vcpu, int *db, int *l) } EXPORT_SYMBOL_GPL(kvm_get_cs_db_l_bits); -static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +static void __get_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) { struct desc_ptr dt; @@ -9878,14 +10132,36 @@ static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) sregs->cr8 = kvm_get_cr8(vcpu); sregs->efer = vcpu->arch.efer; sregs->apic_base = kvm_get_apic_base(vcpu); +} - memset(sregs->interrupt_bitmap, 0, sizeof(sregs->interrupt_bitmap)); +static void __get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + __get_sregs_common(vcpu, sregs); + + if (vcpu->arch.guest_state_protected) + return; if (vcpu->arch.interrupt.injected && !vcpu->arch.interrupt.soft) set_bit(vcpu->arch.interrupt.nr, (unsigned long *)sregs->interrupt_bitmap); } +static void __get_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) +{ + int i; + + __get_sregs_common(vcpu, (struct kvm_sregs *)sregs2); + + if (vcpu->arch.guest_state_protected) + return; + + if (is_pae_paging(vcpu)) { + for (i = 0 ; i < 4 ; i++) + sregs2->pdptrs[i] = kvm_pdptr_read(vcpu, i); + sregs2->flags |= KVM_SREGS2_FLAGS_PDPTRS_VALID; + } +} + int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) { @@ -9898,11 +10174,17 @@ int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, struct kvm_mp_state *mp_state) { + int r; + vcpu_load(vcpu); if (kvm_mpx_supported()) kvm_load_guest_fpu(vcpu); - kvm_apic_accept_events(vcpu); + r = kvm_apic_accept_events(vcpu); + if (r < 0) + goto out; + r = 0; + if ((vcpu->arch.mp_state == KVM_MP_STATE_HALTED || vcpu->arch.mp_state == KVM_MP_STATE_AP_RESET_HOLD) && vcpu->arch.pv.pv_unhalted) @@ -9910,10 +10192,11 @@ int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, else mp_state->mp_state = vcpu->arch.mp_state; +out: if (kvm_mpx_supported()) kvm_put_guest_fpu(vcpu); vcpu_put(vcpu); - return 0; + return r; } int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, @@ -9997,24 +10280,23 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) return kvm_is_valid_cr4(vcpu, sregs->cr4); } -static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, + int *mmu_reset_needed, bool update_pdptrs) { struct msr_data apic_base_msr; - int mmu_reset_needed = 0; - int pending_vec, max_bits, idx; + int idx; struct desc_ptr dt; - int ret = -EINVAL; if (!kvm_is_valid_sregs(vcpu, sregs)) - goto out; + return -EINVAL; apic_base_msr.data = sregs->apic_base; apic_base_msr.host_initiated = true; if (kvm_set_apic_base(vcpu, &apic_base_msr)) - goto out; + return -EINVAL; if (vcpu->arch.guest_state_protected) - goto skip_protected_regs; + return 0; dt.size = sregs->idt.limit; dt.address = sregs->idt.base; @@ -10024,31 +10306,30 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) static_call(kvm_x86_set_gdt)(vcpu, &dt); vcpu->arch.cr2 = sregs->cr2; - mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; + *mmu_reset_needed |= kvm_read_cr3(vcpu) != sregs->cr3; vcpu->arch.cr3 = sregs->cr3; kvm_register_mark_available(vcpu, VCPU_EXREG_CR3); kvm_set_cr8(vcpu, sregs->cr8); - mmu_reset_needed |= vcpu->arch.efer != sregs->efer; + *mmu_reset_needed |= vcpu->arch.efer != sregs->efer; static_call(kvm_x86_set_efer)(vcpu, sregs->efer); - mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; + *mmu_reset_needed |= kvm_read_cr0(vcpu) != sregs->cr0; static_call(kvm_x86_set_cr0)(vcpu, sregs->cr0); vcpu->arch.cr0 = sregs->cr0; - mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; + *mmu_reset_needed |= kvm_read_cr4(vcpu) != sregs->cr4; static_call(kvm_x86_set_cr4)(vcpu, sregs->cr4); - idx = srcu_read_lock(&vcpu->kvm->srcu); - if (is_pae_paging(vcpu)) { - load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); - mmu_reset_needed = 1; + if (update_pdptrs) { + idx = srcu_read_lock(&vcpu->kvm->srcu); + if (is_pae_paging(vcpu)) { + load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu)); + *mmu_reset_needed = 1; + } + srcu_read_unlock(&vcpu->kvm->srcu, idx); } - srcu_read_unlock(&vcpu->kvm->srcu, idx); - - if (mmu_reset_needed) - kvm_mmu_reset_context(vcpu); kvm_set_segment(vcpu, &sregs->cs, VCPU_SREG_CS); kvm_set_segment(vcpu, &sregs->ds, VCPU_SREG_DS); @@ -10068,20 +10349,63 @@ static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) !is_protmode(vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; -skip_protected_regs: + return 0; +} + +static int __set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) +{ + int pending_vec, max_bits; + int mmu_reset_needed = 0; + int ret = __set_sregs_common(vcpu, sregs, &mmu_reset_needed, true); + + if (ret) + return ret; + + if (mmu_reset_needed) + kvm_mmu_reset_context(vcpu); + max_bits = KVM_NR_INTERRUPTS; pending_vec = find_first_bit( (const unsigned long *)sregs->interrupt_bitmap, max_bits); + if (pending_vec < max_bits) { kvm_queue_interrupt(vcpu, pending_vec, false); pr_debug("Set back pending irq %d\n", pending_vec); + kvm_make_request(KVM_REQ_EVENT, vcpu); } + return 0; +} - kvm_make_request(KVM_REQ_EVENT, vcpu); +static int __set_sregs2(struct kvm_vcpu *vcpu, struct kvm_sregs2 *sregs2) +{ + int mmu_reset_needed = 0; + bool valid_pdptrs = sregs2->flags & KVM_SREGS2_FLAGS_PDPTRS_VALID; + bool pae = (sregs2->cr0 & X86_CR0_PG) && (sregs2->cr4 & X86_CR4_PAE) && + !(sregs2->efer & EFER_LMA); + int i, ret; - ret = 0; -out: - return ret; + if (sregs2->flags & ~KVM_SREGS2_FLAGS_PDPTRS_VALID) + return -EINVAL; + + if (valid_pdptrs && (!pae || vcpu->arch.guest_state_protected)) + return -EINVAL; + + ret = __set_sregs_common(vcpu, (struct kvm_sregs *)sregs2, + &mmu_reset_needed, !valid_pdptrs); + if (ret) + return ret; + + if (valid_pdptrs) { + for (i = 0; i < 4 ; i++) + kvm_pdptr_write(vcpu, i, sregs2->pdptrs[i]); + + kvm_register_mark_dirty(vcpu, VCPU_EXREG_PDPTR); + mmu_reset_needed = 1; + vcpu->arch.pdptrs_from_userspace = true; + } + if (mmu_reset_needed) + kvm_mmu_reset_context(vcpu); + return 0; } int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, @@ -10305,13 +10629,13 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) struct page *page; int r; + vcpu->arch.last_vmentry_cpu = -1; + if (!irqchip_in_kernel(vcpu->kvm) || kvm_vcpu_is_reset_bsp(vcpu)) vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE; else vcpu->arch.mp_state = KVM_MP_STATE_UNINITIALIZED; - kvm_set_tsc_khz(vcpu, max_tsc_khz); - r = kvm_mmu_create(vcpu); if (r < 0) return r; @@ -10371,6 +10695,10 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.pending_external_vector = -1; vcpu->arch.preempted_in_kernel = false; +#if IS_ENABLED(CONFIG_HYPERV) + vcpu->arch.hv_root_tdp = INVALID_PAGE; +#endif + r = static_call(kvm_x86_vcpu_create)(vcpu); if (r) goto free_guest_fpu; @@ -10379,8 +10707,9 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) vcpu->arch.msr_platform_info = MSR_PLATFORM_INFO_CPUID_FAULT; kvm_vcpu_mtrr_init(vcpu); vcpu_load(vcpu); + kvm_set_tsc_khz(vcpu, max_tsc_khz); kvm_vcpu_reset(vcpu, false); - kvm_init_mmu(vcpu, false); + kvm_init_mmu(vcpu); vcpu_put(vcpu); return 0; @@ -10454,6 +10783,8 @@ void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) { + unsigned long old_cr0 = kvm_read_cr0(vcpu); + kvm_lapic_reset(vcpu, init_event); vcpu->arch.hflags = 0; @@ -10522,6 +10853,17 @@ void kvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event) vcpu->arch.ia32_xss = 0; static_call(kvm_x86_vcpu_reset)(vcpu, init_event); + + /* + * Reset the MMU context if paging was enabled prior to INIT (which is + * implied if CR0.PG=1 as CR0 will be '0' prior to RESET). Unlike the + * standard CR0/CR4/EFER modification paths, only CR0.PG needs to be + * checked because it is unconditionally cleared on INIT and all other + * paging related bits are ignored if paging is disabled, i.e. CR0.WP, + * CR4, and EFER changes are all irrelevant if CR0.PG was '0'. + */ + if (old_cr0 & X86_CR0_PG) + kvm_mmu_reset_context(vcpu); } void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector) @@ -10639,6 +10981,9 @@ int kvm_arch_hardware_setup(void *opaque) int r; rdmsrl_safe(MSR_EFER, &host_efer); + if (WARN_ON_ONCE(boot_cpu_has(X86_FEATURE_NX) && + !(host_efer & EFER_NX))) + return -EIO; if (boot_cpu_has(X86_FEATURE_XSAVES)) rdmsrl(MSR_IA32_XSS, host_xss); @@ -10754,9 +11099,15 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) kvm->arch.guest_can_read_msr_platform_info = true; +#if IS_ENABLED(CONFIG_HYPERV) + spin_lock_init(&kvm->arch.hv_root_tdp_lock); + kvm->arch.hv_root_tdp = INVALID_PAGE; +#endif + INIT_DELAYED_WORK(&kvm->arch.kvmclock_update_work, kvmclock_update_fn); INIT_DELAYED_WORK(&kvm->arch.kvmclock_sync_work, kvmclock_sync_fn); + kvm_apicv_init(kvm); kvm_hv_init_vm(kvm); kvm_page_track_init(kvm); kvm_mmu_init_vm(kvm); @@ -10917,17 +11268,23 @@ void kvm_arch_destroy_vm(struct kvm *kvm) kvm_hv_destroy_vm(kvm); } -void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) +static void memslot_rmap_free(struct kvm_memory_slot *slot) { int i; for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { kvfree(slot->arch.rmap[i]); slot->arch.rmap[i] = NULL; + } +} - if (i == 0) - continue; +void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) +{ + int i; + memslot_rmap_free(slot); + + for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { kvfree(slot->arch.lpage_info[i - 1]); slot->arch.lpage_info[i - 1] = NULL; } @@ -10935,10 +11292,78 @@ void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) kvm_page_track_free_memslot(slot); } -static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot, +static int memslot_rmap_alloc(struct kvm_memory_slot *slot, + unsigned long npages) +{ + const int sz = sizeof(*slot->arch.rmap[0]); + int i; + + for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { + int level = i + 1; + int lpages = gfn_to_index(slot->base_gfn + npages - 1, + slot->base_gfn, level) + 1; + + WARN_ON(slot->arch.rmap[i]); + + slot->arch.rmap[i] = kvcalloc(lpages, sz, GFP_KERNEL_ACCOUNT); + if (!slot->arch.rmap[i]) { + memslot_rmap_free(slot); + return -ENOMEM; + } + } + + return 0; +} + +int alloc_all_memslots_rmaps(struct kvm *kvm) +{ + struct kvm_memslots *slots; + struct kvm_memory_slot *slot; + int r, i; + + /* + * Check if memslots alreday have rmaps early before acquiring + * the slots_arch_lock below. + */ + if (kvm_memslots_have_rmaps(kvm)) + return 0; + + mutex_lock(&kvm->slots_arch_lock); + + /* + * Read memslots_have_rmaps again, under the slots arch lock, + * before allocating the rmaps + */ + if (kvm_memslots_have_rmaps(kvm)) { + mutex_unlock(&kvm->slots_arch_lock); + return 0; + } + + for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { + slots = __kvm_memslots(kvm, i); + kvm_for_each_memslot(slot, slots) { + r = memslot_rmap_alloc(slot, slot->npages); + if (r) { + mutex_unlock(&kvm->slots_arch_lock); + return r; + } + } + } + + /* + * Ensure that memslots_have_rmaps becomes true strictly after + * all the rmap pointers are set. + */ + smp_store_release(&kvm->arch.memslots_have_rmaps, true); + mutex_unlock(&kvm->slots_arch_lock); + return 0; +} + +static int kvm_alloc_memslot_metadata(struct kvm *kvm, + struct kvm_memory_slot *slot, unsigned long npages) { - int i; + int i, r; /* * Clear out the previous array pointers for the KVM_MR_MOVE case. The @@ -10947,7 +11372,13 @@ static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot, */ memset(&slot->arch, 0, sizeof(slot->arch)); - for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { + if (kvm_memslots_have_rmaps(kvm)) { + r = memslot_rmap_alloc(slot, npages); + if (r) + return r; + } + + for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { struct kvm_lpage_info *linfo; unsigned long ugfn; int lpages; @@ -10956,14 +11387,6 @@ static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot, lpages = gfn_to_index(slot->base_gfn + npages - 1, slot->base_gfn, level) + 1; - slot->arch.rmap[i] = - kvcalloc(lpages, sizeof(*slot->arch.rmap[i]), - GFP_KERNEL_ACCOUNT); - if (!slot->arch.rmap[i]) - goto out_free; - if (i == 0) - continue; - linfo = kvcalloc(lpages, sizeof(*linfo), GFP_KERNEL_ACCOUNT); if (!linfo) goto out_free; @@ -10993,12 +11416,9 @@ static int kvm_alloc_memslot_metadata(struct kvm_memory_slot *slot, return 0; out_free: - for (i = 0; i < KVM_NR_PAGE_SIZES; ++i) { - kvfree(slot->arch.rmap[i]); - slot->arch.rmap[i] = NULL; - if (i == 0) - continue; + memslot_rmap_free(slot); + for (i = 1; i < KVM_NR_PAGE_SIZES; ++i) { kvfree(slot->arch.lpage_info[i - 1]); slot->arch.lpage_info[i - 1] = NULL; } @@ -11027,7 +11447,7 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, enum kvm_mr_change change) { if (change == KVM_MR_CREATE || change == KVM_MR_MOVE) - return kvm_alloc_memslot_metadata(memslot, + return kvm_alloc_memslot_metadata(kvm, memslot, mem->memory_size >> PAGE_SHIFT); return 0; } @@ -11103,36 +11523,19 @@ static void kvm_mmu_slot_apply_flags(struct kvm *kvm, */ kvm_mmu_zap_collapsible_sptes(kvm, new); } else { - /* By default, write-protect everything to log writes. */ - int level = PG_LEVEL_4K; + /* + * Initially-all-set does not require write protecting any page, + * because they're all assumed to be dirty. + */ + if (kvm_dirty_log_manual_protect_and_init_set(kvm)) + return; if (kvm_x86_ops.cpu_dirty_log_size) { - /* - * Clear all dirty bits, unless pages are treated as - * dirty from the get-go. - */ - if (!kvm_dirty_log_manual_protect_and_init_set(kvm)) - kvm_mmu_slot_leaf_clear_dirty(kvm, new); - - /* - * Write-protect large pages on write so that dirty - * logging happens at 4k granularity. No need to - * write-protect small SPTEs since write accesses are - * logged by the CPU via dirty bits. - */ - level = PG_LEVEL_2M; - } else if (kvm_dirty_log_manual_protect_and_init_set(kvm)) { - /* - * If we're with initial-all-set, we don't need - * to write protect any small page because - * they're reported as dirty already. However - * we still need to write-protect huge pages - * so that the page split can happen lazily on - * the first write to the huge page. - */ - level = PG_LEVEL_2M; + kvm_mmu_slot_leaf_clear_dirty(kvm, new); + kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_2M); + } else { + kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K); } - kvm_mmu_slot_remove_write_access(kvm, new, level); } } @@ -11701,8 +12104,6 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) { bool pcid_enabled; struct x86_exception e; - unsigned i; - unsigned long roots_to_free = 0; struct { u64 pcid; u64 gla; @@ -11736,23 +12137,7 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) return 1; } - if (kvm_get_active_pcid(vcpu) == operand.pcid) { - kvm_mmu_sync_roots(vcpu); - kvm_make_request(KVM_REQ_TLB_FLUSH_CURRENT, vcpu); - } - - for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) - if (kvm_get_pcid(vcpu, vcpu->arch.mmu->prev_roots[i].pgd) - == operand.pcid) - roots_to_free |= KVM_MMU_ROOT_PREVIOUS(i); - - kvm_mmu_free_roots(vcpu, vcpu->arch.mmu, roots_to_free); - /* - * If neither the current cr3 nor any of the prev_roots use the - * given PCID, then nothing needs to be done here because a - * resync will happen anyway before switching to any other CR3. - */ - + kvm_invalidate_pcid(vcpu, operand.pcid); return kvm_skip_emulated_instruction(vcpu); case INVPCID_TYPE_ALL_NON_GLOBAL: @@ -11765,7 +12150,7 @@ int kvm_handle_invpcid(struct kvm_vcpu *vcpu, unsigned long type, gva_t gva) fallthrough; case INVPCID_TYPE_ALL_INCL_GLOBAL: - kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); + kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu); return kvm_skip_emulated_instruction(vcpu); default: diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 521f74e5bbf2..44ae10312740 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -157,16 +157,6 @@ static inline bool is_64_bit_mode(struct kvm_vcpu *vcpu) return cs_l; } -static inline bool is_la57_mode(struct kvm_vcpu *vcpu) -{ -#ifdef CONFIG_X86_64 - return (vcpu->arch.efer & EFER_LMA) && - kvm_read_cr4_bits(vcpu, X86_CR4_LA57); -#else - return 0; -#endif -} - static inline bool x86_exception_has_error_code(unsigned int vector) { static u32 exception_has_error_code = BIT(DF_VECTOR) | BIT(TS_VECTOR) | diff --git a/arch/x86/mm/pkeys.c b/arch/x86/mm/pkeys.c index a2332eef66e9..4a67b922bce1 100644 --- a/arch/x86/mm/pkeys.c +++ b/arch/x86/mm/pkeys.c @@ -192,6 +192,10 @@ static const struct file_operations fops_init_pkru = { static int __init create_init_pkru_value(void) { + /* Do not expose the file if pkeys are not supported. */ + if (!cpu_feature_enabled(X86_FEATURE_OSPKE)) + return 0; + debugfs_create_file("init_pkru", S_IRUSR | S_IWUSR, arch_debugfs_dir, NULL, &fops_init_pkru); return 0; diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 18cc82dc4a42..8bd288d2b089 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -411,7 +411,7 @@ int af_alg_make_sg(struct af_alg_sgl *sgl, struct iov_iter *iter, int len) if (n < 0) return n; - npages = (off + n + PAGE_SIZE - 1) >> PAGE_SHIFT; + npages = DIV_ROUND_UP(off + n, PAGE_SIZE); if (WARN_ON(npages == 0)) return -EINVAL; /* Add one extra for linking */ diff --git a/crypto/algapi.c b/crypto/algapi.c index fdabf2675b63..43f999dba4dc 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -868,24 +868,6 @@ const char *crypto_attr_alg_name(struct rtattr *rta) } EXPORT_SYMBOL_GPL(crypto_attr_alg_name); -int crypto_attr_u32(struct rtattr *rta, u32 *num) -{ - struct crypto_attr_u32 *nu32; - - if (!rta) - return -ENOENT; - if (RTA_PAYLOAD(rta) < sizeof(*nu32)) - return -EINVAL; - if (rta->rta_type != CRYPTOA_U32) - return -EINVAL; - - nu32 = RTA_DATA(rta); - *num = nu32->num; - - return 0; -} -EXPORT_SYMBOL_GPL(crypto_attr_u32); - int crypto_inst_setname(struct crypto_instance *inst, const char *name, struct crypto_alg *alg) { diff --git a/crypto/algboss.c b/crypto/algboss.c index 5ebccbd6b74e..1814d2c5188a 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -28,16 +28,9 @@ struct cryptomgr_param { struct crypto_attr_type data; } type; - union { + struct { struct rtattr attr; - struct { - struct rtattr attr; - struct crypto_attr_alg data; - } alg; - struct { - struct rtattr attr; - struct crypto_attr_u32 data; - } nu32; + struct crypto_attr_alg data; } attrs[CRYPTO_MAX_ATTRS]; char template[CRYPTO_MAX_ALG_NAME]; @@ -104,12 +97,10 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval) i = 0; for (;;) { - int notnum = 0; - name = ++p; for (; isalnum(*p) || *p == '-' || *p == '_'; p++) - notnum |= !isdigit(*p); + ; if (*p == '(') { int recursion = 0; @@ -123,7 +114,6 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval) break; } - notnum = 1; p++; } @@ -131,18 +121,9 @@ static int cryptomgr_schedule_probe(struct crypto_larval *larval) if (!len) goto err_free_param; - if (notnum) { - param->attrs[i].alg.attr.rta_len = - sizeof(param->attrs[i].alg); - param->attrs[i].alg.attr.rta_type = CRYPTOA_ALG; - memcpy(param->attrs[i].alg.data.name, name, len); - } else { - param->attrs[i].nu32.attr.rta_len = - sizeof(param->attrs[i].nu32); - param->attrs[i].nu32.attr.rta_type = CRYPTOA_U32; - param->attrs[i].nu32.data.num = - simple_strtol(name, NULL, 0); - } + param->attrs[i].attr.rta_len = sizeof(param->attrs[i]); + param->attrs[i].attr.rta_type = CRYPTOA_ALG; + memcpy(param->attrs[i].data.name, name, len); param->tb[i + 1] = ¶m->attrs[i].attr; i++; diff --git a/crypto/drbg.c b/crypto/drbg.c index 1b4587e0ddad..ea85d4a0fe9e 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -176,18 +176,18 @@ static const struct drbg_core drbg_cores[] = { .blocklen_bytes = 48, .cra_name = "hmac_sha384", .backend_cra_name = "hmac(sha384)", - }, { - .flags = DRBG_HMAC | DRBG_STRENGTH256, - .statelen = 64, /* block length of cipher */ - .blocklen_bytes = 64, - .cra_name = "hmac_sha512", - .backend_cra_name = "hmac(sha512)", }, { .flags = DRBG_HMAC | DRBG_STRENGTH256, .statelen = 32, /* block length of cipher */ .blocklen_bytes = 32, .cra_name = "hmac_sha256", .backend_cra_name = "hmac(sha256)", + }, { + .flags = DRBG_HMAC | DRBG_STRENGTH256, + .statelen = 64, /* block length of cipher */ + .blocklen_bytes = 64, + .cra_name = "hmac_sha512", + .backend_cra_name = "hmac(sha512)", }, #endif /* CONFIG_CRYPTO_DRBG_HMAC */ }; diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 04a427b8c956..c6f61c2211dc 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -141,7 +141,7 @@ static struct kpp_alg ecdh_nist_p192 = { .init = ecdh_nist_p192_init_tfm, .base = { .cra_name = "ecdh-nist-p192", - .cra_driver_name = "ecdh-generic", + .cra_driver_name = "ecdh-nist-p192-generic", .cra_priority = 100, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct ecdh_ctx), @@ -166,7 +166,32 @@ static struct kpp_alg ecdh_nist_p256 = { .init = ecdh_nist_p256_init_tfm, .base = { .cra_name = "ecdh-nist-p256", - .cra_driver_name = "ecdh-generic", + .cra_driver_name = "ecdh-nist-p256-generic", + .cra_priority = 100, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct ecdh_ctx), + }, +}; + +static int ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) +{ + struct ecdh_ctx *ctx = ecdh_get_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P384; + ctx->ndigits = ECC_CURVE_NIST_P384_DIGITS; + + return 0; +} + +static struct kpp_alg ecdh_nist_p384 = { + .set_secret = ecdh_set_secret, + .generate_public_key = ecdh_compute_value, + .compute_shared_secret = ecdh_compute_value, + .max_size = ecdh_max_size, + .init = ecdh_nist_p384_init_tfm, + .base = { + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "ecdh-nist-p384-generic", .cra_priority = 100, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct ecdh_ctx), @@ -179,10 +204,27 @@ static int ecdh_init(void) { int ret; + /* NIST p192 will fail to register in FIPS mode */ ret = crypto_register_kpp(&ecdh_nist_p192); ecdh_nist_p192_registered = ret == 0; - return crypto_register_kpp(&ecdh_nist_p256); + ret = crypto_register_kpp(&ecdh_nist_p256); + if (ret) + goto nist_p256_error; + + ret = crypto_register_kpp(&ecdh_nist_p384); + if (ret) + goto nist_p384_error; + + return 0; + +nist_p384_error: + crypto_unregister_kpp(&ecdh_nist_p256); + +nist_p256_error: + if (ecdh_nist_p192_registered) + crypto_unregister_kpp(&ecdh_nist_p192); + return ret; } static void ecdh_exit(void) @@ -190,6 +232,7 @@ static void ecdh_exit(void) if (ecdh_nist_p192_registered) crypto_unregister_kpp(&ecdh_nist_p192); crypto_unregister_kpp(&ecdh_nist_p256); + crypto_unregister_kpp(&ecdh_nist_p384); } subsys_initcall(ecdh_init); diff --git a/crypto/internal.h b/crypto/internal.h index 976ec9dfc76d..f00869af689f 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -29,6 +29,18 @@ struct crypto_larval { u32 mask; }; +enum { + CRYPTOA_UNSPEC, + CRYPTOA_ALG, + CRYPTOA_TYPE, + __CRYPTOA_MAX, +}; + +#define CRYPTOA_MAX (__CRYPTOA_MAX - 1) + +/* Maximum number of (rtattr) parameters for each template. */ +#define CRYPTO_MAX_ATTRS 32 + extern struct list_head crypto_alg_list; extern struct rw_semaphore crypto_alg_sem; extern struct blocking_notifier_head crypto_chain; diff --git a/crypto/khazad.c b/crypto/khazad.c index 14ca7f1631c7..f19339954c89 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -819,7 +819,7 @@ static void khazad_crypt(const u64 roundKey[KHAZAD_ROUNDS + 1], T6[(int)(state >> 8) & 0xff] ^ T7[(int)(state ) & 0xff] ^ roundKey[r]; - } + } state = (T0[(int)(state >> 56) ] & 0xff00000000000000ULL) ^ (T1[(int)(state >> 48) & 0xff] & 0x00ff000000000000ULL) ^ diff --git a/crypto/shash.c b/crypto/shash.c index 2e3433ad9762..0a0a50cb694f 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -20,12 +20,24 @@ static const struct crypto_type crypto_shash_type; -int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen) +static int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) { return -ENOSYS; } -EXPORT_SYMBOL_GPL(shash_no_setkey); + +/* + * Check whether an shash algorithm has a setkey function. + * + * For CFI compatibility, this must not be an inline function. This is because + * when CFI is enabled, modules won't get the same address for shash_no_setkey + * (if it were exported, which inlining would require) as the core kernel will. + */ +bool crypto_shash_alg_has_setkey(struct shash_alg *alg) +{ + return alg->setkey != shash_no_setkey; +} +EXPORT_SYMBOL_GPL(crypto_shash_alg_has_setkey); static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) diff --git a/crypto/sm2.c b/crypto/sm2.c index b21addc3ac06..db8a4a265669 100644 --- a/crypto/sm2.c +++ b/crypto/sm2.c @@ -79,10 +79,17 @@ static int sm2_ec_ctx_init(struct mpi_ec_ctx *ec) goto free; rc = -ENOMEM; + + ec->Q = mpi_point_new(0); + if (!ec->Q) + goto free; + /* mpi_ec_setup_elliptic_curve */ ec->G = mpi_point_new(0); - if (!ec->G) + if (!ec->G) { + mpi_point_release(ec->Q); goto free; + } mpi_set(ec->G->x, x); mpi_set(ec->G->y, y); @@ -91,6 +98,7 @@ static int sm2_ec_ctx_init(struct mpi_ec_ctx *ec) rc = -EINVAL; ec->n = mpi_scanval(ecp->n); if (!ec->n) { + mpi_point_release(ec->Q); mpi_point_release(ec->G); goto free; } @@ -386,27 +394,15 @@ static int sm2_set_pub_key(struct crypto_akcipher *tfm, MPI a; int rc; - ec->Q = mpi_point_new(0); - if (!ec->Q) - return -ENOMEM; - /* include the uncompressed flag '0x04' */ - rc = -ENOMEM; a = mpi_read_raw_data(key, keylen); if (!a) - goto error; + return -ENOMEM; mpi_normalize(a); rc = sm2_ecc_os2ec(ec->Q, a); mpi_free(a); - if (rc) - goto error; - return 0; - -error: - mpi_point_release(ec->Q); - ec->Q = NULL; return rc; } diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 6b7c158dc508..f8d06da78e4f 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1847,10 +1847,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) ret += tcrypt_test("cts(cbc(aes))"); break; + case 39: + ret += tcrypt_test("xxhash64"); + break; + case 40: ret += tcrypt_test("rmd160"); break; + case 41: + ret += tcrypt_test("blake2s-256"); + break; + + case 42: + ret += tcrypt_test("blake2b-512"); + break; + case 43: ret += tcrypt_test("ecb(seed)"); break; @@ -2356,10 +2368,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_hash_speed("sha224", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; + case 314: + test_hash_speed("xxhash64", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + fallthrough; case 315: test_hash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; fallthrough; + case 316: + test_hash_speed("blake2s-256", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + fallthrough; + case 317: + test_hash_speed("blake2b-512", sec, generic_hash_speed_template); + if (mode > 300 && mode < 400) break; + fallthrough; case 318: klen = 16; test_hash_speed("ghash", sec, generic_hash_speed_template); @@ -2456,10 +2480,22 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("sha224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; + case 414: + test_ahash_speed("xxhash64", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + fallthrough; case 415: test_ahash_speed("rmd160", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; + case 416: + test_ahash_speed("blake2s-256", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + fallthrough; + case 417: + test_ahash_speed("blake2b-512", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + fallthrough; case 418: test_ahash_speed("sha3-224", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 10c5b3b01ec4..1f7f63e836ae 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -4899,21 +4899,25 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { #endif -#ifndef CONFIG_CRYPTO_FIPS .alg = "ecdh-nist-p192", .test = alg_test_kpp, - .fips_allowed = 1, .suite = { .kpp = __VECS(ecdh_p192_tv_template) } }, { -#endif .alg = "ecdh-nist-p256", .test = alg_test_kpp, .fips_allowed = 1, .suite = { .kpp = __VECS(ecdh_p256_tv_template) } + }, { + .alg = "ecdh-nist-p384", + .test = alg_test_kpp, + .fips_allowed = 1, + .suite = { + .kpp = __VECS(ecdh_p384_tv_template) + } }, { .alg = "ecdsa-nist-p192", .test = alg_test_akcipher, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 34e4a3db3991..96eb7ce9f81b 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -2685,7 +2685,6 @@ static const struct kpp_testvec curve25519_tv_template[] = { } }; -#ifndef CONFIG_CRYPTO_FIPS static const struct kpp_testvec ecdh_p192_tv_template[] = { { .secret = @@ -2719,13 +2718,12 @@ static const struct kpp_testvec ecdh_p192_tv_template[] = { "\xf4\x57\xcc\x4f\x1f\x4e\x31\xcc" "\xe3\x40\x60\xc8\x06\x93\xc6\x2e" "\x99\x80\x81\x28\xaf\xc5\x51\x74", - .secret_size = 32, + .secret_size = 30, .b_public_size = 48, .expected_a_public_size = 48, .expected_ss_size = 24 } }; -#endif static const struct kpp_testvec ecdh_p256_tv_template[] = { { @@ -2766,7 +2764,7 @@ static const struct kpp_testvec ecdh_p256_tv_template[] = { "\x9f\x4a\x38\xcc\xc0\x2c\x49\x2f" "\xb1\x32\xbb\xaf\x22\x61\xda\xcb" "\x6f\xdb\xa9\xaa\xfc\x77\x81\xf3", - .secret_size = 40, + .secret_size = 38, .b_public_size = 64, .expected_a_public_size = 64, .expected_ss_size = 32 @@ -2804,8 +2802,8 @@ static const struct kpp_testvec ecdh_p256_tv_template[] = { "\x37\x08\xcc\x40\x5e\x7a\xfd\x6a" "\x6a\x02\x6e\x41\x87\x68\x38\x77" "\xfa\xa9\x44\x43\x2d\xef\x09\xdf", - .secret_size = 8, - .b_secret_size = 40, + .secret_size = 6, + .b_secret_size = 38, .b_public_size = 64, .expected_a_public_size = 64, .expected_ss_size = 32, @@ -2813,6 +2811,67 @@ static const struct kpp_testvec ecdh_p256_tv_template[] = { } }; +/* + * NIST P384 test vectors from RFC5903 + */ +static const struct kpp_testvec ecdh_p384_tv_template[] = { + { + .secret = +#ifdef __LITTLE_ENDIAN + "\x02\x00" /* type */ + "\x36\x00" /* len */ + "\x30\x00" /* key_size */ +#else + "\x00\x02" /* type */ + "\x00\x36" /* len */ + "\x00\x30" /* key_size */ +#endif + "\x09\x9F\x3C\x70\x34\xD4\xA2\xC6" + "\x99\x88\x4D\x73\xA3\x75\xA6\x7F" + "\x76\x24\xEF\x7C\x6B\x3C\x0F\x16" + "\x06\x47\xB6\x74\x14\xDC\xE6\x55" + "\xE3\x5B\x53\x80\x41\xE6\x49\xEE" + "\x3F\xAE\xF8\x96\x78\x3A\xB1\x94", + .b_public = + "\xE5\x58\xDB\xEF\x53\xEE\xCD\xE3" + "\xD3\xFC\xCF\xC1\xAE\xA0\x8A\x89" + "\xA9\x87\x47\x5D\x12\xFD\x95\x0D" + "\x83\xCF\xA4\x17\x32\xBC\x50\x9D" + "\x0D\x1A\xC4\x3A\x03\x36\xDE\xF9" + "\x6F\xDA\x41\xD0\x77\x4A\x35\x71" + "\xDC\xFB\xEC\x7A\xAC\xF3\x19\x64" + "\x72\x16\x9E\x83\x84\x30\x36\x7F" + "\x66\xEE\xBE\x3C\x6E\x70\xC4\x16" + "\xDD\x5F\x0C\x68\x75\x9D\xD1\xFF" + "\xF8\x3F\xA4\x01\x42\x20\x9D\xFF" + "\x5E\xAA\xD9\x6D\xB9\xE6\x38\x6C", + .expected_a_public = + "\x66\x78\x42\xD7\xD1\x80\xAC\x2C" + "\xDE\x6F\x74\xF3\x75\x51\xF5\x57" + "\x55\xC7\x64\x5C\x20\xEF\x73\xE3" + "\x16\x34\xFE\x72\xB4\xC5\x5E\xE6" + "\xDE\x3A\xC8\x08\xAC\xB4\xBD\xB4" + "\xC8\x87\x32\xAE\xE9\x5F\x41\xAA" + "\x94\x82\xED\x1F\xC0\xEE\xB9\xCA" + "\xFC\x49\x84\x62\x5C\xCF\xC2\x3F" + "\x65\x03\x21\x49\xE0\xE1\x44\xAD" + "\xA0\x24\x18\x15\x35\xA0\xF3\x8E" + "\xEB\x9F\xCF\xF3\xC2\xC9\x47\xDA" + "\xE6\x9B\x4C\x63\x45\x73\xA8\x1C", + .expected_ss = + "\x11\x18\x73\x31\xC2\x79\x96\x2D" + "\x93\xD6\x04\x24\x3F\xD5\x92\xCB" + "\x9D\x0A\x92\x6F\x42\x2E\x47\x18" + "\x75\x21\x28\x7E\x71\x56\xC5\xC4" + "\xD6\x03\x13\x55\x69\xB9\xE9\xD0" + "\x9C\xF5\xD4\xA2\x70\xF5\x97\x46", + .secret_size = 54, + .b_public_size = 96, + .expected_a_public_size = 96, + .expected_ss_size = 48 + } +}; + /* * MD4 test vectors from RFC1320 */ diff --git a/crypto/wp512.c b/crypto/wp512.c index feadc13ccae0..bf79fbb2340f 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1066,33 +1066,31 @@ static int wp512_final(struct shash_desc *desc, u8 *out) { struct wp512_ctx *wctx = shash_desc_ctx(desc); int i; - u8 *buffer = wctx->buffer; - u8 *bitLength = wctx->bitLength; - int bufferBits = wctx->bufferBits; - int bufferPos = wctx->bufferPos; + u8 *buffer = wctx->buffer; + u8 *bitLength = wctx->bitLength; + int bufferBits = wctx->bufferBits; + int bufferPos = wctx->bufferPos; __be64 *digest = (__be64 *)out; - buffer[bufferPos] |= 0x80U >> (bufferBits & 7); - bufferPos++; - if (bufferPos > WP512_BLOCK_SIZE - WP512_LENGTHBYTES) { - if (bufferPos < WP512_BLOCK_SIZE) { - memset(&buffer[bufferPos], 0, WP512_BLOCK_SIZE - bufferPos); - } - wp512_process_buffer(wctx); - bufferPos = 0; - } - if (bufferPos < WP512_BLOCK_SIZE - WP512_LENGTHBYTES) { - memset(&buffer[bufferPos], 0, + buffer[bufferPos] |= 0x80U >> (bufferBits & 7); + bufferPos++; + if (bufferPos > WP512_BLOCK_SIZE - WP512_LENGTHBYTES) { + if (bufferPos < WP512_BLOCK_SIZE) + memset(&buffer[bufferPos], 0, WP512_BLOCK_SIZE - bufferPos); + wp512_process_buffer(wctx); + bufferPos = 0; + } + if (bufferPos < WP512_BLOCK_SIZE - WP512_LENGTHBYTES) + memset(&buffer[bufferPos], 0, (WP512_BLOCK_SIZE - WP512_LENGTHBYTES) - bufferPos); - } - bufferPos = WP512_BLOCK_SIZE - WP512_LENGTHBYTES; - memcpy(&buffer[WP512_BLOCK_SIZE - WP512_LENGTHBYTES], + bufferPos = WP512_BLOCK_SIZE - WP512_LENGTHBYTES; + memcpy(&buffer[WP512_BLOCK_SIZE - WP512_LENGTHBYTES], bitLength, WP512_LENGTHBYTES); - wp512_process_buffer(wctx); + wp512_process_buffer(wctx); for (i = 0; i < WP512_DIGEST_SIZE/8; i++) digest[i] = cpu_to_be64(wctx->hash[i]); - wctx->bufferBits = bufferBits; - wctx->bufferPos = bufferPos; + wctx->bufferBits = bufferBits; + wctx->bufferPos = bufferPos; return 0; } diff --git a/drivers/accessibility/braille/braille_console.c b/drivers/accessibility/braille/braille_console.c index 9861302cc7db..359bead4b280 100644 --- a/drivers/accessibility/braille/braille_console.c +++ b/drivers/accessibility/braille/braille_console.c @@ -246,6 +246,7 @@ static int keyboard_notifier_call(struct notifier_block *blk, beep(440); } } + break; case KBD_UNBOUND_KEYCODE: case KBD_UNICODE: case KBD_KEYSYM: diff --git a/drivers/atm/fore200e.c b/drivers/atm/fore200e.c index b508df2ecada..fb2be3574c26 100644 --- a/drivers/atm/fore200e.c +++ b/drivers/atm/fore200e.c @@ -420,6 +420,7 @@ fore200e_shutdown(struct fore200e* fore200e) /* XXX shouldn't we *start* by deregistering the device? */ atm_dev_deregister(fore200e->atm_dev); + fallthrough; case FORE200E_STATE_BLANK: /* nothing to do for that state */ break; diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index 1fe006f3f12f..c11f12d4ab53 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -165,17 +165,17 @@ config HW_RANDOM_IXP4XX config HW_RANDOM_OMAP tristate "OMAP Random Number Generator support" - depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU + depends on ARCH_OMAP16XX || ARCH_OMAP2PLUS || ARCH_MVEBU || ARCH_K3 default HW_RANDOM help - This driver provides kernel-side support for the Random Number + This driver provides kernel-side support for the Random Number Generator hardware found on OMAP16xx, OMAP2/3/4/5, AM33xx/AM43xx multimedia processors, and Marvell Armada 7k/8k SoCs. To compile this driver as a module, choose M here: the module will be called omap-rng. - If unsure, say Y. + If unsure, say Y. config HW_RANDOM_OMAP3_ROM tristate "OMAP3 ROM Random Number Generator support" @@ -485,13 +485,13 @@ config HW_RANDOM_NPCM depends on ARCH_NPCM || COMPILE_TEST default HW_RANDOM help - This driver provides support for the Random Number + This driver provides support for the Random Number Generator hardware available in Nuvoton NPCM SoCs. To compile this driver as a module, choose M here: the module will be called npcm-rng. - If unsure, say Y. + If unsure, say Y. config HW_RANDOM_KEYSTONE depends on ARCH_KEYSTONE || COMPILE_TEST diff --git a/drivers/char/hw_random/amd-rng.c b/drivers/char/hw_random/amd-rng.c index 9959c762da2f..d8d4ef5214a1 100644 --- a/drivers/char/hw_random/amd-rng.c +++ b/drivers/char/hw_random/amd-rng.c @@ -126,7 +126,7 @@ static struct hwrng amd_rng = { static int __init mod_init(void) { - int err = -ENODEV; + int err; struct pci_dev *pdev = NULL; const struct pci_device_id *ent; u32 pmbase; diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index adb3c2bd7783..a3db27916256 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -319,11 +319,11 @@ static int enable_best_rng(void) return ret; } -static ssize_t hwrng_attr_current_store(struct device *dev, - struct device_attribute *attr, - const char *buf, size_t len) +static ssize_t rng_current_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t len) { - int err = -ENODEV; + int err; struct hwrng *rng, *old_rng, *new_rng; err = mutex_lock_interruptible(&rng_mutex); @@ -354,9 +354,9 @@ static ssize_t hwrng_attr_current_store(struct device *dev, return err ? : len; } -static ssize_t hwrng_attr_current_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t rng_current_show(struct device *dev, + struct device_attribute *attr, + char *buf) { ssize_t ret; struct hwrng *rng; @@ -371,9 +371,9 @@ static ssize_t hwrng_attr_current_show(struct device *dev, return ret; } -static ssize_t hwrng_attr_available_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t rng_available_show(struct device *dev, + struct device_attribute *attr, + char *buf) { int err; struct hwrng *rng; @@ -392,22 +392,16 @@ static ssize_t hwrng_attr_available_show(struct device *dev, return strlen(buf); } -static ssize_t hwrng_attr_selected_show(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t rng_selected_show(struct device *dev, + struct device_attribute *attr, + char *buf) { return sysfs_emit(buf, "%d\n", cur_rng_set_by_user); } -static DEVICE_ATTR(rng_current, S_IRUGO | S_IWUSR, - hwrng_attr_current_show, - hwrng_attr_current_store); -static DEVICE_ATTR(rng_available, S_IRUGO, - hwrng_attr_available_show, - NULL); -static DEVICE_ATTR(rng_selected, S_IRUGO, - hwrng_attr_selected_show, - NULL); +static DEVICE_ATTR_RW(rng_current); +static DEVICE_ATTR_RO(rng_available); +static DEVICE_ATTR_RO(rng_selected); static struct attribute *rng_dev_attrs[] = { &dev_attr_rng_current.attr, diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c index 8e1fe3f8dd2d..9cc3d542dd0f 100644 --- a/drivers/char/hw_random/exynos-trng.c +++ b/drivers/char/hw_random/exynos-trng.c @@ -132,7 +132,7 @@ static int exynos_trng_probe(struct platform_device *pdev) return PTR_ERR(trng->mem); pm_runtime_enable(&pdev->dev); - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "Could not get runtime PM.\n"); goto err_pm_get; @@ -165,7 +165,7 @@ static int exynos_trng_probe(struct platform_device *pdev) clk_disable_unprepare(trng->clk); err_clock: - pm_runtime_put_sync(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); err_pm_get: pm_runtime_disable(&pdev->dev); @@ -196,10 +196,9 @@ static int __maybe_unused exynos_trng_resume(struct device *dev) { int ret; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Could not get runtime PM.\n"); - pm_runtime_put_noidle(dev); return ret; } diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c index 8f1d47ff9799..2f2f21f1b659 100644 --- a/drivers/char/hw_random/ks-sa-rng.c +++ b/drivers/char/hw_random/ks-sa-rng.c @@ -241,10 +241,9 @@ static int ks_sa_rng_probe(struct platform_device *pdev) } pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to enable SA power-domain\n"); - pm_runtime_put_noidle(dev); pm_runtime_disable(dev); return ret; } diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index cede9f159102..00ff96703dd2 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -454,10 +454,9 @@ static int omap_rng_probe(struct platform_device *pdev) } pm_runtime_enable(&pdev->dev); - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { dev_err(&pdev->dev, "Failed to runtime_get device: %d\n", ret); - pm_runtime_put_noidle(&pdev->dev); goto err_ioremap; } @@ -543,10 +542,9 @@ static int __maybe_unused omap_rng_resume(struct device *dev) struct omap_rng_dev *priv = dev_get_drvdata(dev); int ret; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to runtime_get device: %d\n", ret); - pm_runtime_put_noidle(dev); return ret; } diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 9a4c275a1335..ebcec460c045 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -266,6 +266,27 @@ config CRYPTO_DEV_NIAGARA2 Group, which can perform encryption, decryption, hashing, checksumming, and raw copies. +config CRYPTO_DEV_SL3516 + tristate "Stormlink SL3516 crypto offloader" + depends on HAS_IOMEM + select CRYPTO_SKCIPHER + select CRYPTO_ENGINE + select CRYPTO_ECB + select CRYPTO_AES + select HW_RANDOM + depends on PM + help + This option allows you to have support for SL3516 crypto offloader. + +config CRYPTO_DEV_SL3516_DEBUG + bool "Enable SL3516 stats" + depends on CRYPTO_DEV_SL3516 + depends on DEBUG_FS + help + Say y to enable SL3516 debug stats. + This will create /sys/kernel/debug/sl3516/stats for displaying + the number of requests per algorithm and other internal stats. + config CRYPTO_DEV_HIFN_795X tristate "Driver HIFN 795x crypto accelerator chips" select CRYPTO_LIB_DES @@ -325,6 +346,11 @@ config CRYPTO_DEV_TALITOS2 config CRYPTO_DEV_IXP4XX tristate "Driver for IXP4xx crypto hardware acceleration" depends on ARCH_IXP4XX && IXP4XX_QMGR && IXP4XX_NPE + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_ECB + select CRYPTO_CBC + select CRYPTO_CTR select CRYPTO_LIB_DES select CRYPTO_AEAD select CRYPTO_AUTHENC @@ -627,6 +653,12 @@ config CRYPTO_DEV_QCE_SHA select CRYPTO_SHA1 select CRYPTO_SHA256 +config CRYPTO_DEV_QCE_AEAD + bool + depends on CRYPTO_DEV_QCE + select CRYPTO_AUTHENC + select CRYPTO_LIB_DES + choice prompt "Algorithms enabled for QCE acceleration" default CRYPTO_DEV_QCE_ENABLE_ALL @@ -647,6 +679,7 @@ choice bool "All supported algorithms" select CRYPTO_DEV_QCE_SKCIPHER select CRYPTO_DEV_QCE_SHA + select CRYPTO_DEV_QCE_AEAD help Enable all supported algorithms: - AES (CBC, CTR, ECB, XTS) @@ -672,6 +705,14 @@ choice - SHA1, HMAC-SHA1 - SHA256, HMAC-SHA256 + config CRYPTO_DEV_QCE_ENABLE_AEAD + bool "AEAD algorithms only" + select CRYPTO_DEV_QCE_AEAD + help + Enable AEAD algorithms only: + - authenc() + - ccm(aes) + - rfc4309(ccm(aes)) endchoice config CRYPTO_DEV_QCE_SW_MAX_LEN diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index fa22cb19e242..1fe5120eb966 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -38,6 +38,7 @@ obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/ obj-$(CONFIG_CRYPTO_DEV_S5P) += s5p-sss.o obj-$(CONFIG_CRYPTO_DEV_SA2UL) += sa2ul.o obj-$(CONFIG_CRYPTO_DEV_SAHARA) += sahara.o +obj-$(CONFIG_CRYPTO_DEV_SL3516) += gemini/ obj-$(CONFIG_ARCH_STM32) += stm32/ obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o obj-$(CONFIG_CRYPTO_DEV_UX500) += ux500/ diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 06ee42e8a245..8c32d0eb8fcf 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -401,7 +401,7 @@ static void cpt_disable_all_cores(struct cpt_device *cpt) cpt_write_csr64(cpt->reg_base, CPTX_PF_EXE_CTL(0), 0); } -/** +/* * Ensure all cores are disengaged from all groups by * calling cpt_disable_all_cores() before calling this * function. diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index 4fe7898c8561..153004bdfb5c 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -9,8 +9,8 @@ /** * get_free_pending_entry - get free entry from pending queue - * @param pqinfo: pending_qinfo structure - * @param qno: queue number + * @q: pending queue + * @qlen: queue length */ static struct pending_entry *get_free_pending_entry(struct pending_queue *q, int qlen) @@ -244,11 +244,7 @@ static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, memcpy(ent, (void *)cmd, qinfo->cmd_size); if (++queue->idx >= queue->qhead->size / 64) { - struct hlist_node *node; - - hlist_for_each(node, &queue->chead) { - chunk = hlist_entry(node, struct command_chunk, - nextchunk); + hlist_for_each_entry(chunk, &queue->chead, nextchunk) { if (chunk == queue->qhead) { continue; } else { diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c index c288c4b51783..f19e520da6d0 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_isr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -307,6 +307,10 @@ int nitrox_register_interrupts(struct nitrox_device *ndev) * Entry 192: NPS_CORE_INT_ACTIVE */ nr_vecs = pci_msix_vec_count(pdev); + if (nr_vecs < 0) { + dev_err(DEV(ndev), "Error in getting vec count %d\n", nr_vecs); + return nr_vecs; + } /* Enable MSI-X */ ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index d385daf2c71c..96bc7b5c6532 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -35,7 +35,7 @@ static LIST_HEAD(ndevlist); static DEFINE_MUTEX(devlist_lock); static unsigned int num_devices; -/** +/* * nitrox_pci_tbl - PCI Device ID Table */ static const struct pci_device_id nitrox_pci_tbl[] = { @@ -65,7 +65,7 @@ struct ucode { u64 code[]; }; -/** +/* * write_to_ucd_unit - Write Firmware to NITROX UCD unit */ static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size, @@ -424,8 +424,7 @@ static int nitrox_probe(struct pci_dev *pdev, err = nitrox_device_flr(pdev); if (err) { dev_err(&pdev->dev, "FLR failed\n"); - pci_disable_device(pdev); - return err; + goto flr_fail; } if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { @@ -434,16 +433,13 @@ static int nitrox_probe(struct pci_dev *pdev, err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (err) { dev_err(&pdev->dev, "DMA configuration failed\n"); - pci_disable_device(pdev); - return err; + goto flr_fail; } } err = pci_request_mem_regions(pdev, nitrox_driver_name); - if (err) { - pci_disable_device(pdev); - return err; - } + if (err) + goto flr_fail; pci_set_master(pdev); ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); @@ -479,7 +475,7 @@ static int nitrox_probe(struct pci_dev *pdev, err = nitrox_pf_sw_init(ndev); if (err) - goto ioremap_err; + goto pf_sw_fail; err = nitrox_pf_hw_init(ndev); if (err) @@ -509,12 +505,15 @@ static int nitrox_probe(struct pci_dev *pdev, smp_mb__after_atomic(); pf_hw_fail: nitrox_pf_sw_cleanup(ndev); +pf_sw_fail: + iounmap(ndev->bar_addr); ioremap_err: nitrox_remove_from_devlist(ndev); kfree(ndev); pci_set_drvdata(pdev, NULL); ndev_fail: pci_release_mem_regions(pdev); +flr_fail: pci_disable_device(pdev); return err; } diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index c1af9d4fca6e..2e9c0d214363 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -8,7 +8,7 @@ #define RING_TO_VFNO(_x, _y) ((_x) / (_y)) -/** +/* * mbx_msg_type - Mailbox message types */ enum mbx_msg_type { @@ -18,7 +18,7 @@ enum mbx_msg_type { MBX_MSG_TYPE_NACK, }; -/** +/* * mbx_msg_opcode - Mailbox message opcodes */ enum mbx_msg_opcode { diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c index df95ba26b414..55c18da4a500 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -19,7 +19,7 @@ #define REQ_BACKLOG 2 #define REQ_POSTED 3 -/** +/* * Response codes from SE microcode * 0x00 - Success * Completion with no error @@ -159,7 +159,7 @@ static int dma_map_inbufs(struct nitrox_softreq *sr, struct se_crypto_request *req) { struct device *dev = DEV(sr->ndev); - struct scatterlist *sg = req->src; + struct scatterlist *sg; int i, nents, ret = 0; nents = dma_map_sg(dev, req->src, sg_nents(req->src), @@ -279,6 +279,7 @@ static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) /** * post_se_instr - Post SE instruction to Packet Input ring * @sr: Request structure + * @cmdq: Command queue structure * * Returns 0 if successful or a negative error code, * if no space in ring. @@ -369,9 +370,11 @@ static int nitrox_enqueue_request(struct nitrox_softreq *sr) } /** - * nitrox_se_request - Send request to SE core + * nitrox_process_se_request - Send request to SE core * @ndev: NITROX device * @req: Crypto request + * @callback: Completion callback + * @cb_arg: Completion callback arguments * * Returns 0 on success, or a negative error code. */ @@ -526,9 +529,8 @@ static bool sr_completed(struct nitrox_softreq *sr) } /** - * process_request_list - process completed requests - * @ndev: N5 device - * @qno: queue to operate + * process_response_list - process completed requests + * @cmdq: Command queue structure * * Returns the number of responses processed. */ @@ -578,7 +580,7 @@ static void process_response_list(struct nitrox_cmdq *cmdq) } } -/** +/* * pkt_slc_resp_tasklet - post processing of SE responses */ void pkt_slc_resp_tasklet(unsigned long data) diff --git a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c index a553ac65f324..248b4fff1c72 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_skcipher.c +++ b/drivers/crypto/cavium/nitrox/nitrox_skcipher.c @@ -20,7 +20,7 @@ struct nitrox_cipher { enum flexi_cipher value; }; -/** +/* * supported cipher list */ static const struct nitrox_cipher flexi_cipher_table[] = { diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index 6777582aa1ce..9ce4b68e9c48 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -470,7 +470,7 @@ int ccp_cmd_queue_thread(void *data) /** * ccp_alloc_struct - allocate and initialize the ccp_device struct * - * @dev: device struct of the CCP + * @sp: sp_device struct of the CCP */ struct ccp_device *ccp_alloc_struct(struct sp_device *sp) { diff --git a/drivers/crypto/ccp/ccp-dmaengine.c b/drivers/crypto/ccp/ccp-dmaengine.c index 0770a83bf1a5..d718db224be4 100644 --- a/drivers/crypto/ccp/ccp-dmaengine.c +++ b/drivers/crypto/ccp/ccp-dmaengine.c @@ -307,8 +307,7 @@ static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) spin_lock_irqsave(&chan->lock, flags); cookie = dma_cookie_assign(tx_desc); - list_del(&desc->entry); - list_add_tail(&desc->entry, &chan->pending); + list_move_tail(&desc->entry, &chan->pending); spin_unlock_irqrestore(&chan->lock, flags); diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 3506b2050fb8..91808402e0bf 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -43,6 +43,10 @@ static int psp_probe_timeout = 5; module_param(psp_probe_timeout, int, 0644); MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe"); +MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */ +MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */ +MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */ + static bool psp_dead; static int psp_timeout; diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index f468594ef8af..6fb6ba35f89d 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -222,7 +222,7 @@ static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (ret) { dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); - goto e_err; + goto free_irqs; } } @@ -230,10 +230,12 @@ static int sp_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) ret = sp_init(sp); if (ret) - goto e_err; + goto free_irqs; return 0; +free_irqs: + sp_free_irqs(sp); e_err: dev_notice(dev, "initialization failed\n"); return ret; diff --git a/drivers/crypto/gemini/Makefile b/drivers/crypto/gemini/Makefile new file mode 100644 index 000000000000..c73c8b69260d --- /dev/null +++ b/drivers/crypto/gemini/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_SL3516) += sl3516-ce.o +sl3516-ce-y += sl3516-ce-core.o sl3516-ce-cipher.o sl3516-ce-rng.o diff --git a/drivers/crypto/gemini/sl3516-ce-cipher.c b/drivers/crypto/gemini/sl3516-ce-cipher.c new file mode 100644 index 000000000000..b41c2f5fc495 --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce-cipher.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sl3516-ce-cipher.c - hardware cryptographic offloader for Stormlink SL3516 SoC + * + * Copyright (C) 2021 Corentin LABBE + * + * This file adds support for AES cipher with 128,192,256 bits keysize in + * ECB mode. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "sl3516-ce.h" + +/* sl3516_ce_need_fallback - check if a request can be handled by the CE */ +static bool sl3516_ce_need_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_dev *ce = op->ce; + struct scatterlist *in_sg = areq->src; + struct scatterlist *out_sg = areq->dst; + struct scatterlist *sg; + + if (areq->cryptlen == 0 || areq->cryptlen % 16) { + ce->fallback_mod16++; + return true; + } + + /* + * check if we have enough descriptors for TX + * Note: TX need one control desc for each SG + */ + if (sg_nents(areq->src) > MAXDESC / 2) { + ce->fallback_sg_count_tx++; + return true; + } + /* check if we have enough descriptors for RX */ + if (sg_nents(areq->dst) > MAXDESC) { + ce->fallback_sg_count_rx++; + return true; + } + + sg = areq->src; + while (sg) { + if ((sg->length % 16) != 0) { + ce->fallback_mod16++; + return true; + } + if ((sg_dma_len(sg) % 16) != 0) { + ce->fallback_mod16++; + return true; + } + if (!IS_ALIGNED(sg->offset, 16)) { + ce->fallback_align16++; + return true; + } + sg = sg_next(sg); + } + sg = areq->dst; + while (sg) { + if ((sg->length % 16) != 0) { + ce->fallback_mod16++; + return true; + } + if ((sg_dma_len(sg) % 16) != 0) { + ce->fallback_mod16++; + return true; + } + if (!IS_ALIGNED(sg->offset, 16)) { + ce->fallback_align16++; + return true; + } + sg = sg_next(sg); + } + + /* need same numbers of SG (with same length) for source and destination */ + in_sg = areq->src; + out_sg = areq->dst; + while (in_sg && out_sg) { + if (in_sg->length != out_sg->length) { + ce->fallback_not_same_len++; + return true; + } + in_sg = sg_next(in_sg); + out_sg = sg_next(out_sg); + } + if (in_sg || out_sg) + return true; + + return false; +} + +static int sl3516_ce_cipher_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sl3516_ce_alg_template *algt; + int err; + + algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher); + algt->stat_fb++; + + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (rctx->op_dir == CE_DECRYPTION) + err = crypto_skcipher_decrypt(&rctx->fallback_req); + else + err = crypto_skcipher_encrypt(&rctx->fallback_req); + return err; +} + +static int sl3516_ce_cipher(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_dev *ce = op->ce; + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + struct sl3516_ce_alg_template *algt; + struct scatterlist *sg; + unsigned int todo, len; + struct pkt_control_ecb *ecb; + int nr_sgs = 0; + int nr_sgd = 0; + int err = 0; + int i; + + algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher); + + dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__, + crypto_tfm_alg_name(areq->base.tfm), + areq->cryptlen, + rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), + op->keylen); + + algt->stat_req++; + + if (areq->src == areq->dst) { + nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_BIDIRECTIONAL); + if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + nr_sgd = nr_sgs; + } else { + nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_TO_DEVICE); + if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); + err = -EINVAL; + goto theend; + } + nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst), + DMA_FROM_DEVICE); + if (nr_sgd <= 0 || nr_sgd > MAXDESC) { + dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); + err = -EINVAL; + goto theend_sgs; + } + } + + len = areq->cryptlen; + i = 0; + sg = areq->src; + while (i < nr_sgs && sg && len) { + if (sg_dma_len(sg) == 0) + goto sgs_next; + rctx->t_src[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_src[i].len = todo; + dev_dbg(ce->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo); + len -= todo; + i++; +sgs_next: + sg = sg_next(sg); + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d/%u nr_sgs=%d\n", len, areq->cryptlen, nr_sgs); + err = -EINVAL; + goto theend_sgs; + } + + len = areq->cryptlen; + i = 0; + sg = areq->dst; + while (i < nr_sgd && sg && len) { + if (sg_dma_len(sg) == 0) + goto sgd_next; + rctx->t_dst[i].addr = sg_dma_address(sg); + todo = min(len, sg_dma_len(sg)); + rctx->t_dst[i].len = todo; + dev_dbg(ce->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n", __func__, + areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo); + len -= todo; + i++; + +sgd_next: + sg = sg_next(sg); + } + if (len > 0) { + dev_err(ce->dev, "remaining len %d\n", len); + err = -EINVAL; + goto theend_sgs; + } + + switch (algt->mode) { + case ECB_AES: + rctx->pctrllen = sizeof(struct pkt_control_ecb); + ecb = (struct pkt_control_ecb *)ce->pctrl; + + rctx->tqflag = TQ0_TYPE_CTRL; + rctx->tqflag |= TQ1_CIPHER; + ecb->control.op_mode = rctx->op_dir; + ecb->control.cipher_algorithm = ECB_AES; + ecb->cipher.header_len = 0; + ecb->cipher.algorithm_len = areq->cryptlen; + cpu_to_be32_array((__be32 *)ecb->key, (u32 *)op->key, op->keylen / 4); + rctx->h = &ecb->cipher; + + rctx->tqflag |= TQ4_KEY0; + rctx->tqflag |= TQ5_KEY4; + rctx->tqflag |= TQ6_KEY6; + ecb->control.aesnk = op->keylen / 4; + break; + } + + rctx->nr_sgs = nr_sgs; + rctx->nr_sgd = nr_sgd; + err = sl3516_ce_run_task(ce, rctx, crypto_tfm_alg_name(areq->base.tfm)); + +theend_sgs: + if (areq->src == areq->dst) { + dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_BIDIRECTIONAL); + } else { + dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src), + DMA_TO_DEVICE); + dma_unmap_sg(ce->dev, areq->dst, sg_nents(areq->dst), + DMA_FROM_DEVICE); + } + +theend: + + return err; +} + +static int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *areq) +{ + int err; + struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); + + err = sl3516_ce_cipher(breq); + crypto_finalize_skcipher_request(engine, breq, err); + + return 0; +} + +int sl3516_ce_skdecrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + + memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx)); + rctx->op_dir = CE_DECRYPTION; + + if (sl3516_ce_need_fallback(areq)) + return sl3516_ce_cipher_fallback(areq); + + engine = op->ce->engine; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sl3516_ce_skencrypt(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(areq); + struct crypto_engine *engine; + + memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx)); + rctx->op_dir = CE_ENCRYPTION; + + if (sl3516_ce_need_fallback(areq)) + return sl3516_ce_cipher_fallback(areq); + + engine = op->ce->engine; + + return crypto_transfer_skcipher_request_to_engine(engine, areq); +} + +int sl3516_ce_cipher_init(struct crypto_tfm *tfm) +{ + struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + struct sl3516_ce_alg_template *algt; + const char *name = crypto_tfm_alg_name(tfm); + struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); + struct skcipher_alg *alg = crypto_skcipher_alg(sktfm); + int err; + + memset(op, 0, sizeof(struct sl3516_ce_cipher_tfm_ctx)); + + algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher); + op->ce = algt->ce; + + op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(op->fallback_tfm)) { + dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(op->fallback_tfm)); + return PTR_ERR(op->fallback_tfm); + } + + sktfm->reqsize = sizeof(struct sl3516_ce_cipher_req_ctx) + + crypto_skcipher_reqsize(op->fallback_tfm); + + dev_info(op->ce->dev, "Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(&sktfm->base), + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm))); + + op->enginectx.op.do_one_request = sl3516_ce_handle_cipher_request; + op->enginectx.op.prepare_request = NULL; + op->enginectx.op.unprepare_request = NULL; + + err = pm_runtime_get_sync(op->ce->dev); + if (err < 0) + goto error_pm; + + return 0; +error_pm: + pm_runtime_put_noidle(op->ce->dev); + crypto_free_skcipher(op->fallback_tfm); + return err; +} + +void sl3516_ce_cipher_exit(struct crypto_tfm *tfm) +{ + struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); + + kfree_sensitive(op->key); + crypto_free_skcipher(op->fallback_tfm); + pm_runtime_put_sync_suspend(op->ce->dev); +} + +int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sl3516_ce_dev *ce = op->ce; + + switch (keylen) { + case 128 / 8: + break; + case 192 / 8: + break; + case 256 / 8: + break; + default: + dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen); + return -EINVAL; + } + kfree_sensitive(op->key); + op->keylen = keylen; + op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); + if (!op->key) + return -ENOMEM; + + crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); +} diff --git a/drivers/crypto/gemini/sl3516-ce-core.c b/drivers/crypto/gemini/sl3516-ce-core.c new file mode 100644 index 000000000000..da6cd529a6c0 --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce-core.c @@ -0,0 +1,535 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sl3516-ce-core.c - hardware cryptographic offloader for Stormlink SL3516 SoC + * + * Copyright (C) 2021 Corentin Labbe + * + * Core file which registers crypto algorithms supported by the CryptoEngine + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "sl3516-ce.h" + +static int sl3516_ce_desc_init(struct sl3516_ce_dev *ce) +{ + const size_t sz = sizeof(struct descriptor) * MAXDESC; + int i; + + ce->tx = dma_alloc_coherent(ce->dev, sz, &ce->dtx, GFP_KERNEL); + if (!ce->tx) + return -ENOMEM; + ce->rx = dma_alloc_coherent(ce->dev, sz, &ce->drx, GFP_KERNEL); + if (!ce->rx) + goto err_rx; + + for (i = 0; i < MAXDESC; i++) { + ce->tx[i].frame_ctrl.bits.own = CE_CPU; + ce->tx[i].next_desc.next_descriptor = ce->dtx + (i + 1) * sizeof(struct descriptor); + } + ce->tx[MAXDESC - 1].next_desc.next_descriptor = ce->dtx; + + for (i = 0; i < MAXDESC; i++) { + ce->rx[i].frame_ctrl.bits.own = CE_CPU; + ce->rx[i].next_desc.next_descriptor = ce->drx + (i + 1) * sizeof(struct descriptor); + } + ce->rx[MAXDESC - 1].next_desc.next_descriptor = ce->drx; + + ce->pctrl = dma_alloc_coherent(ce->dev, sizeof(struct pkt_control_ecb), + &ce->dctrl, GFP_KERNEL); + if (!ce->pctrl) + goto err_pctrl; + + return 0; +err_pctrl: + dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); +err_rx: + dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); + return -ENOMEM; +} + +static void sl3516_ce_free_descs(struct sl3516_ce_dev *ce) +{ + const size_t sz = sizeof(struct descriptor) * MAXDESC; + + dma_free_coherent(ce->dev, sz, ce->tx, ce->dtx); + dma_free_coherent(ce->dev, sz, ce->rx, ce->drx); + dma_free_coherent(ce->dev, sizeof(struct pkt_control_ecb), ce->pctrl, + ce->dctrl); +} + +static void start_dma_tx(struct sl3516_ce_dev *ce) +{ + u32 v; + + v = TXDMA_CTRL_START | TXDMA_CTRL_CHAIN_MODE | TXDMA_CTRL_CONTINUE | \ + TXDMA_CTRL_INT_FAIL | TXDMA_CTRL_INT_PERR | TXDMA_CTRL_BURST_UNK; + + writel(v, ce->base + IPSEC_TXDMA_CTRL); +} + +static void start_dma_rx(struct sl3516_ce_dev *ce) +{ + u32 v; + + v = RXDMA_CTRL_START | RXDMA_CTRL_CHAIN_MODE | RXDMA_CTRL_CONTINUE | \ + RXDMA_CTRL_BURST_UNK | RXDMA_CTRL_INT_FINISH | \ + RXDMA_CTRL_INT_FAIL | RXDMA_CTRL_INT_PERR | \ + RXDMA_CTRL_INT_EOD | RXDMA_CTRL_INT_EOF; + + writel(v, ce->base + IPSEC_RXDMA_CTRL); +} + +static struct descriptor *get_desc_tx(struct sl3516_ce_dev *ce) +{ + struct descriptor *dd; + + dd = &ce->tx[ce->ctx]; + ce->ctx++; + if (ce->ctx >= MAXDESC) + ce->ctx = 0; + return dd; +} + +static struct descriptor *get_desc_rx(struct sl3516_ce_dev *ce) +{ + struct descriptor *rdd; + + rdd = &ce->rx[ce->crx]; + ce->crx++; + if (ce->crx >= MAXDESC) + ce->crx = 0; + return rdd; +} + +int sl3516_ce_run_task(struct sl3516_ce_dev *ce, struct sl3516_ce_cipher_req_ctx *rctx, + const char *name) +{ + struct descriptor *dd, *rdd = NULL; + u32 v; + int i, err = 0; + + ce->stat_req++; + + reinit_completion(&ce->complete); + ce->status = 0; + + for (i = 0; i < rctx->nr_sgd; i++) { + dev_dbg(ce->dev, "%s handle DST SG %d/%d len=%d\n", __func__, + i, rctx->nr_sgd, rctx->t_dst[i].len); + rdd = get_desc_rx(ce); + rdd->buf_adr = rctx->t_dst[i].addr; + rdd->frame_ctrl.bits.buffer_size = rctx->t_dst[i].len; + rdd->frame_ctrl.bits.own = CE_DMA; + } + rdd->next_desc.bits.eofie = 1; + + for (i = 0; i < rctx->nr_sgs; i++) { + dev_dbg(ce->dev, "%s handle SRC SG %d/%d len=%d\n", __func__, + i, rctx->nr_sgs, rctx->t_src[i].len); + rctx->h->algorithm_len = rctx->t_src[i].len; + + dd = get_desc_tx(ce); + dd->frame_ctrl.raw = 0; + dd->flag_status.raw = 0; + dd->frame_ctrl.bits.buffer_size = rctx->pctrllen; + dd->buf_adr = ce->dctrl; + dd->flag_status.tx_flag.tqflag = rctx->tqflag; + dd->next_desc.bits.eofie = 0; + dd->next_desc.bits.dec = 0; + dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; + dd->frame_ctrl.bits.own = CE_DMA; + + dd = get_desc_tx(ce); + dd->frame_ctrl.raw = 0; + dd->flag_status.raw = 0; + dd->frame_ctrl.bits.buffer_size = rctx->t_src[i].len; + dd->buf_adr = rctx->t_src[i].addr; + dd->flag_status.tx_flag.tqflag = 0; + dd->next_desc.bits.eofie = 0; + dd->next_desc.bits.dec = 0; + dd->next_desc.bits.sof_eof = DESC_FIRST | DESC_LAST; + dd->frame_ctrl.bits.own = CE_DMA; + start_dma_tx(ce); + start_dma_rx(ce); + } + wait_for_completion_interruptible_timeout(&ce->complete, + msecs_to_jiffies(5000)); + if (ce->status == 0) { + dev_err(ce->dev, "DMA timeout for %s\n", name); + err = -EFAULT; + } + v = readl(ce->base + IPSEC_STATUS_REG); + if (v & 0xFFF) { + dev_err(ce->dev, "IPSEC_STATUS_REG %x\n", v); + err = -EFAULT; + } + + return err; +} + +static irqreturn_t ce_irq_handler(int irq, void *data) +{ + struct sl3516_ce_dev *ce = (struct sl3516_ce_dev *)data; + u32 v; + + ce->stat_irq++; + + v = readl(ce->base + IPSEC_DMA_STATUS); + writel(v, ce->base + IPSEC_DMA_STATUS); + + if (v & DMA_STATUS_TS_DERR) + dev_err(ce->dev, "AHB bus Error While Tx !!!\n"); + if (v & DMA_STATUS_TS_PERR) + dev_err(ce->dev, "Tx Descriptor Protocol Error !!!\n"); + if (v & DMA_STATUS_RS_DERR) + dev_err(ce->dev, "AHB bus Error While Rx !!!\n"); + if (v & DMA_STATUS_RS_PERR) + dev_err(ce->dev, "Rx Descriptor Protocol Error !!!\n"); + + if (v & DMA_STATUS_TS_EOFI) + ce->stat_irq_tx++; + if (v & DMA_STATUS_RS_EOFI) { + ce->status = 1; + complete(&ce->complete); + ce->stat_irq_rx++; + return IRQ_HANDLED; + } + + return IRQ_HANDLED; +} + +static struct sl3516_ce_alg_template ce_algs[] = { +{ + .type = CRYPTO_ALG_TYPE_SKCIPHER, + .mode = ECB_AES, + .alg.skcipher = { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-sl3516", + .cra_priority = 400, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, + .cra_ctxsize = sizeof(struct sl3516_ce_cipher_tfm_ctx), + .cra_module = THIS_MODULE, + .cra_alignmask = 0xf, + .cra_init = sl3516_ce_cipher_init, + .cra_exit = sl3516_ce_cipher_exit, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = sl3516_ce_aes_setkey, + .encrypt = sl3516_ce_skencrypt, + .decrypt = sl3516_ce_skdecrypt, + } +}, +}; + +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG +static int sl3516_ce_debugfs_show(struct seq_file *seq, void *v) +{ + struct sl3516_ce_dev *ce = seq->private; + unsigned int i; + + seq_printf(seq, "HWRNG %lu %lu\n", + ce->hwrng_stat_req, ce->hwrng_stat_bytes); + seq_printf(seq, "IRQ %lu\n", ce->stat_irq); + seq_printf(seq, "IRQ TX %lu\n", ce->stat_irq_tx); + seq_printf(seq, "IRQ RX %lu\n", ce->stat_irq_rx); + seq_printf(seq, "nreq %lu\n", ce->stat_req); + seq_printf(seq, "fallback SG count TX %lu\n", ce->fallback_sg_count_tx); + seq_printf(seq, "fallback SG count RX %lu\n", ce->fallback_sg_count_rx); + seq_printf(seq, "fallback modulo16 %lu\n", ce->fallback_mod16); + seq_printf(seq, "fallback align16 %lu\n", ce->fallback_align16); + seq_printf(seq, "fallback not same len %lu\n", ce->fallback_not_same_len); + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + if (!ce_algs[i].ce) + continue; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + seq_printf(seq, "%s %s reqs=%lu fallback=%lu\n", + ce_algs[i].alg.skcipher.base.cra_driver_name, + ce_algs[i].alg.skcipher.base.cra_name, + ce_algs[i].stat_req, ce_algs[i].stat_fb); + break; + } + } + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(sl3516_ce_debugfs); +#endif + +static int sl3516_ce_register_algs(struct sl3516_ce_dev *ce) +{ + int err; + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + ce_algs[i].ce = ce; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + dev_info(ce->dev, "DEBUG: Register %s\n", + ce_algs[i].alg.skcipher.base.cra_name); + err = crypto_register_skcipher(&ce_algs[i].alg.skcipher); + if (err) { + dev_err(ce->dev, "Fail to register %s\n", + ce_algs[i].alg.skcipher.base.cra_name); + ce_algs[i].ce = NULL; + return err; + } + break; + default: + ce_algs[i].ce = NULL; + dev_err(ce->dev, "ERROR: tried to register an unknown algo\n"); + } + } + return 0; +} + +static void sl3516_ce_unregister_algs(struct sl3516_ce_dev *ce) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(ce_algs); i++) { + if (!ce_algs[i].ce) + continue; + switch (ce_algs[i].type) { + case CRYPTO_ALG_TYPE_SKCIPHER: + dev_info(ce->dev, "Unregister %d %s\n", i, + ce_algs[i].alg.skcipher.base.cra_name); + crypto_unregister_skcipher(&ce_algs[i].alg.skcipher); + break; + } + } +} + +static void sl3516_ce_start(struct sl3516_ce_dev *ce) +{ + ce->ctx = 0; + ce->crx = 0; + writel(ce->dtx, ce->base + IPSEC_TXDMA_CURR_DESC); + writel(ce->drx, ce->base + IPSEC_RXDMA_CURR_DESC); + writel(0, ce->base + IPSEC_DMA_STATUS); +} + +/* + * Power management strategy: The device is suspended unless a TFM exists for + * one of the algorithms proposed by this driver. + */ +static int sl3516_ce_pm_suspend(struct device *dev) +{ + struct sl3516_ce_dev *ce = dev_get_drvdata(dev); + + reset_control_assert(ce->reset); + clk_disable_unprepare(ce->clks); + return 0; +} + +static int sl3516_ce_pm_resume(struct device *dev) +{ + struct sl3516_ce_dev *ce = dev_get_drvdata(dev); + int err; + + err = clk_prepare_enable(ce->clks); + if (err) { + dev_err(ce->dev, "Cannot prepare_enable\n"); + goto error; + } + err = reset_control_deassert(ce->reset); + if (err) { + dev_err(ce->dev, "Cannot deassert reset control\n"); + goto error; + } + + sl3516_ce_start(ce); + + return 0; +error: + sl3516_ce_pm_suspend(dev); + return err; +} + +static const struct dev_pm_ops sl3516_ce_pm_ops = { + SET_RUNTIME_PM_OPS(sl3516_ce_pm_suspend, sl3516_ce_pm_resume, NULL) +}; + +static int sl3516_ce_pm_init(struct sl3516_ce_dev *ce) +{ + int err; + + pm_runtime_use_autosuspend(ce->dev); + pm_runtime_set_autosuspend_delay(ce->dev, 2000); + + err = pm_runtime_set_suspended(ce->dev); + if (err) + return err; + pm_runtime_enable(ce->dev); + return err; +} + +static void sl3516_ce_pm_exit(struct sl3516_ce_dev *ce) +{ + pm_runtime_disable(ce->dev); +} + +static int sl3516_ce_probe(struct platform_device *pdev) +{ + struct sl3516_ce_dev *ce; + int err, irq; + u32 v; + + ce = devm_kzalloc(&pdev->dev, sizeof(*ce), GFP_KERNEL); + if (!ce) + return -ENOMEM; + + ce->dev = &pdev->dev; + platform_set_drvdata(pdev, ce); + + ce->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ce->base)) + return PTR_ERR(ce->base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + err = devm_request_irq(&pdev->dev, irq, ce_irq_handler, 0, "crypto", ce); + if (err) { + dev_err(ce->dev, "Cannot request Crypto Engine IRQ (err=%d)\n", err); + return err; + } + + ce->reset = devm_reset_control_get(&pdev->dev, NULL); + if (IS_ERR(ce->reset)) + return dev_err_probe(&pdev->dev, PTR_ERR(ce->reset), + "No reset control found\n"); + ce->clks = devm_clk_get(ce->dev, NULL); + if (IS_ERR(ce->clks)) { + err = PTR_ERR(ce->clks); + dev_err(ce->dev, "Cannot get clock err=%d\n", err); + return err; + } + + err = sl3516_ce_desc_init(ce); + if (err) + return err; + + err = sl3516_ce_pm_init(ce); + if (err) + goto error_pm; + + init_completion(&ce->complete); + + ce->engine = crypto_engine_alloc_init(ce->dev, true); + if (!ce->engine) { + dev_err(ce->dev, "Cannot allocate engine\n"); + err = -ENOMEM; + goto error_engine; + } + + err = crypto_engine_start(ce->engine); + if (err) { + dev_err(ce->dev, "Cannot start engine\n"); + goto error_engine; + } + + err = sl3516_ce_register_algs(ce); + if (err) + goto error_alg; + + err = sl3516_ce_rng_register(ce); + if (err) + goto error_rng; + + err = pm_runtime_resume_and_get(ce->dev); + if (err < 0) + goto error_pmuse; + + v = readl(ce->base + IPSEC_ID); + dev_info(ce->dev, "SL3516 dev %lx rev %lx\n", + v & GENMASK(31, 4), + v & GENMASK(3, 0)); + v = readl(ce->base + IPSEC_DMA_DEVICE_ID); + dev_info(ce->dev, "SL3516 DMA dev %lx rev %lx\n", + v & GENMASK(15, 4), + v & GENMASK(3, 0)); + + pm_runtime_put_sync(ce->dev); + +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + /* Ignore error of debugfs */ + ce->dbgfs_dir = debugfs_create_dir("sl3516", NULL); + ce->dbgfs_stats = debugfs_create_file("stats", 0444, + ce->dbgfs_dir, ce, + &sl3516_ce_debugfs_fops); +#endif + + return 0; +error_pmuse: + sl3516_ce_rng_unregister(ce); +error_rng: + sl3516_ce_unregister_algs(ce); +error_alg: + crypto_engine_exit(ce->engine); +error_engine: + sl3516_ce_pm_exit(ce); +error_pm: + sl3516_ce_free_descs(ce); + return err; +} + +static int sl3516_ce_remove(struct platform_device *pdev) +{ + struct sl3516_ce_dev *ce = platform_get_drvdata(pdev); + + sl3516_ce_rng_unregister(ce); + sl3516_ce_unregister_algs(ce); + crypto_engine_exit(ce->engine); + sl3516_ce_pm_exit(ce); + sl3516_ce_free_descs(ce); + +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + debugfs_remove_recursive(ce->dbgfs_dir); +#endif + + return 0; +} + +static const struct of_device_id sl3516_ce_crypto_of_match_table[] = { + { .compatible = "cortina,sl3516-crypto"}, + {} +}; +MODULE_DEVICE_TABLE(of, sl3516_ce_crypto_of_match_table); + +static struct platform_driver sl3516_ce_driver = { + .probe = sl3516_ce_probe, + .remove = sl3516_ce_remove, + .driver = { + .name = "sl3516-crypto", + .pm = &sl3516_ce_pm_ops, + .of_match_table = sl3516_ce_crypto_of_match_table, + }, +}; + +module_platform_driver(sl3516_ce_driver); + +MODULE_DESCRIPTION("SL3516 cryptographic offloader"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Corentin Labbe "); diff --git a/drivers/crypto/gemini/sl3516-ce-rng.c b/drivers/crypto/gemini/sl3516-ce-rng.c new file mode 100644 index 000000000000..76931ec1cec5 --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce-rng.c @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * sl3516-ce-rng.c - hardware cryptographic offloader for SL3516 SoC. + * + * Copyright (C) 2021 Corentin Labbe + * + * This file handle the RNG found in the SL3516 crypto engine + */ +#include "sl3516-ce.h" +#include +#include + +static int sl3516_ce_rng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct sl3516_ce_dev *ce; + u32 *data = buf; + size_t read = 0; + int err; + + ce = container_of(rng, struct sl3516_ce_dev, trng); + +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + ce->hwrng_stat_req++; + ce->hwrng_stat_bytes += max; +#endif + + err = pm_runtime_get_sync(ce->dev); + if (err < 0) { + pm_runtime_put_noidle(ce->dev); + return err; + } + + while (read < max) { + *data = readl(ce->base + IPSEC_RAND_NUM_REG); + data++; + read += 4; + } + + pm_runtime_put(ce->dev); + + return read; +} + +int sl3516_ce_rng_register(struct sl3516_ce_dev *ce) +{ + int ret; + + ce->trng.name = "SL3516 Crypto Engine RNG"; + ce->trng.read = sl3516_ce_rng_read; + ce->trng.quality = 700; + + ret = hwrng_register(&ce->trng); + if (ret) + dev_err(ce->dev, "Fail to register the RNG\n"); + return ret; +} + +void sl3516_ce_rng_unregister(struct sl3516_ce_dev *ce) +{ + hwrng_unregister(&ce->trng); +} diff --git a/drivers/crypto/gemini/sl3516-ce.h b/drivers/crypto/gemini/sl3516-ce.h new file mode 100644 index 000000000000..4c0ec6c920d1 --- /dev/null +++ b/drivers/crypto/gemini/sl3516-ce.h @@ -0,0 +1,347 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * sl3516-ce.h - hardware cryptographic offloader for cortina/gemini SoC + * + * Copyright (C) 2021 Corentin LABBE + * + * General notes on this driver: + * Called either Crypto Acceleration Engine Module, Security Acceleration Engine + * or IPSEC module in the datasheet, it will be called Crypto Engine for short + * in this driver. + * The CE was designed to handle IPSEC and wifi(TKIP WEP) protocol. + * It can handle AES, DES, 3DES, MD5, WEP, TKIP, SHA1, HMAC(MD5), HMAC(SHA1), + * Michael cipher/digest suites. + * It acts the same as a network hw, with both RX and TX chained descriptors. + */ +#include +#include +#include +#include +#include +#include +#include + +#define TQ0_TYPE_DATA 0 +#define TQ0_TYPE_CTRL BIT(0) +#define TQ1_CIPHER BIT(1) +#define TQ2_AUTH BIT(2) +#define TQ3_IV BIT(3) +#define TQ4_KEY0 BIT(4) +#define TQ5_KEY4 BIT(5) +#define TQ6_KEY6 BIT(6) +#define TQ7_AKEY0 BIT(7) +#define TQ8_AKEY2 BIT(8) +#define TQ9_AKEY2 BIT(9) + +#define ECB_AES 0x2 + +#define DESC_LAST 0x01 +#define DESC_FIRST 0x02 + +#define IPSEC_ID 0x0000 +#define IPSEC_STATUS_REG 0x00a8 +#define IPSEC_RAND_NUM_REG 0x00ac +#define IPSEC_DMA_DEVICE_ID 0xff00 +#define IPSEC_DMA_STATUS 0xff04 +#define IPSEC_TXDMA_CTRL 0xff08 +#define IPSEC_TXDMA_FIRST_DESC 0xff0c +#define IPSEC_TXDMA_CURR_DESC 0xff10 +#define IPSEC_RXDMA_CTRL 0xff14 +#define IPSEC_RXDMA_FIRST_DESC 0xff18 +#define IPSEC_RXDMA_CURR_DESC 0xff1c +#define IPSEC_TXDMA_BUF_ADDR 0xff28 +#define IPSEC_RXDMA_BUF_ADDR 0xff38 +#define IPSEC_RXDMA_BUF_SIZE 0xff30 + +#define CE_ENCRYPTION 0x01 +#define CE_DECRYPTION 0x03 + +#define MAXDESC 6 + +#define DMA_STATUS_RS_EOFI BIT(22) +#define DMA_STATUS_RS_PERR BIT(24) +#define DMA_STATUS_RS_DERR BIT(25) +#define DMA_STATUS_TS_EOFI BIT(27) +#define DMA_STATUS_TS_PERR BIT(29) +#define DMA_STATUS_TS_DERR BIT(30) + +#define TXDMA_CTRL_START BIT(31) +#define TXDMA_CTRL_CONTINUE BIT(30) +#define TXDMA_CTRL_CHAIN_MODE BIT(29) +/* the burst value is not documented in the datasheet */ +#define TXDMA_CTRL_BURST_UNK BIT(22) +#define TXDMA_CTRL_INT_FAIL BIT(17) +#define TXDMA_CTRL_INT_PERR BIT(16) + +#define RXDMA_CTRL_START BIT(31) +#define RXDMA_CTRL_CONTINUE BIT(30) +#define RXDMA_CTRL_CHAIN_MODE BIT(29) +/* the burst value is not documented in the datasheet */ +#define RXDMA_CTRL_BURST_UNK BIT(22) +#define RXDMA_CTRL_INT_FINISH BIT(18) +#define RXDMA_CTRL_INT_FAIL BIT(17) +#define RXDMA_CTRL_INT_PERR BIT(16) +#define RXDMA_CTRL_INT_EOD BIT(15) +#define RXDMA_CTRL_INT_EOF BIT(14) + +#define CE_CPU 0 +#define CE_DMA 1 + +/* + * struct sl3516_ce_descriptor - descriptor for CE operations + * @frame_ctrl: Information for the current descriptor + * @flag_status: For send packet, describe flag of operations. + * @buf_adr: pointer to a send/recv buffer for data packet + * @next_desc: control linking to other descriptors + */ +struct descriptor { + union { + u32 raw; + /* + * struct desc_frame_ctrl - Information for the current descriptor + * @buffer_size: the size of buffer at buf_adr + * @desc_count: Upon completion of a DMA operation, DMA + * write the number of descriptors used + * for the current frame + * @checksum: unknown + * @authcomp: unknown + * @perr: Protocol error during processing this descriptor + * @derr: Data error during processing this descriptor + * @own: 0 if owned by CPU, 1 for DMA + */ + struct desc_frame_ctrl { + u32 buffer_size :16; + u32 desc_count :6; + u32 checksum :6; + u32 authcomp :1; + u32 perr :1; + u32 derr :1; + u32 own :1; + } bits; + } frame_ctrl; + + union { + u32 raw; + /* + * struct desc_flag_status - flag for this descriptor + * @tqflag: list of flag describing the type of operation + * to be performed. + */ + struct desc_tx_flag_status { + u32 tqflag :10; + u32 unused :22; + } tx_flag; + } flag_status; + + u32 buf_adr; + + union { + u32 next_descriptor; + /* + * struct desc_next - describe chaining of descriptors + * @sof_eof: does the descriptor is first (0x11), + * the last (0x01), middle of a chan (0x00) + * or the only one (0x11) + * @dec: AHB bus address increase (0), decrease (1) + * @eofie: End of frame interrupt enable + * @ndar: Next descriptor address + */ + struct desc_next { + u32 sof_eof :2; + u32 dec :1; + u32 eofie :1; + u32 ndar :28; + } bits; + } next_desc; +}; + +/* + * struct control - The value of this register is used to set the + * operation mode of the IPSec Module. + * @process_id: Used to identify the process. The number will be copied + * to the descriptor status of the received packet. + * @auth_check_len: Number of 32-bit words to be checked or appended by the + * authentication module + * @auth_algorithm: + * @auth_mode: 0:append 1:Check Authentication Result + * @fcs_stream_copy: 0:enable 1:disable authentication stream copy + * @mix_key_sel: 0:use rCipherKey0-3 1:use Key Mixer + * @aesnk: AES Key Size + * @cipher_algorithm: choice of CBC/ECE and AES/DES/3DES + * @op_mode: Operation Mode for the IPSec Module + */ +struct pkt_control_header { + u32 process_id :8; + u32 auth_check_len :3; + u32 un1 :1; + u32 auth_algorithm :3; + u32 auth_mode :1; + u32 fcs_stream_copy :1; + u32 un2 :2; + u32 mix_key_sel :1; + u32 aesnk :4; + u32 cipher_algorithm :3; + u32 un3 :1; + u32 op_mode :4; +}; + +struct pkt_control_cipher { + u32 algorithm_len :16; + u32 header_len :16; +}; + +/* + * struct pkt_control_ecb - control packet for ECB + */ +struct pkt_control_ecb { + struct pkt_control_header control; + struct pkt_control_cipher cipher; + unsigned char key[AES_MAX_KEY_SIZE]; +}; + +/* + * struct sl3516_ce_dev - main container for all this driver information + * @base: base address + * @clks: clocks used + * @reset: pointer to reset controller + * @dev: the platform device + * @engine: ptr to the crypto/crypto_engine + * @complete: completion for the current task on this flow + * @status: set to 1 by interrupt if task is done + * @dtx: base DMA address for TX descriptors + * @tx base address of TX descriptors + * @drx: base DMA address for RX descriptors + * @rx base address of RX descriptors + * @ctx current used TX descriptor + * @crx current used RX descriptor + * @trng hw_random structure for RNG + * @hwrng_stat_req number of HWRNG requests + * @hwrng_stat_bytes total number of bytes generated by RNG + * @stat_irq number of IRQ handled by CE + * @stat_irq_tx number of TX IRQ handled by CE + * @stat_irq_rx number of RX IRQ handled by CE + * @stat_req number of requests handled by CE + * @fallbak_sg_count_tx number of fallback due to destination SG count + * @fallbak_sg_count_rx number of fallback due to source SG count + * @fallbak_not_same_len number of fallback due to difference in SG length + * @dbgfs_dir: Debugfs dentry for statistic directory + * @dbgfs_stats: Debugfs dentry for statistic counters + */ +struct sl3516_ce_dev { + void __iomem *base; + struct clk *clks; + struct reset_control *reset; + struct device *dev; + struct crypto_engine *engine; + struct completion complete; + int status; + dma_addr_t dtx; + struct descriptor *tx; + dma_addr_t drx; + struct descriptor *rx; + int ctx; + int crx; + struct hwrng trng; + unsigned long hwrng_stat_req; + unsigned long hwrng_stat_bytes; + unsigned long stat_irq; + unsigned long stat_irq_tx; + unsigned long stat_irq_rx; + unsigned long stat_req; + unsigned long fallback_sg_count_tx; + unsigned long fallback_sg_count_rx; + unsigned long fallback_not_same_len; + unsigned long fallback_mod16; + unsigned long fallback_align16; +#ifdef CONFIG_CRYPTO_DEV_SL3516_DEBUG + struct dentry *dbgfs_dir; + struct dentry *dbgfs_stats; +#endif + void *pctrl; + dma_addr_t dctrl; +}; + +struct sginfo { + u32 addr; + u32 len; +}; + +/* + * struct sl3516_ce_cipher_req_ctx - context for a skcipher request + * @t_src: list of mapped SGs with their size + * @t_dst: list of mapped SGs with their size + * @op_dir: direction (encrypt vs decrypt) for this request + * @pctrllen: the length of the ctrl packet + * @tqflag: the TQflag to set in data packet + * @h pointer to the pkt_control_cipher header + * @nr_sgs: number of source SG + * @nr_sgd: number of destination SG + * @fallback_req: request struct for invoking the fallback skcipher TFM + */ +struct sl3516_ce_cipher_req_ctx { + struct sginfo t_src[MAXDESC]; + struct sginfo t_dst[MAXDESC]; + u32 op_dir; + unsigned int pctrllen; + u32 tqflag; + struct pkt_control_cipher *h; + int nr_sgs; + int nr_sgd; + struct skcipher_request fallback_req; // keep at the end +}; + +/* + * struct sl3516_ce_cipher_tfm_ctx - context for a skcipher TFM + * @enginectx: crypto_engine used by this TFM + * @key: pointer to key data + * @keylen: len of the key + * @ce: pointer to the private data of driver handling this TFM + * @fallback_tfm: pointer to the fallback TFM + * + * enginectx must be the first element + */ +struct sl3516_ce_cipher_tfm_ctx { + struct crypto_engine_ctx enginectx; + u32 *key; + u32 keylen; + struct sl3516_ce_dev *ce; + struct crypto_skcipher *fallback_tfm; +}; + +/* + * struct sl3516_ce_alg_template - crypto_alg template + * @type: the CRYPTO_ALG_TYPE for this template + * @mode: value to be used in control packet for this algorithm + * @ce: pointer to the sl3516_ce_dev structure associated with + * this template + * @alg: one of sub struct must be used + * @stat_req: number of request done on this template + * @stat_fb: number of request which has fallbacked + * @stat_bytes: total data size done by this template + */ +struct sl3516_ce_alg_template { + u32 type; + u32 mode; + struct sl3516_ce_dev *ce; + union { + struct skcipher_alg skcipher; + } alg; + unsigned long stat_req; + unsigned long stat_fb; + unsigned long stat_bytes; +}; + +int sl3516_ce_enqueue(struct crypto_async_request *areq, u32 type); + +int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen); +int sl3516_ce_cipher_init(struct crypto_tfm *tfm); +void sl3516_ce_cipher_exit(struct crypto_tfm *tfm); +int sl3516_ce_skdecrypt(struct skcipher_request *areq); +int sl3516_ce_skencrypt(struct skcipher_request *areq); + +int sl3516_ce_run_task(struct sl3516_ce_dev *ce, + struct sl3516_ce_cipher_req_ctx *rctx, const char *name); + +int sl3516_ce_rng_register(struct sl3516_ce_dev *ce); +void sl3516_ce_rng_unregister(struct sl3516_ce_dev *ce); diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index a380087c83f7..a032c192ef1d 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,6 @@ struct hpre_ctx; #define HPRE_DH_G_FLAG 0x02 #define HPRE_TRY_SEND_TIMES 100 #define HPRE_INVLD_REQ_ID (-1) -#define HPRE_DEV(ctx) (&((ctx)->qp->qm->pdev->dev)) #define HPRE_SQE_ALG_BITS 5 #define HPRE_SQE_DONE_SHIFT 30 @@ -39,12 +39,17 @@ struct hpre_ctx; #define HPRE_DFX_SEC_TO_US 1000000 #define HPRE_DFX_US_TO_NS 1000 +/* due to nist p521 */ +#define HPRE_ECC_MAX_KSZ 66 + /* size in bytes of the n prime */ #define HPRE_ECC_NIST_P192_N_SIZE 24 #define HPRE_ECC_NIST_P256_N_SIZE 32 +#define HPRE_ECC_NIST_P384_N_SIZE 48 /* size in bytes */ #define HPRE_ECC_HW256_KSZ_B 32 +#define HPRE_ECC_HW384_KSZ_B 48 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); @@ -102,6 +107,7 @@ struct hpre_curve25519_ctx { struct hpre_ctx { struct hisi_qp *qp; + struct device *dev; struct hpre_asym_request **req_list; struct hpre *hpre; spinlock_t req_lock; @@ -214,8 +220,7 @@ static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, struct scatterlist *data, unsigned int len, int is_src, dma_addr_t *tmp) { - struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = hpre_req->ctx->dev; enum dma_data_direction dma_dir; if (is_src) { @@ -239,7 +244,7 @@ static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, int is_src, dma_addr_t *tmp) { struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; void *ptr; int shift; @@ -293,11 +298,13 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, struct scatterlist *dst, struct scatterlist *src) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; struct hpre_sqe *sqe = &req->req; dma_addr_t tmp; tmp = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, tmp))) + return; if (src) { if (req->src) @@ -307,6 +314,8 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, } tmp = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, tmp))) + return; if (req->dst) { if (dst) @@ -321,16 +330,15 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { - struct device *dev = HPRE_DEV(ctx); struct hpre_asym_request *req; unsigned int err, done, alg; int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 -#define HREE_HW_ERR_MASK 0x7ff -#define HREE_SQE_DONE_MASK 0x3 -#define HREE_ALG_TYPE_MASK 0x1f +#define HREE_HW_ERR_MASK GENMASK(10, 0) +#define HREE_SQE_DONE_MASK GENMASK(1, 0) +#define HREE_ALG_TYPE_MASK GENMASK(4, 0) id = (int)le16_to_cpu(sqe->tag); req = ctx->req_list[id]; hpre_rm_req_from_ctx(req); @@ -346,7 +354,7 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, return 0; alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; - dev_err_ratelimited(dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", + dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", alg, done, err); return -EINVAL; @@ -361,6 +369,7 @@ static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) spin_lock_init(&ctx->req_lock); ctx->qp = qp; + ctx->dev = &qp->qm->pdev->dev; hpre = container_of(ctx->qp->qm, struct hpre, qm); ctx->hpre = hpre; @@ -524,6 +533,8 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) msg->key = cpu_to_le64(ctx->dh.dma_xa_p); } + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; @@ -618,14 +629,14 @@ static int hpre_is_dh_params_length_valid(unsigned int key_sz) case _HPRE_DH_GRP15: case _HPRE_DH_GRP16: return 0; + default: + return -EINVAL; } - - return -EINVAL; } static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz; if (params->p_size > HPRE_DH_MAX_P_SZ) @@ -664,7 +675,7 @@ static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; if (is_clear_all) @@ -877,18 +888,18 @@ static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, if (!hpre_rsa_key_size_is_support(ctx->key_sz)) return 0; - ctx->rsa.pubkey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1, + ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_pubkey, GFP_KERNEL); if (!ctx->rsa.pubkey) return -ENOMEM; if (private) { - ctx->rsa.prikey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1, + ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_prikey, GFP_KERNEL); if (!ctx->rsa.prikey) { - dma_free_coherent(HPRE_DEV(ctx), vlen << 1, + dma_free_coherent(ctx->dev, vlen << 1, ctx->rsa.pubkey, ctx->rsa.dma_pubkey); ctx->rsa.pubkey = NULL; @@ -950,7 +961,7 @@ static int hpre_crt_para_get(char *para, size_t para_sz, static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) { unsigned int hlf_ksz = ctx->key_sz >> 1; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; u64 offset; int ret; @@ -1008,7 +1019,7 @@ static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) { unsigned int half_key_sz = ctx->key_sz >> 1; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; if (is_clear_all) hisi_qm_stop_qp(ctx->qp); @@ -1179,7 +1190,7 @@ static void hpre_key_to_big_end(u8 *data, int len) static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, bool is_ecdh) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; @@ -1202,12 +1213,21 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, hpre_ctx_clear(ctx, is_clear_all); } +/* + * The bits of 192/224/256/384/521 are supported by HPRE, + * and convert the bits like: + * bits<=256, bits=256; 256dev; unsigned int sz, shift, curve_sz; int ret; @@ -1328,11 +1350,32 @@ static bool hpre_key_is_zero(char *key, unsigned short key_sz) return true; } +static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params) +{ + struct device *dev = ctx->dev; + int ret; + + ret = crypto_get_default_rng(); + if (ret) { + dev_err(dev, "failed to get default rng, ret = %d!\n", ret); + return ret; + } + + ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key, + params->key_size); + crypto_put_default_rng(); + if (ret) + dev_err(dev, "failed to get rng, ret = %d!\n", ret); + + return ret; +} + static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; + char key[HPRE_ECC_MAX_KSZ]; unsigned int sz, sz_shift; struct ecdh params; int ret; @@ -1342,6 +1385,15 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, return -EINVAL; } + /* Use stdrng to generate private key */ + if (!params.key || !params.key_size) { + params.key = key; + params.key_size = hpre_ecdh_get_curvesz(ctx->curve_id); + ret = ecdh_gen_privkey(ctx, ¶ms); + if (ret) + return ret; + } + if (hpre_key_is_zero(params.key, params.key_size)) { dev_err(dev, "Invalid hpre key!\n"); return -EINVAL; @@ -1367,16 +1419,20 @@ static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, struct scatterlist *dst, struct scatterlist *src) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; struct hpre_sqe *sqe = &req->req; dma_addr_t dma; dma = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (src && req->src) dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); dma = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (req->dst) dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); @@ -1431,6 +1487,8 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, h_req->areq.ecdh = req; msg = &h_req->req; memset(msg, 0, sizeof(*msg)); + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); msg->key = cpu_to_le64(ctx->ecdh.dma_p); msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); @@ -1450,7 +1508,7 @@ static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int tmpshift; dma_addr_t dma = 0; void *ptr; @@ -1480,8 +1538,8 @@ static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); - dma_addr_t dma = 0; + struct device *dev = ctx->dev; + dma_addr_t dma; if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { dev_err(dev, "data or data length is illegal!\n"); @@ -1503,7 +1561,7 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; void *tmp = kpp_request_ctx(req); struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); struct hpre_sqe *msg = &hpre_req->req; @@ -1568,6 +1626,15 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); } +static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P384; + + return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); +} + static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); @@ -1609,7 +1676,7 @@ static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf, static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf, unsigned int len) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; @@ -1634,7 +1701,7 @@ static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; int ret = -EINVAL; if (len != CURVE25519_KEY_SIZE || @@ -1662,16 +1729,20 @@ static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, struct scatterlist *dst, struct scatterlist *src) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; struct hpre_sqe *sqe = &req->req; dma_addr_t dma; dma = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (src && req->src) dma_free_coherent(dev, ctx->key_sz, req->src, dma); dma = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (req->dst) dma_free_coherent(dev, ctx->key_sz, req->dst, dma); @@ -1722,6 +1793,8 @@ static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, h_req->areq.curve25519 = req; msg = &h_req->req; memset(msg, 0, sizeof(*msg)); + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); msg->key = cpu_to_le64(ctx->curve25519.dma_p); msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); @@ -1752,7 +1825,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; u8 p[CURVE25519_KEY_SIZE] = { 0 }; const struct ecc_curve *curve; dma_addr_t dma = 0; @@ -1790,8 +1863,12 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p, * we get its modulus to p, and then use it. */ - if (memcmp(ptr, p, ctx->key_sz) >= 0) + if (memcmp(ptr, p, ctx->key_sz) == 0) { + dev_err(dev, "gx is p!\n"); + return -EINVAL; + } else if (memcmp(ptr, p, ctx->key_sz) > 0) { hpre_curve25519_src_modulo_p(ptr); + } hpre_req->src = ptr; msg->in = cpu_to_le64(dma); @@ -1807,8 +1884,8 @@ static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); - dma_addr_t dma = 0; + struct device *dev = ctx->dev; + dma_addr_t dma; if (!data || !sg_is_last(data) || len != ctx->key_sz) { dev_err(dev, "data or data length is illegal!\n"); @@ -1830,7 +1907,7 @@ static int hpre_curve25519_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; void *tmp = kpp_request_ctx(req); struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); struct hpre_sqe *msg = &hpre_req->req; @@ -1940,7 +2017,7 @@ static struct kpp_alg ecdh_nist_p192 = { .cra_ctxsize = sizeof(struct hpre_ctx), .cra_priority = HPRE_CRYPTO_ALG_PRI, .cra_name = "ecdh-nist-p192", - .cra_driver_name = "hpre-ecdh", + .cra_driver_name = "hpre-ecdh-nist-p192", .cra_module = THIS_MODULE, }, }; @@ -1957,7 +2034,24 @@ static struct kpp_alg ecdh_nist_p256 = { .cra_ctxsize = sizeof(struct hpre_ctx), .cra_priority = HPRE_CRYPTO_ALG_PRI, .cra_name = "ecdh-nist-p256", - .cra_driver_name = "hpre-ecdh", + .cra_driver_name = "hpre-ecdh-nist-p256", + .cra_module = THIS_MODULE, + }, +}; + +static struct kpp_alg ecdh_nist_p384 = { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p384_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "hpre-ecdh-nist-p384", .cra_module = THIS_MODULE, }, }; @@ -1989,16 +2083,25 @@ static int hpre_register_ecdh(void) return ret; ret = crypto_register_kpp(&ecdh_nist_p256); - if (ret) { - crypto_unregister_kpp(&ecdh_nist_p192); - return ret; - } + if (ret) + goto unregister_ecdh_p192; + + ret = crypto_register_kpp(&ecdh_nist_p384); + if (ret) + goto unregister_ecdh_p256; return 0; + +unregister_ecdh_p256: + crypto_unregister_kpp(&ecdh_nist_p256); +unregister_ecdh_p192: + crypto_unregister_kpp(&ecdh_nist_p192); + return ret; } static void hpre_unregister_ecdh(void) { + crypto_unregister_kpp(&ecdh_nist_p384); crypto_unregister_kpp(&ecdh_nist_p256); crypto_unregister_kpp(&ecdh_nist_p192); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 046bc962c8b2..8b0640fb04be 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -36,7 +36,7 @@ #define HPRE_INT_MASK 0x301400 #define HPRE_INT_STATUS 0x301800 #define HPRE_CORE_INT_ENABLE 0 -#define HPRE_CORE_INT_DISABLE 0x003fffff +#define HPRE_CORE_INT_DISABLE GENMASK(21, 0) #define HPRE_RDCHN_INI_ST 0x301a00 #define HPRE_CLSTR_BASE 0x302000 #define HPRE_CORE_EN_OFFSET 0x04 @@ -50,6 +50,7 @@ #define HPRE_RAS_NFE_ENB 0x301414 #define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe #define HPRE_RAS_FE_ENB 0x301418 +#define HPRE_OOO_SHUTDOWN_SEL 0x301a3c #define HPRE_HAC_RAS_FE_ENABLE 0 #define HPRE_CORE_ENB (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET) @@ -57,7 +58,6 @@ #define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET) #define HPRE_HAC_ECC1_CNT 0x301a04 #define HPRE_HAC_ECC2_CNT 0x301a08 -#define HPRE_HAC_INT_STATUS 0x301800 #define HPRE_HAC_SOURCE_INT 0x301600 #define HPRE_CLSTR_ADDR_INTRVL 0x1000 #define HPRE_CLUSTER_INQURY 0x100 @@ -69,13 +69,17 @@ #define HPRE_DBGFS_VAL_MAX_LEN 20 #define HPRE_PCI_DEVICE_ID 0xa258 #define HPRE_PCI_VF_DEVICE_ID 0xa259 -#define HPRE_ADDR(qm, offset) ((qm)->io_base + (offset)) -#define HPRE_QM_USR_CFG_MASK 0xfffffffe -#define HPRE_QM_AXI_CFG_MASK 0xffff -#define HPRE_QM_VFG_AX_MASK 0xff -#define HPRE_BD_USR_MASK 0x3 -#define HPRE_CLUSTER_CORE_MASK_V2 0xf -#define HPRE_CLUSTER_CORE_MASK_V3 0xff +#define HPRE_QM_USR_CFG_MASK GENMASK(31, 1) +#define HPRE_QM_AXI_CFG_MASK GENMASK(15, 0) +#define HPRE_QM_VFG_AX_MASK GENMASK(7, 0) +#define HPRE_BD_USR_MASK GENMASK(1, 0) +#define HPRE_CLUSTER_CORE_MASK_V2 GENMASK(3, 0) +#define HPRE_CLUSTER_CORE_MASK_V3 GENMASK(7, 0) +#define HPRE_PREFETCH_CFG 0x301130 +#define HPRE_SVA_PREFTCH_DFX 0x30115C +#define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30))) +#define HPRE_PREFETCH_DISABLE BIT(30) +#define HPRE_SVA_DISABLE_READY (BIT(4) | BIT(8)) #define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044 #define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0) @@ -88,11 +92,7 @@ #define HPRE_QM_PM_FLR BIT(11) #define HPRE_QM_SRIOV_FLR BIT(12) -#define HPRE_CLUSTERS_NUM(qm) \ - (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : HPRE_CLUSTERS_NUM_V2) -#define HPRE_CLUSTER_CORE_MASK(qm) \ - (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTER_CORE_MASK_V3 :\ - HPRE_CLUSTER_CORE_MASK_V2) +#define HPRE_SHAPER_TYPE_RATE 128 #define HPRE_VIA_MSI_DSM 1 #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 @@ -123,21 +123,49 @@ static const char * const hpre_debug_file_name[] = { }; static const struct hpre_hw_error hpre_hw_errors[] = { - { .int_msk = BIT(0), .msg = "core_ecc_1bit_err_int_set" }, - { .int_msk = BIT(1), .msg = "core_ecc_2bit_err_int_set" }, - { .int_msk = BIT(2), .msg = "dat_wb_poison_int_set" }, - { .int_msk = BIT(3), .msg = "dat_rd_poison_int_set" }, - { .int_msk = BIT(4), .msg = "bd_rd_poison_int_set" }, - { .int_msk = BIT(5), .msg = "ooo_ecc_2bit_err_int_set" }, - { .int_msk = BIT(6), .msg = "cluster1_shb_timeout_int_set" }, - { .int_msk = BIT(7), .msg = "cluster2_shb_timeout_int_set" }, - { .int_msk = BIT(8), .msg = "cluster3_shb_timeout_int_set" }, - { .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" }, - { .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" }, - { .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" }, - { .int_msk = BIT(22), .msg = "pt_rng_timeout_int_set"}, - { .int_msk = BIT(23), .msg = "sva_fsm_timeout_int_set"}, { + .int_msk = BIT(0), + .msg = "core_ecc_1bit_err_int_set" + }, { + .int_msk = BIT(1), + .msg = "core_ecc_2bit_err_int_set" + }, { + .int_msk = BIT(2), + .msg = "dat_wb_poison_int_set" + }, { + .int_msk = BIT(3), + .msg = "dat_rd_poison_int_set" + }, { + .int_msk = BIT(4), + .msg = "bd_rd_poison_int_set" + }, { + .int_msk = BIT(5), + .msg = "ooo_ecc_2bit_err_int_set" + }, { + .int_msk = BIT(6), + .msg = "cluster1_shb_timeout_int_set" + }, { + .int_msk = BIT(7), + .msg = "cluster2_shb_timeout_int_set" + }, { + .int_msk = BIT(8), + .msg = "cluster3_shb_timeout_int_set" + }, { + .int_msk = BIT(9), + .msg = "cluster4_shb_timeout_int_set" + }, { + .int_msk = GENMASK(15, 10), + .msg = "ooo_rdrsp_err_int_set" + }, { + .int_msk = GENMASK(21, 16), + .msg = "ooo_wrrsp_err_int_set" + }, { + .int_msk = BIT(22), + .msg = "pt_rng_timeout_int_set" + }, { + .int_msk = BIT(23), + .msg = "sva_fsm_timeout_int_set" + }, { /* sentinel */ } }; @@ -224,6 +252,18 @@ static u32 vfs_num; module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); +static inline int hpre_cluster_num(struct hisi_qm *qm) +{ + return (qm->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : + HPRE_CLUSTERS_NUM_V2; +} + +static inline int hpre_cluster_core_mask(struct hisi_qm *qm) +{ + return (qm->ver >= QM_HW_V3) ? + HPRE_CLUSTER_CORE_MASK_V3 : HPRE_CLUSTER_CORE_MASK_V2; +} + struct hisi_qp *hpre_create_qp(u8 type) { int node = cpu_to_node(smp_processor_id()); @@ -290,8 +330,8 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm) static int hpre_set_cluster(struct hisi_qm *qm) { - u32 cluster_core_mask = HPRE_CLUSTER_CORE_MASK(qm); - u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + u32 cluster_core_mask = hpre_cluster_core_mask(qm); + u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; unsigned long offset; u32 val = 0; @@ -302,10 +342,10 @@ static int hpre_set_cluster(struct hisi_qm *qm) /* clusters initiating */ writel(cluster_core_mask, - HPRE_ADDR(qm, offset + HPRE_CORE_ENB)); - writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG)); - ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset + - HPRE_CORE_INI_STATUS), val, + qm->io_base + offset + HPRE_CORE_ENB); + writel(0x1, qm->io_base + offset + HPRE_CORE_INI_CFG); + ret = readl_relaxed_poll_timeout(qm->io_base + offset + + HPRE_CORE_INI_STATUS, val, ((val & cluster_core_mask) == cluster_core_mask), HPRE_REG_RD_INTVRL_US, @@ -329,11 +369,52 @@ static void disable_flr_of_bme(struct hisi_qm *qm) { u32 val; - val = readl(HPRE_ADDR(qm, QM_PEH_AXUSER_CFG)); + val = readl(qm->io_base + QM_PEH_AXUSER_CFG); val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR); val |= HPRE_QM_PM_FLR; - writel(val, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG)); - writel(PEH_AXUSER_CFG_ENABLE, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG_ENABLE)); + writel(val, qm->io_base + QM_PEH_AXUSER_CFG); + writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); +} + +static void hpre_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); + val &= HPRE_PREFETCH_ENABLE; + writel(val, qm->io_base + HPRE_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG, + val, !(val & HPRE_PREFETCH_DISABLE), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to open sva prefetch\n"); +} + +static void hpre_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); + val |= HPRE_PREFETCH_DISABLE; + writel(val, qm->io_base + HPRE_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX, + val, !(val & HPRE_SVA_DISABLE_READY), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); } static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) @@ -342,33 +423,33 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) u32 val; int ret; - writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE)); - writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE)); - writel_relaxed(HPRE_QM_AXI_CFG_MASK, HPRE_ADDR(qm, QM_AXI_M_CFG)); + writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_ARUSER_M_CFG_ENABLE); + writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_AWUSER_M_CFG_ENABLE); + writel_relaxed(HPRE_QM_AXI_CFG_MASK, qm->io_base + QM_AXI_M_CFG); /* HPRE need more time, we close this interrupt */ - val = readl_relaxed(HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); + val = readl_relaxed(qm->io_base + HPRE_QM_ABNML_INT_MASK); val |= BIT(HPRE_TIMEOUT_ABNML_BIT); - writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); + writel_relaxed(val, qm->io_base + HPRE_QM_ABNML_INT_MASK); if (qm->ver >= QM_HW_V3) writel(HPRE_RSA_ENB | HPRE_ECC_ENB, - HPRE_ADDR(qm, HPRE_TYPES_ENB)); + qm->io_base + HPRE_TYPES_ENB); else - writel(HPRE_RSA_ENB, HPRE_ADDR(qm, HPRE_TYPES_ENB)); + writel(HPRE_RSA_ENB, qm->io_base + HPRE_TYPES_ENB); - writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE)); - writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN)); - writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK)); - writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS)); - writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE)); - writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS)); + writel(HPRE_QM_VFG_AX_MASK, qm->io_base + HPRE_VFG_AXCACHE); + writel(0x0, qm->io_base + HPRE_BD_ENDIAN); + writel(0x0, qm->io_base + HPRE_INT_MASK); + writel(0x0, qm->io_base + HPRE_POISON_BYPASS); + writel(0x0, qm->io_base + HPRE_COMM_CNT_CLR_CE); + writel(0x0, qm->io_base + HPRE_ECC_BYPASS); - writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_ARUSR_CFG)); - writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_AWUSR_CFG)); - writel(0x1, HPRE_ADDR(qm, HPRE_RDCHN_INI_CFG)); - ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, HPRE_RDCHN_INI_ST), val, - val & BIT(0), + writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_ARUSR_CFG); + writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_AWUSR_CFG); + writel(0x1, qm->io_base + HPRE_RDCHN_INI_CFG); + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_RDCHN_INI_ST, val, + val & BIT(0), HPRE_REG_RD_INTVRL_US, HPRE_REG_RD_TMOUT_US); if (ret) { @@ -397,7 +478,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) static void hpre_cnt_regs_clear(struct hisi_qm *qm) { - u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + u8 clusters_num = hpre_cluster_num(qm); unsigned long offset; int i; @@ -413,36 +494,49 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm) hisi_qm_debug_regs_clear(qm); } +static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val1, val2; + + val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + if (enable) { + val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE; + val2 = HPRE_HAC_RAS_NFE_ENABLE; + } else { + val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + HPRE_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); +} + static void hpre_hw_error_disable(struct hisi_qm *qm) { - u32 val; - /* disable hpre hw error interrupts */ writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK); - /* disable HPRE block master OOO when m-bit error occur */ - val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); - val &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ + hpre_master_ooo_ctrl(qm, false); } static void hpre_hw_error_enable(struct hisi_qm *qm) { - u32 val; - /* clear HPRE hw error source if having */ writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT); - /* enable hpre hw error interrupts */ - writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK); + /* configure error type */ writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB); writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB); writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB); - /* enable HPRE block master OOO when m-bit error occur */ - val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); - val |= HPRE_AM_OOO_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */ + hpre_master_ooo_ctrl(qm, true); + + /* enable hpre hw error interrupts */ + writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK); } static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) @@ -650,7 +744,7 @@ static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm) static int hpre_cluster_debugfs_init(struct hisi_qm *qm) { - u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; char buf[HPRE_DBGFS_VAL_MAX_LEN]; struct debugfs_regset32 *regset; @@ -788,7 +882,7 @@ static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts) static u32 hpre_get_hw_err_status(struct hisi_qm *qm) { - return readl(qm->io_base + HPRE_HAC_INT_STATUS); + return readl(qm->io_base + HPRE_INT_STATUS); } static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) @@ -802,9 +896,9 @@ static void hpre_open_axi_master_ooo(struct hisi_qm *qm) value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE, - HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB)); + qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE, - HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB)); + qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); } static void hpre_err_info_init(struct hisi_qm *qm) @@ -829,6 +923,8 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .clear_dev_hw_err_status = hpre_clear_hw_err_status, .log_dev_hw_err = hpre_log_hw_error, .open_axi_master_ooo = hpre_open_axi_master_ooo, + .open_sva_prefetch = hpre_open_sva_prefetch, + .close_sva_prefetch = hpre_close_sva_prefetch, .err_info_init = hpre_err_info_init, }; @@ -841,6 +937,8 @@ static int hpre_pf_probe_init(struct hpre *hpre) if (ret) return ret; + hpre_open_sva_prefetch(qm); + qm->err_ini = &hpre_err_ini; qm->err_ini->err_info_init(qm); hisi_qm_dev_err_init(qm); @@ -850,6 +948,7 @@ static int hpre_pf_probe_init(struct hpre *hpre) static int hpre_probe_init(struct hpre *hpre) { + u32 type_rate = HPRE_SHAPER_TYPE_RATE; struct hisi_qm *qm = &hpre->qm; int ret; @@ -857,6 +956,11 @@ static int hpre_probe_init(struct hpre *hpre) ret = hpre_pf_probe_init(hpre); if (ret) return ret; + /* Enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + qm->type_rate = type_rate; + } } return 0; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index ce439a0c66c9..1d67f94a1d56 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -25,9 +25,11 @@ #define QM_IRQ_NUM_V1 1 #define QM_IRQ_NUM_PF_V2 4 #define QM_IRQ_NUM_VF_V2 2 +#define QM_IRQ_NUM_VF_V3 3 #define QM_EQ_EVENT_IRQ_VECTOR 0 #define QM_AEQ_EVENT_IRQ_VECTOR 1 +#define QM_CMD_EVENT_IRQ_VECTOR 2 #define QM_ABNORMAL_EVENT_IRQ_VECTOR 3 /* mailbox */ @@ -39,6 +41,8 @@ #define QM_MB_CMD_CQC_BT 0x5 #define QM_MB_CMD_SQC_VFT_V2 0x6 #define QM_MB_CMD_STOP_QP 0x8 +#define QM_MB_CMD_SRC 0xc +#define QM_MB_CMD_DST 0xd #define QM_MB_CMD_SEND_BASE 0x300 #define QM_MB_EVENT_SHIFT 8 @@ -46,6 +50,9 @@ #define QM_MB_OP_SHIFT 14 #define QM_MB_CMD_DATA_ADDR_L 0x304 #define QM_MB_CMD_DATA_ADDR_H 0x308 +#define QM_MB_PING_ALL_VFS 0xffff +#define QM_MB_CMD_DATA_SHIFT 32 +#define QM_MB_CMD_DATA_MASK GENMASK(31, 0) /* sqc shift */ #define QM_SQ_HOP_NUM_SHIFT 0 @@ -95,6 +102,7 @@ #define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000 #define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000 #define QM_QUE_ISO_CFG_V 0x0030 +#define QM_PAGE_SIZE 0x0034 #define QM_QUE_ISO_EN 0x100154 #define QM_CAPBILITY 0x100158 #define QM_QP_NUN_MASK GENMASK(10, 0) @@ -155,11 +163,15 @@ #define QM_RAS_CE_THRESHOLD 0x1000f8 #define QM_RAS_CE_TIMES_PER_IRQ 1 #define QM_RAS_MSI_INT_SEL 0x1040f4 +#define QM_OOO_SHUTDOWN_SEL 0x1040f8 #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 #define ACC_VENDOR_ID_VALUE 0x5a5a #define QM_PEH_DFX_INFO0 0x1000fc +#define QM_PEH_DFX_INFO1 0x100100 +#define QM_PEH_DFX_MASK (BIT(0) | BIT(2)) +#define QM_PEH_MSI_FINISH_MASK GENMASK(19, 16) #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3 #define ACC_PEH_MSI_DISABLE GENMASK(31, 0) #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1 @@ -170,6 +182,31 @@ #define QM_RAS_NFE_MBIT_DISABLE ~QM_ECC_MBIT #define ACC_AM_ROB_ECC_INT_STS 0x300104 #define ACC_ROB_ECC_ERR_MULTPL BIT(1) +#define QM_MSI_CAP_ENABLE BIT(16) + +/* interfunction communication */ +#define QM_IFC_READY_STATUS 0x100128 +#define QM_IFC_C_STS_M 0x10012C +#define QM_IFC_INT_SET_P 0x100130 +#define QM_IFC_INT_CFG 0x100134 +#define QM_IFC_INT_SOURCE_P 0x100138 +#define QM_IFC_INT_SOURCE_V 0x0020 +#define QM_IFC_INT_MASK 0x0024 +#define QM_IFC_INT_STATUS 0x0028 +#define QM_IFC_INT_SET_V 0x002C +#define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) +#define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) +#define QM_IFC_INT_SOURCE_MASK BIT(0) +#define QM_IFC_INT_DISABLE BIT(0) +#define QM_IFC_INT_STATUS_MASK BIT(0) +#define QM_IFC_INT_SET_MASK BIT(0) +#define QM_WAIT_DST_ACK 10 +#define QM_MAX_PF_WAIT_COUNT 10 +#define QM_MAX_VF_WAIT_COUNT 40 +#define QM_VF_RESET_WAIT_US 20000 +#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF_RESET_WAIT_TIMEOUT_US \ + (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) #define QM_DFX_MB_CNT_VF 0x104010 #define QM_DFX_DB_CNT_VF 0x104020 @@ -205,6 +242,33 @@ #define QM_DRIVER_REMOVING 0 #define QM_RST_SCHED 1 #define QM_RESETTING 2 +#define QM_QOS_PARAM_NUM 2 +#define QM_QOS_VAL_NUM 1 +#define QM_QOS_BDF_PARAM_NUM 4 +#define QM_QOS_MAX_VAL 1000 +#define QM_QOS_RATE 100 +#define QM_QOS_EXPAND_RATE 1000 +#define QM_SHAPER_CIR_B_MASK GENMASK(7, 0) +#define QM_SHAPER_CIR_U_MASK GENMASK(10, 8) +#define QM_SHAPER_CIR_S_MASK GENMASK(14, 11) +#define QM_SHAPER_FACTOR_CIR_U_SHIFT 8 +#define QM_SHAPER_FACTOR_CIR_S_SHIFT 11 +#define QM_SHAPER_FACTOR_CBS_B_SHIFT 15 +#define QM_SHAPER_FACTOR_CBS_S_SHIFT 19 +#define QM_SHAPER_CBS_B 1 +#define QM_SHAPER_CBS_S 16 +#define QM_SHAPER_VFT_OFFSET 6 +#define WAIT_FOR_QOS_VF 100 +#define QM_QOS_MIN_ERROR_RATE 5 +#define QM_QOS_TYPICAL_NUM 8 +#define QM_SHAPER_MIN_CBS_S 8 +#define QM_QOS_TICK 0x300U +#define QM_QOS_DIVISOR_CLK 0x1f40U +#define QM_QOS_MAX_CIR_B 200 +#define QM_QOS_MIN_CIR_B 100 +#define QM_QOS_MAX_CIR_U 6 +#define QM_QOS_MAX_CIR_S 11 +#define QM_QOS_VAL_MAX_LEN 32 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ @@ -245,6 +309,7 @@ enum vft_type { SQC_VFT = 0, CQC_VFT, + SHAPER_VFT, }; enum acc_err_result { @@ -253,6 +318,23 @@ enum acc_err_result { ACC_ERR_RECOVERED, }; +enum qm_alg_type { + ALG_TYPE_0, + ALG_TYPE_1, +}; + +enum qm_mb_cmd { + QM_PF_FLR_PREPARE = 0x01, + QM_PF_SRST_PREPARE, + QM_PF_RESET_DONE, + QM_VF_PREPARE_DONE, + QM_VF_PREPARE_FAIL, + QM_VF_START_DONE, + QM_VF_START_FAIL, + QM_PF_SET_QOS, + QM_VF_GET_QOS, +}; + struct qm_cqe { __le32 rsvd0; __le16 cmd_id; @@ -351,6 +433,9 @@ struct hisi_qm_hw_ops { void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); int (*stop_qp)(struct hisi_qp *qp); + int (*set_msi)(struct hisi_qm *qm, bool set); + int (*ping_all_vfs)(struct hisi_qm *qm, u64 cmd); + int (*ping_pf)(struct hisi_qm *qm, u64 cmd); }; struct qm_dfx_item { @@ -412,6 +497,11 @@ static const char * const qp_s[] = { "none", "init", "start", "stop", "close", }; +static const u32 typical_qos_val[QM_QOS_TYPICAL_NUM] = {100, 250, 500, 1000, + 10000, 25000, 50000, 100000}; +static const u32 typical_qos_cbs_s[QM_QOS_TYPICAL_NUM] = {9, 10, 11, 12, 16, + 17, 18, 19}; + static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new) { enum qm_state curr = atomic_read(&qm->status.flags); @@ -491,6 +581,18 @@ static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp, return avail; } +static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd, + u64 base, u16 queue, bool op) +{ + mailbox->w0 = cpu_to_le16((cmd) | + ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) | + (0x1 << QM_MB_BUSY_SHIFT)); + mailbox->queue_num = cpu_to_le16(queue); + mailbox->base_l = cpu_to_le32(lower_32_bits(base)); + mailbox->base_h = cpu_to_le32(upper_32_bits(base)); + mailbox->rsvd = 0; +} + /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */ static int qm_wait_mb_ready(struct hisi_qm *qm) { @@ -523,44 +625,42 @@ static void qm_mb_write(struct hisi_qm *qm, const void *src) : "memory"); } +static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) +{ + if (unlikely(qm_wait_mb_ready(qm))) { + dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); + goto mb_busy; + } + + qm_mb_write(qm, mailbox); + + if (unlikely(qm_wait_mb_ready(qm))) { + dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); + goto mb_busy; + } + + return 0; + +mb_busy: + atomic64_inc(&qm->debug.dfx.mb_err_cnt); + return -EBUSY; +} + static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, bool op) { struct qm_mailbox mailbox; - int ret = 0; + int ret; dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n", queue, cmd, (unsigned long long)dma_addr); - mailbox.w0 = cpu_to_le16(cmd | - (op ? 0x1 << QM_MB_OP_SHIFT : 0) | - (0x1 << QM_MB_BUSY_SHIFT)); - mailbox.queue_num = cpu_to_le16(queue); - mailbox.base_l = cpu_to_le32(lower_32_bits(dma_addr)); - mailbox.base_h = cpu_to_le32(upper_32_bits(dma_addr)); - mailbox.rsvd = 0; + qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); mutex_lock(&qm->mailbox_lock); - - if (unlikely(qm_wait_mb_ready(qm))) { - ret = -EBUSY; - dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); - goto busy_unlock; - } - - qm_mb_write(qm, &mailbox); - - if (unlikely(qm_wait_mb_ready(qm))) { - ret = -EBUSY; - dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); - goto busy_unlock; - } - -busy_unlock: + ret = qm_mb_nolock(qm, &mailbox); mutex_unlock(&qm->mailbox_lock); - if (ret) - atomic64_inc(&qm->debug.dfx.mb_err_cnt); return ret; } @@ -626,6 +726,14 @@ static u32 qm_get_irq_num_v2(struct hisi_qm *qm) return QM_IRQ_NUM_VF_V2; } +static u32 qm_get_irq_num_v3(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_PF) + return QM_IRQ_NUM_PF_V2; + + return QM_IRQ_NUM_VF_V3; +} + static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe) { u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; @@ -730,6 +838,21 @@ static irqreturn_t qm_irq(int irq, void *data) return IRQ_NONE; } +static irqreturn_t qm_mb_cmd_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_STATUS); + val &= QM_IFC_INT_STATUS_MASK; + if (!val) + return IRQ_NONE; + + schedule_work(&qm->cmd_process); + + return IRQ_HANDLED; +} + static irqreturn_t qm_aeq_irq(int irq, void *data) { struct hisi_qm *qm = data; @@ -770,14 +893,16 @@ static void qm_irq_unregister(struct hisi_qm *qm) free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); - if (qm->ver == QM_HW_V1) - return; + if (qm->ver > QM_HW_V1) { + free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); - free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); + if (qm->fun_type == QM_HW_PF) + free_irq(pci_irq_vector(pdev, + QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); + } - if (qm->fun_type == QM_HW_PF) - free_irq(pci_irq_vector(pdev, - QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); + if (qm->ver > QM_HW_V2) + free_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), qm); } static void qm_init_qp_status(struct hisi_qp *qp) @@ -790,8 +915,95 @@ static void qm_init_qp_status(struct hisi_qp *qp) atomic_set(&qp_status->used, 0); } +static void qm_init_prefetch(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 page_type = 0x0; + + if (qm->ver < QM_HW_V3) + return; + + switch (PAGE_SIZE) { + case SZ_4K: + page_type = 0x0; + break; + case SZ_16K: + page_type = 0x1; + break; + case SZ_64K: + page_type = 0x2; + break; + default: + dev_err(dev, "system page size is not support: %lu, default set to 4KB", + PAGE_SIZE); + } + + writel(page_type, qm->io_base + QM_PAGE_SIZE); +} + +/* + * the formula: + * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps + * + * IR_b * (2 ^ IR_u) * 8 + * IR(Mbps) * 10 ^ -3 = ------------------------- + * Tick * (2 ^ IR_s) + */ +static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) +{ + return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) / + (QM_QOS_TICK * (1 << cir_s)); +} + +static u32 acc_shaper_calc_cbs_s(u32 ir) +{ + int i; + + if (ir < typical_qos_val[0]) + return QM_SHAPER_MIN_CBS_S; + + for (i = 1; i < QM_QOS_TYPICAL_NUM; i++) { + if (ir >= typical_qos_val[i - 1] && ir < typical_qos_val[i]) + return typical_qos_cbs_s[i - 1]; + } + + return typical_qos_cbs_s[QM_QOS_TYPICAL_NUM - 1]; +} + +static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) +{ + u32 cir_b, cir_u, cir_s, ir_calc; + u32 error_rate; + + factor->cbs_s = acc_shaper_calc_cbs_s(ir); + + for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) { + for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) { + for (cir_s = 0; cir_s <= QM_QOS_MAX_CIR_S; cir_s++) { + /** the formula is changed to: + * IR_b * (2 ^ IR_u) * DIVISOR_CLK + * IR(Mbps) = ------------------------- + * 768 * (2 ^ IR_s) + */ + ir_calc = acc_shaper_para_calc(cir_b, cir_u, + cir_s); + error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; + if (error_rate <= QM_QOS_MIN_ERROR_RATE) { + factor->cir_b = cir_b; + factor->cir_u = cir_u; + factor->cir_s = cir_s; + + return 0; + } + } + } + } + + return -EINVAL; +} + static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, - u32 number) + u32 number, struct qm_shaper_factor *factor) { u64 tmp = 0; @@ -820,6 +1032,15 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, tmp = QM_CQC_VFT_VALID; } break; + case SHAPER_VFT: + if (qm->ver >= QM_HW_V3) { + tmp = factor->cir_b | + (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) | + (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) | + (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) | + (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); + } + break; } } @@ -830,6 +1051,7 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, u32 fun_num, u32 base, u32 number) { + struct qm_shaper_factor *factor = &qm->factor[fun_num]; unsigned int val; int ret; @@ -841,9 +1063,12 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR); writel(type, qm->io_base + QM_VFT_CFG_TYPE); + if (type == SHAPER_VFT) + fun_num |= base << QM_SHAPER_VFT_OFFSET; + writel(fun_num, qm->io_base + QM_VFT_CFG); - qm_vft_data_cfg(qm, type, base, number); + qm_vft_data_cfg(qm, type, base, number, factor); writel(0x0, qm->io_base + QM_VFT_CFG_RDY); writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); @@ -853,6 +1078,27 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, POLL_TIMEOUT); } +static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) +{ + int ret, i; + + qm->factor[fun_num].func_qos = QM_QOS_MAX_VAL; + ret = qm_get_shaper_para(QM_QOS_MAX_VAL * QM_QOS_RATE, &qm->factor[fun_num]); + if (ret) { + dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n"); + return ret; + } + writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG); + for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { + /* The base number of queue reuse for different alg type */ + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1); + if (ret) + return ret; + } + + return 0; +} + /* The config should be conducted after qm_dev_mem_reset() */ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number) @@ -865,7 +1111,21 @@ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, return ret; } + /* init default shaper qos val */ + if (qm->ver >= QM_HW_V3) { + ret = qm_shaper_init_vft(qm, fun_num); + if (ret) + goto back_sqc_cqc; + } + return 0; +back_sqc_cqc: + for (i = SQC_VFT; i <= CQC_VFT; i++) { + ret = qm_set_vft_common(qm, i, fun_num, 0, 0); + if (ret) + return ret; + } + return ret; } static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) @@ -1570,16 +1830,9 @@ static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer, if (count > QM_DBG_WRITE_LEN) return -ENOSPC; - cmd_buf = kzalloc(count + 1, GFP_KERNEL); - if (!cmd_buf) - return -ENOMEM; - - if (copy_from_user(cmd_buf, buffer, count)) { - kfree(cmd_buf); - return -EFAULT; - } - - cmd_buf[count] = '\0'; + cmd_buf = memdup_user_nul(buffer, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); cmd_buf_tmp = strchr(cmd_buf, '\n'); if (cmd_buf_tmp) { @@ -1623,13 +1876,9 @@ static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); } -static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_cfg(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) { - u32 irq_enable = ce | nfe | fe; - u32 irq_unmask = ~irq_enable; - qm->error_mask = ce | nfe | fe; - /* clear QM hw residual error source */ writel(QM_ABNORMAL_INT_SOURCE_CLR, qm->io_base + QM_ABNORMAL_INT_SOURCE); @@ -1639,6 +1888,14 @@ static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE); writel(fe, qm->io_base + QM_RAS_FE_ENABLE); +} + +static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +{ + u32 irq_enable = ce | nfe | fe; + u32 irq_unmask = ~irq_enable; + + qm_hw_error_cfg(qm, ce, nfe, fe); irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); @@ -1649,6 +1906,28 @@ static void qm_hw_error_uninit_v2(struct hisi_qm *qm) writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); } +static void qm_hw_error_init_v3(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +{ + u32 irq_enable = ce | nfe | fe; + u32 irq_unmask = ~irq_enable; + + qm_hw_error_cfg(qm, ce, nfe, fe); + + /* enable close master ooo when hardware error happened */ + writel(nfe & (~QM_DB_RANDOM_INVALID), qm->io_base + QM_OOO_SHUTDOWN_SEL); + + irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); + writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); +} + +static void qm_hw_error_uninit_v3(struct hisi_qm *qm) +{ + writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); + + /* disable close master ooo when hardware error happened */ + writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); +} + static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) { const struct hisi_qm_hw_error *err; @@ -1715,15 +1994,371 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } +static u32 qm_get_hw_error_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); +} + +static u32 qm_get_dev_err_status(struct hisi_qm *qm) +{ + return qm->err_ini->get_dev_hw_err_status(qm); +} + +/* Check if the error causes the master ooo block */ +static int qm_check_dev_error(struct hisi_qm *qm) +{ + u32 val, dev_val; + + if (qm->fun_type == QM_HW_VF) + return 0; + + val = qm_get_hw_error_status(qm); + dev_val = qm_get_dev_err_status(qm); + + if (qm->ver < QM_HW_V3) + return (val & QM_ECC_MBIT) || + (dev_val & qm->err_info.ecc_2bits_mask); + + return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) || + (dev_val & (~qm->err_info.dev_ce_mask)); +} + +static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num) +{ + struct qm_mailbox mailbox; + int ret; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox); + if (ret) + goto err_unlock; + + *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | + ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); + +err_unlock: + mutex_unlock(&qm->mailbox_lock); + return ret; +} + +static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) +{ + u32 val; + + if (qm->fun_type == QM_HW_PF) + writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P); + + val = readl(qm->io_base + QM_IFC_INT_SOURCE_V); + val |= QM_IFC_INT_SOURCE_MASK; + writel(val, qm->io_base + QM_IFC_INT_SOURCE_V); +} + +static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) +{ + struct device *dev = &qm->pdev->dev; + u32 cmd; + u64 msg; + int ret; + + ret = qm_get_mb_cmd(qm, &msg, vf_id); + if (ret) { + dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id); + return; + } + + cmd = msg & QM_MB_CMD_DATA_MASK; + switch (cmd) { + case QM_VF_PREPARE_FAIL: + dev_err(dev, "failed to stop VF(%u)!\n", vf_id); + break; + case QM_VF_START_FAIL: + dev_err(dev, "failed to start VF(%u)!\n", vf_id); + break; + case QM_VF_PREPARE_DONE: + case QM_VF_START_DONE: + break; + default: + dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id); + break; + } +} + +static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 vfs_num = qm->vfs_num; + int cnt = 0; + int ret = 0; + u64 val; + u32 i; + + if (!qm->vfs_num || qm->ver < QM_HW_V3) + return 0; + + while (true) { + val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); + /* All VFs send command to PF, break */ + if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1)) + break; + + if (++cnt > QM_MAX_PF_WAIT_COUNT) { + ret = -EBUSY; + break; + } + + msleep(QM_WAIT_DST_ACK); + } + + /* PF check VFs msg */ + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + qm_handle_vf_msg(qm, i); + else + dev_err(dev, "VF(%u) not ping PF!\n", i); + } + + /* PF clear interrupt to ack VFs */ + qm_clear_cmd_interrupt(qm, val); + + return ret; +} + +static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num) +{ + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_CFG); + val &= ~QM_IFC_SEND_ALL_VFS; + val |= fun_num; + writel(val, qm->io_base + QM_IFC_INT_CFG); + + val = readl(qm->io_base + QM_IFC_INT_SET_P); + val |= QM_IFC_INT_SET_MASK; + writel(val, qm->io_base + QM_IFC_INT_SET_P); +} + +static void qm_trigger_pf_interrupt(struct hisi_qm *qm) +{ + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_SET_V); + val |= QM_IFC_INT_SET_MASK; + writel(val, qm->io_base + QM_IFC_INT_SET_V); +} + +static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + struct qm_mailbox mailbox; + int cnt = 0; + u64 val; + int ret; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox); + if (ret) { + dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); + goto err_unlock; + } + + qm_trigger_vf_interrupt(qm, fun_num); + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readq(qm->io_base + QM_IFC_READY_STATUS); + /* if VF respond, PF notifies VF successfully. */ + if (!(val & BIT(fun_num))) + goto err_unlock; + + if (++cnt > QM_MAX_PF_WAIT_COUNT) { + dev_err(dev, "failed to get response from VF(%u)!\n", fun_num); + ret = -ETIMEDOUT; + break; + } + } + +err_unlock: + mutex_unlock(&qm->mailbox_lock); + return ret; +} + +static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) +{ + struct device *dev = &qm->pdev->dev; + u32 vfs_num = qm->vfs_num; + struct qm_mailbox mailbox; + u64 val = 0; + int cnt = 0; + int ret; + u32 i; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0); + mutex_lock(&qm->mailbox_lock); + /* PF sends command to all VFs by mailbox */ + ret = qm_mb_nolock(qm, &mailbox); + if (ret) { + dev_err(dev, "failed to send command to VFs!\n"); + mutex_unlock(&qm->mailbox_lock); + return ret; + } + + qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS); + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readq(qm->io_base + QM_IFC_READY_STATUS); + /* If all VFs acked, PF notifies VFs successfully. */ + if (!(val & GENMASK(vfs_num, 1))) { + mutex_unlock(&qm->mailbox_lock); + return 0; + } + + if (++cnt > QM_MAX_PF_WAIT_COUNT) + break; + } + + mutex_unlock(&qm->mailbox_lock); + + /* Check which vf respond timeout. */ + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + dev_err(dev, "failed to get response from VF(%u)!\n", i); + } + + return -ETIMEDOUT; +} + +static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) +{ + struct qm_mailbox mailbox; + int cnt = 0; + u32 val; + int ret; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0); + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox); + if (ret) { + dev_err(&qm->pdev->dev, "failed to send command to PF!\n"); + goto unlock; + } + + qm_trigger_pf_interrupt(qm); + /* Waiting for PF response */ + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readl(qm->io_base + QM_IFC_INT_SET_V); + if (!(val & QM_IFC_INT_STATUS_MASK)) + break; + + if (++cnt > QM_MAX_VF_WAIT_COUNT) { + ret = -ETIMEDOUT; + break; + } + } + +unlock: + mutex_unlock(&qm->mailbox_lock); + return ret; +} + static int qm_stop_qp(struct hisi_qp *qp) { return qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); } +static int qm_set_msi(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + + if (set) { + pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, + 0); + } else { + pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, + ACC_PEH_MSI_DISABLE); + if (qm->err_status.is_qm_ecc_mbit || + qm->err_status.is_dev_ecc_mbit) + return 0; + + mdelay(1); + if (readl(qm->io_base + QM_PEH_DFX_INFO0)) + return -EFAULT; + } + + return 0; +} + +static void qm_wait_msi_finish(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 cmd = ~0; + int cnt = 0; + u32 val; + int ret; + + while (true) { + pci_read_config_dword(pdev, pdev->msi_cap + + PCI_MSI_PENDING_64, &cmd); + if (!cmd) + break; + + if (++cnt > MAX_WAIT_COUNTS) { + pci_warn(pdev, "failed to empty MSI PENDING!\n"); + break; + } + + udelay(1); + } + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0, + val, !(val & QM_PEH_DFX_MASK), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + pci_warn(pdev, "failed to empty PEH MSI!\n"); + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1, + val, !(val & QM_PEH_MSI_FINISH_MASK), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + pci_warn(pdev, "failed to finish MSI operation!\n"); +} + +static int qm_set_msi_v3(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + int ret = -ETIMEDOUT; + u32 cmd, i; + + pci_read_config_dword(pdev, pdev->msi_cap, &cmd); + if (set) + cmd |= QM_MSI_CAP_ENABLE; + else + cmd &= ~QM_MSI_CAP_ENABLE; + + pci_write_config_dword(pdev, pdev->msi_cap, cmd); + if (set) { + for (i = 0; i < MAX_WAIT_COUNTS; i++) { + pci_read_config_dword(pdev, pdev->msi_cap, &cmd); + if (cmd & QM_MSI_CAP_ENABLE) + return 0; + + udelay(1); + } + } else { + udelay(WAIT_PERIOD_US_MIN); + qm_wait_msi_finish(qm); + ret = 0; + } + + return ret; +} + static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, .get_irq_num = qm_get_irq_num_v1, .hw_error_init = qm_hw_error_init_v1, + .set_msi = qm_set_msi, }; static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { @@ -1733,16 +2368,20 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { .hw_error_init = qm_hw_error_init_v2, .hw_error_uninit = qm_hw_error_uninit_v2, .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi, }; static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .get_vft = qm_get_vft_v2, .qm_db = qm_db_v2, - .get_irq_num = qm_get_irq_num_v2, - .hw_error_init = qm_hw_error_init_v2, - .hw_error_uninit = qm_hw_error_uninit_v2, + .get_irq_num = qm_get_irq_num_v3, + .hw_error_init = qm_hw_error_init_v3, + .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, .stop_qp = qm_stop_qp, + .set_msi = qm_set_msi_v3, + .ping_all_vfs = qm_ping_all_vfs, + .ping_pf = qm_ping_pf, }; static void *qm_get_avail_sqe(struct hisi_qp *qp) @@ -2017,11 +2656,8 @@ static int qm_drain_qp(struct hisi_qp *qp) int ret = 0, i = 0; void *addr; - /* - * No need to judge if ECC multi-bit error occurs because the - * master OOO will be blocked. - */ - if (qm->err_status.is_qm_ecc_mbit || qm->err_status.is_dev_ecc_mbit) + /* No need to judge if master OOO is blocked. */ + if (qm_check_dev_error(qm)) return 0; /* Kunpeng930 supports drain qp by device */ @@ -2290,6 +2926,23 @@ static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) hisi_qm_stop_qp(q->priv); } +static int hisi_qm_is_q_updated(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + int updated = 0; + + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { + /* make sure to read data from memory */ + dma_rmb(); + qm_cq_head_update(qp); + cqe = qp->cqe + qp->qp_status.cq_head; + updated = 1; + } + + return updated; +} + static void qm_set_sqctype(struct uacce_queue *q, u16 type) { struct hisi_qm *qm = q->uacce->priv; @@ -2335,6 +2988,7 @@ static const struct uacce_ops uacce_qm_ops = { .stop_queue = hisi_qm_uacce_stop_queue, .mmap = hisi_qm_uacce_mmap, .ioctl = hisi_qm_uacce_ioctl, + .is_q_updated = hisi_qm_is_q_updated, }; static int qm_alloc_uacce(struct hisi_qm *qm) @@ -2530,62 +3184,6 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) return 0; } -static int hisi_qm_memory_init(struct hisi_qm *qm) -{ - struct device *dev = &qm->pdev->dev; - size_t qp_dma_size, off = 0; - int i, ret = 0; - -#define QM_INIT_BUF(qm, type, num) do { \ - (qm)->type = ((qm)->qdma.va + (off)); \ - (qm)->type##_dma = (qm)->qdma.dma + (off); \ - off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ -} while (0) - - idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + - QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + - QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + - QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); - qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, - GFP_ATOMIC); - dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); - if (!qm->qdma.va) - return -ENOMEM; - - QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); - QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); - QM_INIT_BUF(qm, sqc, qm->qp_num); - QM_INIT_BUF(qm, cqc, qm->qp_num); - - qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); - if (!qm->qp_array) { - ret = -ENOMEM; - goto err_alloc_qp_array; - } - - /* one more page for device or qp statuses */ - qp_dma_size = qm->sqe_size * QM_Q_DEPTH + - sizeof(struct qm_cqe) * QM_Q_DEPTH; - qp_dma_size = PAGE_ALIGN(qp_dma_size); - for (i = 0; i < qm->qp_num; i++) { - ret = hisi_qp_memory_init(qm, qp_dma_size, i); - if (ret) - goto err_init_qp_mem; - - dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); - } - - return ret; - -err_init_qp_mem: - hisi_qp_memory_uninit(qm, i); -err_alloc_qp_array: - dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); - - return ret; -} - static void hisi_qm_pre_init(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -2604,6 +3202,34 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->misc_ctl = false; } +static void qm_cmd_uninit(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl(qm->io_base + QM_IFC_INT_MASK); + val |= QM_IFC_INT_DISABLE; + writel(val, qm->io_base + QM_IFC_INT_MASK); +} + +static void qm_cmd_init(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + /* Clear communication interrupt source */ + qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR); + + /* Enable pf to vf communication reg. */ + val = readl(qm->io_base + QM_IFC_INT_MASK); + val &= ~QM_IFC_INT_DISABLE; + writel(val, qm->io_base + QM_IFC_INT_MASK); +} + static void qm_put_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -2635,6 +3261,8 @@ void hisi_qm_uninit(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; + qm_cmd_uninit(qm); + kfree(qm->factor); down_write(&qm->qps_lock); if (!qm_avail_state(qm, QM_CLOSE)) { @@ -2826,6 +3454,8 @@ static int __hisi_qm_start(struct hisi_qm *qm) if (ret) return ret; + qm_init_prefetch(qm); + writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); @@ -3034,79 +3664,6 @@ static int qm_debugfs_atomic64_get(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, qm_debugfs_atomic64_set, "%llu\n"); -/** - * hisi_qm_debug_init() - Initialize qm related debugfs files. - * @qm: The qm for which we want to add debugfs files. - * - * Create qm related debugfs files. - */ -void hisi_qm_debug_init(struct hisi_qm *qm) -{ - struct qm_dfx *dfx = &qm->debug.dfx; - struct dentry *qm_d; - void *data; - int i; - - qm_d = debugfs_create_dir("qm", qm->debug.debug_root); - qm->debug.qm_d = qm_d; - - /* only show this in PF */ - if (qm->fun_type == QM_HW_PF) { - qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM); - for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) - qm_create_debugfs_file(qm, qm_d, i); - } - - debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); - - debugfs_create_file("cmd", 0444, qm->debug.qm_d, qm, &qm_cmd_fops); - - debugfs_create_file("status", 0444, qm->debug.qm_d, qm, - &qm_status_fops); - for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) { - data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset); - debugfs_create_file(qm_dfx_files[i].name, - 0644, - qm_d, - data, - &qm_atomic64_ops); - } -} -EXPORT_SYMBOL_GPL(hisi_qm_debug_init); - -/** - * hisi_qm_debug_regs_clear() - clear qm debug related registers. - * @qm: The qm for which we want to clear its debug registers. - */ -void hisi_qm_debug_regs_clear(struct hisi_qm *qm) -{ - struct qm_dfx_registers *regs; - int i; - - /* clear current_qm */ - writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); - writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); - - /* clear current_q */ - writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); - writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); - - /* - * these registers are reading and clearing, so clear them after - * reading them. - */ - writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE); - - regs = qm_dfx_regs; - for (i = 0; i < CNT_CYC_REGS_NUM; i++) { - readl(qm->io_base + regs->reg_offset); - regs++; - } - - writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE); -} -EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); - static void qm_hw_error_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -3362,6 +3919,360 @@ static int qm_clear_vft_config(struct hisi_qm *qm) return 0; } +static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) +{ + struct device *dev = &qm->pdev->dev; + u32 ir = qos * QM_QOS_RATE; + int ret, total_vfs, i; + + total_vfs = pci_sriov_get_totalvfs(qm->pdev); + if (fun_index > total_vfs) + return -EINVAL; + + qm->factor[fun_index].func_qos = qos; + + ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); + if (ret) { + dev_err(dev, "failed to calculate shaper parameter!\n"); + return -EINVAL; + } + + for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { + /* The base number of queue reuse for different alg type */ + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) { + dev_err(dev, "type: %d, failed to set shaper vft!\n", i); + return -EINVAL; + } + } + + return 0; +} + +static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) +{ + u64 cir_u = 0, cir_b = 0, cir_s = 0; + u64 shaper_vft, ir_calc, ir; + unsigned int val; + u32 error_rate; + int ret; + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); + if (ret) + return 0; + + writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR); + writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE); + writel(fun_index, qm->io_base + QM_VFT_CFG); + + writel(0x0, qm->io_base + QM_VFT_CFG_RDY); + writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); + if (ret) + return 0; + + shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) | + ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32); + + cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK; + cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK; + cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT; + + cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK; + cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT; + + ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); + + ir = qm->factor[fun_index].func_qos * QM_QOS_RATE; + + error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; + if (error_rate > QM_QOS_MIN_ERROR_RATE) { + pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate); + return 0; + } + + return ir; +} + +static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + u64 mb_cmd; + u32 qos; + int ret; + + qos = qm_get_shaper_vft_qos(qm, fun_num); + if (!qos) { + dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num); + return; + } + + mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT; + ret = qm_ping_single_vf(qm, mb_cmd, fun_num); + if (ret) + dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num); +} + +static int qm_vf_read_qos(struct hisi_qm *qm) +{ + int cnt = 0; + int ret; + + /* reset mailbox qos val */ + qm->mb_qos = 0; + + /* vf ping pf to get function qos */ + if (qm->ops->ping_pf) { + ret = qm->ops->ping_pf(qm, QM_VF_GET_QOS); + if (ret) { + pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); + return ret; + } + } + + while (true) { + msleep(QM_WAIT_DST_ACK); + if (qm->mb_qos) + break; + + if (++cnt > QM_MAX_VF_WAIT_COUNT) { + pci_err(qm->pdev, "PF ping VF timeout!\n"); + return -ETIMEDOUT; + } + } + + return ret; +} + +static ssize_t qm_algqos_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char tbuf[QM_DBG_READ_LEN]; + u32 qos_val, ir; + int ret; + + /* Mailbox and reset cannot be operated at the same time */ + if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + pci_err(qm->pdev, "dev resetting, read alg qos failed!\n"); + return -EAGAIN; + } + + if (qm->fun_type == QM_HW_PF) { + ir = qm_get_shaper_vft_qos(qm, 0); + } else { + ret = qm_vf_read_qos(qm); + if (ret) + goto err_get_status; + ir = qm->mb_qos; + } + + qos_val = ir / QM_QOS_RATE; + ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val); + + ret = simple_read_from_buffer(buf, count, pos, tbuf, ret); + +err_get_status: + clear_bit(QM_RESETTING, &qm->misc_ctl); + return ret; +} + +static ssize_t qm_qos_value_init(const char *buf, unsigned long *val) +{ + int buflen = strlen(buf); + int ret, i; + + for (i = 0; i < buflen; i++) { + if (!isdigit(buf[i])) + return -EINVAL; + } + + ret = sscanf(buf, "%ld", val); + if (ret != QM_QOS_VAL_NUM) + return -EINVAL; + + return 0; +} + +static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char tbuf[QM_DBG_READ_LEN]; + int tmp1, bus, device, function; + char tbuf_bdf[QM_DBG_READ_LEN] = {0}; + char val_buf[QM_QOS_VAL_MAX_LEN] = {0}; + unsigned int fun_index; + unsigned long val = 0; + int len, ret; + + if (qm->fun_type == QM_HW_VF) + return -EINVAL; + + /* Mailbox and reset cannot be operated at the same time */ + if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + pci_err(qm->pdev, "dev resetting, write alg qos failed!\n"); + return -EAGAIN; + } + + if (*pos != 0) { + ret = 0; + goto err_get_status; + } + + if (count >= QM_DBG_READ_LEN) { + ret = -ENOSPC; + goto err_get_status; + } + + len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count); + if (len < 0) { + ret = len; + goto err_get_status; + } + + tbuf[len] = '\0'; + ret = sscanf(tbuf, "%s %s", tbuf_bdf, val_buf); + if (ret != QM_QOS_PARAM_NUM) { + ret = -EINVAL; + goto err_get_status; + } + + ret = qm_qos_value_init(val_buf, &val); + if (val == 0 || val > QM_QOS_MAX_VAL || ret) { + pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); + ret = -EINVAL; + goto err_get_status; + } + + ret = sscanf(tbuf_bdf, "%d:%x:%d.%d", &tmp1, &bus, &device, &function); + if (ret != QM_QOS_BDF_PARAM_NUM) { + pci_err(qm->pdev, "input pci bdf value is error!\n"); + ret = -EINVAL; + goto err_get_status; + } + + fun_index = device * 8 + function; + + ret = qm_func_shaper_enable(qm, fun_index, val); + if (ret) { + pci_err(qm->pdev, "failed to enable function shaper!\n"); + ret = -EINVAL; + goto err_get_status; + } + + ret = count; + +err_get_status: + clear_bit(QM_RESETTING, &qm->misc_ctl); + return ret; +} + +static const struct file_operations qm_algqos_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = qm_algqos_read, + .write = qm_algqos_write, +}; + +/** + * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. + * @qm: The qm for which we want to add debugfs files. + * + * Create function qos debugfs files. + */ +static void hisi_qm_set_algqos_init(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_PF) + debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, + qm, &qm_algqos_fops); + else + debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, + qm, &qm_algqos_fops); +} + +/** + * hisi_qm_debug_init() - Initialize qm related debugfs files. + * @qm: The qm for which we want to add debugfs files. + * + * Create qm related debugfs files. + */ +void hisi_qm_debug_init(struct hisi_qm *qm) +{ + struct qm_dfx *dfx = &qm->debug.dfx; + struct dentry *qm_d; + void *data; + int i; + + qm_d = debugfs_create_dir("qm", qm->debug.debug_root); + qm->debug.qm_d = qm_d; + + /* only show this in PF */ + if (qm->fun_type == QM_HW_PF) { + qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM); + for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) + qm_create_debugfs_file(qm, qm->debug.qm_d, i); + } + + debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); + + debugfs_create_file("cmd", 0600, qm->debug.qm_d, qm, &qm_cmd_fops); + + debugfs_create_file("status", 0444, qm->debug.qm_d, qm, + &qm_status_fops); + for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) { + data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset); + debugfs_create_file(qm_dfx_files[i].name, + 0644, + qm_d, + data, + &qm_atomic64_ops); + } + + if (qm->ver >= QM_HW_V3) + hisi_qm_set_algqos_init(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_debug_init); + +/** + * hisi_qm_debug_regs_clear() - clear qm debug related registers. + * @qm: The qm for which we want to clear its debug registers. + */ +void hisi_qm_debug_regs_clear(struct hisi_qm *qm) +{ + struct qm_dfx_registers *regs; + int i; + + /* clear current_qm */ + writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); + writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + + /* clear current_q */ + writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); + writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); + + /* + * these registers are reading and clearing, so clear them after + * reading them. + */ + writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE); + + regs = qm_dfx_regs; + for (i = 0; i < CNT_CYC_REGS_NUM; i++) { + readl(qm->io_base + regs->reg_offset); + regs++; + } + + /* clear clear_enable */ + writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE); +} +EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); + /** * hisi_qm_sriov_enable() - enable virtual functions * @pdev: the PCIe device @@ -3416,6 +4327,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); + int total_vfs = pci_sriov_get_totalvfs(qm->pdev); if (pci_vfs_assigned(pdev)) { pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n"); @@ -3429,6 +4341,9 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) } pci_disable_sriov(pdev); + /* clear vf function shaper configure array */ + memset(qm->factor + 1, 0, sizeof(struct qm_shaper_factor) * total_vfs); + return qm_clear_vft_config(qm); } EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); @@ -3527,17 +4442,15 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); -static u32 qm_get_hw_error_status(struct hisi_qm *qm) -{ - return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); -} - static int qm_check_req_recv(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; int ret; u32 val; + if (qm->ver >= QM_HW_V3) + return 0; + writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID); ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, (val == ACC_VENDOR_ID_VALUE), @@ -3608,28 +4521,6 @@ static int qm_set_vf_mse(struct hisi_qm *qm, bool set) return -ETIMEDOUT; } -static int qm_set_msi(struct hisi_qm *qm, bool set) -{ - struct pci_dev *pdev = qm->pdev; - - if (set) { - pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, - 0); - } else { - pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, - ACC_PEH_MSI_DISABLE); - if (qm->err_status.is_qm_ecc_mbit || - qm->err_status.is_dev_ecc_mbit) - return 0; - - mdelay(1); - if (readl(qm->io_base + QM_PEH_DFX_INFO0)) - return -EFAULT; - } - - return 0; -} - static int qm_vf_reset_prepare(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { @@ -3660,14 +4551,35 @@ static int qm_vf_reset_prepare(struct hisi_qm *qm, return ret; } -static int qm_reset_prepare_ready(struct hisi_qm *qm) +static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, + enum qm_stop_reason stop_reason) { struct pci_dev *pdev = qm->pdev; - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + int ret; + + if (!qm->vfs_num) + return 0; + + /* Kunpeng930 supports to notify VFs to stop before PF reset */ + if (qm->ops->ping_all_vfs) { + ret = qm->ops->ping_all_vfs(qm, cmd); + if (ret) + pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); + } else { + ret = qm_vf_reset_prepare(qm, stop_reason); + if (ret) + pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret); + } + + return ret; +} + +static int qm_wait_reset_finish(struct hisi_qm *qm) +{ int delay = 0; /* All reset requests need to be queued for processing */ - while (test_and_set_bit(QM_RESETTING, &pf_qm->misc_ctl)) { + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { msleep(++delay); if (delay > QM_RESET_WAIT_TIMEOUT) return -EBUSY; @@ -3676,6 +4588,32 @@ static int qm_reset_prepare_ready(struct hisi_qm *qm) return 0; } +static int qm_reset_prepare_ready(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + /* + * PF and VF on host doesnot support resetting at the + * same time on Kunpeng920. + */ + if (qm->ver < QM_HW_V3) + return qm_wait_reset_finish(pf_qm); + + return qm_wait_reset_finish(qm); +} + +static void qm_reset_bit_clear(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + if (qm->ver < QM_HW_V3) + clear_bit(QM_RESETTING, &pf_qm->misc_ctl); + + clear_bit(QM_RESETTING, &qm->misc_ctl); +} + static int qm_controller_reset_prepare(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -3687,22 +4625,25 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) return ret; } - if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); - if (ret) { - pci_err(pdev, "Fails to stop VFs!\n"); - clear_bit(QM_RESETTING, &qm->misc_ctl); - return ret; - } - } + /* PF obtains the information of VF by querying the register. */ + qm_cmd_uninit(qm); + + /* Whether VFs stop successfully, soft reset will continue. */ + ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET); + if (ret) + pci_err(pdev, "failed to stop vfs by pf in soft reset.\n"); ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return ret; } + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to stop by vfs in soft reset!\n"); + clear_bit(QM_RST_SCHED, &qm->misc_ctl); return 0; @@ -3712,6 +4653,10 @@ static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) { u32 nfe_enb = 0; + /* Kunpeng930 hardware automatically close master ooo when NFE occurs */ + if (qm->ver >= QM_HW_V3) + return; + if (!qm->err_status.is_dev_ecc_mbit && qm->err_status.is_qm_ecc_mbit && qm->err_ini->close_axi_master_ooo) { @@ -3748,7 +4693,7 @@ static int qm_soft_reset(struct hisi_qm *qm) } } - ret = qm_set_msi(qm, false); + ret = qm->ops->set_msi(qm, false); if (ret) { pci_err(pdev, "Fails to disable PEH MSI bit.\n"); return ret; @@ -3770,6 +4715,9 @@ static int qm_soft_reset(struct hisi_qm *qm) return ret; } + if (qm->err_ini->close_sva_prefetch) + qm->err_ini->close_sva_prefetch(qm); + ret = qm_set_pf_mse(qm, false); if (ret) { pci_err(pdev, "Fails to disable pf MSE bit.\n"); @@ -3830,9 +4778,32 @@ static int qm_vf_reset_done(struct hisi_qm *qm) return ret; } -static u32 qm_get_dev_err_status(struct hisi_qm *qm) +static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) { - return qm->err_ini->get_dev_hw_err_status(qm); + struct pci_dev *pdev = qm->pdev; + int ret; + + if (!qm->vfs_num) + return 0; + + ret = qm_vf_q_assign(qm, qm->vfs_num); + if (ret) { + pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret); + return ret; + } + + /* Kunpeng930 supports to notify VFs to start after PF reset. */ + if (qm->ops->ping_all_vfs) { + ret = qm->ops->ping_all_vfs(qm, cmd); + if (ret) + pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); + } else { + ret = qm_vf_reset_done(qm); + if (ret) + pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret); + } + + return ret; } static int qm_dev_hw_init(struct hisi_qm *qm) @@ -3844,6 +4815,12 @@ static void qm_restart_prepare(struct hisi_qm *qm) { u32 value; + if (qm->err_ini->open_sva_prefetch) + qm->err_ini->open_sva_prefetch(qm); + + if (qm->ver >= QM_HW_V3) + return; + if (!qm->err_status.is_qm_ecc_mbit && !qm->err_status.is_dev_ecc_mbit) return; @@ -3863,15 +4840,15 @@ static void qm_restart_prepare(struct hisi_qm *qm) /* clear AM Reorder Buffer ecc mbit source */ writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); - - if (qm->err_ini->open_axi_master_ooo) - qm->err_ini->open_axi_master_ooo(qm); } static void qm_restart_done(struct hisi_qm *qm) { u32 value; + if (qm->ver >= QM_HW_V3) + goto clear_flags; + if (!qm->err_status.is_qm_ecc_mbit && !qm->err_status.is_dev_ecc_mbit) return; @@ -3881,6 +4858,7 @@ static void qm_restart_done(struct hisi_qm *qm) value |= qm->err_info.msi_wr_port; writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN); +clear_flags: qm->err_status.is_qm_ecc_mbit = false; qm->err_status.is_dev_ecc_mbit = false; } @@ -3890,7 +4868,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; int ret; - ret = qm_set_msi(qm, true); + ret = qm->ops->set_msi(qm, true); if (ret) { pci_err(pdev, "Fails to enable PEH MSI bit!\n"); return ret; @@ -3917,6 +4895,9 @@ static int qm_controller_reset_done(struct hisi_qm *qm) } qm_restart_prepare(qm); + hisi_qm_dev_err_init(qm); + if (qm->err_ini->open_axi_master_ooo) + qm->err_ini->open_axi_master_ooo(qm); ret = qm_restart(qm); if (ret) { @@ -3924,24 +4905,18 @@ static int qm_controller_reset_done(struct hisi_qm *qm) return ret; } - if (qm->vfs_num) { - ret = qm_vf_q_assign(qm, qm->vfs_num); - if (ret) { - pci_err(pdev, "Failed to assign queue!\n"); - return ret; - } - } + ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); + if (ret) + pci_err(pdev, "failed to start vfs by pf in soft reset.\n"); - ret = qm_vf_reset_done(qm); - if (ret) { - pci_err(pdev, "Failed to start VFs!\n"); - return -EPERM; - } + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to start by vfs in soft reset!\n"); - hisi_qm_dev_err_init(qm); + qm_cmd_init(qm); qm_restart_done(qm); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return 0; } @@ -3962,13 +4937,13 @@ static int qm_controller_reset(struct hisi_qm *qm) ret = qm_soft_reset(qm); if (ret) { pci_err(pdev, "Controller reset failed (%d)\n", ret); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return ret; } ret = qm_controller_reset_done(qm); if (ret) { - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return ret; } @@ -4005,21 +4980,6 @@ pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset); -/* check the interrupt is ecc-mbit error or not */ -static int qm_check_dev_error(struct hisi_qm *qm) -{ - int ret; - - if (qm->fun_type == QM_HW_VF) - return 0; - - ret = qm_get_hw_error_status(qm) & QM_ECC_MBIT; - if (ret) - return ret; - - return (qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask); -} - void hisi_qm_reset_prepare(struct pci_dev *pdev) { struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); @@ -4045,14 +5005,13 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) return; } - if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm, QM_FLR); - if (ret) { - pci_err(pdev, "Failed to prepare reset, ret = %d.\n", - ret); - return; - } - } + /* PF obtains the information of VF by querying the register. */ + if (qm->fun_type == QM_HW_PF) + qm_cmd_uninit(qm); + + ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_FLR); + if (ret) + pci_err(pdev, "failed to stop vfs by pf in FLR.\n"); ret = hisi_qm_stop(qm, QM_FLR); if (ret) { @@ -4060,6 +5019,10 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) return; } + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to stop by vfs in FLR!\n"); + pci_info(pdev, "FLR resetting...\n"); } EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare); @@ -4085,6 +5048,14 @@ void hisi_qm_reset_done(struct pci_dev *pdev) struct hisi_qm *qm = pci_get_drvdata(pdev); int ret; + if (qm->fun_type == QM_HW_PF) { + ret = qm_dev_hw_init(qm); + if (ret) { + pci_err(pdev, "Failed to init PF, ret = %d.\n", ret); + goto flr_done; + } + } + hisi_qm_dev_err_init(pf_qm); ret = qm_restart(qm); @@ -4093,34 +5064,22 @@ void hisi_qm_reset_done(struct pci_dev *pdev) goto flr_done; } - if (qm->fun_type == QM_HW_PF) { - ret = qm_dev_hw_init(qm); - if (ret) { - pci_err(pdev, "Failed to init PF, ret = %d.\n", ret); - goto flr_done; - } + ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); + if (ret) + pci_err(pdev, "failed to start vfs by pf in FLR.\n"); - if (!qm->vfs_num) - goto flr_done; - - ret = qm_vf_q_assign(qm, qm->vfs_num); - if (ret) { - pci_err(pdev, "Failed to assign VFs, ret = %d.\n", ret); - goto flr_done; - } - - ret = qm_vf_reset_done(qm); - if (ret) { - pci_err(pdev, "Failed to start VFs, ret = %d.\n", ret); - goto flr_done; - } - } + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to start by vfs in FLR!\n"); flr_done: + if (qm->fun_type == QM_HW_PF) + qm_cmd_init(qm); + if (qm_flr_reset_complete(pdev)) pci_info(pdev, "FLR reset complete\n"); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); } EXPORT_SYMBOL_GPL(hisi_qm_reset_done); @@ -4149,7 +5108,7 @@ static int qm_irq_register(struct hisi_qm *qm) if (ret) return ret; - if (qm->ver != QM_HW_V1) { + if (qm->ver > QM_HW_V1) { ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm_aeq_irq, 0, qm->dev_name, qm); if (ret) @@ -4164,8 +5123,18 @@ static int qm_irq_register(struct hisi_qm *qm) } } + if (qm->ver > QM_HW_V2) { + ret = request_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), + qm_mb_cmd_irq, 0, qm->dev_name, qm); + if (ret) + goto err_mb_cmd_irq; + } + return 0; +err_mb_cmd_irq: + if (qm->fun_type == QM_HW_PF) + free_irq(pci_irq_vector(pdev, QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); err_abonormal_irq: free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); err_aeq_irq: @@ -4202,6 +5171,183 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) } +static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) +{ + enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE; + struct pci_dev *pdev = qm->pdev; + int ret; + + ret = qm_reset_prepare_ready(qm); + if (ret) { + dev_err(&pdev->dev, "reset prepare not ready!\n"); + atomic_set(&qm->status.flags, QM_STOP); + cmd = QM_VF_PREPARE_FAIL; + goto err_prepare; + } + + ret = hisi_qm_stop(qm, stop_reason); + if (ret) { + dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret); + atomic_set(&qm->status.flags, QM_STOP); + cmd = QM_VF_PREPARE_FAIL; + goto err_prepare; + } + +err_prepare: + pci_save_state(pdev); + ret = qm->ops->ping_pf(qm, cmd); + if (ret) + dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n"); +} + +static void qm_pf_reset_vf_done(struct hisi_qm *qm) +{ + enum qm_mb_cmd cmd = QM_VF_START_DONE; + struct pci_dev *pdev = qm->pdev; + int ret; + + pci_restore_state(pdev); + ret = hisi_qm_start(qm); + if (ret) { + dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret); + cmd = QM_VF_START_FAIL; + } + + ret = qm->ops->ping_pf(qm, cmd); + if (ret) + dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); + + qm_reset_bit_clear(qm); +} + +static int qm_wait_pf_reset_finish(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 val, cmd; + u64 msg; + int ret; + + /* Wait for reset to finish */ + ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val, + val == BIT(0), QM_VF_RESET_WAIT_US, + QM_VF_RESET_WAIT_TIMEOUT_US); + /* hardware completion status should be available by this time */ + if (ret) { + dev_err(dev, "couldn't get reset done status from PF, timeout!\n"); + return -ETIMEDOUT; + } + + /* + * Whether message is got successfully, + * VF needs to ack PF by clearing the interrupt. + */ + ret = qm_get_mb_cmd(qm, &msg, 0); + qm_clear_cmd_interrupt(qm, 0); + if (ret) { + dev_err(dev, "failed to get msg from PF in reset done!\n"); + return ret; + } + + cmd = msg & QM_MB_CMD_DATA_MASK; + if (cmd != QM_PF_RESET_DONE) { + dev_err(dev, "the cmd(%u) is not reset done!\n", cmd); + ret = -EINVAL; + } + + return ret; +} + +static void qm_pf_reset_vf_process(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + dev_info(dev, "device reset start...\n"); + + /* The message is obtained by querying the register during resetting */ + qm_cmd_uninit(qm); + qm_pf_reset_vf_prepare(qm, stop_reason); + + ret = qm_wait_pf_reset_finish(qm); + if (ret) + goto err_get_status; + + qm_pf_reset_vf_done(qm); + qm_cmd_init(qm); + + dev_info(dev, "device reset done.\n"); + + return; + +err_get_status: + qm_cmd_init(qm); + qm_reset_bit_clear(qm); +} + +static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + u64 msg; + u32 cmd; + int ret; + + /* + * Get the msg from source by sending mailbox. Whether message is got + * successfully, destination needs to ack source by clearing the interrupt. + */ + ret = qm_get_mb_cmd(qm, &msg, fun_num); + qm_clear_cmd_interrupt(qm, BIT(fun_num)); + if (ret) { + dev_err(dev, "failed to get msg from source!\n"); + return; + } + + cmd = msg & QM_MB_CMD_DATA_MASK; + switch (cmd) { + case QM_PF_FLR_PREPARE: + qm_pf_reset_vf_process(qm, QM_FLR); + break; + case QM_PF_SRST_PREPARE: + qm_pf_reset_vf_process(qm, QM_SOFT_RESET); + break; + case QM_VF_GET_QOS: + qm_vf_get_qos(qm, fun_num); + break; + case QM_PF_SET_QOS: + qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT; + break; + default: + dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num); + break; + } +} + +static void qm_cmd_process(struct work_struct *cmd_process) +{ + struct hisi_qm *qm = container_of(cmd_process, + struct hisi_qm, cmd_process); + u32 vfs_num = qm->vfs_num; + u64 val; + u32 i; + + if (qm->fun_type == QM_HW_PF) { + val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); + if (!val) + return; + + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + qm_handle_cmd_msg(qm, i); + } + + return; + } + + qm_handle_cmd_msg(qm, 0); +} + /** * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list. * @qm: The qm needs add. @@ -4212,11 +5358,9 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) */ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { + struct device *dev = &qm->pdev->dev; int flag = 0; int ret = 0; - /* HW V2 not support both use uacce sva mode and hardware crypto algs */ - if (qm->ver <= QM_HW_V2 && qm->use_sva) - return 0; mutex_lock(&qm_list->lock); if (list_empty(&qm_list->list)) @@ -4224,6 +5368,11 @@ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) list_add_tail(&qm->list, &qm_list->list); mutex_unlock(&qm_list->lock); + if (qm->ver <= QM_HW_V2 && qm->use_sva) { + dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n"); + return 0; + } + if (flag) { ret = qm_list->register_to_crypto(qm); if (ret) { @@ -4248,13 +5397,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_alg_register); */ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { - if (qm->ver <= QM_HW_V2 && qm->use_sva) - return; - mutex_lock(&qm_list->lock); list_del(&qm->list); mutex_unlock(&qm_list->lock); + if (qm->ver <= QM_HW_V2 && qm->use_sva) + return; + if (list_empty(&qm_list->list)) qm_list->unregister_from_crypto(qm); } @@ -4389,6 +5538,94 @@ static int hisi_qm_pci_init(struct hisi_qm *qm) return ret; } +static void hisi_qm_init_work(struct hisi_qm *qm) +{ + INIT_WORK(&qm->work, qm_work_process); + if (qm->fun_type == QM_HW_PF) + INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); + + if (qm->ver > QM_HW_V2) + INIT_WORK(&qm->cmd_process, qm_cmd_process); +} + +static int hisi_qp_alloc_memory(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + size_t qp_dma_size; + int i, ret; + + qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); + if (!qm->qp_array) + return -ENOMEM; + + /* one more page for device or qp statuses */ + qp_dma_size = qm->sqe_size * QM_Q_DEPTH + + sizeof(struct qm_cqe) * QM_Q_DEPTH; + qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; + for (i = 0; i < qm->qp_num; i++) { + ret = hisi_qp_memory_init(qm, qp_dma_size, i); + if (ret) + goto err_init_qp_mem; + + dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); + } + + return 0; +err_init_qp_mem: + hisi_qp_memory_uninit(qm, i); + + return ret; +} + +static int hisi_qm_memory_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret, total_vfs; + size_t off = 0; + + total_vfs = pci_sriov_get_totalvfs(qm->pdev); + qm->factor = kcalloc(total_vfs + 1, sizeof(struct qm_shaper_factor), GFP_KERNEL); + if (!qm->factor) + return -ENOMEM; + +#define QM_INIT_BUF(qm, type, num) do { \ + (qm)->type = ((qm)->qdma.va + (off)); \ + (qm)->type##_dma = (qm)->qdma.dma + (off); \ + off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ +} while (0) + + idr_init(&qm->qp_idr); + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + + QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); + qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, + GFP_ATOMIC); + dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); + if (!qm->qdma.va) { + ret = -ENOMEM; + goto err_alloc_qdma; + } + + QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); + QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, sqc, qm->qp_num); + QM_INIT_BUF(qm, cqc, qm->qp_num); + + ret = hisi_qp_alloc_memory(qm); + if (ret) + goto err_alloc_qp_array; + + return 0; + +err_alloc_qp_array: + dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); +err_alloc_qdma: + kfree(qm->factor); + + return ret; +} + /** * hisi_qm_init() - Initialize configures about qm. * @qm: The qm needing init. @@ -4426,10 +5663,8 @@ int hisi_qm_init(struct hisi_qm *qm) if (ret) goto err_alloc_uacce; - INIT_WORK(&qm->work, qm_work_process); - if (qm->fun_type == QM_HW_PF) - INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); - + hisi_qm_init_work(qm); + qm_cmd_init(qm); atomic_set(&qm->status.flags, QM_INIT); return 0; diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index acefdf8b3a50..035eaf8c442d 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -76,6 +76,9 @@ #define QM_Q_DEPTH 1024 #define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 +#define QM_SHAPER_CFG 0x100164 +#define QM_SHAPER_ENABLE BIT(30) +#define QM_SHAPER_TYPE1_OFFSET 10 /* page number for queue file region */ #define QM_DOORBELL_PAGE_NR 1 @@ -148,6 +151,14 @@ struct qm_debug { struct debugfs_file files[DEBUG_FILE_NUM]; }; +struct qm_shaper_factor { + u32 func_qos; + u64 cir_b; + u64 cir_u; + u64 cir_s; + u64 cbs_s; +}; + struct qm_dma { void *va; dma_addr_t dma; @@ -188,6 +199,8 @@ struct hisi_qm_err_ini { void (*clear_dev_hw_err_status)(struct hisi_qm *qm, u32 err_sts); void (*open_axi_master_ooo)(struct hisi_qm *qm); void (*close_axi_master_ooo)(struct hisi_qm *qm); + void (*open_sva_prefetch)(struct hisi_qm *qm); + void (*close_sva_prefetch)(struct hisi_qm *qm); void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts); void (*err_info_init)(struct hisi_qm *qm); }; @@ -248,6 +261,7 @@ struct hisi_qm { struct workqueue_struct *wq; struct work_struct work; struct work_struct rst_work; + struct work_struct cmd_process; const char *algs; bool use_sva; @@ -259,6 +273,9 @@ struct hisi_qm { resource_size_t db_phys_base; struct uacce_device *uacce; int mode; + struct qm_shaper_factor *factor; + u32 mb_qos; + u32 type_rate; }; struct hisi_qp_status { diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index dfdce2f21e65..018415b9840a 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -13,14 +13,14 @@ struct sec_alg_res { dma_addr_t pbuf_dma; u8 *c_ivin; dma_addr_t c_ivin_dma; + u8 *a_ivin; + dma_addr_t a_ivin_dma; u8 *out_mac; dma_addr_t out_mac_dma; }; /* Cipher request of SEC private */ struct sec_cipher_req { - struct hisi_acc_hw_sgl *c_in; - dma_addr_t c_in_dma; struct hisi_acc_hw_sgl *c_out; dma_addr_t c_out_dma; u8 *c_ivin; @@ -33,15 +33,25 @@ struct sec_cipher_req { struct sec_aead_req { u8 *out_mac; dma_addr_t out_mac_dma; + u8 *a_ivin; + dma_addr_t a_ivin_dma; struct aead_request *aead_req; }; /* SEC request of Crypto */ struct sec_req { - struct sec_sqe sec_sqe; + union { + struct sec_sqe sec_sqe; + struct sec_sqe3 sec_sqe3; + }; struct sec_ctx *ctx; struct sec_qp_ctx *qp_ctx; + /** + * Common parameter of the SEC request. + */ + struct hisi_acc_hw_sgl *in; + dma_addr_t in_dma; struct sec_cipher_req c_req; struct sec_aead_req aead_req; struct list_head backlog_head; @@ -81,7 +91,9 @@ struct sec_auth_ctx { u8 a_key_len; u8 mac_len; u8 a_alg; + bool fallback; struct crypto_shash *hash_tfm; + struct crypto_aead *fallback_aead_tfm; }; /* SEC cipher context which cipher's relatives */ @@ -94,6 +106,10 @@ struct sec_cipher_ctx { u8 c_mode; u8 c_alg; u8 c_key_len; + + /* add software support */ + bool fallback; + struct crypto_sync_skcipher *fbtfm; }; /* SEC queue context which defines queue's relatives */ @@ -137,6 +153,7 @@ struct sec_ctx { bool pbuf_supported; struct sec_cipher_ctx c_ctx; struct sec_auth_ctx a_ctx; + u8 type_supported; struct device *dev; }; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 133aede8bf07..6a45bd23b363 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -2,6 +2,7 @@ /* Copyright (c) 2019 HiSilicon Limited. */ #include +#include #include #include #include @@ -21,6 +22,7 @@ #define SEC_PRIORITY 4001 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) +#define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) @@ -37,10 +39,23 @@ #define SEC_AEAD_ALG_OFFSET 11 #define SEC_AUTH_OFFSET 6 +#define SEC_DE_OFFSET_V3 9 +#define SEC_SCENE_OFFSET_V3 5 +#define SEC_CKEY_OFFSET_V3 13 +#define SEC_SRC_SGL_OFFSET_V3 11 +#define SEC_DST_SGL_OFFSET_V3 14 +#define SEC_CALG_OFFSET_V3 4 +#define SEC_AKEY_OFFSET_V3 9 +#define SEC_MAC_OFFSET_V3 4 +#define SEC_AUTH_ALG_OFFSET_V3 15 +#define SEC_CIPHER_AUTH_V3 0xbf +#define SEC_AUTH_CIPHER_V3 0x40 #define SEC_FLAG_OFFSET 7 #define SEC_FLAG_MASK 0x0780 #define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 +#define SEC_ICV_MASK 0x000E +#define SEC_SQE_LEN_RATE_MASK 0x3 #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) #define SEC_SGL_SGE_NR 128 @@ -66,6 +81,25 @@ #define SEC_SQE_CFLAG 2 #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 +#define SEC_ICV_ERR 0x2 +#define MIN_MAC_LEN 4 +#define MAC_LEN_MASK 0x1U +#define MAX_INPUT_DATA_LEN 0xFFFE00 +#define BITS_MASK 0xFF +#define BYTE_BITS 0x8 +#define SEC_XTS_NAME_SZ 0x3 +#define IV_CM_CAL_NUM 2 +#define IV_CL_MASK 0x7 +#define IV_CL_MIN 2 +#define IV_CL_MID 4 +#define IV_CL_MAX 8 +#define IV_FLAGS_OFFSET 0x6 +#define IV_CM_OFFSET 0x3 +#define IV_LAST_BYTE1 1 +#define IV_LAST_BYTE2 2 +#define IV_LAST_BYTE_MASK 0xFF +#define IV_CTR_INIT 0x1 +#define IV_BYTE_OFFSET 0x8 /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) @@ -124,22 +158,59 @@ static void sec_free_req_id(struct sec_req *req) mutex_unlock(&qp_ctx->req_lock); } -static int sec_aead_verify(struct sec_req *req) +static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) { - struct aead_request *aead_req = req->aead_req.aead_req; - struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); - size_t authsize = crypto_aead_authsize(tfm); - u8 *mac_out = req->aead_req.out_mac; - u8 *mac = mac_out + SEC_MAX_MAC_LEN; - struct scatterlist *sgl = aead_req->src; - size_t sz; + struct sec_sqe *bd = resp; - sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize, - aead_req->cryptlen + aead_req->assoclen - - authsize); - if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { - dev_err(req->ctx->dev, "aead verify failure!\n"); - return -EBADMSG; + status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; + status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1; + status->flag = (le16_to_cpu(bd->type2.done_flag) & + SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; + status->tag = le16_to_cpu(bd->type2.tag); + status->err_type = bd->type2.error_type; + + return bd->type_cipher_auth & SEC_TYPE_MASK; +} + +static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) +{ + struct sec_sqe3 *bd3 = resp; + + status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; + status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1; + status->flag = (le16_to_cpu(bd3->done_flag) & + SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; + status->tag = le64_to_cpu(bd3->tag); + status->err_type = bd3->error_type; + + return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; +} + +static int sec_cb_status_check(struct sec_req *req, + struct bd_status *status) +{ + struct sec_ctx *ctx = req->ctx; + + if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { + dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", + req->err_type, status->done); + return -EIO; + } + + if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { + if (unlikely(status->flag != SEC_SQE_CFLAG)) { + dev_err_ratelimited(ctx->dev, "flag[%u]\n", + status->flag); + return -EIO; + } + } else if (unlikely(ctx->alg_type == SEC_AEAD)) { + if (unlikely(status->flag != SEC_SQE_AEAD_FLAG || + status->icv == SEC_ICV_ERR)) { + dev_err_ratelimited(ctx->dev, + "flag[%u], icv[%u]\n", + status->flag, status->icv); + return -EBADMSG; + } } return 0; @@ -149,43 +220,38 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) { struct sec_qp_ctx *qp_ctx = qp->qp_ctx; struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; - struct sec_sqe *bd = resp; + u8 type_supported = qp_ctx->ctx->type_supported; + struct bd_status status; struct sec_ctx *ctx; struct sec_req *req; - u16 done, flag; - int err = 0; + int err; u8 type; - type = bd->type_cipher_auth & SEC_TYPE_MASK; - if (unlikely(type != SEC_BD_TYPE2)) { + if (type_supported == SEC_BD_TYPE2) { + type = pre_parse_finished_bd(&status, resp); + req = qp_ctx->req_list[status.tag]; + } else { + type = pre_parse_finished_bd3(&status, resp); + req = (void *)(uintptr_t)status.tag; + } + + if (unlikely(type != type_supported)) { atomic64_inc(&dfx->err_bd_cnt); pr_err("err bd type [%d]\n", type); return; } - req = qp_ctx->req_list[le16_to_cpu(bd->type2.tag)]; if (unlikely(!req)) { atomic64_inc(&dfx->invalid_req_cnt); atomic_inc(&qp->qp_status.used); return; } - req->err_type = bd->type2.error_type; - ctx = req->ctx; - done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; - flag = (le16_to_cpu(bd->type2.done_flag) & - SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; - if (unlikely(req->err_type || done != SEC_SQE_DONE || - (ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) || - (ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) { - dev_err_ratelimited(ctx->dev, - "err_type[%d],done[%d],flag[%d]\n", - req->err_type, done, flag); - err = -EIO; - atomic64_inc(&dfx->done_flag_cnt); - } - if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) - err = sec_aead_verify(req); + req->err_type = status.err_type; + ctx = req->ctx; + err = sec_cb_status_check(req, &status); + if (err) + atomic64_inc(&dfx->done_flag_cnt); atomic64_inc(&dfx->recv_cnt); @@ -253,6 +319,30 @@ static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) res->c_ivin, res->c_ivin_dma); } +static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) +{ + int i; + + res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, + &res->a_ivin_dma, GFP_KERNEL); + if (!res->a_ivin) + return -ENOMEM; + + for (i = 1; i < QM_Q_DEPTH; i++) { + res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; + res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; + } + + return 0; +} + +static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->a_ivin) + dma_free_coherent(dev, SEC_TOTAL_IV_SZ, + res->a_ivin, res->a_ivin_dma); +} + static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) { int i; @@ -335,9 +425,13 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, return ret; if (ctx->alg_type == SEC_AEAD) { + ret = sec_alloc_aiv_resource(dev, res); + if (ret) + goto alloc_aiv_fail; + ret = sec_alloc_mac_resource(dev, res); if (ret) - goto alloc_fail; + goto alloc_mac_fail; } if (ctx->pbuf_supported) { ret = sec_alloc_pbuf_resource(dev, res); @@ -352,7 +446,10 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, alloc_pbuf_fail: if (ctx->alg_type == SEC_AEAD) sec_free_mac_resource(dev, qp_ctx->res); -alloc_fail: +alloc_mac_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_aiv_resource(dev, res); +alloc_aiv_fail: sec_free_civ_resource(dev, res); return ret; } @@ -382,10 +479,11 @@ static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, qp = ctx->qps[qp_ctx_id]; qp->req_type = 0; qp->qp_ctx = qp_ctx; - qp->req_cb = sec_req_cb; qp_ctx->qp = qp; qp_ctx->ctx = ctx; + qp->req_cb = sec_req_cb; + mutex_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); INIT_LIST_HEAD(&qp_ctx->backlog); @@ -536,6 +634,26 @@ static void sec_auth_uninit(struct sec_ctx *ctx) a_ctx->a_key, a_ctx->a_key_dma); } +static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) +{ + const char *alg = crypto_tfm_alg_name(&tfm->base); + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + + c_ctx->fallback = false; + if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) + return 0; + + c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(c_ctx->fbtfm)) { + pr_err("failed to alloc fallback tfm!\n"); + return PTR_ERR(c_ctx->fbtfm); + } + + return 0; +} + static int sec_skcipher_init(struct crypto_skcipher *tfm) { struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -557,8 +675,14 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) if (ret) goto err_cipher_init; + ret = sec_skcipher_fbtfm_init(tfm); + if (ret) + goto err_fbtfm_init; + return 0; +err_fbtfm_init: + sec_cipher_uninit(ctx); err_cipher_init: sec_ctx_base_uninit(ctx); return ret; @@ -568,6 +692,9 @@ static void sec_skcipher_uninit(struct crypto_skcipher *tfm) { struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + if (ctx->c_ctx.fbtfm) + crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); + sec_cipher_uninit(ctx); sec_ctx_base_uninit(ctx); } @@ -607,6 +734,9 @@ static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, case SEC_XTS_MIN_KEY_SIZE: c_ctx->c_key_len = SEC_CKEY_128BIT; break; + case SEC_XTS_MID_KEY_SIZE: + c_ctx->fallback = true; + break; case SEC_XTS_MAX_KEY_SIZE: c_ctx->c_key_len = SEC_CKEY_256BIT; break; @@ -615,19 +745,25 @@ static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, return -EINVAL; } } else { - switch (keylen) { - case AES_KEYSIZE_128: - c_ctx->c_key_len = SEC_CKEY_128BIT; - break; - case AES_KEYSIZE_192: - c_ctx->c_key_len = SEC_CKEY_192BIT; - break; - case AES_KEYSIZE_256: - c_ctx->c_key_len = SEC_CKEY_256BIT; - break; - default: - pr_err("hisi_sec2: aes key error!\n"); + if (c_ctx->c_alg == SEC_CALG_SM4 && + keylen != AES_KEYSIZE_128) { + pr_err("hisi_sec2: sm4 key error!\n"); return -EINVAL; + } else { + switch (keylen) { + case AES_KEYSIZE_128: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case AES_KEYSIZE_192: + c_ctx->c_key_len = SEC_CKEY_192BIT; + break; + case AES_KEYSIZE_256: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: aes key error!\n"); + return -EINVAL; + } } } @@ -672,7 +808,13 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - + if (c_ctx->fallback) { + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); + if (ret) { + dev_err(dev, "failed to set fallback skcipher key!\n"); + return ret; + } + } return 0; } @@ -686,22 +828,30 @@ static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) - +GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB) +GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB) +GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) - GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) +GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB) +GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB) +GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src) { - struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aead_req = a_req->aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; struct device *dev = ctx->dev; int copy_size, pbuf_length; int req_id = req->req_id; + struct crypto_aead *tfm; + size_t authsize; + u8 *mac_offset; if (ctx->alg_type == SEC_AEAD) copy_size = aead_req->cryptlen + aead_req->assoclen; @@ -709,15 +859,20 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, copy_size = c_req->c_len; pbuf_length = sg_copy_to_buffer(src, sg_nents(src), - qp_ctx->res[req_id].pbuf, - copy_size); + qp_ctx->res[req_id].pbuf, copy_size); if (unlikely(pbuf_length != copy_size)) { dev_err(dev, "copy src data to pbuf error!\n"); return -EINVAL; } + if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { + tfm = crypto_aead_reqtfm(aead_req); + authsize = crypto_aead_authsize(tfm); + mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize; + memcpy(a_req->out_mac, mac_offset, authsize); + } - c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; - c_req->c_out_dma = c_req->c_in_dma; + req->in_dma = qp_ctx->res[req_id].pbuf_dma; + c_req->c_out_dma = req->in_dma; return 0; } @@ -728,7 +883,6 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, struct aead_request *aead_req = req->aead_req.aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct device *dev = ctx->dev; int copy_size, pbuf_length; int req_id = req->req_id; @@ -738,10 +892,29 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, copy_size = c_req->c_len; pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), - qp_ctx->res[req_id].pbuf, - copy_size); + qp_ctx->res[req_id].pbuf, copy_size); if (unlikely(pbuf_length != copy_size)) - dev_err(dev, "copy pbuf data to dst error!\n"); + dev_err(ctx->dev, "copy pbuf data to dst error!\n"); +} + +static int sec_aead_mac_init(struct sec_aead_req *req) +{ + struct aead_request *aead_req = req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); + u8 *mac_out = req->out_mac; + struct scatterlist *sgl = aead_req->src; + size_t copy_size; + off_t skip_size; + + /* Copy input mac */ + skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; + copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, + authsize, skip_size); + if (unlikely(copy_size != authsize)) + return -EINVAL; + + return 0; } static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, @@ -755,37 +928,48 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, int ret; if (req->use_pbuf) { - ret = sec_cipher_pbuf_map(ctx, req, src); c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; if (ctx->alg_type == SEC_AEAD) { + a_req->a_ivin = res->a_ivin; + a_req->a_ivin_dma = res->a_ivin_dma; a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; a_req->out_mac_dma = res->pbuf_dma + SEC_PBUF_MAC_OFFSET; } + ret = sec_cipher_pbuf_map(ctx, req, src); return ret; } c_req->c_ivin = res->c_ivin; c_req->c_ivin_dma = res->c_ivin_dma; if (ctx->alg_type == SEC_AEAD) { + a_req->a_ivin = res->a_ivin; + a_req->a_ivin_dma = res->a_ivin_dma; a_req->out_mac = res->out_mac; a_req->out_mac_dma = res->out_mac_dma; } - c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, - qp_ctx->c_in_pool, - req->req_id, - &c_req->c_in_dma); - - if (IS_ERR(c_req->c_in)) { + req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, + qp_ctx->c_in_pool, + req->req_id, + &req->in_dma); + if (IS_ERR(req->in)) { dev_err(dev, "fail to dma map input sgl buffers!\n"); - return PTR_ERR(c_req->c_in); + return PTR_ERR(req->in); + } + + if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { + ret = sec_aead_mac_init(a_req); + if (unlikely(ret)) { + dev_err(dev, "fail to init mac data for ICV!\n"); + return ret; + } } if (dst == src) { - c_req->c_out = c_req->c_in; - c_req->c_out_dma = c_req->c_in_dma; + c_req->c_out = req->in; + c_req->c_out_dma = req->in_dma; } else { c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, qp_ctx->c_out_pool, @@ -794,7 +978,7 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, if (IS_ERR(c_req->c_out)) { dev_err(dev, "fail to dma map output sgl buffers!\n"); - hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); + hisi_acc_sg_buf_unmap(dev, src, req->in); return PTR_ERR(c_req->c_out); } } @@ -812,7 +996,7 @@ static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, sec_cipher_pbuf_unmap(ctx, req, dst); } else { if (dst != src) - hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); + hisi_acc_sg_buf_unmap(dev, src, req->in); hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); } @@ -883,6 +1067,28 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, return 0; } +static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct sec_ctx *ctx = crypto_tfm_ctx(tfm); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + + if (unlikely(a_ctx->fallback_aead_tfm)) + return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); + + return 0; +} + +static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, + struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(a_ctx->fallback_aead_tfm, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); +} + static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, const u32 keylen, const enum sec_hash_alg a_alg, const enum sec_calg c_alg, @@ -891,6 +1097,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, { struct sec_ctx *ctx = crypto_aead_ctx(tfm); struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; struct device *dev = ctx->dev; struct crypto_authenc_keys keys; int ret; @@ -900,6 +1107,23 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, ctx->a_ctx.mac_len = mac_len; c_ctx->c_mode = c_mode; + if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { + ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); + if (ret) { + dev_err(dev, "set sec aes ccm cipher key err!\n"); + return ret; + } + memcpy(c_ctx->c_key, key, keylen); + + if (unlikely(a_ctx->fallback_aead_tfm)) { + ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + if (ret) + return ret; + } + + return 0; + } + if (crypto_authenc_extractkeys(&keys, key, keylen)) goto bad_key; @@ -915,6 +1139,12 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, goto bad_key; } + if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || + (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { + dev_err(dev, "MAC or AUTH key length error!\n"); + goto bad_key; + } + return 0; bad_key: @@ -936,6 +1166,14 @@ GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, + SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, + SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, + SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, + SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) { @@ -998,7 +1236,7 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); - sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); + sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << @@ -1014,29 +1252,86 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; sec_sqe->type_cipher_auth = bd_type | cipher; - if (req->use_pbuf) + /* Set destination and source address type */ + if (req->use_pbuf) { sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; - else + da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; + } else { sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; + da_type = SEC_SGL << SEC_DST_SGL_OFFSET; + } + + sec_sqe->sdm_addr_type |= da_type; scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; - if (c_req->c_in_dma != c_req->c_out_dma) + if (req->in_dma != c_req->c_out_dma) de = 0x1 << SEC_DE_OFFSET; sec_sqe->sds_sa_type = (de | scene | sa_type); - /* Just set DST address type */ - if (req->use_pbuf) - da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; - else - da_type = SEC_SGL << SEC_DST_SGL_OFFSET; - sec_sqe->sdm_addr_type |= da_type; - sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); return 0; } +static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_cipher_req *c_req = &req->c_req; + u32 bd_param = 0; + u16 cipher; + + memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); + + sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); + sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); + sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); + sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); + + sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | + c_ctx->c_mode; + sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << + SEC_CKEY_OFFSET_V3); + + if (c_req->encrypt) + cipher = SEC_CIPHER_ENC; + else + cipher = SEC_CIPHER_DEC; + sec_sqe3->c_icv_key |= cpu_to_le16(cipher); + + if (req->use_pbuf) { + bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; + bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; + } else { + bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; + bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; + } + + bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; + if (req->in_dma != c_req->c_out_dma) + bd_param |= 0x1 << SEC_DE_OFFSET_V3; + + bd_param |= SEC_BD_TYPE3; + sec_sqe3->bd_param = cpu_to_le32(bd_param); + + sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); + sec_sqe3->tag = cpu_to_le64(req); + + return 0; +} + +/* increment counter (128-bit int) */ +static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) +{ + do { + --bits; + nums += counter[bits]; + counter[bits] = nums & BITS_MASK; + nums >>= BYTE_BITS; + } while (bits && nums); +} + static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) { struct aead_request *aead_req = req->aead_req.aead_req; @@ -1060,10 +1355,17 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) cryptlen = aead_req->cryptlen; } - sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, - cryptlen - iv_size); - if (unlikely(sz != iv_size)) - dev_err(req->ctx->dev, "copy output iv error!\n"); + if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { + sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, + cryptlen - iv_size); + if (unlikely(sz != iv_size)) + dev_err(req->ctx->dev, "copy output iv error!\n"); + } else { + sz = cryptlen / iv_size; + if (cryptlen % iv_size) + sz += 1; + ctr_iv_inc(iv, iv_size, sz); + } } static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, @@ -1094,8 +1396,9 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, sec_free_req_id(req); - /* IV output at encrypto of CBC mode */ - if (!err && ctx->c_ctx.c_mode == SEC_CMODE_CBC && req->c_req.encrypt) + /* IV output at encrypto of CBC/CTR mode */ + if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || + ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) sec_update_iv(req, SEC_SKCIPHER); while (1) { @@ -1112,12 +1415,125 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, sk_req->base.complete(&sk_req->base, err); } -static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req) +static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) { struct aead_request *aead_req = req->aead_req.aead_req; struct sec_cipher_req *c_req = &req->c_req; + struct sec_aead_req *a_req = &req->aead_req; + size_t authsize = ctx->a_ctx.mac_len; + u32 data_size = aead_req->cryptlen; + u8 flage = 0; + u8 cm, cl; + + /* the specification has been checked in aead_iv_demension_check() */ + cl = c_req->c_ivin[0] + 1; + c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; + memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); + c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; + + /* the last 3bit is L' */ + flage |= c_req->c_ivin[0] & IV_CL_MASK; + + /* the M' is bit3~bit5, the Flags is bit6 */ + cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; + flage |= cm << IV_CM_OFFSET; + if (aead_req->assoclen) + flage |= 0x01 << IV_FLAGS_OFFSET; + + memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); + a_req->a_ivin[0] = flage; + + /* + * the last 32bit is counter's initial number, + * but the nonce uses the first 16bit + * the tail 16bit fill with the cipher length + */ + if (!c_req->encrypt) + data_size = aead_req->cryptlen - authsize; + + a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = + data_size & IV_LAST_BYTE_MASK; + data_size >>= IV_BYTE_OFFSET; + a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = + data_size & IV_LAST_BYTE_MASK; +} + +static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); + struct sec_cipher_req *c_req = &req->c_req; + struct sec_aead_req *a_req = &req->aead_req; memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); + + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { + /* + * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, + * the counter must set to 0x01 + */ + ctx->a_ctx.mac_len = authsize; + /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ + set_aead_auth_iv(ctx, req); + } + + /* GCM 12Byte Cipher_IV == Auth_IV */ + if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { + ctx->a_ctx.mac_len = authsize; + memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); + } +} + +static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe *sec_sqe) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aq = a_req->aead_req; + + /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ + sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len); + + /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ + sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; + sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); + sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; + + if (dir) + sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; + else + sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; + + sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); + sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); + sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe3 *sqe3) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aq = a_req->aead_req; + + /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ + sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3); + + /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ + sqe3->a_key_addr = sqe3->c_key_addr; + sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); + sqe3->auth_mac_key |= SEC_NO_AUTH; + + if (dir) + sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; + else + sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; + + sqe3->a_len_key = cpu_to_le32(aq->assoclen); + sqe3->auth_src_offset = cpu_to_le16(0x0); + sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); } static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, @@ -1139,13 +1555,13 @@ static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, sec_sqe->type2.mac_key_alg |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); - sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; - - if (dir) + if (dir) { + sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; - else + } else { + sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET; sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; - + } sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); @@ -1165,7 +1581,68 @@ static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) return ret; } - sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || + ctx->c_ctx.c_mode == SEC_CMODE_GCM) + sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); + else + sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); + + return 0; +} + +static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe3 *sqe3) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct aead_request *aq = a_req->aead_req; + + sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)(ctx->mac_len / + SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)(ctx->a_key_len / + SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); + + if (dir) { + sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); + sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; + } else { + sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); + sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; + } + sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); + + sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; + struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; + int ret; + + ret = sec_skcipher_bd_fill_v3(ctx, req); + if (unlikely(ret)) { + dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); + return ret; + } + + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || + ctx->c_ctx.c_mode == SEC_CMODE_GCM) + sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, + req, sec_sqe3); + else + sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, + req, sec_sqe3); return 0; } @@ -1254,7 +1731,8 @@ static int sec_process(struct sec_ctx *ctx, struct sec_req *req) goto err_uninit_req; /* Output IV as decrypto */ - if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) + if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || + ctx->c_ctx.c_mode == SEC_CMODE_CTR)) sec_update_iv(req, ctx->alg_type); ret = ctx->req_op->bd_send(ctx, req); @@ -1296,20 +1774,51 @@ static const struct sec_req_op sec_skcipher_req_ops = { static const struct sec_req_op sec_aead_req_ops = { .buf_map = sec_aead_sgl_map, .buf_unmap = sec_aead_sgl_unmap, - .do_transfer = sec_aead_copy_iv, + .do_transfer = sec_aead_set_iv, .bd_fill = sec_aead_bd_fill, .bd_send = sec_bd_send, .callback = sec_aead_callback, .process = sec_process, }; +static const struct sec_req_op sec_skcipher_req_ops_v3 = { + .buf_map = sec_skcipher_sgl_map, + .buf_unmap = sec_skcipher_sgl_unmap, + .do_transfer = sec_skcipher_copy_iv, + .bd_fill = sec_skcipher_bd_fill_v3, + .bd_send = sec_bd_send, + .callback = sec_skcipher_callback, + .process = sec_process, +}; + +static const struct sec_req_op sec_aead_req_ops_v3 = { + .buf_map = sec_aead_sgl_map, + .buf_unmap = sec_aead_sgl_unmap, + .do_transfer = sec_aead_set_iv, + .bd_fill = sec_aead_bd_fill_v3, + .bd_send = sec_bd_send, + .callback = sec_aead_callback, + .process = sec_process, +}; + static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) { struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; - ctx->req_op = &sec_skcipher_req_ops; + ret = sec_skcipher_init(tfm); + if (ret) + return ret; - return sec_skcipher_init(tfm); + if (ctx->sec->qm.ver < QM_HW_V3) { + ctx->type_supported = SEC_BD_TYPE2; + ctx->req_op = &sec_skcipher_req_ops; + } else { + ctx->type_supported = SEC_BD_TYPE3; + ctx->req_op = &sec_skcipher_req_ops_v3; + } + + return ret; } static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) @@ -1325,15 +1834,22 @@ static int sec_aead_init(struct crypto_aead *tfm) crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); ctx->alg_type = SEC_AEAD; ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); - if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { - dev_err(ctx->dev, "get error aead iv size!\n"); + if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || + ctx->c_ctx.ivsize > SEC_IV_SIZE) { + pr_err("get error aead iv size!\n"); return -EINVAL; } - ctx->req_op = &sec_aead_req_ops; ret = sec_ctx_base_init(ctx); if (ret) return ret; + if (ctx->sec->qm.ver < QM_HW_V3) { + ctx->type_supported = SEC_BD_TYPE2; + ctx->req_op = &sec_aead_req_ops; + } else { + ctx->type_supported = SEC_BD_TYPE3; + ctx->req_op = &sec_aead_req_ops_v3; + } ret = sec_auth_init(ctx); if (ret) @@ -1391,6 +1907,41 @@ static void sec_aead_ctx_exit(struct crypto_aead *tfm) sec_aead_exit(tfm); } +static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + const char *aead_name = alg->base.cra_name; + int ret; + + ret = sec_aead_init(tfm); + if (ret) { + dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); + return ret; + } + + a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(a_ctx->fallback_aead_tfm)) { + dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); + sec_aead_exit(tfm); + return PTR_ERR(a_ctx->fallback_aead_tfm); + } + a_ctx->fallback = false; + + return 0; +} + +static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); + sec_aead_exit(tfm); +} + static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) { return sec_aead_ctx_init(tfm, "sha1"); @@ -1429,6 +1980,14 @@ static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, ret = -EINVAL; } break; + case SEC_CMODE_CFB: + case SEC_CMODE_OFB: + case SEC_CMODE_CTR: + if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { + dev_err(dev, "skcipher HW version error!\n"); + ret = -EINVAL; + } + break; default: ret = -EINVAL; } @@ -1442,7 +2001,8 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; - if (unlikely(!sk_req->src || !sk_req->dst)) { + if (unlikely(!sk_req->src || !sk_req->dst || + sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { dev_err(dev, "skcipher input param error!\n"); return -EINVAL; } @@ -1468,6 +2028,37 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } +static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, + struct skcipher_request *sreq, bool encrypt) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct device *dev = ctx->dev; + int ret; + + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); + + if (!c_ctx->fbtfm) { + dev_err(dev, "failed to check fallback tfm\n"); + return -EINVAL; + } + + skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); + + /* software need sync mode to do crypto */ + skcipher_request_set_callback(subreq, sreq->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, + sreq->cryptlen, sreq->iv); + if (encrypt) + ret = crypto_skcipher_encrypt(subreq); + else + ret = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return ret; +} + static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); @@ -1475,8 +2066,11 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - if (!sk_req->cryptlen) + if (!sk_req->cryptlen) { + if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) + return -EINVAL; return 0; + } req->flag = sk_req->base.flags; req->c_req.sk_req = sk_req; @@ -1487,6 +2081,9 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) if (unlikely(ret)) return -EINVAL; + if (unlikely(ctx->c_ctx.fallback)) + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); + return ctx->req_op->process(ctx, req); } @@ -1507,7 +2104,9 @@ static int sec_skcipher_decrypt(struct skcipher_request *sk_req) .cra_name = sec_cra_name,\ .cra_driver_name = "hisi_sec_"sec_cra_name,\ .cra_priority = SEC_PRIORITY,\ - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ + .cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_ALLOCATES_MEMORY |\ + CRYPTO_ALG_NEED_FALLBACK,\ .cra_blocksize = blk_size,\ .cra_ctxsize = sizeof(struct sec_ctx),\ .cra_module = THIS_MODULE,\ @@ -1541,11 +2140,11 @@ static struct skcipher_alg sec_skciphers[] = { AES_BLOCK_SIZE, AES_BLOCK_SIZE) SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, - SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0) SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, - SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, @@ -1557,6 +2156,90 @@ static struct skcipher_alg sec_skciphers[] = { AES_BLOCK_SIZE, AES_BLOCK_SIZE) }; +static struct skcipher_alg sec_skciphers_v3[] = { + SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, + AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, + AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, + AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) +}; + +static int aead_iv_demension_check(struct aead_request *aead_req) +{ + u8 cl; + + cl = aead_req->iv[0] + 1; + if (cl < IV_CL_MIN || cl > IV_CL_MAX) + return -EINVAL; + + if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) + return -EOVERFLOW; + + return 0; +} + +static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) +{ + struct aead_request *req = sreq->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + size_t authsize = crypto_aead_authsize(tfm); + u8 c_mode = ctx->c_ctx.c_mode; + struct device *dev = ctx->dev; + int ret; + + if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || + req->assoclen > SEC_MAX_AAD_LEN)) { + dev_err(dev, "aead input spec error!\n"); + return -EINVAL; + } + + if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) || + (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN || + authsize & MAC_LEN_MASK)))) { + dev_err(dev, "aead input mac length error!\n"); + return -EINVAL; + } + + if (c_mode == SEC_CMODE_CCM) { + ret = aead_iv_demension_check(req); + if (ret) { + dev_err(dev, "aead input iv param error!\n"); + return ret; + } + } + + if (sreq->c_req.encrypt) + sreq->c_req.c_len = req->cryptlen; + else + sreq->c_req.c_len = req->cryptlen - authsize; + if (c_mode == SEC_CMODE_CBC) { + if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "aead crypto length error!\n"); + return -EINVAL; + } + } + + return 0; +} + static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) { struct aead_request *req = sreq->aead_req.aead_req; @@ -1565,36 +2248,63 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; - if (unlikely(!req->src || !req->dst || !req->cryptlen || - req->assoclen > SEC_MAX_AAD_LEN)) { + if (unlikely(!req->src || !req->dst)) { dev_err(dev, "aead input param error!\n"); return -EINVAL; } + if (ctx->sec->qm.ver == QM_HW_V2) { + if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && + req->cryptlen <= authsize))) { + dev_err(dev, "Kunpeng920 not support 0 length!\n"); + ctx->a_ctx.fallback = true; + return -EINVAL; + } + } + + /* Support AES or SM4 */ + if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { + dev_err(dev, "aead crypto alg error!\n"); + return -EINVAL; + } + + if (unlikely(sec_aead_spec_check(ctx, sreq))) + return -EINVAL; + if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= SEC_PBUF_SZ) sreq->use_pbuf = true; else sreq->use_pbuf = false; - /* Support AES only */ - if (unlikely(c_alg != SEC_CALG_AES)) { - dev_err(dev, "aead crypto alg error!\n"); - return -EINVAL; - } - if (sreq->c_req.encrypt) - sreq->c_req.c_len = req->cryptlen; - else - sreq->c_req.c_len = req->cryptlen - authsize; - - if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { - dev_err(dev, "aead crypto length error!\n"); - return -EINVAL; - } - return 0; } +static int sec_aead_soft_crypto(struct sec_ctx *ctx, + struct aead_request *aead_req, + bool encrypt) +{ + struct aead_request *subreq = aead_request_ctx(aead_req); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + struct device *dev = ctx->dev; + + /* Kunpeng920 aead mode not support input 0 size */ + if (!a_ctx->fallback_aead_tfm) { + dev_err(dev, "aead fallback tfm is NULL!\n"); + return -EINVAL; + } + + aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); + aead_request_set_callback(subreq, aead_req->base.flags, + aead_req->base.complete, aead_req->base.data); + aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, + aead_req->cryptlen, aead_req->iv); + aead_request_set_ad(subreq, aead_req->assoclen); + + return encrypt ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); +} + static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) { struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); @@ -1608,8 +2318,11 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) req->ctx = ctx; ret = sec_aead_param_check(ctx, req); - if (unlikely(ret)) + if (unlikely(ret)) { + if (ctx->a_ctx.fallback) + return sec_aead_soft_crypto(ctx, a_req, encrypt); return -EINVAL; + } return ctx->req_op->process(ctx, req); } @@ -1624,14 +2337,16 @@ static int sec_aead_decrypt(struct aead_request *a_req) return sec_aead_crypto(a_req, false); } -#define SEC_AEAD_GEN_ALG(sec_cra_name, sec_set_key, ctx_init,\ +#define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ ctx_exit, blk_size, iv_size, max_authsize)\ {\ .base = {\ .cra_name = sec_cra_name,\ .cra_driver_name = "hisi_sec_"sec_cra_name,\ .cra_priority = SEC_PRIORITY,\ - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ + .cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_ALLOCATES_MEMORY |\ + CRYPTO_ALG_NEED_FALLBACK,\ .cra_blocksize = blk_size,\ .cra_ctxsize = sizeof(struct sec_ctx),\ .cra_module = THIS_MODULE,\ @@ -1639,28 +2354,46 @@ static int sec_aead_decrypt(struct aead_request *a_req) .init = ctx_init,\ .exit = ctx_exit,\ .setkey = sec_set_key,\ + .setauthsize = sec_aead_setauthsize,\ .decrypt = sec_aead_decrypt,\ .encrypt = sec_aead_encrypt,\ .ivsize = iv_size,\ .maxauthsize = max_authsize,\ } -#define SEC_AEAD_ALG(algname, keyfunc, aead_init, blksize, ivsize, authsize)\ - SEC_AEAD_GEN_ALG(algname, keyfunc, aead_init,\ - sec_aead_ctx_exit, blksize, ivsize, authsize) - static struct aead_alg sec_aeads[] = { SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, - AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), + sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, - AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), + sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, - AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + + SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + AES_BLOCK_SIZE, AES_BLOCK_SIZE), + + SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + SEC_AIV_SIZE, AES_BLOCK_SIZE) +}; + +static struct aead_alg sec_aeads_v3[] = { + SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + AES_BLOCK_SIZE, AES_BLOCK_SIZE), + + SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + SEC_AIV_SIZE, AES_BLOCK_SIZE) }; int sec_register_to_crypto(struct hisi_qm *qm) @@ -1673,16 +2406,45 @@ int sec_register_to_crypto(struct hisi_qm *qm) if (ret) return ret; + if (qm->ver > QM_HW_V2) { + ret = crypto_register_skciphers(sec_skciphers_v3, + ARRAY_SIZE(sec_skciphers_v3)); + if (ret) + goto reg_skcipher_fail; + } + ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); if (ret) - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); + goto reg_aead_fail; + if (qm->ver > QM_HW_V2) { + ret = crypto_register_aeads(sec_aeads_v3, ARRAY_SIZE(sec_aeads_v3)); + if (ret) + goto reg_aead_v3_fail; + } + return ret; + +reg_aead_v3_fail: + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); +reg_aead_fail: + if (qm->ver > QM_HW_V2) + crypto_unregister_skciphers(sec_skciphers_v3, + ARRAY_SIZE(sec_skciphers_v3)); +reg_skcipher_fail: + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); return ret; } void sec_unregister_from_crypto(struct hisi_qm *qm) { + if (qm->ver > QM_HW_V2) + crypto_unregister_aeads(sec_aeads_v3, + ARRAY_SIZE(sec_aeads_v3)); + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + + if (qm->ver > QM_HW_V2) + crypto_unregister_skciphers(sec_skciphers_v3, + ARRAY_SIZE(sec_skciphers_v3)); crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers)); - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); } diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h index 9c78edac56a4..9f71c358a6d3 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.h +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -4,9 +4,11 @@ #ifndef __HISI_SEC_V2_CRYPTO_H #define __HISI_SEC_V2_CRYPTO_H +#define SEC_AIV_SIZE 12 #define SEC_IV_SIZE 24 #define SEC_MAX_KEY_SIZE 64 #define SEC_COMM_SCENE 0 +#define SEC_MIN_BLOCK_SZ 1 enum sec_calg { SEC_CALG_3DES = 0x1, @@ -21,6 +23,11 @@ enum sec_hash_alg { }; enum sec_mac_len { + SEC_HMAC_CCM_MAC = 16, + SEC_HMAC_GCM_MAC = 16, + SEC_SM3_MAC = 32, + SEC_HMAC_SM3_MAC = 32, + SEC_HMAC_MD5_MAC = 16, SEC_HMAC_SHA1_MAC = 20, SEC_HMAC_SHA256_MAC = 32, SEC_HMAC_SHA512_MAC = 64, @@ -29,7 +36,11 @@ enum sec_mac_len { enum sec_cmode { SEC_CMODE_ECB = 0x0, SEC_CMODE_CBC = 0x1, + SEC_CMODE_CFB = 0x2, + SEC_CMODE_OFB = 0x3, SEC_CMODE_CTR = 0x4, + SEC_CMODE_CCM = 0x5, + SEC_CMODE_GCM = 0x6, SEC_CMODE_XTS = 0x7, }; @@ -44,6 +55,7 @@ enum sec_ckey_type { enum sec_bd_type { SEC_BD_TYPE1 = 0x1, SEC_BD_TYPE2 = 0x2, + SEC_BD_TYPE3 = 0x3, }; enum sec_auth { @@ -63,6 +75,24 @@ enum sec_addr_type { SEC_PRP = 0x2, }; +struct bd_status { + u64 tag; + u8 done; + u8 err_type; + u16 flag; + u16 icv; +}; + +enum { + AUTHPAD_PAD, + AUTHPAD_NOPAD, +}; + +enum { + AIGEN_GEN, + AIGEN_NOGEN, +}; + struct sec_sqe_type2 { /* * mac_len: 0~4 bits @@ -209,6 +239,169 @@ struct sec_sqe { struct sec_sqe_type2 type2; }; +struct bd3_auth_ivin { + __le64 a_ivin_addr; + __le32 rsvd0; + __le32 rsvd1; +} __packed __aligned(4); + +struct bd3_skip_data { + __le32 rsvd0; + + /* + * gran_num: 0~15 bits + * reserved: 16~31 bits + */ + __le32 gran_num; + + /* + * src_skip_data_len: 0~24 bits + * reserved: 25~31 bits + */ + __le32 src_skip_data_len; + + /* + * dst_skip_data_len: 0~24 bits + * reserved: 25~31 bits + */ + __le32 dst_skip_data_len; +}; + +struct bd3_stream_scene { + __le64 c_ivin_addr; + __le64 long_a_data_len; + + /* + * auth_pad: 0~1 bits + * stream_protocol: 2~4 bits + * reserved: 5~7 bits + */ + __u8 stream_auth_pad; + __u8 plaintext_type; + __le16 pad_len_1p3; +} __packed __aligned(4); + +struct bd3_no_scene { + __le64 c_ivin_addr; + __le32 rsvd0; + __le32 rsvd1; + __le32 rsvd2; +} __packed __aligned(4); + +struct bd3_check_sum { + __u8 rsvd0; + __u8 hac_sva_status; + __le16 check_sum_i; +}; + +struct bd3_tls_type_back { + __u8 tls_1p3_type_back; + __u8 hac_sva_status; + __le16 pad_len_1p3_back; +}; + +struct sec_sqe3 { + /* + * type: 0~3 bit + * bd_invalid: 4 bit + * scene: 5~8 bit + * de: 9~10 bit + * src_addr_type: 11~13 bit + * dst_addr_type: 14~16 bit + * mac_addr_type: 17~19 bit + * reserved: 20~31 bits + */ + __le32 bd_param; + + /* + * cipher: 0~1 bits + * ci_gen: 2~3 bit + * c_icv_len: 4~9 bit + * c_width: 10~12 bits + * c_key_len: 13~15 bits + */ + __le16 c_icv_key; + + /* + * c_mode : 0~3 bits + * c_alg : 4~7 bits + */ + __u8 c_mode_alg; + + /* + * nonce_len : 0~3 bits + * huk : 4 bits + * cal_iv_addr_en : 5 bits + * seq : 6 bits + * reserved : 7 bits + */ + __u8 huk_iv_seq; + + __le64 tag; + __le64 data_src_addr; + __le64 a_key_addr; + union { + struct bd3_auth_ivin auth_ivin; + struct bd3_skip_data skip_data; + }; + + __le64 c_key_addr; + + /* + * auth: 0~1 bits + * ai_gen: 2~3 bits + * mac_len: 4~8 bits + * akey_len: 9~14 bits + * a_alg: 15~20 bits + * key_sel: 21~24 bits + * updata_key: 25 bits + * reserved: 26~31 bits + */ + __le32 auth_mac_key; + __le32 salt; + __le16 auth_src_offset; + __le16 cipher_src_offset; + + /* + * auth_len: 0~23 bit + * auth_key_offset: 24~31 bits + */ + __le32 a_len_key; + + /* + * cipher_len: 0~23 bit + * auth_ivin_offset: 24~31 bits + */ + __le32 c_len_ivin; + __le64 data_dst_addr; + __le64 mac_addr; + union { + struct bd3_stream_scene stream_scene; + struct bd3_no_scene no_scene; + }; + + /* + * done: 0 bit + * icv: 1~3 bit + * csc: 4~6 bit + * flag: 7~10 bit + * reserved: 11~15 bit + */ + __le16 done_flag; + __u8 error_type; + __u8 warning_type; + union { + __le32 mac_i; + __le32 kek_key_addr_l; + }; + union { + __le32 kek_key_addr_h; + struct bd3_check_sum check_sum; + struct bd3_tls_type_back tls_type_back; + }; + __le32 counter; +} __packed __aligned(4); + int sec_register_to_crypto(struct hisi_qm *qm); void sec_unregister_from_crypto(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 6f0062d4408c..d120ce3e34ed 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -52,6 +52,7 @@ #define SEC_RAS_CE_ENB_MSK 0x88 #define SEC_RAS_FE_ENB_MSK 0x0 #define SEC_RAS_NFE_ENB_MSK 0x7c177 +#define SEC_OOO_SHUTDOWN_SEL 0x301014 #define SEC_RAS_DISABLE 0x0 #define SEC_MEM_START_INIT_REG 0x301100 #define SEC_MEM_INIT_DONE_REG 0x301104 @@ -84,6 +85,12 @@ #define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET) #define SEC_CORE_INT_STATUS_M_ECC BIT(2) +#define SEC_PREFETCH_CFG 0x301130 +#define SEC_SVA_TRANS 0x301EC4 +#define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11))) +#define SEC_PREFETCH_DISABLE BIT(1) +#define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11)) + #define SEC_DELAY_10_US 10 #define SEC_POLL_TIMEOUT_US 1000 #define SEC_DBGFS_VAL_MAX_LEN 20 @@ -91,6 +98,7 @@ #define SEC_SQE_MASK_OFFSET 64 #define SEC_SQE_MASK_LEN 48 +#define SEC_SHAPER_TYPE_RATE 128 struct sec_hw_error { u32 int_msk; @@ -331,6 +339,42 @@ static u8 sec_get_endian(struct hisi_qm *qm) return SEC_64BE; } +static void sec_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val &= SEC_PREFETCH_ENABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, + val, !(val & SEC_PREFETCH_DISABLE), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to open sva prefetch\n"); +} + +static void sec_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val |= SEC_PREFETCH_DISABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, + val, !(val & SEC_SVA_DISABLE_READY), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); +} + static int sec_engine_init(struct hisi_qm *qm) { int ret; @@ -430,53 +474,60 @@ static void sec_debug_regs_clear(struct hisi_qm *qm) hisi_qm_debug_regs_clear(qm); } +static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val1, val2; + + val1 = readl(qm->io_base + SEC_CONTROL_REG); + if (enable) { + val1 |= SEC_AXI_SHUTDOWN_ENABLE; + val2 = SEC_RAS_NFE_ENB_MSK; + } else { + val1 &= SEC_AXI_SHUTDOWN_DISABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + SEC_CONTROL_REG); +} + static void sec_hw_error_enable(struct hisi_qm *qm) { - u32 val; - if (qm->ver == QM_HW_V1) { writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); pci_info(qm->pdev, "V1 not support hw error handle\n"); return; } - val = readl(qm->io_base + SEC_CONTROL_REG); - /* clear SEC hw error source if having */ writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE); - /* enable SEC hw error interrupts */ - writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); - /* enable RAS int */ writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG); writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG); - /* enable SEC block master OOO when m-bit error occur */ - val = val | SEC_AXI_SHUTDOWN_ENABLE; + /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ + sec_master_ooo_ctrl(qm, true); - writel(val, qm->io_base + SEC_CONTROL_REG); + /* enable SEC hw error interrupts */ + writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); } static void sec_hw_error_disable(struct hisi_qm *qm) { - u32 val; + /* disable SEC hw error interrupts */ + writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); - val = readl(qm->io_base + SEC_CONTROL_REG); + /* disable SEC block master OOO when nfe occurs on Kunpeng930 */ + sec_master_ooo_ctrl(qm, false); /* disable RAS int */ writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG); writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG); - - /* disable SEC hw error interrupts */ - writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); - - /* disable SEC block master OOO when m-bit error occur */ - val = val & SEC_AXI_SHUTDOWN_DISABLE; - - writel(val, qm->io_base + SEC_CONTROL_REG); } static u32 sec_clear_enable_read(struct sec_debug_file *file) @@ -743,6 +794,8 @@ static const struct hisi_qm_err_ini sec_err_ini = { .clear_dev_hw_err_status = sec_clear_hw_err_status, .log_dev_hw_err = sec_log_hw_error, .open_axi_master_ooo = sec_open_axi_master_ooo, + .open_sva_prefetch = sec_open_sva_prefetch, + .close_sva_prefetch = sec_close_sva_prefetch, .err_info_init = sec_err_info_init, }; @@ -758,6 +811,7 @@ static int sec_pf_probe_init(struct sec_dev *sec) if (ret) return ret; + sec_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); sec_debug_regs_clear(qm); @@ -821,6 +875,7 @@ static void sec_qm_uninit(struct hisi_qm *qm) static int sec_probe_init(struct sec_dev *sec) { + u32 type_rate = SEC_SHAPER_TYPE_RATE; struct hisi_qm *qm = &sec->qm; int ret; @@ -828,6 +883,11 @@ static int sec_probe_init(struct sec_dev *sec) ret = sec_pf_probe_init(sec); if (ret) return ret; + /* enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + qm->type_rate = type_rate; + } } return 0; diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 2178b40e9f82..f8482ceebf2a 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -68,6 +68,7 @@ #define HZIP_CORE_INT_RAS_CE_ENABLE 0x1 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168 +#define HZIP_OOO_SHUTDOWN_SEL 0x30120C #define HZIP_CORE_INT_RAS_NFE_ENABLE 0x1FFE #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 @@ -96,6 +97,16 @@ #define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \ HZIP_RO_CNT_CLR_CE_EN) +#define HZIP_PREFETCH_CFG 0x3011B0 +#define HZIP_SVA_TRANS 0x3011C4 +#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0))) +#define HZIP_SVA_PREFETCH_DISABLE BIT(26) +#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30)) +#define HZIP_SHAPER_RATE_COMPRESS 252 +#define HZIP_SHAPER_RATE_DECOMPRESS 229 +#define HZIP_DELAY_1_US 1 +#define HZIP_POLL_TIMEOUT_US 1000 + static const char hisi_zip_name[] = "hisi_zip"; static struct dentry *hzip_debugfs_root; @@ -262,6 +273,45 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); } +static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); + val &= HZIP_PREFETCH_ENABLE; + writel(val, qm->io_base + HZIP_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG, + val, !(val & HZIP_SVA_PREFETCH_DISABLE), + HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to open sva prefetch\n"); +} + +static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); + val |= HZIP_SVA_PREFETCH_DISABLE; + writel(val, qm->io_base + HZIP_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS, + val, !(val & HZIP_SVA_DISABLE_READY), + HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); +} + static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) { void __iomem *base = qm->io_base; @@ -312,10 +362,27 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) return 0; } +static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) +{ + u32 val1, val2; + + val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + if (enable) { + val1 |= HZIP_AXI_SHUTDOWN_ENABLE; + val2 = HZIP_CORE_INT_RAS_NFE_ENABLE; + } else { + val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); +} + static void hisi_zip_hw_error_enable(struct hisi_qm *qm) { - u32 val; - if (qm->ver == QM_HW_V1) { writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); @@ -333,26 +400,20 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) writel(HZIP_CORE_INT_RAS_NFE_ENABLE, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + /* enable ZIP block master OOO when nfe occurs on Kunpeng930 */ + hisi_zip_master_ooo_ctrl(qm, true); + /* enable ZIP hw error interrupts */ writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); - - /* enable ZIP block master OOO when m-bit error occur */ - val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); - val = val | HZIP_AXI_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); } static void hisi_zip_hw_error_disable(struct hisi_qm *qm) { - u32 val; - /* disable ZIP hw error interrupts */ writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); - /* disable ZIP block master OOO when m-bit error occur */ - val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); - val = val & ~HZIP_AXI_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + /* disable ZIP block master OOO when nfe occurs on Kunpeng930 */ + hisi_zip_master_ooo_ctrl(qm, false); } static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) @@ -684,6 +745,8 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .log_dev_hw_err = hisi_zip_log_hw_error, .open_axi_master_ooo = hisi_zip_open_axi_master_ooo, .close_axi_master_ooo = hisi_zip_close_axi_master_ooo, + .open_sva_prefetch = hisi_zip_open_sva_prefetch, + .close_sva_prefetch = hisi_zip_close_sva_prefetch, .err_info_init = hisi_zip_err_info_init, }; @@ -702,6 +765,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) qm->err_ini->err_info_init(qm); hisi_zip_set_user_domain_and_cache(qm); + hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); @@ -761,6 +825,7 @@ static void hisi_zip_qm_uninit(struct hisi_qm *qm) static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) { + u32 type_rate = HZIP_SHAPER_RATE_COMPRESS; struct hisi_qm *qm = &hisi_zip->qm; int ret; @@ -768,6 +833,14 @@ static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) ret = hisi_zip_pf_probe_init(hisi_zip); if (ret) return ret; + /* enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + + /* ZIP need to enable shaper type 1 */ + type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET; + qm->type_rate = type_rate; + } } return 0; diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 0616e369522e..35fc5ee70491 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -71,15 +72,11 @@ #define MOD_AES256 (0x0a00 | KEYLEN_256) #define MAX_IVLEN 16 -#define NPE_ID 2 /* NPE C */ #define NPE_QLEN 16 /* Space for registering when the first * NPE_QLEN crypt_ctl are busy */ #define NPE_QLEN_TOTAL 64 -#define SEND_QID 29 -#define RECV_QID 30 - #define CTL_FLAG_UNUSED 0x0000 #define CTL_FLAG_USED 0x1000 #define CTL_FLAG_PERFORM_ABLK 0x0001 @@ -136,7 +133,7 @@ struct crypt_ctl { u32 crypto_ctx; /* NPE Crypto Param structure address */ /* Used by Host: 4*4 bytes*/ - unsigned ctl_flags; + unsigned int ctl_flags; union { struct skcipher_request *ablk_req; struct aead_request *aead_req; @@ -149,6 +146,9 @@ struct crypt_ctl { struct ablk_ctx { struct buffer_desc *src; struct buffer_desc *dst; + u8 iv[MAX_IVLEN]; + bool encrypt; + struct skcipher_request fallback_req; // keep at the end }; struct aead_ctx { @@ -181,9 +181,10 @@ struct ixp_ctx { u8 enckey[MAX_KEYLEN]; u8 salt[MAX_IVLEN]; u8 nonce[CTR_RFC3686_NONCE_SIZE]; - unsigned salted; + unsigned int salted; atomic_t configuring; struct completion completion; + struct crypto_skcipher *fallback_tfm; }; struct ixp_alg { @@ -209,6 +210,7 @@ static const struct ix_hash_algo hash_alg_md5 = { .icv = "\x01\x23\x45\x67\x89\xAB\xCD\xEF" "\xFE\xDC\xBA\x98\x76\x54\x32\x10", }; + static const struct ix_hash_algo hash_alg_sha1 = { .cfgword = 0x00000005, .icv = "\x67\x45\x23\x01\xEF\xCD\xAB\x89\x98\xBA" @@ -216,16 +218,17 @@ static const struct ix_hash_algo hash_alg_sha1 = { }; static struct npe *npe_c; -static struct dma_pool *buffer_pool = NULL; -static struct dma_pool *ctx_pool = NULL; -static struct crypt_ctl *crypt_virt = NULL; +static unsigned int send_qid; +static unsigned int recv_qid; +static struct dma_pool *buffer_pool; +static struct dma_pool *ctx_pool; + +static struct crypt_ctl *crypt_virt; static dma_addr_t crypt_phys; static int support_aes = 1; -#define DRIVER_NAME "ixp4xx_crypto" - static struct platform_device *pdev; static inline dma_addr_t crypt_virt2phys(struct crypt_ctl *virt) @@ -240,12 +243,12 @@ static inline struct crypt_ctl *crypt_phys2virt(dma_addr_t phys) static inline u32 cipher_cfg_enc(struct crypto_tfm *tfm) { - return container_of(tfm->__crt_alg, struct ixp_alg,crypto.base)->cfg_enc; + return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_enc; } static inline u32 cipher_cfg_dec(struct crypto_tfm *tfm) { - return container_of(tfm->__crt_alg, struct ixp_alg,crypto.base)->cfg_dec; + return container_of(tfm->__crt_alg, struct ixp_alg, crypto.base)->cfg_dec; } static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm) @@ -256,6 +259,7 @@ static inline const struct ix_hash_algo *ix_hash(struct crypto_tfm *tfm) static int setup_crypt_desc(void) { struct device *dev = &pdev->dev; + BUILD_BUG_ON(sizeof(struct crypt_ctl) != 64); crypt_virt = dma_alloc_coherent(dev, NPE_QLEN * sizeof(struct crypt_ctl), @@ -269,7 +273,7 @@ static DEFINE_SPINLOCK(desc_lock); static struct crypt_ctl *get_crypt_desc(void) { int i; - static int idx = 0; + static int idx; unsigned long flags; spin_lock_irqsave(&desc_lock, flags); @@ -286,7 +290,7 @@ static struct crypt_ctl *get_crypt_desc(void) idx = 0; crypt_virt[i].ctl_flags = CTL_FLAG_USED; spin_unlock_irqrestore(&desc_lock, flags); - return crypt_virt +i; + return crypt_virt + i; } else { spin_unlock_irqrestore(&desc_lock, flags); return NULL; @@ -314,7 +318,7 @@ static struct crypt_ctl *get_crypt_desc_emerg(void) idx = NPE_QLEN; crypt_virt[i].ctl_flags = CTL_FLAG_USED; spin_unlock_irqrestore(&emerg_lock, flags); - return crypt_virt +i; + return crypt_virt + i; } else { spin_unlock_irqrestore(&emerg_lock, flags); return NULL; @@ -330,7 +334,7 @@ static void free_buf_chain(struct device *dev, struct buffer_desc *buf, buf1 = buf->next; phys1 = buf->phys_next; - dma_unmap_single(dev, buf->phys_next, buf->buf_len, buf->dir); + dma_unmap_single(dev, buf->phys_addr, buf->buf_len, buf->dir); dma_pool_free(buffer_pool, buf, phys); buf = buf1; phys = phys1; @@ -348,8 +352,8 @@ static void finish_scattered_hmac(struct crypt_ctl *crypt) int decryptlen = req->assoclen + req->cryptlen - authsize; if (req_ctx->encrypt) { - scatterwalk_map_and_copy(req_ctx->hmac_virt, - req->dst, decryptlen, authsize, 1); + scatterwalk_map_and_copy(req_ctx->hmac_virt, req->dst, + decryptlen, authsize, 1); } dma_pool_free(buffer_pool, req_ctx->hmac_virt, crypt->icv_rev_aes); } @@ -372,19 +376,33 @@ static void one_packet(dma_addr_t phys) free_buf_chain(dev, req_ctx->src, crypt->src_buf); free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); - if (req_ctx->hmac_virt) { + if (req_ctx->hmac_virt) finish_scattered_hmac(crypt); - } + req->base.complete(&req->base, failed); break; } case CTL_FLAG_PERFORM_ABLK: { struct skcipher_request *req = crypt->data.ablk_req; struct ablk_ctx *req_ctx = skcipher_request_ctx(req); + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + unsigned int ivsize = crypto_skcipher_ivsize(tfm); + unsigned int offset; - if (req_ctx->dst) { - free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); + if (ivsize > 0) { + offset = req->cryptlen - ivsize; + if (req_ctx->encrypt) { + scatterwalk_map_and_copy(req->iv, req->dst, + offset, ivsize, 0); + } else { + memcpy(req->iv, req_ctx->iv, ivsize); + memzero_explicit(req_ctx->iv, ivsize); + } } + + if (req_ctx->dst) + free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); + free_buf_chain(dev, req_ctx->src, crypt->src_buf); req->base.complete(&req->base, failed); break; @@ -392,14 +410,14 @@ static void one_packet(dma_addr_t phys) case CTL_FLAG_GEN_ICV: ctx = crypto_tfm_ctx(crypt->data.tfm); dma_pool_free(ctx_pool, crypt->regist_ptr, - crypt->regist_buf->phys_addr); + crypt->regist_buf->phys_addr); dma_pool_free(buffer_pool, crypt->regist_buf, crypt->src_buf); if (atomic_dec_and_test(&ctx->configuring)) complete(&ctx->completion); break; case CTL_FLAG_GEN_REVAES: ctx = crypto_tfm_ctx(crypt->data.tfm); - *(u32*)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR); + *(u32 *)ctx->decrypt.npe_ctx &= cpu_to_be32(~CIPH_ENCR); if (atomic_dec_and_test(&ctx->configuring)) complete(&ctx->completion); break; @@ -418,8 +436,8 @@ static void crypto_done_action(unsigned long arg) { int i; - for(i=0; i<4; i++) { - dma_addr_t phys = qmgr_get_entry(RECV_QID); + for (i = 0; i < 4; i++) { + dma_addr_t phys = qmgr_get_entry(recv_qid); if (!phys) return; one_packet(phys); @@ -429,15 +447,52 @@ static void crypto_done_action(unsigned long arg) static int init_ixp_crypto(struct device *dev) { - int ret = -ENODEV; + struct device_node *np = dev->of_node; u32 msg[2] = { 0, 0 }; + int ret = -ENODEV; + u32 npe_id; - if (! ( ~(*IXP4XX_EXP_CFG2) & (IXP4XX_FEATURE_HASH | - IXP4XX_FEATURE_AES | IXP4XX_FEATURE_DES))) { - printk(KERN_ERR "ixp_crypto: No HW crypto available\n"); - return ret; + dev_info(dev, "probing...\n"); + + /* Locate the NPE and queue manager to use from device tree */ + if (IS_ENABLED(CONFIG_OF) && np) { + struct of_phandle_args queue_spec; + struct of_phandle_args npe_spec; + + ret = of_parse_phandle_with_fixed_args(np, "intel,npe-handle", + 1, 0, &npe_spec); + if (ret) { + dev_err(dev, "no NPE engine specified\n"); + return -ENODEV; + } + npe_id = npe_spec.args[0]; + + ret = of_parse_phandle_with_fixed_args(np, "queue-rx", 1, 0, + &queue_spec); + if (ret) { + dev_err(dev, "no rx queue phandle\n"); + return -ENODEV; + } + recv_qid = queue_spec.args[0]; + + ret = of_parse_phandle_with_fixed_args(np, "queue-txready", 1, 0, + &queue_spec); + if (ret) { + dev_err(dev, "no txready queue phandle\n"); + return -ENODEV; + } + send_qid = queue_spec.args[0]; + } else { + /* + * Hardcoded engine when using platform data, this goes away + * when we switch to using DT only. + */ + npe_id = 2; + send_qid = 29; + recv_qid = 30; } - npe_c = npe_request(NPE_ID); + + npe_c = npe_request(npe_id); if (!npe_c) return ret; @@ -455,10 +510,9 @@ static int init_ixp_crypto(struct device *dev) goto npe_error; } - switch ((msg[1]>>16) & 0xff) { + switch ((msg[1] >> 16) & 0xff) { case 3: - printk(KERN_WARNING "Firmware of %s lacks AES support\n", - npe_name(npe_c)); + dev_warn(dev, "Firmware of %s lacks AES support\n", npe_name(npe_c)); support_aes = 0; break; case 4: @@ -466,8 +520,7 @@ static int init_ixp_crypto(struct device *dev) support_aes = 1; break; default: - printk(KERN_ERR "Firmware of %s lacks crypto support\n", - npe_name(npe_c)); + dev_err(dev, "Firmware of %s lacks crypto support\n", npe_name(npe_c)); ret = -ENODEV; goto npe_release; } @@ -475,35 +528,34 @@ static int init_ixp_crypto(struct device *dev) * so assure it is large enough */ BUILD_BUG_ON(SHA1_DIGEST_SIZE > sizeof(struct buffer_desc)); - buffer_pool = dma_pool_create("buffer", dev, - sizeof(struct buffer_desc), 32, 0); + buffer_pool = dma_pool_create("buffer", dev, sizeof(struct buffer_desc), + 32, 0); ret = -ENOMEM; - if (!buffer_pool) { + if (!buffer_pool) goto err; - } - ctx_pool = dma_pool_create("context", dev, - NPE_CTX_LEN, 16, 0); - if (!ctx_pool) { + + ctx_pool = dma_pool_create("context", dev, NPE_CTX_LEN, 16, 0); + if (!ctx_pool) goto err; - } - ret = qmgr_request_queue(SEND_QID, NPE_QLEN_TOTAL, 0, 0, + + ret = qmgr_request_queue(send_qid, NPE_QLEN_TOTAL, 0, 0, "ixp_crypto:out", NULL); if (ret) goto err; - ret = qmgr_request_queue(RECV_QID, NPE_QLEN, 0, 0, + ret = qmgr_request_queue(recv_qid, NPE_QLEN, 0, 0, "ixp_crypto:in", NULL); if (ret) { - qmgr_release_queue(SEND_QID); + qmgr_release_queue(send_qid); goto err; } - qmgr_set_irq(RECV_QID, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL); + qmgr_set_irq(recv_qid, QUEUE_IRQ_SRC_NOT_EMPTY, irqhandler, NULL); tasklet_init(&crypto_done_tasklet, crypto_done_action, 0); - qmgr_enable_irq(RECV_QID); + qmgr_enable_irq(recv_qid); return 0; npe_error: - printk(KERN_ERR "%s not responding\n", npe_name(npe_c)); + dev_err(dev, "%s not responding\n", npe_name(npe_c)); ret = -EIO; err: dma_pool_destroy(ctx_pool); @@ -515,22 +567,20 @@ static int init_ixp_crypto(struct device *dev) static void release_ixp_crypto(struct device *dev) { - qmgr_disable_irq(RECV_QID); + qmgr_disable_irq(recv_qid); tasklet_kill(&crypto_done_tasklet); - qmgr_release_queue(SEND_QID); - qmgr_release_queue(RECV_QID); + qmgr_release_queue(send_qid); + qmgr_release_queue(recv_qid); dma_pool_destroy(ctx_pool); dma_pool_destroy(buffer_pool); npe_release(npe_c); - if (crypt_virt) { - dma_free_coherent(dev, - NPE_QLEN * sizeof(struct crypt_ctl), - crypt_virt, crypt_phys); - } + if (crypt_virt) + dma_free_coherent(dev, NPE_QLEN * sizeof(struct crypt_ctl), + crypt_virt, crypt_phys); } static void reset_sa_dir(struct ix_sa_dir *dir) @@ -543,9 +593,9 @@ static void reset_sa_dir(struct ix_sa_dir *dir) static int init_sa_dir(struct ix_sa_dir *dir) { dir->npe_ctx = dma_pool_alloc(ctx_pool, GFP_KERNEL, &dir->npe_ctx_phys); - if (!dir->npe_ctx) { + if (!dir->npe_ctx) return -ENOMEM; - } + reset_sa_dir(dir); return 0; } @@ -566,15 +616,31 @@ static int init_tfm(struct crypto_tfm *tfm) if (ret) return ret; ret = init_sa_dir(&ctx->decrypt); - if (ret) { + if (ret) free_sa_dir(&ctx->encrypt); - } + return ret; } static int init_tfm_ablk(struct crypto_skcipher *tfm) { - crypto_skcipher_set_reqsize(tfm, sizeof(struct ablk_ctx)); + struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm); + struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm); + const char *name = crypto_tfm_alg_name(ctfm); + + ctx->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->fallback_tfm)) { + pr_err("ERROR: Cannot allocate fallback for %s %ld\n", + name, PTR_ERR(ctx->fallback_tfm)); + return PTR_ERR(ctx->fallback_tfm); + } + + pr_info("Fallback for %s is %s\n", + crypto_tfm_alg_driver_name(&tfm->base), + crypto_tfm_alg_driver_name(crypto_skcipher_tfm(ctx->fallback_tfm)) + ); + + crypto_skcipher_set_reqsize(tfm, sizeof(struct ablk_ctx) + crypto_skcipher_reqsize(ctx->fallback_tfm)); return init_tfm(crypto_skcipher_tfm(tfm)); } @@ -587,12 +653,17 @@ static int init_tfm_aead(struct crypto_aead *tfm) static void exit_tfm(struct crypto_tfm *tfm) { struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); + free_sa_dir(&ctx->encrypt); free_sa_dir(&ctx->decrypt); } static void exit_tfm_ablk(struct crypto_skcipher *tfm) { + struct crypto_tfm *ctfm = crypto_skcipher_tfm(tfm); + struct ixp_ctx *ctx = crypto_tfm_ctx(ctfm); + + crypto_free_skcipher(ctx->fallback_tfm); exit_tfm(crypto_skcipher_tfm(tfm)); } @@ -602,7 +673,8 @@ static void exit_tfm_aead(struct crypto_aead *tfm) } static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, - int init_len, u32 ctx_addr, const u8 *key, int key_len) + int init_len, u32 ctx_addr, const u8 *key, + int key_len) { struct ixp_ctx *ctx = crypto_tfm_ctx(tfm); struct crypt_ctl *crypt; @@ -629,9 +701,8 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, memcpy(pad, key, key_len); memset(pad + key_len, 0, HMAC_PAD_BLOCKLEN - key_len); - for (i = 0; i < HMAC_PAD_BLOCKLEN; i++) { + for (i = 0; i < HMAC_PAD_BLOCKLEN; i++) pad[i] ^= xpad; - } crypt->data.tfm = tfm; crypt->regist_ptr = pad; @@ -652,13 +723,13 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, buf->phys_addr = pad_phys; atomic_inc(&ctx->configuring); - qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt)); - BUG_ON(qmgr_stat_overflow(SEND_QID)); + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); return 0; } -static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize, - const u8 *key, int key_len, unsigned digest_len) +static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned int authsize, + const u8 *key, int key_len, unsigned int digest_len) { u32 itarget, otarget, npe_ctx_addr; unsigned char *cinfo; @@ -673,11 +744,11 @@ static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize, algo = ix_hash(tfm); /* write cfg word to cryptinfo */ - cfgword = algo->cfgword | ( authsize << 6); /* (authsize/4) << 8 */ + cfgword = algo->cfgword | (authsize << 6); /* (authsize/4) << 8 */ #ifndef __ARMEB__ cfgword ^= 0xAA000000; /* change the "byte swap" flags */ #endif - *(u32*)cinfo = cpu_to_be32(cfgword); + *(u32 *)cinfo = cpu_to_be32(cfgword); cinfo += sizeof(cfgword); /* write ICV to cryptinfo */ @@ -697,11 +768,11 @@ static int setup_auth(struct crypto_tfm *tfm, int encrypt, unsigned authsize, dir->npe_mode |= NPE_OP_HASH_VERIFY; ret = register_chain_var(tfm, HMAC_OPAD_VALUE, otarget, - init_len, npe_ctx_addr, key, key_len); + init_len, npe_ctx_addr, key, key_len); if (ret) return ret; return register_chain_var(tfm, HMAC_IPAD_VALUE, itarget, - init_len, npe_ctx_addr, key, key_len); + init_len, npe_ctx_addr, key, key_len); } static int gen_rev_aes_key(struct crypto_tfm *tfm) @@ -711,10 +782,10 @@ static int gen_rev_aes_key(struct crypto_tfm *tfm) struct ix_sa_dir *dir = &ctx->decrypt; crypt = get_crypt_desc_emerg(); - if (!crypt) { + if (!crypt) return -EAGAIN; - } - *(u32*)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR); + + *(u32 *)dir->npe_ctx |= cpu_to_be32(CIPH_ENCR); crypt->data.tfm = tfm; crypt->crypt_offs = 0; @@ -727,13 +798,13 @@ static int gen_rev_aes_key(struct crypto_tfm *tfm) crypt->ctl_flags |= CTL_FLAG_GEN_REVAES; atomic_inc(&ctx->configuring); - qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt)); - BUG_ON(qmgr_stat_overflow(SEND_QID)); + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); return 0; } -static int setup_cipher(struct crypto_tfm *tfm, int encrypt, - const u8 *key, int key_len) +static int setup_cipher(struct crypto_tfm *tfm, int encrypt, const u8 *key, + int key_len) { u8 *cinfo; u32 cipher_cfg; @@ -753,9 +824,15 @@ static int setup_cipher(struct crypto_tfm *tfm, int encrypt, } if (cipher_cfg & MOD_AES) { switch (key_len) { - case 16: keylen_cfg = MOD_AES128; break; - case 24: keylen_cfg = MOD_AES192; break; - case 32: keylen_cfg = MOD_AES256; break; + case 16: + keylen_cfg = MOD_AES128; + break; + case 24: + keylen_cfg = MOD_AES192; + break; + case 32: + keylen_cfg = MOD_AES256; + break; default: return -EINVAL; } @@ -766,31 +843,31 @@ static int setup_cipher(struct crypto_tfm *tfm, int encrypt, return err; } /* write cfg word to cryptinfo */ - *(u32*)cinfo = cpu_to_be32(cipher_cfg); + *(u32 *)cinfo = cpu_to_be32(cipher_cfg); cinfo += sizeof(cipher_cfg); /* write cipher key to cryptinfo */ memcpy(cinfo, key, key_len); /* NPE wants keylen set to DES3_EDE_KEY_SIZE even for single DES */ if (key_len < DES3_EDE_KEY_SIZE && !(cipher_cfg & MOD_AES)) { - memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE -key_len); + memset(cinfo + key_len, 0, DES3_EDE_KEY_SIZE - key_len); key_len = DES3_EDE_KEY_SIZE; } dir->npe_ctx_idx = sizeof(cipher_cfg) + key_len; dir->npe_mode |= NPE_OP_CRYPT_ENABLE; - if ((cipher_cfg & MOD_AES) && !encrypt) { + if ((cipher_cfg & MOD_AES) && !encrypt) return gen_rev_aes_key(tfm); - } + return 0; } static struct buffer_desc *chainup_buffers(struct device *dev, - struct scatterlist *sg, unsigned nbytes, + struct scatterlist *sg, unsigned int nbytes, struct buffer_desc *buf, gfp_t flags, enum dma_data_direction dir) { for (; nbytes > 0; sg = sg_next(sg)) { - unsigned len = min(nbytes, sg->length); + unsigned int len = min(nbytes, sg->length); struct buffer_desc *next_buf; dma_addr_t next_buf_phys; void *ptr; @@ -817,7 +894,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev, } static int ablk_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int key_len) + unsigned int key_len) { struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; @@ -838,7 +915,12 @@ static int ablk_setkey(struct crypto_skcipher *tfm, const u8 *key, out: if (!atomic_dec_and_test(&ctx->configuring)) wait_for_completion(&ctx->completion); - return ret; + if (ret) + return ret; + crypto_skcipher_clear_flags(ctx->fallback_tfm, CRYPTO_TFM_REQ_MASK); + crypto_skcipher_set_flags(ctx->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); + + return crypto_skcipher_setkey(ctx->fallback_tfm, key, key_len); } static int ablk_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, @@ -849,7 +931,7 @@ static int ablk_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, } static int ablk_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key, - unsigned int key_len) + unsigned int key_len) { struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -858,17 +940,36 @@ static int ablk_rfc3686_setkey(struct crypto_skcipher *tfm, const u8 *key, return -EINVAL; memcpy(ctx->nonce, key + (key_len - CTR_RFC3686_NONCE_SIZE), - CTR_RFC3686_NONCE_SIZE); + CTR_RFC3686_NONCE_SIZE); key_len -= CTR_RFC3686_NONCE_SIZE; return ablk_setkey(tfm, key, key_len); } +static int ixp4xx_cipher_fallback(struct skcipher_request *areq, int encrypt) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct ixp_ctx *op = crypto_skcipher_ctx(tfm); + struct ablk_ctx *rctx = skcipher_request_ctx(areq); + int err; + + skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); + skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, + areq->base.complete, areq->base.data); + skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (encrypt) + err = crypto_skcipher_encrypt(&rctx->fallback_req); + else + err = crypto_skcipher_decrypt(&rctx->fallback_req); + return err; +} + static int ablk_perform(struct skcipher_request *req, int encrypt) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct ixp_ctx *ctx = crypto_skcipher_ctx(tfm); - unsigned ivsize = crypto_skcipher_ivsize(tfm); + unsigned int ivsize = crypto_skcipher_ivsize(tfm); struct ix_sa_dir *dir; struct crypt_ctl *crypt; unsigned int nbytes = req->cryptlen; @@ -876,15 +977,20 @@ static int ablk_perform(struct skcipher_request *req, int encrypt) struct ablk_ctx *req_ctx = skcipher_request_ctx(req); struct buffer_desc src_hook; struct device *dev = &pdev->dev; + unsigned int offset; gfp_t flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC; - if (qmgr_stat_full(SEND_QID)) + if (sg_nents(req->src) > 1 || sg_nents(req->dst) > 1) + return ixp4xx_cipher_fallback(req, encrypt); + + if (qmgr_stat_full(send_qid)) return -EAGAIN; if (atomic_read(&ctx->configuring)) return -EAGAIN; dir = encrypt ? &ctx->encrypt : &ctx->decrypt; + req_ctx->encrypt = encrypt; crypt = get_crypt_desc(); if (!crypt) @@ -900,14 +1006,19 @@ static int ablk_perform(struct skcipher_request *req, int encrypt) BUG_ON(ivsize && !req->iv); memcpy(crypt->iv, req->iv, ivsize); + if (ivsize > 0 && !encrypt) { + offset = req->cryptlen - ivsize; + scatterwalk_map_and_copy(req_ctx->iv, req->src, offset, ivsize, 0); + } if (req->src != req->dst) { struct buffer_desc dst_hook; + crypt->mode |= NPE_OP_NOT_IN_PLACE; /* This was never tested by Intel * for more than one dst buffer, I think. */ req_ctx->dst = NULL; if (!chainup_buffers(dev, req->dst, nbytes, &dst_hook, - flags, DMA_FROM_DEVICE)) + flags, DMA_FROM_DEVICE)) goto free_buf_dest; src_direction = DMA_TO_DEVICE; req_ctx->dst = dst_hook.next; @@ -916,23 +1027,23 @@ static int ablk_perform(struct skcipher_request *req, int encrypt) req_ctx->dst = NULL; } req_ctx->src = NULL; - if (!chainup_buffers(dev, req->src, nbytes, &src_hook, - flags, src_direction)) + if (!chainup_buffers(dev, req->src, nbytes, &src_hook, flags, + src_direction)) goto free_buf_src; req_ctx->src = src_hook.next; crypt->src_buf = src_hook.phys_next; crypt->ctl_flags |= CTL_FLAG_PERFORM_ABLK; - qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt)); - BUG_ON(qmgr_stat_overflow(SEND_QID)); + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); return -EINPROGRESS; free_buf_src: free_buf_chain(dev, req_ctx->src, crypt->src_buf); free_buf_dest: - if (req->src != req->dst) { + if (req->src != req->dst) free_buf_chain(dev, req_ctx->dst, crypt->dst_buf); - } + crypt->ctl_flags = CTL_FLAG_UNUSED; return -ENOMEM; } @@ -956,7 +1067,7 @@ static int ablk_rfc3686_crypt(struct skcipher_request *req) int ret; /* set up counter block */ - memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE); + memcpy(iv, ctx->nonce, CTR_RFC3686_NONCE_SIZE); memcpy(iv + CTR_RFC3686_NONCE_SIZE, info, CTR_RFC3686_IV_SIZE); /* initialize counter portion of counter block */ @@ -970,12 +1081,12 @@ static int ablk_rfc3686_crypt(struct skcipher_request *req) } static int aead_perform(struct aead_request *req, int encrypt, - int cryptoffset, int eff_cryptlen, u8 *iv) + int cryptoffset, int eff_cryptlen, u8 *iv) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct ixp_ctx *ctx = crypto_aead_ctx(tfm); - unsigned ivsize = crypto_aead_ivsize(tfm); - unsigned authsize = crypto_aead_authsize(tfm); + unsigned int ivsize = crypto_aead_ivsize(tfm); + unsigned int authsize = crypto_aead_authsize(tfm); struct ix_sa_dir *dir; struct crypt_ctl *crypt; unsigned int cryptlen; @@ -987,7 +1098,7 @@ static int aead_perform(struct aead_request *req, int encrypt, enum dma_data_direction src_direction = DMA_BIDIRECTIONAL; unsigned int lastlen; - if (qmgr_stat_full(SEND_QID)) + if (qmgr_stat_full(send_qid)) return -EAGAIN; if (atomic_read(&ctx->configuring)) return -EAGAIN; @@ -998,7 +1109,7 @@ static int aead_perform(struct aead_request *req, int encrypt, } else { dir = &ctx->decrypt; /* req->cryptlen includes the authsize when decrypting */ - cryptlen = req->cryptlen -authsize; + cryptlen = req->cryptlen - authsize; eff_cryptlen -= authsize; } crypt = get_crypt_desc(); @@ -1058,12 +1169,12 @@ static int aead_perform(struct aead_request *req, int encrypt, /* The 12 hmac bytes are scattered, * we need to copy them into a safe buffer */ req_ctx->hmac_virt = dma_pool_alloc(buffer_pool, flags, - &crypt->icv_rev_aes); + &crypt->icv_rev_aes); if (unlikely(!req_ctx->hmac_virt)) goto free_buf_dst; if (!encrypt) { scatterwalk_map_and_copy(req_ctx->hmac_virt, - req->src, cryptlen, authsize, 0); + req->src, cryptlen, authsize, 0); } req_ctx->encrypt = encrypt; } else { @@ -1071,8 +1182,8 @@ static int aead_perform(struct aead_request *req, int encrypt, } crypt->ctl_flags |= CTL_FLAG_PERFORM_AEAD; - qmgr_put_entry(SEND_QID, crypt_virt2phys(crypt)); - BUG_ON(qmgr_stat_overflow(SEND_QID)); + qmgr_put_entry(send_qid, crypt_virt2phys(crypt)); + BUG_ON(qmgr_stat_overflow(send_qid)); return -EINPROGRESS; free_buf_dst: @@ -1086,7 +1197,7 @@ static int aead_perform(struct aead_request *req, int encrypt, static int aead_setup(struct crypto_aead *tfm, unsigned int authsize) { struct ixp_ctx *ctx = crypto_aead_ctx(tfm); - unsigned digest_len = crypto_aead_maxauthsize(tfm); + unsigned int digest_len = crypto_aead_maxauthsize(tfm); int ret; if (!ctx->enckey_len && !ctx->authkey_len) @@ -1104,11 +1215,11 @@ static int aead_setup(struct crypto_aead *tfm, unsigned int authsize) if (ret) goto out; ret = setup_auth(&tfm->base, 0, authsize, ctx->authkey, - ctx->authkey_len, digest_len); + ctx->authkey_len, digest_len); if (ret) goto out; ret = setup_auth(&tfm->base, 1, authsize, ctx->authkey, - ctx->authkey_len, digest_len); + ctx->authkey_len, digest_len); out: if (!atomic_dec_and_test(&ctx->configuring)) wait_for_completion(&ctx->completion); @@ -1119,13 +1230,13 @@ static int aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) { int max = crypto_aead_maxauthsize(tfm) >> 2; - if ((authsize>>2) < 1 || (authsize>>2) > max || (authsize & 3)) + if ((authsize >> 2) < 1 || (authsize >> 2) > max || (authsize & 3)) return -EINVAL; return aead_setup(tfm, authsize); } static int aead_setkey(struct crypto_aead *tfm, const u8 *key, - unsigned int keylen) + unsigned int keylen) { struct ixp_ctx *ctx = crypto_aead_ctx(tfm); struct crypto_authenc_keys keys; @@ -1364,43 +1475,33 @@ static struct ixp_aead_alg ixp4xx_aeads[] = { #define IXP_POSTFIX "-ixp4xx" -static const struct platform_device_info ixp_dev_info __initdata = { - .name = DRIVER_NAME, - .id = 0, - .dma_mask = DMA_BIT_MASK(32), -}; - -static int __init ixp_module_init(void) +static int ixp_crypto_probe(struct platform_device *_pdev) { + struct device *dev = &_pdev->dev; int num = ARRAY_SIZE(ixp4xx_algos); int i, err; - pdev = platform_device_register_full(&ixp_dev_info); - if (IS_ERR(pdev)) - return PTR_ERR(pdev); + pdev = _pdev; - err = init_ixp_crypto(&pdev->dev); - if (err) { - platform_device_unregister(pdev); + err = init_ixp_crypto(dev); + if (err) return err; - } - for (i=0; i< num; i++) { + + for (i = 0; i < num; i++) { struct skcipher_alg *cra = &ixp4xx_algos[i].crypto; if (snprintf(cra->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "%s"IXP_POSTFIX, cra->base.cra_name) >= - CRYPTO_MAX_ALG_NAME) - { + "%s"IXP_POSTFIX, cra->base.cra_name) >= + CRYPTO_MAX_ALG_NAME) continue; - } - if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) { + if (!support_aes && (ixp4xx_algos[i].cfg_enc & MOD_AES)) continue; - } /* block ciphers */ cra->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | CRYPTO_ALG_ASYNC | - CRYPTO_ALG_ALLOCATES_MEMORY; + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_NEED_FALLBACK; if (!cra->setkey) cra->setkey = ablk_setkey; if (!cra->encrypt) @@ -1415,7 +1516,7 @@ static int __init ixp_module_init(void) cra->base.cra_alignmask = 3; cra->base.cra_priority = 300; if (crypto_register_skcipher(cra)) - printk(KERN_ERR "Failed to register '%s'\n", + dev_err(&pdev->dev, "Failed to register '%s'\n", cra->base.cra_name); else ixp4xx_algos[i].registered = 1; @@ -1448,7 +1549,7 @@ static int __init ixp_module_init(void) cra->base.cra_priority = 300; if (crypto_register_aead(cra)) - printk(KERN_ERR "Failed to register '%s'\n", + dev_err(&pdev->dev, "Failed to register '%s'\n", cra->base.cra_driver_name); else ixp4xx_aeads[i].registered = 1; @@ -1456,7 +1557,7 @@ static int __init ixp_module_init(void) return 0; } -static void __exit ixp_module_exit(void) +static int ixp_crypto_remove(struct platform_device *pdev) { int num = ARRAY_SIZE(ixp4xx_algos); int i; @@ -1466,16 +1567,30 @@ static void __exit ixp_module_exit(void) crypto_unregister_aead(&ixp4xx_aeads[i].crypto); } - for (i=0; i< num; i++) { + for (i = 0; i < num; i++) { if (ixp4xx_algos[i].registered) crypto_unregister_skcipher(&ixp4xx_algos[i].crypto); } release_ixp_crypto(&pdev->dev); - platform_device_unregister(pdev); -} -module_init(ixp_module_init); -module_exit(ixp_module_exit); + return 0; +} +static const struct of_device_id ixp4xx_crypto_of_match[] = { + { + .compatible = "intel,ixp4xx-crypto", + }, + {}, +}; + +static struct platform_driver ixp_crypto_driver = { + .probe = ixp_crypto_probe, + .remove = ixp_crypto_remove, + .driver = { + .name = "ixp4xx_crypto", + .of_match_table = ixp4xx_crypto_of_match, + }, +}; +module_platform_driver(ixp_crypto_driver); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Christian Hohnstaedt "); diff --git a/drivers/crypto/marvell/cesa/cesa.h b/drivers/crypto/marvell/cesa/cesa.h index c1007f2ba79c..d215a6bed6bc 100644 --- a/drivers/crypto/marvell/cesa/cesa.h +++ b/drivers/crypto/marvell/cesa/cesa.h @@ -66,7 +66,7 @@ #define CESA_SA_ST_ACT_1 BIT(1) /* - * CESA_SA_FPGA_INT_STATUS looks like a FPGA leftover and is documented only + * CESA_SA_FPGA_INT_STATUS looks like an FPGA leftover and is documented only * in Errata 4.12. It looks like that it was part of an IRQ-controller in FPGA * and someone forgot to remove it while switching to the core and moving to * CESA_SA_INT_STATUS. diff --git a/drivers/crypto/marvell/octeontx2/Makefile b/drivers/crypto/marvell/octeontx2/Makefile index b9c6201019e0..c242d22008c3 100644 --- a/drivers/crypto/marvell/octeontx2/Makefile +++ b/drivers/crypto/marvell/octeontx2/Makefile @@ -1,10 +1,11 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += octeontx2-cpt.o octeontx2-cptvf.o +obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += rvu_cptpf.o rvu_cptvf.o -octeontx2-cpt-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \ - otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o -octeontx2-cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \ - otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \ - otx2_cptvf_algs.o +rvu_cptpf-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \ + otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o \ + cn10k_cpt.o +rvu_cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \ + otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \ + otx2_cptvf_algs.o cn10k_cpt.o ccflags-y += -I$(srctree)/drivers/net/ethernet/marvell/octeontx2/af diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c new file mode 100644 index 000000000000..1499ef75b5c2 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2021 Marvell. */ + +#include +#include "otx2_cptpf.h" +#include "otx2_cptvf.h" +#include "otx2_cptlf.h" +#include "cn10k_cpt.h" + +static struct cpt_hw_ops otx2_hw_ops = { + .send_cmd = otx2_cpt_send_cmd, + .cpt_get_compcode = otx2_cpt_get_compcode, + .cpt_get_uc_compcode = otx2_cpt_get_uc_compcode, +}; + +static struct cpt_hw_ops cn10k_hw_ops = { + .send_cmd = cn10k_cpt_send_cmd, + .cpt_get_compcode = cn10k_cpt_get_compcode, + .cpt_get_uc_compcode = cn10k_cpt_get_uc_compcode, +}; + +void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, + struct otx2_cptlf_info *lf) +{ + void __iomem *lmtline = lf->lmtline; + u64 val = (lf->slot & 0x7FF); + u64 tar_addr = 0; + + /* tar_addr<6:4> = Size of first LMTST - 1 in units of 128b. */ + tar_addr |= (__force u64)lf->ioreg | + (((OTX2_CPT_INST_SIZE/16) - 1) & 0x7) << 4; + /* + * Make sure memory areas pointed in CPT_INST_S + * are flushed before the instruction is sent to CPT + */ + dma_wmb(); + + /* Copy CPT command to LMTLINE */ + memcpy_toio(lmtline, cptinst, insts_num * OTX2_CPT_INST_SIZE); + cn10k_lmt_flush(val, tar_addr); +} + +int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf) +{ + struct pci_dev *pdev = cptpf->pdev; + resource_size_t size; + u64 lmt_base; + + if (!test_bit(CN10K_LMTST, &cptpf->cap_flag)) { + cptpf->lfs.ops = &otx2_hw_ops; + return 0; + } + + cptpf->lfs.ops = &cn10k_hw_ops; + lmt_base = readq(cptpf->reg_base + RVU_PF_LMTLINE_ADDR); + if (!lmt_base) { + dev_err(&pdev->dev, "PF LMTLINE address not configured\n"); + return -ENOMEM; + } + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + size -= ((1 + cptpf->max_vfs) * MBOX_SIZE); + cptpf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, lmt_base, size); + if (!cptpf->lfs.lmt_base) { + dev_err(&pdev->dev, + "Mapping of PF LMTLINE address failed\n"); + return -ENOMEM; + } + + return 0; +} + +int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf) +{ + struct pci_dev *pdev = cptvf->pdev; + resource_size_t offset, size; + + if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) { + cptvf->lfs.ops = &otx2_hw_ops; + return 0; + } + + cptvf->lfs.ops = &cn10k_hw_ops; + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + /* Map VF LMILINE region */ + cptvf->lfs.lmt_base = devm_ioremap_wc(&pdev->dev, offset, size); + if (!cptvf->lfs.lmt_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + return -ENOMEM; + } + + return 0; +} diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h new file mode 100644 index 000000000000..c091392b47e0 --- /dev/null +++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * Copyright (C) 2021 Marvell. + */ +#ifndef __CN10K_CPT_H +#define __CN10K_CPT_H + +#include "otx2_cpt_common.h" +#include "otx2_cptpf.h" +#include "otx2_cptvf.h" + +static inline u8 cn10k_cpt_get_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn10k_cpt_res_s *)result)->compcode; +} + +static inline u8 cn10k_cpt_get_uc_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn10k_cpt_res_s *)result)->uc_compcode; +} + +static inline u8 otx2_cpt_get_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn9k_cpt_res_s *)result)->compcode; +} + +static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result) +{ + return ((struct cn9k_cpt_res_s *)result)->uc_compcode; +} + +void cn10k_cpt_send_cmd(union otx2_cpt_inst_s *cptinst, u32 insts_num, + struct otx2_cptlf_info *lf); +int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf); +int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf); + +#endif /* __CN10K_CPTLF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h index ecedd91a8d85..c5445b05f53c 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h @@ -25,6 +25,10 @@ #define OTX2_CPT_NAME_LENGTH 64 #define OTX2_CPT_DMA_MINALIGN 128 +/* HW capability flags */ +#define CN10K_MBOX 0 +#define CN10K_LMTST 1 + #define BAD_OTX2_CPT_ENG_TYPE OTX2_CPT_MAX_ENG_TYPES enum otx2_cpt_eng_type { @@ -116,6 +120,25 @@ static inline u64 otx2_cpt_read64(void __iomem *reg_base, u64 blk, u64 slot, OTX2_CPT_RVU_FUNC_ADDR_S(blk, slot, offs)); } +static inline bool is_dev_otx2(struct pci_dev *pdev) +{ + if (pdev->device == OTX2_CPT_PCI_PF_DEVICE_ID || + pdev->device == OTX2_CPT_PCI_VF_DEVICE_ID) + return true; + + return false; +} + +static inline void otx2_cpt_set_hw_caps(struct pci_dev *pdev, + unsigned long *cap_flag) +{ + if (!is_dev_otx2(pdev)) { + __set_bit(CN10K_MBOX, cap_flag); + __set_bit(CN10K_LMTST, cap_flag); + } +} + + int otx2_cpt_send_ready_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); int otx2_cpt_send_mbox_msg(struct otx2_mbox *mbox, struct pci_dev *pdev); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h index ecafc42f37a2..6f947978e4e8 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h @@ -10,6 +10,8 @@ /* Device IDs */ #define OTX2_CPT_PCI_PF_DEVICE_ID 0xA0FD #define OTX2_CPT_PCI_VF_DEVICE_ID 0xA0FE +#define CN10K_CPT_PCI_PF_DEVICE_ID 0xA0F2 +#define CN10K_CPT_PCI_VF_DEVICE_ID 0xA0F3 /* Mailbox interrupts offset */ #define OTX2_CPT_PF_MBOX_INT 6 @@ -25,6 +27,7 @@ */ #define OTX2_CPT_VF_MSIX_VECTORS 1 #define OTX2_CPT_VF_INTR_MBOX_MASK BIT(0) +#define CN10K_CPT_VF_MBOX_REGION (0xC0000) /* CPT LF MSIX vectors */ #define OTX2_CPT_LF_MSIX_VECTORS 2 @@ -135,7 +138,7 @@ enum otx2_cpt_comp_e { OTX2_CPT_COMP_E_FAULT = 0x02, OTX2_CPT_COMP_E_HWERR = 0x04, OTX2_CPT_COMP_E_INSTERR = 0x05, - OTX2_CPT_COMP_E_LAST_ENTRY = 0x06 + OTX2_CPT_COMP_E_WARN = 0x06 }; /* @@ -266,13 +269,22 @@ union otx2_cpt_inst_s { union otx2_cpt_res_s { u64 u[2]; - struct { + struct cn9k_cpt_res_s { u64 compcode:8; u64 uc_compcode:8; u64 doneint:1; u64 reserved_17_63:47; u64 reserved_64_127; } s; + + struct cn10k_cpt_res_s { + u64 compcode:7; + u64 doneint:1; + u64 uc_compcode:8; + u64 rlen:16; + u64 spi:32; + u64 esn; + } cn10k; }; /* diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c index 34aba1532761..c8350fcd60fa 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c @@ -379,9 +379,14 @@ int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri, for (slot = 0; slot < lfs->lfs_num; slot++) { lfs->lf[slot].lfs = lfs; lfs->lf[slot].slot = slot; - lfs->lf[slot].lmtline = lfs->reg_base + - OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, + if (lfs->lmt_base) + lfs->lf[slot].lmtline = lfs->lmt_base + + (slot * LMTLINE_SIZE); + else + lfs->lf[slot].lmtline = lfs->reg_base + + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_LMT, slot, OTX2_CPT_LMT_LF_LMTLINEX(0)); + lfs->lf[slot].ioreg = lfs->reg_base + OTX2_CPT_RVU_FUNC_ADDR_S(BLKADDR_CPT0, slot, OTX2_CPT_LF_NQX(0)); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h index ab1678fc564d..b691b6c1d5c4 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h @@ -84,12 +84,22 @@ struct otx2_cptlf_info { struct otx2_cptlf_wqe *wqe; /* Tasklet work info */ }; +struct cpt_hw_ops { + void (*send_cmd)(union otx2_cpt_inst_s *cptinst, u32 insts_num, + struct otx2_cptlf_info *lf); + u8 (*cpt_get_compcode)(union otx2_cpt_res_s *result); + u8 (*cpt_get_uc_compcode)(union otx2_cpt_res_s *result); +}; + struct otx2_cptlfs_info { /* Registers start address of VF/PF LFs are attached to */ void __iomem *reg_base; +#define LMTLINE_SIZE 128 + void __iomem *lmt_base; struct pci_dev *pdev; /* Device LFs are attached to */ struct otx2_cptlf_info lf[OTX2_CPT_MAX_LFS_NUM]; struct otx2_mbox *mbox; + struct cpt_hw_ops *ops; u8 are_lfs_attached; /* Whether CPT LFs are attached */ u8 lfs_num; /* Number of CPT LFs */ u8 kcrypto_eng_grp_num; /* Kernel crypto engine group number */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h index e19af1356f12..5ebba86c65d9 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h @@ -47,6 +47,7 @@ struct otx2_cptpf_dev { struct workqueue_struct *flr_wq; struct cptpf_flr_work *flr_work; + unsigned long cap_flag; u8 pf_id; /* RVU PF number */ u8 max_vfs; /* Maximum number of VFs supported by CPT */ u8 enabled_vfs; /* Number of enabled VFs */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c index 58f47e3ab62e..146a55ac4b9b 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c @@ -6,10 +6,11 @@ #include "otx2_cpt_common.h" #include "otx2_cptpf_ucode.h" #include "otx2_cptpf.h" +#include "cn10k_cpt.h" #include "rvu_reg.h" -#define OTX2_CPT_DRV_NAME "octeontx2-cpt" -#define OTX2_CPT_DRV_STRING "Marvell OcteonTX2 CPT Physical Function Driver" +#define OTX2_CPT_DRV_NAME "rvu_cptpf" +#define OTX2_CPT_DRV_STRING "Marvell RVU CPT Physical Function Driver" static void cptpf_enable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) @@ -62,45 +63,66 @@ static void cptpf_disable_vfpf_mbox_intr(struct otx2_cptpf_dev *cptpf, } } -static void cptpf_enable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf) +static void cptpf_enable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf, + int num_vfs) { - /* Clear interrupt if any */ + /* Clear FLR interrupt if any */ otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0), - ~0x0ULL); - otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), - ~0x0ULL); + INTR_MASK(num_vfs)); /* Enable VF FLR interrupts */ otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, - RVU_PF_VFFLR_INT_ENA_W1SX(0), ~0x0ULL); + RVU_PF_VFFLR_INT_ENA_W1SX(0), INTR_MASK(num_vfs)); + /* Clear ME interrupt if any */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(0), + INTR_MASK(num_vfs)); + /* Enable VF ME interrupts */ otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, - RVU_PF_VFFLR_INT_ENA_W1SX(1), ~0x0ULL); + RVU_PF_VFME_INT_ENA_W1SX(0), INTR_MASK(num_vfs)); + + if (num_vfs <= 64) + return; + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), + INTR_MASK(num_vfs - 64)); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64)); + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFME_INTX(1), + INTR_MASK(num_vfs - 64)); + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INT_ENA_W1SX(1), INTR_MASK(num_vfs - 64)); } -static void cptpf_disable_vf_flr_intrs(struct otx2_cptpf_dev *cptpf, +static void cptpf_disable_vf_flr_me_intrs(struct otx2_cptpf_dev *cptpf, int num_vfs) { int vector; /* Disable VF FLR interrupts */ otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, - RVU_PF_VFFLR_INT_ENA_W1CX(0), ~0x0ULL); - otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, - RVU_PF_VFFLR_INT_ENA_W1CX(1), ~0x0ULL); - - /* Clear interrupt if any */ - otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(0), - ~0x0ULL); - otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, RVU_PF_VFFLR_INTX(1), - ~0x0ULL); - + RVU_PF_VFFLR_INT_ENA_W1CX(0), INTR_MASK(num_vfs)); vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR0); free_irq(vector, cptpf); - if (num_vfs > 64) { - vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1); - free_irq(vector, cptpf); - } + /* Disable VF ME interrupts */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INT_ENA_W1CX(0), INTR_MASK(num_vfs)); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME0); + free_irq(vector, cptpf); + + if (num_vfs <= 64) + return; + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFFLR_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64)); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFFLR1); + free_irq(vector, cptpf); + + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INT_ENA_W1CX(1), INTR_MASK(num_vfs - 64)); + vector = pci_irq_vector(cptpf->pdev, RVU_PF_INT_VEC_VFME1); + free_irq(vector, cptpf); } static void cptpf_flr_wq_handler(struct work_struct *work) @@ -172,11 +194,38 @@ static irqreturn_t cptpf_vf_flr_intr(int __always_unused irq, void *arg) return IRQ_HANDLED; } +static irqreturn_t cptpf_vf_me_intr(int __always_unused irq, void *arg) +{ + struct otx2_cptpf_dev *cptpf = arg; + int reg, vf, num_reg = 1; + u64 intr; + + if (cptpf->max_vfs > 64) + num_reg = 2; + + for (reg = 0; reg < num_reg; reg++) { + intr = otx2_cpt_read64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INTX(reg)); + if (!intr) + continue; + for (vf = 0; vf < 64; vf++) { + if (!(intr & BIT_ULL(vf))) + continue; + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFTRPENDX(reg), BIT_ULL(vf)); + /* Clear interrupt */ + otx2_cpt_write64(cptpf->reg_base, BLKADDR_RVUM, 0, + RVU_PF_VFME_INTX(reg), BIT_ULL(vf)); + } + } + return IRQ_HANDLED; +} + static void cptpf_unregister_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) { cptpf_disable_vfpf_mbox_intr(cptpf, num_vfs); - cptpf_disable_vf_flr_intrs(cptpf, num_vfs); + cptpf_disable_vf_flr_me_intrs(cptpf, num_vfs); } static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) @@ -202,6 +251,15 @@ static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) "IRQ registration failed for VFFLR0 irq\n"); goto free_mbox0_irq; } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0); + /* Register VF ME interrupt handler */ + ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME0", cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for PFVF mbox0 irq\n"); + goto free_flr0_irq; + } + if (num_vfs > 64) { vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); ret = request_irq(vector, otx2_cptpf_vfpf_mbox_intr, 0, @@ -209,7 +267,7 @@ static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) if (ret) { dev_err(dev, "IRQ registration failed for PFVF mbox1 irq\n"); - goto free_flr0_irq; + goto free_me0_irq; } vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1); /* Register VF FLR interrupt handler */ @@ -220,15 +278,30 @@ static int cptpf_register_vfpf_intr(struct otx2_cptpf_dev *cptpf, int num_vfs) "IRQ registration failed for VFFLR1 irq\n"); goto free_mbox1_irq; } + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME1); + /* Register VF FLR interrupt handler */ + ret = request_irq(vector, cptpf_vf_me_intr, 0, "CPTPF ME1", + cptpf); + if (ret) { + dev_err(dev, + "IRQ registration failed for VFFLR1 irq\n"); + goto free_flr1_irq; + } } cptpf_enable_vfpf_mbox_intr(cptpf, num_vfs); - cptpf_enable_vf_flr_intrs(cptpf); + cptpf_enable_vf_flr_me_intrs(cptpf, num_vfs); return 0; +free_flr1_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR1); + free_irq(vector, cptpf); free_mbox1_irq: vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFPF_MBOX1); free_irq(vector, cptpf); +free_me0_irq: + vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFME0); + free_irq(vector, cptpf); free_flr0_irq: vector = pci_irq_vector(pdev, RVU_PF_INT_VEC_VFFLR0); free_irq(vector, cptpf); @@ -284,7 +357,11 @@ static int cptpf_vfpf_mbox_init(struct otx2_cptpf_dev *cptpf, int num_vfs) return -ENOMEM; /* Map VF-PF mailbox memory */ - vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR); + if (test_bit(CN10K_MBOX, &cptpf->cap_flag)) + vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_MBOX_ADDR); + else + vfpf_mbox_base = readq(cptpf->reg_base + RVU_PF_VF_BAR4_ADDR); + if (!vfpf_mbox_base) { dev_err(dev, "VF-PF mailbox address not configured\n"); err = -ENOMEM; @@ -365,6 +442,8 @@ static int cptpf_register_afpf_mbox_intr(struct otx2_cptpf_dev *cptpf) static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf) { + struct pci_dev *pdev = cptpf->pdev; + resource_size_t offset; int err; cptpf->afpf_mbox_wq = alloc_workqueue("cpt_afpf_mailbox", @@ -373,8 +452,17 @@ static int cptpf_afpf_mbox_init(struct otx2_cptpf_dev *cptpf) if (!cptpf->afpf_mbox_wq) return -ENOMEM; + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + /* Map AF-PF mailbox memory */ + cptpf->afpf_mbox_base = devm_ioremap_wc(&pdev->dev, offset, MBOX_SIZE); + if (!cptpf->afpf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + err = -ENOMEM; + goto error; + } + err = otx2_mbox_init(&cptpf->afpf_mbox, cptpf->afpf_mbox_base, - cptpf->pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1); + pdev, cptpf->reg_base, MBOX_DIR_PFAF, 1); if (err) goto error; @@ -570,7 +658,7 @@ static int cptpf_sriov_enable(struct pci_dev *pdev, int num_vfs) if (ret) goto disable_intr; - ret = otx2_cpt_create_eng_grps(cptpf->pdev, &cptpf->eng_grps); + ret = otx2_cpt_create_eng_grps(cptpf, &cptpf->eng_grps); if (ret) goto disable_intr; @@ -607,7 +695,6 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct device *dev = &pdev->dev; - resource_size_t offset, size; struct otx2_cptpf_dev *cptpf; int err; @@ -644,15 +731,6 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, if (err) goto clear_drvdata; - offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); - size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); - /* Map AF-PF mailbox memory */ - cptpf->afpf_mbox_base = devm_ioremap_wc(dev, offset, size); - if (!cptpf->afpf_mbox_base) { - dev_err(&pdev->dev, "Unable to map BAR4\n"); - err = -ENODEV; - goto clear_drvdata; - } err = pci_alloc_irq_vectors(pdev, RVU_PF_INT_VEC_CNT, RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX); if (err < 0) { @@ -660,6 +738,7 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, RVU_PF_INT_VEC_CNT); goto clear_drvdata; } + otx2_cpt_set_hw_caps(pdev, &cptpf->cap_flag); /* Initialize AF-PF mailbox */ err = cptpf_afpf_mbox_init(cptpf); if (err) @@ -671,6 +750,10 @@ static int otx2_cptpf_probe(struct pci_dev *pdev, cptpf->max_vfs = pci_sriov_get_totalvfs(pdev); + err = cn10k_cptpf_lmtst_init(cptpf); + if (err) + goto unregister_intr; + /* Initialize CPT PF device */ err = cptpf_device_init(cptpf); if (err) @@ -719,6 +802,7 @@ static void otx2_cptpf_remove(struct pci_dev *pdev) /* Supported devices */ static const struct pci_device_id otx2_cpt_id_table[] = { { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, OTX2_CPT_PCI_PF_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, CN10K_CPT_PCI_PF_DEVICE_ID) }, { 0, } /* end of table */ }; diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index a531f4c8b441..dff34b3ec09e 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -16,6 +16,8 @@ #define LOADFVC_MAJOR_OP 0x01 #define LOADFVC_MINOR_OP 0x08 +#define CTX_FLUSH_TIMER_CNT 0xFFFFFF + struct fw_info_t { struct list_head ucodes; }; @@ -666,7 +668,8 @@ static int reserve_engines(struct device *dev, static void ucode_unload(struct device *dev, struct otx2_cpt_ucode *ucode) { if (ucode->va) { - dma_free_coherent(dev, ucode->size, ucode->va, ucode->dma); + dma_free_coherent(dev, OTX2_CPT_UCODE_SZ, ucode->va, + ucode->dma); ucode->va = NULL; ucode->dma = 0; ucode->size = 0; @@ -685,7 +688,7 @@ static int copy_ucode_to_dma_mem(struct device *dev, u32 i; /* Allocate DMAable space */ - ucode->va = dma_alloc_coherent(dev, ucode->size, &ucode->dma, + ucode->va = dma_alloc_coherent(dev, OTX2_CPT_UCODE_SZ, &ucode->dma, GFP_KERNEL); if (!ucode->va) return -ENOMEM; @@ -1100,11 +1103,12 @@ int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type) return eng_grp_num; } -int otx2_cpt_create_eng_grps(struct pci_dev *pdev, +int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, struct otx2_cpt_eng_grps *eng_grps) { struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = { }; struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} }; + struct pci_dev *pdev = cptpf->pdev; struct fw_info_t fw_info; int ret; @@ -1180,6 +1184,23 @@ int otx2_cpt_create_eng_grps(struct pci_dev *pdev, eng_grps->is_grps_created = true; cpt_ucode_release_fw(&fw_info); + + if (is_dev_otx2(pdev)) + return 0; + /* + * Configure engine group mask to allow context prefetching + * for the groups. + */ + otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTL, + OTX2_CPT_ALL_ENG_GRPS_MASK << 3 | BIT_ULL(16), + BLKADDR_CPT0); + /* + * Set interval to periodically flush dirty data for the next + * CTX cache entry. Set the interval count to maximum supported + * value. + */ + otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER, + CTX_FLUSH_TIMER_CNT, BLKADDR_CPT0); return 0; delete_eng_grp: @@ -1460,9 +1481,10 @@ int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf) iq_cmd.cptr.s.grp = otx2_cpt_get_eng_grp(&cptpf->eng_grps, etype); otx2_cpt_fill_inst(&inst, &iq_cmd, rptr_baddr); - otx2_cpt_send_cmd(&inst, 1, &cptpf->lfs.lf[0]); + lfs->ops->send_cmd(&inst, 1, &cptpf->lfs.lf[0]); - while (result->s.compcode == OTX2_CPT_COMPLETION_CODE_INIT) + while (lfs->ops->cpt_get_compcode(result) == + OTX2_CPT_COMPLETION_CODE_INIT) cpu_relax(); cptpf->eng_caps[etype].u = be64_to_cpup(rptr); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h index 6b0d432de0af..fe019ab730b2 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h @@ -23,11 +23,13 @@ /* Microcode version string length */ #define OTX2_CPT_UCODE_VER_STR_SZ 44 -/* Maximum number of supported engines/cores on OcteonTX2 platform */ -#define OTX2_CPT_MAX_ENGINES 128 +/* Maximum number of supported engines/cores on OcteonTX2/CN10K platform */ +#define OTX2_CPT_MAX_ENGINES 144 #define OTX2_CPT_ENGS_BITMASK_LEN BITS_TO_LONGS(OTX2_CPT_MAX_ENGINES) +#define OTX2_CPT_UCODE_SZ (64 * 1024) + /* Microcode types */ enum otx2_cpt_ucode_type { OTX2_CPT_AE_UC_TYPE = 1, /* AE-MAIN */ @@ -153,7 +155,7 @@ int otx2_cpt_init_eng_grps(struct pci_dev *pdev, struct otx2_cpt_eng_grps *eng_grps); void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev, struct otx2_cpt_eng_grps *eng_grps); -int otx2_cpt_create_eng_grps(struct pci_dev *pdev, +int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf, struct otx2_cpt_eng_grps *eng_grps); int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf); int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h index 4f0a169fddbd..4207e2236903 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf.h +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf.h @@ -19,11 +19,14 @@ struct otx2_cptvf_dev { struct otx2_mbox pfvf_mbox; struct work_struct pfvf_mbox_work; struct workqueue_struct *pfvf_mbox_wq; + void *bbuf_base; + unsigned long cap_flag; }; irqreturn_t otx2_cptvf_pfvf_mbox_intr(int irq, void *arg); void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work); int otx2_cptvf_send_eng_grp_num_msg(struct otx2_cptvf_dev *cptvf, int eng_type); int otx2_cptvf_send_kvf_limits_msg(struct otx2_cptvf_dev *cptvf); +int otx2_cpt_mbox_bbuf_init(struct otx2_cptvf_dev *cptvf, struct pci_dev *pdev); #endif /* __OTX2_CPTVF_H */ diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c index 47f378731024..3411e664cf50 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c @@ -5,9 +5,10 @@ #include "otx2_cptvf.h" #include "otx2_cptlf.h" #include "otx2_cptvf_algs.h" +#include "cn10k_cpt.h" #include -#define OTX2_CPTVF_DRV_NAME "octeontx2-cptvf" +#define OTX2_CPTVF_DRV_NAME "rvu_cptvf" static void cptvf_enable_pfvf_mbox_intrs(struct otx2_cptvf_dev *cptvf) { @@ -70,6 +71,8 @@ static int cptvf_register_interrupts(struct otx2_cptvf_dev *cptvf) static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf) { + struct pci_dev *pdev = cptvf->pdev; + resource_size_t offset, size; int ret; cptvf->pfvf_mbox_wq = alloc_workqueue("cpt_pfvf_mailbox", @@ -78,14 +81,39 @@ static int cptvf_pfvf_mbox_init(struct otx2_cptvf_dev *cptvf) if (!cptvf->pfvf_mbox_wq) return -ENOMEM; + if (test_bit(CN10K_MBOX, &cptvf->cap_flag)) { + /* For cn10k platform, VF mailbox region is in its BAR2 + * register space + */ + cptvf->pfvf_mbox_base = cptvf->reg_base + + CN10K_CPT_VF_MBOX_REGION; + } else { + offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); + size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); + /* Map PF-VF mailbox memory */ + cptvf->pfvf_mbox_base = devm_ioremap_wc(&pdev->dev, offset, + size); + if (!cptvf->pfvf_mbox_base) { + dev_err(&pdev->dev, "Unable to map BAR4\n"); + ret = -ENOMEM; + goto free_wqe; + } + } + ret = otx2_mbox_init(&cptvf->pfvf_mbox, cptvf->pfvf_mbox_base, - cptvf->pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1); + pdev, cptvf->reg_base, MBOX_DIR_VFPF, 1); if (ret) goto free_wqe; + ret = otx2_cpt_mbox_bbuf_init(cptvf, pdev); + if (ret) + goto destroy_mbox; + INIT_WORK(&cptvf->pfvf_mbox_work, otx2_cptvf_pfvf_mbox_handler); return 0; +destroy_mbox: + otx2_mbox_destroy(&cptvf->pfvf_mbox); free_wqe: destroy_workqueue(cptvf->pfvf_mbox_wq); return ret; @@ -305,7 +333,6 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct device *dev = &pdev->dev; - resource_size_t offset, size; struct otx2_cptvf_dev *cptvf; int ret; @@ -337,15 +364,12 @@ static int otx2_cptvf_probe(struct pci_dev *pdev, cptvf->reg_base = pcim_iomap_table(pdev)[PCI_PF_REG_BAR_NUM]; - offset = pci_resource_start(pdev, PCI_MBOX_BAR_NUM); - size = pci_resource_len(pdev, PCI_MBOX_BAR_NUM); - /* Map PF-VF mailbox memory */ - cptvf->pfvf_mbox_base = devm_ioremap_wc(dev, offset, size); - if (!cptvf->pfvf_mbox_base) { - dev_err(&pdev->dev, "Unable to map BAR4\n"); - ret = -ENODEV; + otx2_cpt_set_hw_caps(pdev, &cptvf->cap_flag); + + ret = cn10k_cptvf_lmtst_init(cptvf); + if (ret) goto clear_drvdata; - } + /* Initialize PF<=>VF mailbox */ ret = cptvf_pfvf_mbox_init(cptvf); if (ret) @@ -392,6 +416,7 @@ static void otx2_cptvf_remove(struct pci_dev *pdev) /* Supported devices */ static const struct pci_device_id otx2_cptvf_id_table[] = { {PCI_VDEVICE(CAVIUM, OTX2_CPT_PCI_VF_DEVICE_ID), 0}, + {PCI_VDEVICE(CAVIUM, CN10K_CPT_PCI_VF_DEVICE_ID), 0}, { 0, } /* end of table */ }; @@ -405,6 +430,6 @@ static struct pci_driver otx2_cptvf_pci_driver = { module_pci_driver(otx2_cptvf_pci_driver); MODULE_AUTHOR("Marvell"); -MODULE_DESCRIPTION("Marvell OcteonTX2 CPT Virtual Function Driver"); +MODULE_DESCRIPTION("Marvell RVU CPT Virtual Function Driver"); MODULE_LICENSE("GPL v2"); MODULE_DEVICE_TABLE(pci, otx2_cptvf_id_table); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c index 5d73b711cba6..02cb9e44afd8 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_mbox.c @@ -5,6 +5,48 @@ #include "otx2_cptvf.h" #include +int otx2_cpt_mbox_bbuf_init(struct otx2_cptvf_dev *cptvf, struct pci_dev *pdev) +{ + struct otx2_mbox_dev *mdev; + struct otx2_mbox *otx2_mbox; + + cptvf->bbuf_base = devm_kmalloc(&pdev->dev, MBOX_SIZE, GFP_KERNEL); + if (!cptvf->bbuf_base) + return -ENOMEM; + /* + * Overwrite mbox mbase to point to bounce buffer, so that PF/VF + * prepare all mbox messages in bounce buffer instead of directly + * in hw mbox memory. + */ + otx2_mbox = &cptvf->pfvf_mbox; + mdev = &otx2_mbox->dev[0]; + mdev->mbase = cptvf->bbuf_base; + + return 0; +} + +static void otx2_cpt_sync_mbox_bbuf(struct otx2_mbox *mbox, int devid) +{ + u16 msgs_offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); + void *hw_mbase = mbox->hwbase + (devid * MBOX_SIZE); + struct otx2_mbox_dev *mdev = &mbox->dev[devid]; + struct mbox_hdr *hdr; + u64 msg_size; + + if (mdev->mbase == hw_mbase) + return; + + hdr = hw_mbase + mbox->rx_start; + msg_size = hdr->msg_size; + + if (msg_size > mbox->rx_size - msgs_offset) + msg_size = mbox->rx_size - msgs_offset; + + /* Copy mbox messages from mbox memory to bounce buffer */ + memcpy(mdev->mbase + mbox->rx_start, + hw_mbase + mbox->rx_start, msg_size + msgs_offset); +} + irqreturn_t otx2_cptvf_pfvf_mbox_intr(int __always_unused irq, void *arg) { struct otx2_cptvf_dev *cptvf = arg; @@ -106,6 +148,7 @@ void otx2_cptvf_pfvf_mbox_handler(struct work_struct *work) cptvf = container_of(work, struct otx2_cptvf_dev, pfvf_mbox_work); pfvf_mbox = &cptvf->pfvf_mbox; + otx2_cpt_sync_mbox_bbuf(pfvf_mbox, 0); mdev = &pfvf_mbox->dev[0]; rsp_hdr = (struct mbox_hdr *)(mdev->mbase + pfvf_mbox->rx_start); if (rsp_hdr->num_msgs == 0) diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c index d5c1c1b7c7e4..811ded72ce5f 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c @@ -320,7 +320,7 @@ static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, cpt_req->dlen, false); /* Send CPT command */ - otx2_cpt_send_cmd(&cptinst, 1, lf); + lf->lfs->ops->send_cmd(&cptinst, 1, lf); /* * We allocate and prepare pending queue entry in critical section @@ -349,13 +349,14 @@ int otx2_cpt_do_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req, &lfs->lf[cpu_num]); } -static int cpt_process_ccode(struct pci_dev *pdev, +static int cpt_process_ccode(struct otx2_cptlfs_info *lfs, union otx2_cpt_res_s *cpt_status, struct otx2_cpt_inst_info *info, u32 *res_code) { - u8 uc_ccode = cpt_status->s.uc_compcode; - u8 ccode = cpt_status->s.compcode; + u8 uc_ccode = lfs->ops->cpt_get_uc_compcode(cpt_status); + u8 ccode = lfs->ops->cpt_get_compcode(cpt_status); + struct pci_dev *pdev = lfs->pdev; switch (ccode) { case OTX2_CPT_COMP_E_FAULT: @@ -389,6 +390,7 @@ static int cpt_process_ccode(struct pci_dev *pdev, return 1; case OTX2_CPT_COMP_E_GOOD: + case OTX2_CPT_COMP_E_WARN: /* * Check microcode completion code, it is only valid * when completion code is CPT_COMP_E::GOOD @@ -426,7 +428,7 @@ static int cpt_process_ccode(struct pci_dev *pdev, return 0; } -static inline void process_pending_queue(struct pci_dev *pdev, +static inline void process_pending_queue(struct otx2_cptlfs_info *lfs, struct otx2_cpt_pending_queue *pqueue) { struct otx2_cpt_pending_entry *resume_pentry = NULL; @@ -436,6 +438,7 @@ static inline void process_pending_queue(struct pci_dev *pdev, struct otx2_cpt_inst_info *info = NULL; struct otx2_cpt_req_info *req = NULL; struct crypto_async_request *areq; + struct pci_dev *pdev = lfs->pdev; u32 res_code, resume_index; while (1) { @@ -476,7 +479,7 @@ static inline void process_pending_queue(struct pci_dev *pdev, goto process_pentry; } - if (cpt_process_ccode(pdev, cpt_status, info, &res_code)) { + if (cpt_process_ccode(lfs, cpt_status, info, &res_code)) { spin_unlock_bh(&pqueue->lock); return; } @@ -529,7 +532,7 @@ static inline void process_pending_queue(struct pci_dev *pdev, void otx2_cpt_post_process(struct otx2_cptlf_wqe *wqe) { - process_pending_queue(wqe->lfs->pdev, + process_pending_queue(wqe->lfs, &wqe->lfs->lf[wqe->lf_num].pqueue); } diff --git a/drivers/crypto/nx/nx-842-pseries.c b/drivers/crypto/nx/nx-842-pseries.c index cc8dd3072b8b..1491cbfbc071 100644 --- a/drivers/crypto/nx/nx-842-pseries.c +++ b/drivers/crypto/nx/nx-842-pseries.c @@ -264,8 +264,8 @@ static int nx842_validate_result(struct device *dev, * @inlen: Length of input buffer * @out: Pointer to output buffer * @outlen: Length of output buffer - * @wrkmem: ptr to buffer for working memory, size determined by - * nx842_pseries_driver.workmem_size + * @wmem: ptr to buffer for working memory, size determined by + * nx842_pseries_driver.workmem_size * * Returns: * 0 Success, output of length @outlen stored in the buffer at @out @@ -393,8 +393,8 @@ static int nx842_pseries_compress(const unsigned char *in, unsigned int inlen, * @inlen: Length of input buffer * @out: Pointer to output buffer * @outlen: Length of output buffer - * @wrkmem: ptr to buffer for working memory, size determined by - * nx842_pseries_driver.workmem_size + * @wmem: ptr to buffer for working memory, size determined by + * nx842_pseries_driver.workmem_size * * Returns: * 0 Success, output of length @outlen stored in the buffer at @out @@ -513,7 +513,7 @@ static int nx842_pseries_decompress(const unsigned char *in, unsigned int inlen, /** * nx842_OF_set_defaults -- Set default (disabled) values for devdata * - * @devdata - struct nx842_devdata to update + * @devdata: struct nx842_devdata to update * * Returns: * 0 on success @@ -538,13 +538,15 @@ static int nx842_OF_set_defaults(struct nx842_devdata *devdata) * The status field indicates if the device is enabled when the status * is 'okay'. Otherwise the device driver will be disabled. * - * @prop - struct property point containing the maxsyncop for the update + * @devdata: struct nx842_devdata to use for dev_info + * @prop: struct property point containing the maxsyncop for the update * * Returns: * 0 - Device is available * -ENODEV - Device is not available */ -static int nx842_OF_upd_status(struct property *prop) +static int nx842_OF_upd_status(struct nx842_devdata *devdata, + struct property *prop) { const char *status = (const char *)prop->value; @@ -571,8 +573,8 @@ static int nx842_OF_upd_status(struct property *prop) * In this example, the maximum byte length of a scatter list is * 0x0ff0 (4,080). * - * @devdata - struct nx842_devdata to update - * @prop - struct property point containing the maxsyncop for the update + * @devdata: struct nx842_devdata to update + * @prop: struct property point containing the maxsyncop for the update * * Returns: * 0 on success @@ -619,8 +621,8 @@ static int nx842_OF_upd_maxsglen(struct nx842_devdata *devdata, * 0x1000 (4,096) data byte length and 0x1f3 (510) total scatter list * elements. * - * @devdata - struct nx842_devdata to update - * @prop - struct property point containing the maxsyncop for the update + * @devdata: struct nx842_devdata to update + * @prop: struct property point containing the maxsyncop for the update * * Returns: * 0 on success @@ -689,7 +691,6 @@ static int nx842_OF_upd_maxsyncop(struct nx842_devdata *devdata, } /** - * * nx842_OF_upd -- Handle OF properties updates for the device. * * Set all properties from the OF tree. Optionally, a new property @@ -758,7 +759,7 @@ static int nx842_OF_upd(struct property *new_prop) goto out; /* Perform property updates */ - ret = nx842_OF_upd_status(status); + ret = nx842_OF_upd_status(new_devdata, status); if (ret) goto error_out; @@ -812,8 +813,7 @@ static int nx842_OF_upd(struct property *new_prop) * * @np: notifier block * @action: notifier action - * @update: struct pSeries_reconfig_prop_update pointer if action is - * PSERIES_UPDATE_PROPERTY + * @data: struct of_reconfig_data pointer * * Returns: * NOTIFY_OK on success @@ -1069,6 +1069,7 @@ static const struct vio_device_id nx842_vio_driver_ids[] = { {"ibm,compression-v1", "ibm,compression"}, {"", ""}, }; +MODULE_DEVICE_TABLE(vio, nx842_vio_driver_ids); static struct vio_driver nx842_vio_driver = { .name = KBUILD_MODNAME, diff --git a/drivers/crypto/nx/nx-aes-cbc.c b/drivers/crypto/nx/nx-aes-cbc.c index d6314ea9ae89..0e440f704a8f 100644 --- a/drivers/crypto/nx/nx-aes-cbc.c +++ b/drivers/crypto/nx/nx-aes-cbc.c @@ -88,7 +88,7 @@ static int cbc_aes_nx_crypt(struct skcipher_request *req, memcpy(req->iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE); atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(csbcpb->csb.processed_byte_count, + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), &(nx_ctx->stats->aes_bytes)); processed += to_process; diff --git a/drivers/crypto/nx/nx-aes-ccm.c b/drivers/crypto/nx/nx-aes-ccm.c index e7384d107573..3793885f928d 100644 --- a/drivers/crypto/nx/nx-aes-ccm.c +++ b/drivers/crypto/nx/nx-aes-ccm.c @@ -391,7 +391,7 @@ static int ccm_nx_decrypt(struct aead_request *req, /* update stats */ atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(csbcpb->csb.processed_byte_count, + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), &(nx_ctx->stats->aes_bytes)); processed += to_process; @@ -460,7 +460,7 @@ static int ccm_nx_encrypt(struct aead_request *req, /* update stats */ atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(csbcpb->csb.processed_byte_count, + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), &(nx_ctx->stats->aes_bytes)); processed += to_process; diff --git a/drivers/crypto/nx/nx-aes-ctr.c b/drivers/crypto/nx/nx-aes-ctr.c index 13f518802343..dfa3ad1a12f2 100644 --- a/drivers/crypto/nx/nx-aes-ctr.c +++ b/drivers/crypto/nx/nx-aes-ctr.c @@ -102,7 +102,7 @@ static int ctr_aes_nx_crypt(struct skcipher_request *req, u8 *iv) memcpy(iv, csbcpb->cpb.aes_cbc.cv, AES_BLOCK_SIZE); atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(csbcpb->csb.processed_byte_count, + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), &(nx_ctx->stats->aes_bytes)); processed += to_process; @@ -118,7 +118,7 @@ static int ctr3686_aes_nx_crypt(struct skcipher_request *req) struct nx_crypto_ctx *nx_ctx = crypto_skcipher_ctx(tfm); u8 iv[16]; - memcpy(iv, nx_ctx->priv.ctr.nonce, CTR_RFC3686_IV_SIZE); + memcpy(iv, nx_ctx->priv.ctr.nonce, CTR_RFC3686_NONCE_SIZE); memcpy(iv + CTR_RFC3686_NONCE_SIZE, req->iv, CTR_RFC3686_IV_SIZE); iv[12] = iv[13] = iv[14] = 0; iv[15] = 1; diff --git a/drivers/crypto/nx/nx-aes-ecb.c b/drivers/crypto/nx/nx-aes-ecb.c index 7a729dc2bc17..502a565074e9 100644 --- a/drivers/crypto/nx/nx-aes-ecb.c +++ b/drivers/crypto/nx/nx-aes-ecb.c @@ -86,7 +86,7 @@ static int ecb_aes_nx_crypt(struct skcipher_request *req, goto out; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(csbcpb->csb.processed_byte_count, + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), &(nx_ctx->stats->aes_bytes)); processed += to_process; diff --git a/drivers/crypto/nx/nx-aes-gcm.c b/drivers/crypto/nx/nx-aes-gcm.c index fc9baca13920..4a796318b430 100644 --- a/drivers/crypto/nx/nx-aes-gcm.c +++ b/drivers/crypto/nx/nx-aes-gcm.c @@ -382,7 +382,7 @@ static int gcm_aes_nx_crypt(struct aead_request *req, int enc, NX_CPB_FDM(csbcpb) |= NX_FDM_CONTINUATION; atomic_inc(&(nx_ctx->stats->aes_ops)); - atomic64_add(csbcpb->csb.processed_byte_count, + atomic64_add(be32_to_cpu(csbcpb->csb.processed_byte_count), &(nx_ctx->stats->aes_bytes)); processed += to_process; diff --git a/drivers/crypto/nx/nx-common-powernv.c b/drivers/crypto/nx/nx-common-powernv.c index 446f611726df..655361ba9107 100644 --- a/drivers/crypto/nx/nx-common-powernv.c +++ b/drivers/crypto/nx/nx-common-powernv.c @@ -660,8 +660,8 @@ static int nx842_powernv_compress(const unsigned char *in, unsigned int inlen, * @inlen: input buffer size * @out: output buffer pointer * @outlenp: output buffer size pointer - * @workmem: working memory buffer pointer, size determined by - * nx842_powernv_driver.workmem_size + * @wmem: working memory buffer pointer, size determined by + * nx842_powernv_driver.workmem_size * * Returns: see @nx842_powernv_exec() */ diff --git a/drivers/crypto/nx/nx-sha256.c b/drivers/crypto/nx/nx-sha256.c index b0ad665e4bda..c3bebf0feabe 100644 --- a/drivers/crypto/nx/nx-sha256.c +++ b/drivers/crypto/nx/nx-sha256.c @@ -16,6 +16,11 @@ #include "nx_csbcpb.h" #include "nx.h" +struct sha256_state_be { + __be32 state[SHA256_DIGEST_SIZE / 4]; + u64 count; + u8 buf[SHA256_BLOCK_SIZE]; +}; static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) { @@ -36,7 +41,7 @@ static int nx_crypto_ctx_sha256_init(struct crypto_tfm *tfm) } static int nx_sha256_init(struct shash_desc *desc) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct sha256_state_be *sctx = shash_desc_ctx(desc); memset(sctx, 0, sizeof *sctx); @@ -56,7 +61,7 @@ static int nx_sha256_init(struct shash_desc *desc) { static int nx_sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct sha256_state_be *sctx = shash_desc_ctx(desc); struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *out_sg; @@ -175,7 +180,7 @@ static int nx_sha256_update(struct shash_desc *desc, const u8 *data, static int nx_sha256_final(struct shash_desc *desc, u8 *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct sha256_state_be *sctx = shash_desc_ctx(desc); struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; @@ -245,7 +250,7 @@ static int nx_sha256_final(struct shash_desc *desc, u8 *out) static int nx_sha256_export(struct shash_desc *desc, void *out) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct sha256_state_be *sctx = shash_desc_ctx(desc); memcpy(out, sctx, sizeof(*sctx)); @@ -254,7 +259,7 @@ static int nx_sha256_export(struct shash_desc *desc, void *out) static int nx_sha256_import(struct shash_desc *desc, const void *in) { - struct sha256_state *sctx = shash_desc_ctx(desc); + struct sha256_state_be *sctx = shash_desc_ctx(desc); memcpy(sctx, in, sizeof(*sctx)); @@ -268,8 +273,8 @@ struct shash_alg nx_shash_sha256_alg = { .final = nx_sha256_final, .export = nx_sha256_export, .import = nx_sha256_import, - .descsize = sizeof(struct sha256_state), - .statesize = sizeof(struct sha256_state), + .descsize = sizeof(struct sha256_state_be), + .statesize = sizeof(struct sha256_state_be), .base = { .cra_name = "sha256", .cra_driver_name = "sha256-nx", diff --git a/drivers/crypto/nx/nx-sha512.c b/drivers/crypto/nx/nx-sha512.c index c29103a1a0b6..1ffb40d2c324 100644 --- a/drivers/crypto/nx/nx-sha512.c +++ b/drivers/crypto/nx/nx-sha512.c @@ -15,6 +15,11 @@ #include "nx_csbcpb.h" #include "nx.h" +struct sha512_state_be { + __be64 state[SHA512_DIGEST_SIZE / 8]; + u64 count[2]; + u8 buf[SHA512_BLOCK_SIZE]; +}; static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm) { @@ -36,7 +41,7 @@ static int nx_crypto_ctx_sha512_init(struct crypto_tfm *tfm) static int nx_sha512_init(struct shash_desc *desc) { - struct sha512_state *sctx = shash_desc_ctx(desc); + struct sha512_state_be *sctx = shash_desc_ctx(desc); memset(sctx, 0, sizeof *sctx); @@ -56,7 +61,7 @@ static int nx_sha512_init(struct shash_desc *desc) static int nx_sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha512_state *sctx = shash_desc_ctx(desc); + struct sha512_state_be *sctx = shash_desc_ctx(desc); struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *out_sg; @@ -178,7 +183,7 @@ static int nx_sha512_update(struct shash_desc *desc, const u8 *data, static int nx_sha512_final(struct shash_desc *desc, u8 *out) { - struct sha512_state *sctx = shash_desc_ctx(desc); + struct sha512_state_be *sctx = shash_desc_ctx(desc); struct nx_crypto_ctx *nx_ctx = crypto_tfm_ctx(&desc->tfm->base); struct nx_csbcpb *csbcpb = (struct nx_csbcpb *)nx_ctx->csbcpb; struct nx_sg *in_sg, *out_sg; @@ -251,7 +256,7 @@ static int nx_sha512_final(struct shash_desc *desc, u8 *out) static int nx_sha512_export(struct shash_desc *desc, void *out) { - struct sha512_state *sctx = shash_desc_ctx(desc); + struct sha512_state_be *sctx = shash_desc_ctx(desc); memcpy(out, sctx, sizeof(*sctx)); @@ -260,7 +265,7 @@ static int nx_sha512_export(struct shash_desc *desc, void *out) static int nx_sha512_import(struct shash_desc *desc, const void *in) { - struct sha512_state *sctx = shash_desc_ctx(desc); + struct sha512_state_be *sctx = shash_desc_ctx(desc); memcpy(sctx, in, sizeof(*sctx)); @@ -274,8 +279,8 @@ struct shash_alg nx_shash_sha512_alg = { .final = nx_sha512_final, .export = nx_sha512_export, .import = nx_sha512_import, - .descsize = sizeof(struct sha512_state), - .statesize = sizeof(struct sha512_state), + .descsize = sizeof(struct sha512_state_be), + .statesize = sizeof(struct sha512_state_be), .base = { .cra_name = "sha512", .cra_driver_name = "sha512-nx", diff --git a/drivers/crypto/nx/nx_csbcpb.h b/drivers/crypto/nx/nx_csbcpb.h index 493f8490ff94..e64f7e36fb92 100644 --- a/drivers/crypto/nx/nx_csbcpb.h +++ b/drivers/crypto/nx/nx_csbcpb.h @@ -140,8 +140,8 @@ struct cop_status_block { u8 crb_seq_number; u8 completion_code; u8 completion_extension; - u32 processed_byte_count; - u64 address; + __be32 processed_byte_count; + __be64 address; } __packed; /* Nest accelerator workbook section 4.4 */ diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index c9d38bcfd1c7..bc8631363d72 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -229,9 +229,8 @@ static int omap_des_hw_init(struct omap_des_dev *dd) * It may be long delays between requests. * Device might go to off mode to save power. */ - err = pm_runtime_get_sync(dd->dev); + err = pm_runtime_resume_and_get(dd->dev); if (err < 0) { - pm_runtime_put_noidle(dd->dev); dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err); return err; } @@ -994,9 +993,8 @@ static int omap_des_probe(struct platform_device *pdev) pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); pm_runtime_enable(dev); - err = pm_runtime_get_sync(dev); + err = pm_runtime_resume_and_get(dev); if (err < 0) { - pm_runtime_put_noidle(dev); dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err); goto err_get; } @@ -1124,9 +1122,8 @@ static int omap_des_resume(struct device *dev) { int err; - err = pm_runtime_get_sync(dev); + err = pm_runtime_resume_and_get(dev); if (err < 0) { - pm_runtime_put_noidle(dev); dev_err(dev, "%s: failed to get_sync(%d)\n", __func__, err); return err; } diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index ae0d320d3c60..dd53ad9987b0 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -372,7 +372,7 @@ static int omap_sham_hw_init(struct omap_sham_dev *dd) { int err; - err = pm_runtime_get_sync(dd->dev); + err = pm_runtime_resume_and_get(dd->dev); if (err < 0) { dev_err(dd->dev, "failed to get sync: %d\n", err); return err; @@ -2244,7 +2244,7 @@ static int omap_sham_suspend(struct device *dev) static int omap_sham_resume(struct device *dev) { - int err = pm_runtime_get_sync(dev); + int err = pm_runtime_resume_and_get(dev); if (err < 0) { dev_err(dev, "failed to get sync: %d\n", err); return err; diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index b8f3463be6ef..7eb5daef4f88 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -24,7 +24,7 @@ struct icp_qat_fw_loader_hal_handle { }; struct icp_qat_fw_loader_chip_info { - bool sram_visible; + int mmp_sram_size; bool nn; bool lm2lm3; u32 lm_size; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index bd3028126cbe..12ca6b8764aa 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -696,7 +696,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->pci_dev = pci_info->pci_dev; switch (handle->pci_dev->device) { case ADF_4XXX_PCI_DEVICE_ID: - handle->chip_info->sram_visible = false; + handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = false; handle->chip_info->lm2lm3 = true; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG_2X; @@ -730,7 +730,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, break; case PCI_DEVICE_ID_INTEL_QAT_C62X: case PCI_DEVICE_ID_INTEL_QAT_C3XXX: - handle->chip_info->sram_visible = false; + handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; @@ -763,7 +763,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, + LOCAL_TO_XFER_REG_OFFSET); break; case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: - handle->chip_info->sram_visible = true; + handle->chip_info->mmp_sram_size = 0x40000; handle->chip_info->nn = true; handle->chip_info->lm2lm3 = false; handle->chip_info->lm_size = ICP_QAT_UCLO_MAX_LMEM_REG; @@ -800,7 +800,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, goto out_err; } - if (handle->chip_info->sram_visible) { + if (handle->chip_info->mmp_sram_size > 0) { sram_bar = &pci_info->pci_bars[hw_data->get_sram_bar_id(hw_data)]; handle->hal_sram_addr_v = sram_bar->virt_addr; @@ -1417,7 +1417,11 @@ static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle, pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr); return -EINVAL; } - qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); + status = qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval); + if (status) { + pr_err("QAT: failed to read register"); + return status; + } gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum); data16low = 0xffff & data; data16hi = 0xffff & (data >> 0x10); diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index 1fb5fc852f6b..2026cc6be8f0 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -342,7 +342,6 @@ static int qat_uclo_init_umem_seg(struct icp_qat_fw_loader_handle *handle, return 0; } -#define ICP_DH895XCC_PESRAM_BAR_SIZE 0x80000 static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, struct icp_qat_uof_initmem *init_mem) { @@ -1546,15 +1545,14 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, int status = 0; if (handle->chip_info->fw_auth) { - if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc)) + status = qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc); + if (!status) status = qat_uclo_auth_fw(handle, desc); qat_uclo_ummap_auth_fw(handle, &desc); } else { - if (!handle->chip_info->sram_visible) { - dev_dbg(&handle->pci_dev->dev, - "QAT MMP fw not loaded for device 0x%x", - handle->pci_dev->device); - return status; + if (handle->chip_info->mmp_sram_size < mem_size) { + pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); + return -EFBIG; } qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); } diff --git a/drivers/crypto/qce/Makefile b/drivers/crypto/qce/Makefile index 14ade8a7d664..2cf8984e1b85 100644 --- a/drivers/crypto/qce/Makefile +++ b/drivers/crypto/qce/Makefile @@ -6,3 +6,4 @@ qcrypto-objs := core.o \ qcrypto-$(CONFIG_CRYPTO_DEV_QCE_SHA) += sha.o qcrypto-$(CONFIG_CRYPTO_DEV_QCE_SKCIPHER) += skcipher.o +qcrypto-$(CONFIG_CRYPTO_DEV_QCE_AEAD) += aead.o diff --git a/drivers/crypto/qce/aead.c b/drivers/crypto/qce/aead.c new file mode 100644 index 000000000000..290e2446a2f3 --- /dev/null +++ b/drivers/crypto/qce/aead.c @@ -0,0 +1,847 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (C) 2021, Linaro Limited. All rights reserved. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "aead.h" + +#define CCM_NONCE_ADATA_SHIFT 6 +#define CCM_NONCE_AUTHSIZE_SHIFT 3 +#define MAX_CCM_ADATA_HEADER_LEN 6 + +static LIST_HEAD(aead_algs); + +static void qce_aead_done(void *data) +{ + struct crypto_async_request *async_req = data; + struct aead_request *req = aead_request_cast(async_req); + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + struct qce_result_dump *result_buf = qce->dma.result_buf; + enum dma_data_direction dir_src, dir_dst; + bool diff_dst; + int error; + u32 status; + unsigned int totallen; + unsigned char tag[SHA256_DIGEST_SIZE] = {0}; + int ret = 0; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + error = qce_dma_terminate_all(&qce->dma); + if (error) + dev_dbg(qce->dev, "aead dma termination error (%d)\n", + error); + if (diff_dst) + dma_unmap_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); + + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); + + if (IS_CCM(rctx->flags)) { + if (req->assoclen) { + sg_free_table(&rctx->src_tbl); + if (diff_dst) + sg_free_table(&rctx->dst_tbl); + } else { + if (!(IS_DECRYPT(rctx->flags) && !diff_dst)) + sg_free_table(&rctx->dst_tbl); + } + } else { + sg_free_table(&rctx->dst_tbl); + } + + error = qce_check_status(qce, &status); + if (error < 0 && (error != -EBADMSG)) + dev_err(qce->dev, "aead operation error (%x)\n", status); + + if (IS_ENCRYPT(rctx->flags)) { + totallen = req->cryptlen + req->assoclen; + if (IS_CCM(rctx->flags)) + scatterwalk_map_and_copy(rctx->ccmresult_buf, req->dst, + totallen, ctx->authsize, 1); + else + scatterwalk_map_and_copy(result_buf->auth_iv, req->dst, + totallen, ctx->authsize, 1); + + } else if (!IS_CCM(rctx->flags)) { + totallen = req->cryptlen + req->assoclen - ctx->authsize; + scatterwalk_map_and_copy(tag, req->src, totallen, ctx->authsize, 0); + ret = memcmp(result_buf->auth_iv, tag, ctx->authsize); + if (ret) { + pr_err("Bad message error\n"); + error = -EBADMSG; + } + } + + qce->async_req_done(qce, error); +} + +static struct scatterlist * +qce_aead_prepare_result_buf(struct sg_table *tbl, struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + + sg_init_one(&rctx->result_sg, qce->dma.result_buf, QCE_RESULT_BUF_SZ); + return qce_sgtable_add(tbl, &rctx->result_sg, QCE_RESULT_BUF_SZ); +} + +static struct scatterlist * +qce_aead_prepare_ccm_result_buf(struct sg_table *tbl, struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + + sg_init_one(&rctx->result_sg, rctx->ccmresult_buf, QCE_BAM_BURST_SIZE); + return qce_sgtable_add(tbl, &rctx->result_sg, QCE_BAM_BURST_SIZE); +} + +static struct scatterlist * +qce_aead_prepare_dst_buf(struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + struct scatterlist *sg, *msg_sg, __sg[2]; + gfp_t gfp; + unsigned int assoclen = req->assoclen; + unsigned int totallen; + int ret; + + totallen = rctx->cryptlen + assoclen; + rctx->dst_nents = sg_nents_for_len(req->dst, totallen); + if (rctx->dst_nents < 0) { + dev_err(qce->dev, "Invalid numbers of dst SG.\n"); + return ERR_PTR(-EINVAL); + } + if (IS_CCM(rctx->flags)) + rctx->dst_nents += 2; + else + rctx->dst_nents += 1; + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + ret = sg_alloc_table(&rctx->dst_tbl, rctx->dst_nents, gfp); + if (ret) + return ERR_PTR(ret); + + if (IS_CCM(rctx->flags) && assoclen) { + /* Get the dst buffer */ + msg_sg = scatterwalk_ffwd(__sg, req->dst, assoclen); + + sg = qce_sgtable_add(&rctx->dst_tbl, &rctx->adata_sg, + rctx->assoclen); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto dst_tbl_free; + } + /* dst buffer */ + sg = qce_sgtable_add(&rctx->dst_tbl, msg_sg, rctx->cryptlen); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto dst_tbl_free; + } + totallen = rctx->cryptlen + rctx->assoclen; + } else { + if (totallen) { + sg = qce_sgtable_add(&rctx->dst_tbl, req->dst, totallen); + if (IS_ERR(sg)) + goto dst_tbl_free; + } + } + if (IS_CCM(rctx->flags)) + sg = qce_aead_prepare_ccm_result_buf(&rctx->dst_tbl, req); + else + sg = qce_aead_prepare_result_buf(&rctx->dst_tbl, req); + + if (IS_ERR(sg)) + goto dst_tbl_free; + + sg_mark_end(sg); + rctx->dst_sg = rctx->dst_tbl.sgl; + rctx->dst_nents = sg_nents_for_len(rctx->dst_sg, totallen) + 1; + + return sg; + +dst_tbl_free: + sg_free_table(&rctx->dst_tbl); + return sg; +} + +static int +qce_aead_ccm_prepare_buf_assoclen(struct aead_request *req) +{ + struct scatterlist *sg, *msg_sg, __sg[2]; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned int assoclen = rctx->assoclen; + unsigned int adata_header_len, cryptlen, totallen; + gfp_t gfp; + bool diff_dst; + int ret; + + if (IS_DECRYPT(rctx->flags)) + cryptlen = rctx->cryptlen + ctx->authsize; + else + cryptlen = rctx->cryptlen; + totallen = cryptlen + req->assoclen; + + /* Get the msg */ + msg_sg = scatterwalk_ffwd(__sg, req->src, req->assoclen); + + rctx->adata = kzalloc((ALIGN(assoclen, 16) + MAX_CCM_ADATA_HEADER_LEN) * + sizeof(unsigned char), GFP_ATOMIC); + if (!rctx->adata) + return -ENOMEM; + + /* + * Format associated data (RFC3610 and NIST 800-38C) + * Even though specification allows for AAD to be up to 2^64 - 1 bytes, + * the assoclen field in aead_request is unsigned int and thus limits + * the AAD to be up to 2^32 - 1 bytes. So we handle only two scenarios + * while forming the header for AAD. + */ + if (assoclen < 0xff00) { + adata_header_len = 2; + *(__be16 *)rctx->adata = cpu_to_be16(assoclen); + } else { + adata_header_len = 6; + *(__be16 *)rctx->adata = cpu_to_be16(0xfffe); + *(__be32 *)(rctx->adata + 2) = cpu_to_be32(assoclen); + } + + /* Copy the associated data */ + if (sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, assoclen), + rctx->adata + adata_header_len, + assoclen) != assoclen) + return -EINVAL; + + /* Pad associated data to block size */ + rctx->assoclen = ALIGN(assoclen + adata_header_len, 16); + + diff_dst = (req->src != req->dst) ? true : false; + + if (diff_dst) + rctx->src_nents = sg_nents_for_len(req->src, totallen) + 1; + else + rctx->src_nents = sg_nents_for_len(req->src, totallen) + 2; + + gfp = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; + ret = sg_alloc_table(&rctx->src_tbl, rctx->src_nents, gfp); + if (ret) + return ret; + + /* Associated Data */ + sg_init_one(&rctx->adata_sg, rctx->adata, rctx->assoclen); + sg = qce_sgtable_add(&rctx->src_tbl, &rctx->adata_sg, + rctx->assoclen); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + /* src msg */ + sg = qce_sgtable_add(&rctx->src_tbl, msg_sg, cryptlen); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + if (!diff_dst) { + /* + * For decrypt, when src and dst buffers are same, there is already space + * in the buffer for padded 0's which is output in lieu of + * the MAC that is input. So skip the below. + */ + if (!IS_DECRYPT(rctx->flags)) { + sg = qce_aead_prepare_ccm_result_buf(&rctx->src_tbl, req); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + } + } + sg_mark_end(sg); + rctx->src_sg = rctx->src_tbl.sgl; + totallen = cryptlen + rctx->assoclen; + rctx->src_nents = sg_nents_for_len(rctx->src_sg, totallen); + + if (diff_dst) { + sg = qce_aead_prepare_dst_buf(req); + if (IS_ERR(sg)) { + ret = PTR_ERR(sg); + goto err_free; + } + } else { + if (IS_ENCRYPT(rctx->flags)) + rctx->dst_nents = rctx->src_nents + 1; + else + rctx->dst_nents = rctx->src_nents; + rctx->dst_sg = rctx->src_sg; + } + + return 0; +err_free: + sg_free_table(&rctx->src_tbl); + return ret; +} + +static int qce_aead_prepare_buf(struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + struct scatterlist *sg; + bool diff_dst = (req->src != req->dst) ? true : false; + unsigned int totallen; + + totallen = rctx->cryptlen + rctx->assoclen; + + sg = qce_aead_prepare_dst_buf(req); + if (IS_ERR(sg)) + return PTR_ERR(sg); + if (diff_dst) { + rctx->src_nents = sg_nents_for_len(req->src, totallen); + if (rctx->src_nents < 0) { + dev_err(qce->dev, "Invalid numbers of src SG.\n"); + return -EINVAL; + } + rctx->src_sg = req->src; + } else { + rctx->src_nents = rctx->dst_nents - 1; + rctx->src_sg = rctx->dst_sg; + } + return 0; +} + +static int qce_aead_ccm_prepare_buf(struct aead_request *req) +{ + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct scatterlist *sg; + bool diff_dst = (req->src != req->dst) ? true : false; + unsigned int cryptlen; + + if (rctx->assoclen) + return qce_aead_ccm_prepare_buf_assoclen(req); + + if (IS_ENCRYPT(rctx->flags)) + return qce_aead_prepare_buf(req); + + cryptlen = rctx->cryptlen + ctx->authsize; + if (diff_dst) { + rctx->src_nents = sg_nents_for_len(req->src, cryptlen); + rctx->src_sg = req->src; + sg = qce_aead_prepare_dst_buf(req); + if (IS_ERR(sg)) + return PTR_ERR(sg); + } else { + rctx->src_nents = sg_nents_for_len(req->src, cryptlen); + rctx->src_sg = req->src; + rctx->dst_nents = rctx->src_nents; + rctx->dst_sg = rctx->src_sg; + } + + return 0; +} + +static int qce_aead_create_ccm_nonce(struct qce_aead_reqctx *rctx, struct qce_aead_ctx *ctx) +{ + unsigned int msglen_size, ivsize; + u8 msg_len[4]; + int i; + + if (!rctx || !rctx->iv) + return -EINVAL; + + msglen_size = rctx->iv[0] + 1; + + /* Verify that msg len size is valid */ + if (msglen_size < 2 || msglen_size > 8) + return -EINVAL; + + ivsize = rctx->ivsize; + + /* + * Clear the msglen bytes in IV. + * Else the h/w engine and nonce will use any stray value pending there. + */ + if (!IS_CCM_RFC4309(rctx->flags)) { + for (i = 0; i < msglen_size; i++) + rctx->iv[ivsize - i - 1] = 0; + } + + /* + * The crypto framework encodes cryptlen as unsigned int. Thus, even though + * spec allows for upto 8 bytes to encode msg_len only 4 bytes are needed. + */ + if (msglen_size > 4) + msglen_size = 4; + + memcpy(&msg_len[0], &rctx->cryptlen, 4); + + memcpy(&rctx->ccm_nonce[0], rctx->iv, rctx->ivsize); + if (rctx->assoclen) + rctx->ccm_nonce[0] |= 1 << CCM_NONCE_ADATA_SHIFT; + rctx->ccm_nonce[0] |= ((ctx->authsize - 2) / 2) << + CCM_NONCE_AUTHSIZE_SHIFT; + for (i = 0; i < msglen_size; i++) + rctx->ccm_nonce[QCE_MAX_NONCE - i - 1] = msg_len[i]; + + return 0; +} + +static int +qce_aead_async_req_handle(struct crypto_async_request *async_req) +{ + struct aead_request *req = aead_request_cast(async_req); + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + enum dma_data_direction dir_src, dir_dst; + bool diff_dst; + int dst_nents, src_nents, ret; + + if (IS_CCM_RFC4309(rctx->flags)) { + memset(rctx->ccm_rfc4309_iv, 0, QCE_MAX_IV_SIZE); + rctx->ccm_rfc4309_iv[0] = 3; + memcpy(&rctx->ccm_rfc4309_iv[1], ctx->ccm4309_salt, QCE_CCM4309_SALT_SIZE); + memcpy(&rctx->ccm_rfc4309_iv[4], req->iv, 8); + rctx->iv = rctx->ccm_rfc4309_iv; + rctx->ivsize = AES_BLOCK_SIZE; + } else { + rctx->iv = req->iv; + rctx->ivsize = crypto_aead_ivsize(tfm); + } + if (IS_CCM_RFC4309(rctx->flags)) + rctx->assoclen = req->assoclen - 8; + else + rctx->assoclen = req->assoclen; + + diff_dst = (req->src != req->dst) ? true : false; + dir_src = diff_dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL; + dir_dst = diff_dst ? DMA_FROM_DEVICE : DMA_BIDIRECTIONAL; + + if (IS_CCM(rctx->flags)) { + ret = qce_aead_create_ccm_nonce(rctx, ctx); + if (ret) + return ret; + } + if (IS_CCM(rctx->flags)) + ret = qce_aead_ccm_prepare_buf(req); + else + ret = qce_aead_prepare_buf(req); + + if (ret) + return ret; + dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); + if (dst_nents < 0) { + ret = dst_nents; + goto error_free; + } + + if (diff_dst) { + src_nents = dma_map_sg(qce->dev, rctx->src_sg, rctx->src_nents, dir_src); + if (src_nents < 0) { + ret = src_nents; + goto error_unmap_dst; + } + } else { + if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) + src_nents = dst_nents; + else + src_nents = dst_nents - 1; + } + + ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, rctx->dst_sg, dst_nents, + qce_aead_done, async_req); + if (ret) + goto error_unmap_src; + + qce_dma_issue_pending(&qce->dma); + + ret = qce_start(async_req, tmpl->crypto_alg_type); + if (ret) + goto error_terminate; + + return 0; + +error_terminate: + qce_dma_terminate_all(&qce->dma); +error_unmap_src: + if (diff_dst) + dma_unmap_sg(qce->dev, req->src, rctx->src_nents, dir_src); +error_unmap_dst: + dma_unmap_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); +error_free: + if (IS_CCM(rctx->flags) && rctx->assoclen) { + sg_free_table(&rctx->src_tbl); + if (diff_dst) + sg_free_table(&rctx->dst_tbl); + } else { + sg_free_table(&rctx->dst_tbl); + } + return ret; +} + +static int qce_aead_crypt(struct aead_request *req, int encrypt) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(tfm); + unsigned int blocksize = crypto_aead_blocksize(tfm); + + rctx->flags = tmpl->alg_flags; + rctx->flags |= encrypt ? QCE_ENCRYPT : QCE_DECRYPT; + + if (encrypt) + rctx->cryptlen = req->cryptlen; + else + rctx->cryptlen = req->cryptlen - ctx->authsize; + + /* CE does not handle 0 length messages */ + if (!rctx->cryptlen) { + if (!(IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags))) + ctx->need_fallback = true; + } + + /* If fallback is needed, schedule and exit */ + if (ctx->need_fallback) { + /* Reset need_fallback in case the same ctx is used for another transaction */ + ctx->need_fallback = false; + + aead_request_set_tfm(&rctx->fallback_req, ctx->fallback); + aead_request_set_callback(&rctx->fallback_req, req->base.flags, + req->base.complete, req->base.data); + aead_request_set_crypt(&rctx->fallback_req, req->src, + req->dst, req->cryptlen, req->iv); + aead_request_set_ad(&rctx->fallback_req, req->assoclen); + + return encrypt ? crypto_aead_encrypt(&rctx->fallback_req) : + crypto_aead_decrypt(&rctx->fallback_req); + } + + /* + * CBC algorithms require message lengths to be + * multiples of block size. + */ + if (IS_CBC(rctx->flags) && !IS_ALIGNED(rctx->cryptlen, blocksize)) + return -EINVAL; + + /* RFC4309 supported AAD size 16 bytes/20 bytes */ + if (IS_CCM_RFC4309(rctx->flags)) + if (crypto_ipsec_check_assoclen(req->assoclen)) + return -EINVAL; + + return tmpl->qce->async_req_enqueue(tmpl->qce, &req->base); +} + +static int qce_aead_encrypt(struct aead_request *req) +{ + return qce_aead_crypt(req, 1); +} + +static int qce_aead_decrypt(struct aead_request *req) +{ + return qce_aead_crypt(req, 0); +} + +static int qce_aead_ccm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned long flags = to_aead_tmpl(tfm)->alg_flags; + + if (IS_CCM_RFC4309(flags)) { + if (keylen < QCE_CCM4309_SALT_SIZE) + return -EINVAL; + keylen -= QCE_CCM4309_SALT_SIZE; + memcpy(ctx->ccm4309_salt, key + keylen, QCE_CCM4309_SALT_SIZE); + } + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_256 && keylen != AES_KEYSIZE_192) + return -EINVAL; + + ctx->enc_keylen = keylen; + ctx->auth_keylen = keylen; + + memcpy(ctx->enc_key, key, keylen); + memcpy(ctx->auth_key, key, keylen); + + if (keylen == AES_KEYSIZE_192) + ctx->need_fallback = true; + + return IS_CCM_RFC4309(flags) ? + crypto_aead_setkey(ctx->fallback, key, keylen + QCE_CCM4309_SALT_SIZE) : + crypto_aead_setkey(ctx->fallback, key, keylen); +} + +static int qce_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct crypto_authenc_keys authenc_keys; + unsigned long flags = to_aead_tmpl(tfm)->alg_flags; + u32 _key[6]; + int err; + + err = crypto_authenc_extractkeys(&authenc_keys, key, keylen); + if (err) + return err; + + if (authenc_keys.enckeylen > QCE_MAX_KEY_SIZE || + authenc_keys.authkeylen > QCE_MAX_KEY_SIZE) + return -EINVAL; + + if (IS_DES(flags)) { + err = verify_aead_des_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen); + if (err) + return err; + } else if (IS_3DES(flags)) { + err = verify_aead_des3_key(tfm, authenc_keys.enckey, authenc_keys.enckeylen); + if (err) + return err; + /* + * The crypto engine does not support any two keys + * being the same for triple des algorithms. The + * verify_skcipher_des3_key does not check for all the + * below conditions. Schedule fallback in this case. + */ + memcpy(_key, authenc_keys.enckey, DES3_EDE_KEY_SIZE); + if (!((_key[0] ^ _key[2]) | (_key[1] ^ _key[3])) || + !((_key[2] ^ _key[4]) | (_key[3] ^ _key[5])) || + !((_key[0] ^ _key[4]) | (_key[1] ^ _key[5]))) + ctx->need_fallback = true; + } else if (IS_AES(flags)) { + /* No random key sizes */ + if (authenc_keys.enckeylen != AES_KEYSIZE_128 && + authenc_keys.enckeylen != AES_KEYSIZE_192 && + authenc_keys.enckeylen != AES_KEYSIZE_256) + return -EINVAL; + if (authenc_keys.enckeylen == AES_KEYSIZE_192) + ctx->need_fallback = true; + } + + ctx->enc_keylen = authenc_keys.enckeylen; + ctx->auth_keylen = authenc_keys.authkeylen; + + memcpy(ctx->enc_key, authenc_keys.enckey, authenc_keys.enckeylen); + + memset(ctx->auth_key, 0, sizeof(ctx->auth_key)); + memcpy(ctx->auth_key, authenc_keys.authkey, authenc_keys.authkeylen); + + return crypto_aead_setkey(ctx->fallback, key, keylen); +} + +static int qce_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + unsigned long flags = to_aead_tmpl(tfm)->alg_flags; + + if (IS_CCM(flags)) { + if (authsize < 4 || authsize > 16 || authsize % 2) + return -EINVAL; + if (IS_CCM_RFC4309(flags) && (authsize < 8 || authsize % 4)) + return -EINVAL; + } + ctx->authsize = authsize; + + return crypto_aead_setauthsize(ctx->fallback, authsize); +} + +static int qce_aead_init(struct crypto_aead *tfm) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + + ctx->need_fallback = false; + ctx->fallback = crypto_alloc_aead(crypto_tfm_alg_name(&tfm->base), + 0, CRYPTO_ALG_NEED_FALLBACK); + + if (IS_ERR(ctx->fallback)) + return PTR_ERR(ctx->fallback); + + crypto_aead_set_reqsize(tfm, sizeof(struct qce_aead_reqctx) + + crypto_aead_reqsize(ctx->fallback)); + return 0; +} + +static void qce_aead_exit(struct crypto_aead *tfm) +{ + struct qce_aead_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->fallback); +} + +struct qce_aead_def { + unsigned long flags; + const char *name; + const char *drv_name; + unsigned int blocksize; + unsigned int chunksize; + unsigned int ivsize; + unsigned int maxauthsize; +}; + +static const struct qce_aead_def aead_def[] = { + { + .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, + .name = "authenc(hmac(sha1),cbc(des))", + .drv_name = "authenc-hmac-sha1-cbc-des-qce", + .blocksize = DES_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + { + .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA1_HMAC, + .name = "authenc(hmac(sha1),cbc(des3_ede))", + .drv_name = "authenc-hmac-sha1-cbc-3des-qce", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA1_DIGEST_SIZE, + }, + { + .flags = QCE_ALG_DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, + .name = "authenc(hmac(sha256),cbc(des))", + .drv_name = "authenc-hmac-sha256-cbc-des-qce", + .blocksize = DES_BLOCK_SIZE, + .ivsize = DES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + { + .flags = QCE_ALG_3DES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, + .name = "authenc(hmac(sha256),cbc(des3_ede))", + .drv_name = "authenc-hmac-sha256-cbc-3des-qce", + .blocksize = DES3_EDE_BLOCK_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CBC | QCE_HASH_SHA256_HMAC, + .name = "authenc(hmac(sha256),cbc(aes))", + .drv_name = "authenc-hmac-sha256-cbc-aes-qce", + .blocksize = AES_BLOCK_SIZE, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = SHA256_DIGEST_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CCM, + .name = "ccm(aes)", + .drv_name = "ccm-aes-qce", + .blocksize = 1, + .ivsize = AES_BLOCK_SIZE, + .maxauthsize = AES_BLOCK_SIZE, + }, + { + .flags = QCE_ALG_AES | QCE_MODE_CCM | QCE_MODE_CCM_RFC4309, + .name = "rfc4309(ccm(aes))", + .drv_name = "rfc4309-ccm-aes-qce", + .blocksize = 1, + .ivsize = 8, + .maxauthsize = AES_BLOCK_SIZE, + }, +}; + +static int qce_aead_register_one(const struct qce_aead_def *def, struct qce_device *qce) +{ + struct qce_alg_template *tmpl; + struct aead_alg *alg; + int ret; + + tmpl = kzalloc(sizeof(*tmpl), GFP_KERNEL); + if (!tmpl) + return -ENOMEM; + + alg = &tmpl->alg.aead; + + snprintf(alg->base.cra_name, CRYPTO_MAX_ALG_NAME, "%s", def->name); + snprintf(alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", + def->drv_name); + + alg->base.cra_blocksize = def->blocksize; + alg->chunksize = def->chunksize; + alg->ivsize = def->ivsize; + alg->maxauthsize = def->maxauthsize; + if (IS_CCM(def->flags)) + alg->setkey = qce_aead_ccm_setkey; + else + alg->setkey = qce_aead_setkey; + alg->setauthsize = qce_aead_setauthsize; + alg->encrypt = qce_aead_encrypt; + alg->decrypt = qce_aead_decrypt; + alg->init = qce_aead_init; + alg->exit = qce_aead_exit; + + alg->base.cra_priority = 300; + alg->base.cra_flags = CRYPTO_ALG_ASYNC | + CRYPTO_ALG_ALLOCATES_MEMORY | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK; + alg->base.cra_ctxsize = sizeof(struct qce_aead_ctx); + alg->base.cra_alignmask = 0; + alg->base.cra_module = THIS_MODULE; + + INIT_LIST_HEAD(&tmpl->entry); + tmpl->crypto_alg_type = CRYPTO_ALG_TYPE_AEAD; + tmpl->alg_flags = def->flags; + tmpl->qce = qce; + + ret = crypto_register_aead(alg); + if (ret) { + kfree(tmpl); + dev_err(qce->dev, "%s registration failed\n", alg->base.cra_name); + return ret; + } + + list_add_tail(&tmpl->entry, &aead_algs); + dev_dbg(qce->dev, "%s is registered\n", alg->base.cra_name); + return 0; +} + +static void qce_aead_unregister(struct qce_device *qce) +{ + struct qce_alg_template *tmpl, *n; + + list_for_each_entry_safe(tmpl, n, &aead_algs, entry) { + crypto_unregister_aead(&tmpl->alg.aead); + list_del(&tmpl->entry); + kfree(tmpl); + } +} + +static int qce_aead_register(struct qce_device *qce) +{ + int ret, i; + + for (i = 0; i < ARRAY_SIZE(aead_def); i++) { + ret = qce_aead_register_one(&aead_def[i], qce); + if (ret) + goto err; + } + + return 0; +err: + qce_aead_unregister(qce); + return ret; +} + +const struct qce_algo_ops aead_ops = { + .type = CRYPTO_ALG_TYPE_AEAD, + .register_algs = qce_aead_register, + .unregister_algs = qce_aead_unregister, + .async_req_handle = qce_aead_async_req_handle, +}; diff --git a/drivers/crypto/qce/aead.h b/drivers/crypto/qce/aead.h new file mode 100644 index 000000000000..efb8477cc088 --- /dev/null +++ b/drivers/crypto/qce/aead.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, Linaro Limited. All rights reserved. + */ + +#ifndef _AEAD_H_ +#define _AEAD_H_ + +#include "common.h" +#include "core.h" + +#define QCE_MAX_KEY_SIZE 64 +#define QCE_CCM4309_SALT_SIZE 3 + +struct qce_aead_ctx { + u8 enc_key[QCE_MAX_KEY_SIZE]; + u8 auth_key[QCE_MAX_KEY_SIZE]; + u8 ccm4309_salt[QCE_CCM4309_SALT_SIZE]; + unsigned int enc_keylen; + unsigned int auth_keylen; + unsigned int authsize; + bool need_fallback; + struct crypto_aead *fallback; +}; + +struct qce_aead_reqctx { + unsigned long flags; + u8 *iv; + unsigned int ivsize; + int src_nents; + int dst_nents; + struct scatterlist result_sg; + struct scatterlist adata_sg; + struct sg_table dst_tbl; + struct sg_table src_tbl; + struct scatterlist *dst_sg; + struct scatterlist *src_sg; + unsigned int cryptlen; + unsigned int assoclen; + unsigned char *adata; + u8 ccm_nonce[QCE_MAX_NONCE]; + u8 ccmresult_buf[QCE_BAM_BURST_SIZE]; + u8 ccm_rfc4309_iv[QCE_MAX_IV_SIZE]; + struct aead_request fallback_req; +}; + +static inline struct qce_alg_template *to_aead_tmpl(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + + return container_of(alg, struct qce_alg_template, alg.aead); +} + +extern const struct qce_algo_ops aead_ops; + +#endif /* _AEAD_H_ */ diff --git a/drivers/crypto/qce/common.c b/drivers/crypto/qce/common.c index dceb9579d87a..7c612ba5068f 100644 --- a/drivers/crypto/qce/common.c +++ b/drivers/crypto/qce/common.c @@ -15,6 +15,7 @@ #include "core.h" #include "regs-v5.h" #include "sha.h" +#include "aead.h" static inline u32 qce_read(struct qce_device *qce, u32 offset) { @@ -88,17 +89,20 @@ static void qce_setup_config(struct qce_device *qce) qce_write(qce, REG_CONFIG, config); } -static inline void qce_crypto_go(struct qce_device *qce) +static inline void qce_crypto_go(struct qce_device *qce, bool result_dump) { - qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT)); + if (result_dump) + qce_write(qce, REG_GOPROC, BIT(GO_SHIFT) | BIT(RESULTS_DUMP_SHIFT)); + else + qce_write(qce, REG_GOPROC, BIT(GO_SHIFT)); } -#ifdef CONFIG_CRYPTO_DEV_QCE_SHA -static u32 qce_auth_cfg(unsigned long flags, u32 key_size) +#if defined(CONFIG_CRYPTO_DEV_QCE_SHA) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD) +static u32 qce_auth_cfg(unsigned long flags, u32 key_size, u32 auth_size) { u32 cfg = 0; - if (IS_AES(flags) && (IS_CCM(flags) || IS_CMAC(flags))) + if (IS_CCM(flags) || IS_CMAC(flags)) cfg |= AUTH_ALG_AES << AUTH_ALG_SHIFT; else cfg |= AUTH_ALG_SHA << AUTH_ALG_SHIFT; @@ -116,15 +120,16 @@ static u32 qce_auth_cfg(unsigned long flags, u32 key_size) cfg |= AUTH_SIZE_SHA256 << AUTH_SIZE_SHIFT; else if (IS_CMAC(flags)) cfg |= AUTH_SIZE_ENUM_16_BYTES << AUTH_SIZE_SHIFT; + else if (IS_CCM(flags)) + cfg |= (auth_size - 1) << AUTH_SIZE_SHIFT; if (IS_SHA1(flags) || IS_SHA256(flags)) cfg |= AUTH_MODE_HASH << AUTH_MODE_SHIFT; - else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags) || - IS_CBC(flags) || IS_CTR(flags)) + else if (IS_SHA1_HMAC(flags) || IS_SHA256_HMAC(flags)) cfg |= AUTH_MODE_HMAC << AUTH_MODE_SHIFT; - else if (IS_AES(flags) && IS_CCM(flags)) + else if (IS_CCM(flags)) cfg |= AUTH_MODE_CCM << AUTH_MODE_SHIFT; - else if (IS_AES(flags) && IS_CMAC(flags)) + else if (IS_CMAC(flags)) cfg |= AUTH_MODE_CMAC << AUTH_MODE_SHIFT; if (IS_SHA(flags) || IS_SHA_HMAC(flags)) @@ -133,13 +138,11 @@ static u32 qce_auth_cfg(unsigned long flags, u32 key_size) if (IS_CCM(flags)) cfg |= QCE_MAX_NONCE_WORDS << AUTH_NONCE_NUM_WORDS_SHIFT; - if (IS_CBC(flags) || IS_CTR(flags) || IS_CCM(flags) || - IS_CMAC(flags)) - cfg |= BIT(AUTH_LAST_SHIFT) | BIT(AUTH_FIRST_SHIFT); - return cfg; } +#endif +#ifdef CONFIG_CRYPTO_DEV_QCE_SHA static int qce_setup_regs_ahash(struct crypto_async_request *async_req) { struct ahash_request *req = ahash_request_cast(async_req); @@ -168,7 +171,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req) qce_clear_array(qce, REG_AUTH_KEY0, 16); qce_clear_array(qce, REG_AUTH_BYTECNT0, 4); - auth_cfg = qce_auth_cfg(rctx->flags, rctx->authklen); + auth_cfg = qce_auth_cfg(rctx->flags, rctx->authklen, digestsize); } if (IS_SHA_HMAC(rctx->flags) || IS_CMAC(rctx->flags)) { @@ -196,7 +199,7 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req) qce_write_array(qce, REG_AUTH_BYTECNT0, (u32 *)rctx->byte_count, 2); - auth_cfg = qce_auth_cfg(rctx->flags, 0); + auth_cfg = qce_auth_cfg(rctx->flags, 0, digestsize); if (rctx->last_blk) auth_cfg |= BIT(AUTH_LAST_SHIFT); @@ -219,13 +222,13 @@ static int qce_setup_regs_ahash(struct crypto_async_request *async_req) config = qce_config_reg(qce, 1); qce_write(qce, REG_CONFIG, config); - qce_crypto_go(qce); + qce_crypto_go(qce, true); return 0; } #endif -#ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER +#if defined(CONFIG_CRYPTO_DEV_QCE_SKCIPHER) || defined(CONFIG_CRYPTO_DEV_QCE_AEAD) static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size) { u32 cfg = 0; @@ -271,7 +274,9 @@ static u32 qce_encr_cfg(unsigned long flags, u32 aes_key_size) return cfg; } +#endif +#ifdef CONFIG_CRYPTO_DEV_QCE_SKCIPHER static void qce_xts_swapiv(__be32 *dst, const u8 *src, unsigned int ivsize) { u8 swap[QCE_AES_IV_LENGTH]; @@ -380,7 +385,156 @@ static int qce_setup_regs_skcipher(struct crypto_async_request *async_req) config = qce_config_reg(qce, 1); qce_write(qce, REG_CONFIG, config); - qce_crypto_go(qce); + qce_crypto_go(qce, true); + + return 0; +} +#endif + +#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD +static const u32 std_iv_sha1[SHA256_DIGEST_SIZE / sizeof(u32)] = { + SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, 0, 0, 0 +}; + +static const u32 std_iv_sha256[SHA256_DIGEST_SIZE / sizeof(u32)] = { + SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, + SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7 +}; + +static unsigned int qce_be32_to_cpu_array(u32 *dst, const u8 *src, unsigned int len) +{ + u32 *d = dst; + const u8 *s = src; + unsigned int n; + + n = len / sizeof(u32); + for (; n > 0; n--) { + *d = be32_to_cpup((const __be32 *)s); + s += sizeof(u32); + d++; + } + return DIV_ROUND_UP(len, sizeof(u32)); +} + +static int qce_setup_regs_aead(struct crypto_async_request *async_req) +{ + struct aead_request *req = aead_request_cast(async_req); + struct qce_aead_reqctx *rctx = aead_request_ctx(req); + struct qce_aead_ctx *ctx = crypto_tfm_ctx(async_req->tfm); + struct qce_alg_template *tmpl = to_aead_tmpl(crypto_aead_reqtfm(req)); + struct qce_device *qce = tmpl->qce; + u32 enckey[QCE_MAX_CIPHER_KEY_SIZE / sizeof(u32)] = {0}; + u32 enciv[QCE_MAX_IV_SIZE / sizeof(u32)] = {0}; + u32 authkey[QCE_SHA_HMAC_KEY_SIZE / sizeof(u32)] = {0}; + u32 authiv[SHA256_DIGEST_SIZE / sizeof(u32)] = {0}; + u32 authnonce[QCE_MAX_NONCE / sizeof(u32)] = {0}; + unsigned int enc_keylen = ctx->enc_keylen; + unsigned int auth_keylen = ctx->auth_keylen; + unsigned int enc_ivsize = rctx->ivsize; + unsigned int auth_ivsize = 0; + unsigned int enckey_words, enciv_words; + unsigned int authkey_words, authiv_words, authnonce_words; + unsigned long flags = rctx->flags; + u32 encr_cfg, auth_cfg, config, totallen; + u32 iv_last_word; + + qce_setup_config(qce); + + /* Write encryption key */ + enckey_words = qce_be32_to_cpu_array(enckey, ctx->enc_key, enc_keylen); + qce_write_array(qce, REG_ENCR_KEY0, enckey, enckey_words); + + /* Write encryption iv */ + enciv_words = qce_be32_to_cpu_array(enciv, rctx->iv, enc_ivsize); + qce_write_array(qce, REG_CNTR0_IV0, enciv, enciv_words); + + if (IS_CCM(rctx->flags)) { + iv_last_word = enciv[enciv_words - 1]; + qce_write(qce, REG_CNTR3_IV3, iv_last_word + 1); + qce_write_array(qce, REG_ENCR_CCM_INT_CNTR0, (u32 *)enciv, enciv_words); + qce_write(qce, REG_CNTR_MASK, ~0); + qce_write(qce, REG_CNTR_MASK0, ~0); + qce_write(qce, REG_CNTR_MASK1, ~0); + qce_write(qce, REG_CNTR_MASK2, ~0); + } + + /* Clear authentication IV and KEY registers of previous values */ + qce_clear_array(qce, REG_AUTH_IV0, 16); + qce_clear_array(qce, REG_AUTH_KEY0, 16); + + /* Clear byte count */ + qce_clear_array(qce, REG_AUTH_BYTECNT0, 4); + + /* Write authentication key */ + authkey_words = qce_be32_to_cpu_array(authkey, ctx->auth_key, auth_keylen); + qce_write_array(qce, REG_AUTH_KEY0, (u32 *)authkey, authkey_words); + + /* Write initial authentication IV only for HMAC algorithms */ + if (IS_SHA_HMAC(rctx->flags)) { + /* Write default authentication iv */ + if (IS_SHA1_HMAC(rctx->flags)) { + auth_ivsize = SHA1_DIGEST_SIZE; + memcpy(authiv, std_iv_sha1, auth_ivsize); + } else if (IS_SHA256_HMAC(rctx->flags)) { + auth_ivsize = SHA256_DIGEST_SIZE; + memcpy(authiv, std_iv_sha256, auth_ivsize); + } + authiv_words = auth_ivsize / sizeof(u32); + qce_write_array(qce, REG_AUTH_IV0, (u32 *)authiv, authiv_words); + } else if (IS_CCM(rctx->flags)) { + /* Write nonce for CCM algorithms */ + authnonce_words = qce_be32_to_cpu_array(authnonce, rctx->ccm_nonce, QCE_MAX_NONCE); + qce_write_array(qce, REG_AUTH_INFO_NONCE0, authnonce, authnonce_words); + } + + /* Set up ENCR_SEG_CFG */ + encr_cfg = qce_encr_cfg(flags, enc_keylen); + if (IS_ENCRYPT(flags)) + encr_cfg |= BIT(ENCODE_SHIFT); + qce_write(qce, REG_ENCR_SEG_CFG, encr_cfg); + + /* Set up AUTH_SEG_CFG */ + auth_cfg = qce_auth_cfg(rctx->flags, auth_keylen, ctx->authsize); + auth_cfg |= BIT(AUTH_LAST_SHIFT); + auth_cfg |= BIT(AUTH_FIRST_SHIFT); + if (IS_ENCRYPT(flags)) { + if (IS_CCM(rctx->flags)) + auth_cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT; + else + auth_cfg |= AUTH_POS_AFTER << AUTH_POS_SHIFT; + } else { + if (IS_CCM(rctx->flags)) + auth_cfg |= AUTH_POS_AFTER << AUTH_POS_SHIFT; + else + auth_cfg |= AUTH_POS_BEFORE << AUTH_POS_SHIFT; + } + qce_write(qce, REG_AUTH_SEG_CFG, auth_cfg); + + totallen = rctx->cryptlen + rctx->assoclen; + + /* Set the encryption size and start offset */ + if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) + qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen + ctx->authsize); + else + qce_write(qce, REG_ENCR_SEG_SIZE, rctx->cryptlen); + qce_write(qce, REG_ENCR_SEG_START, rctx->assoclen & 0xffff); + + /* Set the authentication size and start offset */ + qce_write(qce, REG_AUTH_SEG_SIZE, totallen); + qce_write(qce, REG_AUTH_SEG_START, 0); + + /* Write total length */ + if (IS_CCM(rctx->flags) && IS_DECRYPT(rctx->flags)) + qce_write(qce, REG_SEG_SIZE, totallen + ctx->authsize); + else + qce_write(qce, REG_SEG_SIZE, totallen); + + /* get little endianness */ + config = qce_config_reg(qce, 1); + qce_write(qce, REG_CONFIG, config); + + /* Start the process */ + qce_crypto_go(qce, !IS_CCM(flags)); return 0; } @@ -396,6 +550,10 @@ int qce_start(struct crypto_async_request *async_req, u32 type) #ifdef CONFIG_CRYPTO_DEV_QCE_SHA case CRYPTO_ALG_TYPE_AHASH: return qce_setup_regs_ahash(async_req); +#endif +#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD + case CRYPTO_ALG_TYPE_AEAD: + return qce_setup_regs_aead(async_req); #endif default: return -EINVAL; @@ -419,6 +577,8 @@ int qce_check_status(struct qce_device *qce, u32 *status) */ if (*status & STATUS_ERRORS || !(*status & BIT(OPERATION_DONE_SHIFT))) ret = -ENXIO; + else if (*status & BIT(MAC_FAILED_SHIFT)) + ret = -EBADMSG; return ret; } diff --git a/drivers/crypto/qce/common.h b/drivers/crypto/qce/common.h index 3bc244bcca2d..02e63ad9f245 100644 --- a/drivers/crypto/qce/common.h +++ b/drivers/crypto/qce/common.h @@ -11,6 +11,7 @@ #include #include #include +#include /* xts du size */ #define QCE_SECTOR_SIZE 512 @@ -51,9 +52,11 @@ #define QCE_MODE_CCM BIT(12) #define QCE_MODE_MASK GENMASK(12, 8) +#define QCE_MODE_CCM_RFC4309 BIT(13) + /* cipher encryption/decryption operations */ -#define QCE_ENCRYPT BIT(13) -#define QCE_DECRYPT BIT(14) +#define QCE_ENCRYPT BIT(30) +#define QCE_DECRYPT BIT(31) #define IS_DES(flags) (flags & QCE_ALG_DES) #define IS_3DES(flags) (flags & QCE_ALG_3DES) @@ -73,6 +76,7 @@ #define IS_CTR(mode) (mode & QCE_MODE_CTR) #define IS_XTS(mode) (mode & QCE_MODE_XTS) #define IS_CCM(mode) (mode & QCE_MODE_CCM) +#define IS_CCM_RFC4309(mode) ((mode) & QCE_MODE_CCM_RFC4309) #define IS_ENCRYPT(dir) (dir & QCE_ENCRYPT) #define IS_DECRYPT(dir) (dir & QCE_DECRYPT) @@ -85,6 +89,7 @@ struct qce_alg_template { union { struct skcipher_alg skcipher; struct ahash_alg ahash; + struct aead_alg aead; } alg; struct qce_device *qce; const u8 *hash_zero; diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index 80b75085c265..d3780be44a76 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -17,6 +17,7 @@ #include "core.h" #include "cipher.h" #include "sha.h" +#include "aead.h" #define QCE_MAJOR_VERSION5 0x05 #define QCE_QUEUE_LENGTH 1 @@ -28,6 +29,9 @@ static const struct qce_algo_ops *qce_ops[] = { #ifdef CONFIG_CRYPTO_DEV_QCE_SHA &ahash_ops, #endif +#ifdef CONFIG_CRYPTO_DEV_QCE_AEAD + &aead_ops, +#endif }; static void qce_unregister_algs(struct qce_device *qce) diff --git a/drivers/crypto/qce/skcipher.c b/drivers/crypto/qce/skcipher.c index c0a0d8c4fce1..8ff10928f581 100644 --- a/drivers/crypto/qce/skcipher.c +++ b/drivers/crypto/qce/skcipher.c @@ -72,7 +72,7 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req) struct scatterlist *sg; bool diff_dst; gfp_t gfp; - int ret; + int dst_nents, src_nents, ret; rctx->iv = req->iv; rctx->ivsize = crypto_skcipher_ivsize(skcipher); @@ -123,21 +123,26 @@ qce_skcipher_async_req_handle(struct crypto_async_request *async_req) sg_mark_end(sg); rctx->dst_sg = rctx->dst_tbl.sgl; - ret = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); - if (ret < 0) + dst_nents = dma_map_sg(qce->dev, rctx->dst_sg, rctx->dst_nents, dir_dst); + if (dst_nents < 0) { + ret = dst_nents; goto error_free; + } if (diff_dst) { - ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); - if (ret < 0) + src_nents = dma_map_sg(qce->dev, req->src, rctx->src_nents, dir_src); + if (src_nents < 0) { + ret = src_nents; goto error_unmap_dst; + } rctx->src_sg = req->src; } else { rctx->src_sg = rctx->dst_sg; + src_nents = dst_nents - 1; } - ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, rctx->src_nents, - rctx->dst_sg, rctx->dst_nents, + ret = qce_dma_prep_sgs(&qce->dma, rctx->src_sg, src_nents, + rctx->dst_sg, dst_nents, qce_skcipher_done, async_req); if (ret) goto error_unmap_src; diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 1c6929fb3a13..544d7040cfc5 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -1698,7 +1698,6 @@ static void sa_aead_dma_in_callback(void *data) size_t pl, ml; int i; int err = 0; - u16 auth_len; u32 *mdptr; sa_sync_from_device(rxd); @@ -1711,13 +1710,10 @@ static void sa_aead_dma_in_callback(void *data) for (i = 0; i < (authsize / 4); i++) mdptr[i + 4] = swab32(mdptr[i + 4]); - auth_len = req->assoclen + req->cryptlen; - if (rxd->enc) { scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1); } else { - auth_len -= authsize; start -= authsize; scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 0); @@ -2300,9 +2296,9 @@ static int sa_dma_init(struct sa_crypto_data *dd) dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); if (IS_ERR(dd->dma_rx2)) { - dma_release_channel(dd->dma_rx1); - return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), - "Unable to request rx2 DMA channel\n"); + ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), + "Unable to request rx2 DMA channel\n"); + goto err_dma_rx2; } dd->dma_tx = dma_request_chan(dd->dev, "tx"); @@ -2323,28 +2319,31 @@ static int sa_dma_init(struct sa_crypto_data *dd) if (ret) { dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", ret); - return ret; + goto err_dma_config; } ret = dmaengine_slave_config(dd->dma_rx2, &cfg); if (ret) { dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", ret); - return ret; + goto err_dma_config; } ret = dmaengine_slave_config(dd->dma_tx, &cfg); if (ret) { dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", ret); - return ret; + goto err_dma_config; } return 0; +err_dma_config: + dma_release_channel(dd->dma_tx); err_dma_tx: - dma_release_channel(dd->dma_rx1); dma_release_channel(dd->dma_rx2); +err_dma_rx2: + dma_release_channel(dd->dma_rx1); return ret; } @@ -2385,10 +2384,8 @@ MODULE_DEVICE_TABLE(of, of_match); static int sa_ul_probe(struct platform_device *pdev) { - const struct of_device_id *match; struct device *dev = &pdev->dev; struct device_node *node = dev->of_node; - struct resource *res; static void __iomem *saul_base; struct sa_crypto_data *dev_data; int ret; @@ -2397,9 +2394,18 @@ static int sa_ul_probe(struct platform_device *pdev) if (!dev_data) return -ENOMEM; + dev_data->match_data = of_device_get_match_data(dev); + if (!dev_data->match_data) + return -ENODEV; + + saul_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(saul_base)) + return PTR_ERR(saul_base); + sa_k3_dev = dev; dev_data->dev = dev; dev_data->pdev = pdev; + dev_data->base = saul_base; platform_set_drvdata(pdev, dev_data); dev_set_drvdata(sa_k3_dev, dev_data); @@ -2408,26 +2414,16 @@ static int sa_ul_probe(struct platform_device *pdev) if (ret < 0) { dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, ret); + pm_runtime_disable(dev); return ret; } sa_init_mem(dev_data); ret = sa_dma_init(dev_data); if (ret) - goto disable_pm_runtime; - - match = of_match_node(of_match, dev->of_node); - if (!match) { - dev_err(dev, "No compatible match found\n"); - return -ENODEV; - } - dev_data->match_data = match->data; + goto destroy_dma_pool; spin_lock_init(&dev_data->scid_lock); - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - saul_base = devm_ioremap_resource(dev, res); - - dev_data->base = saul_base; if (!dev_data->match_data->skip_engine_control) { u32 val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | @@ -2454,9 +2450,9 @@ static int sa_ul_probe(struct platform_device *pdev) dma_release_channel(dev_data->dma_rx1); dma_release_channel(dev_data->dma_tx); +destroy_dma_pool: dma_pool_destroy(dev_data->sc_pool); -disable_pm_runtime: pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); @@ -2467,6 +2463,8 @@ static int sa_ul_remove(struct platform_device *pdev) { struct sa_crypto_data *dev_data = platform_get_drvdata(pdev); + of_platform_depopulate(&pdev->dev); + sa_unregister_algos(&pdev->dev); dma_release_channel(dev_data->dma_rx2); diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index ecb7412e84e3..51a6e1a42434 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -1011,6 +1011,7 @@ static int hash_hw_final(struct ahash_request *req) goto out; } } else if (req->nbytes == 0 && ctx->keylen > 0) { + ret = -EPERM; dev_err(device_data->dev, "%s: Empty message with keylength > 0, NOT supported\n", __func__); goto out; diff --git a/drivers/firewire/core-topology.c b/drivers/firewire/core-topology.c index ec68ed27b0a5..b63d55f5ebd3 100644 --- a/drivers/firewire/core-topology.c +++ b/drivers/firewire/core-topology.c @@ -58,6 +58,7 @@ static u32 *count_ports(u32 *sid, int *total_port_count, int *child_port_count) case SELFID_PORT_PARENT: case SELFID_PORT_NCONN: (*total_port_count)++; + fallthrough; case SELFID_PORT_NONE: break; } diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c index 18a47c9d5b02..cfb448eabdaa 100644 --- a/drivers/firmware/psci/psci.c +++ b/drivers/firmware/psci/psci.c @@ -343,10 +343,15 @@ int psci_cpu_suspend_enter(u32 state) { int ret; - if (!psci_power_state_loses_context(state)) + if (!psci_power_state_loses_context(state)) { + struct arm_cpuidle_irq_context context; + + arm_cpuidle_save_irq_context(&context); ret = psci_ops.cpu_suspend(state, 0); - else + arm_cpuidle_restore_irq_context(&context); + } else { ret = cpu_suspend(state, psci_suspend_finisher); + } return ret; } diff --git a/drivers/firmware/smccc/smccc.c b/drivers/firmware/smccc/smccc.c index 028f81d702cc..9f937b125ab0 100644 --- a/drivers/firmware/smccc/smccc.c +++ b/drivers/firmware/smccc/smccc.c @@ -15,6 +15,7 @@ static u32 smccc_version = ARM_SMCCC_VERSION_1_0; static enum arm_smccc_conduit smccc_conduit = SMCCC_CONDUIT_NONE; bool __ro_after_init smccc_trng_available = false; +u64 __ro_after_init smccc_has_sve_hint = false; void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit) { @@ -22,6 +23,9 @@ void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit) smccc_conduit = conduit; smccc_trng_available = smccc_probe_trng(); + if (IS_ENABLED(CONFIG_ARM64_SVE) && + smccc_version >= ARM_SMCCC_VERSION_1_3) + smccc_has_sve_hint = true; } enum arm_smccc_conduit arm_smccc_1_1_get_conduit(void) diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c index 170aba99a110..cf11febf60c0 100644 --- a/drivers/gpu/drm/nouveau/nouveau_bo.c +++ b/drivers/gpu/drm/nouveau/nouveau_bo.c @@ -440,6 +440,7 @@ nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig) break; case TTM_PL_TT: error |= !(domain & NOUVEAU_GEM_DOMAIN_GART); + break; default: break; } diff --git a/drivers/gpu/drm/nouveau/nouveau_connector.c b/drivers/gpu/drm/nouveau/nouveau_connector.c index 61e6d7412505..eb844cdcaec2 100644 --- a/drivers/gpu/drm/nouveau/nouveau_connector.c +++ b/drivers/gpu/drm/nouveau/nouveau_connector.c @@ -157,6 +157,7 @@ nouveau_conn_atomic_set_property(struct drm_connector *connector, default: break; } + break; case DRM_MODE_SCALE_FULLSCREEN: case DRM_MODE_SCALE_CENTER: case DRM_MODE_SCALE_ASPECT: diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/nv50.c index 83067763c0ec..e1d31c62f9ec 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/nv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/nv50.c @@ -313,6 +313,7 @@ nv50_clk_read(struct nvkm_clk *base, enum nv_clk_src src) default: break; } + break; default: break; } diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c index 2b031d4eaeb6..684aff7437ee 100644 --- a/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/subdev/therm/gf119.c @@ -41,6 +41,7 @@ pwm_info(struct nvkm_therm *therm, int line) default: break; } + break; default: break; } diff --git a/drivers/hwmon/corsair-cpro.c b/drivers/hwmon/corsair-cpro.c index 591929ec217a..fa6aa4fc8b52 100644 --- a/drivers/hwmon/corsair-cpro.c +++ b/drivers/hwmon/corsair-cpro.c @@ -310,6 +310,7 @@ static int ccp_write(struct device *dev, enum hwmon_sensor_types type, default: break; } + break; default: break; } diff --git a/drivers/hwmon/max6621.c b/drivers/hwmon/max6621.c index 367855d5edae..7821132e17fa 100644 --- a/drivers/hwmon/max6621.c +++ b/drivers/hwmon/max6621.c @@ -156,7 +156,7 @@ max6621_is_visible(const void *data, enum hwmon_sensor_types type, u32 attr, default: break; } - + break; default: break; } diff --git a/drivers/input/keyboard/Kconfig b/drivers/input/keyboard/Kconfig index 32d15809ae58..40a070a2e7f5 100644 --- a/drivers/input/keyboard/Kconfig +++ b/drivers/input/keyboard/Kconfig @@ -67,9 +67,6 @@ config KEYBOARD_AMIGA To compile this driver as a module, choose M here: the module will be called amikbd. -config ATARI_KBD_CORE - bool - config KEYBOARD_APPLESPI tristate "Apple SPI keyboard and trackpad" depends on ACPI && EFI diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index c179e27062fd..b8c06bd8659e 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -50,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -787,6 +788,12 @@ static int aic_init_cpu(unsigned int cpu) return 0; } +static struct gic_kvm_info vgic_info __initdata = { + .type = GIC_V3, + .no_maint_irq_mask = true, + .no_hw_deactivation = true, +}; + static int __init aic_of_ic_init(struct device_node *node, struct device_node *parent) { int i; @@ -843,6 +850,8 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p "irqchip/apple-aic/ipi:starting", aic_init_cpu, NULL); + vgic_set_kvm_info(&vgic_info); + pr_info("Initialized with %d IRQs, %d FIQs, %d vIPIs\n", irqc->nr_hw, AIC_NR_FIQ, AIC_NR_SWIPI); diff --git a/drivers/irqchip/irq-gic-common.c b/drivers/irqchip/irq-gic-common.c index f47b41dfd023..a610821c8ff2 100644 --- a/drivers/irqchip/irq-gic-common.c +++ b/drivers/irqchip/irq-gic-common.c @@ -12,19 +12,6 @@ static DEFINE_RAW_SPINLOCK(irq_controller_lock); -static const struct gic_kvm_info *gic_kvm_info; - -const struct gic_kvm_info *gic_get_kvm_info(void) -{ - return gic_kvm_info; -} - -void gic_set_kvm_info(const struct gic_kvm_info *info) -{ - BUG_ON(gic_kvm_info != NULL); - gic_kvm_info = info; -} - void gic_enable_of_quirks(const struct device_node *np, const struct gic_quirk *quirks, void *data) { diff --git a/drivers/irqchip/irq-gic-common.h b/drivers/irqchip/irq-gic-common.h index ccba8b0fe0f5..27e3d4ed4f32 100644 --- a/drivers/irqchip/irq-gic-common.h +++ b/drivers/irqchip/irq-gic-common.h @@ -28,6 +28,4 @@ void gic_enable_quirks(u32 iidr, const struct gic_quirk *quirks, void gic_enable_of_quirks(const struct device_node *np, const struct gic_quirk *quirks, void *data); -void gic_set_kvm_info(const struct gic_kvm_info *info); - #endif /* _IRQ_GIC_COMMON_H */ diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 12c2cc5d53b5..a71545fc8f10 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -109,7 +109,7 @@ EXPORT_SYMBOL(gic_nonsecure_priorities); /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ static refcount_t *ppi_nmi_refs; -static struct gic_kvm_info gic_v3_kvm_info; +static struct gic_kvm_info gic_v3_kvm_info __initdata; static DEFINE_PER_CPU(bool, has_rss); #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) @@ -1919,7 +1919,7 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; - gic_set_kvm_info(&gic_v3_kvm_info); + vgic_set_kvm_info(&gic_v3_kvm_info); } static int __init gic_of_init(struct device_node *node, struct device_node *parent) @@ -2235,7 +2235,7 @@ static void __init gic_acpi_setup_kvm_info(void) gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; gic_v3_kvm_info.has_v4_1 = gic_data.rdists.has_rvpeid; - gic_set_kvm_info(&gic_v3_kvm_info); + vgic_set_kvm_info(&gic_v3_kvm_info); } static int __init diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index b1d9c22caf2e..2de9ec8ece0c 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -119,7 +119,7 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); static struct gic_chip_data gic_data[CONFIG_ARM_GIC_MAX_NR] __read_mostly; -static struct gic_kvm_info gic_v2_kvm_info; +static struct gic_kvm_info gic_v2_kvm_info __initdata; static DEFINE_PER_CPU(u32, sgi_intid); @@ -1451,7 +1451,7 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) return; if (static_branch_likely(&supports_deactivate_key)) - gic_set_kvm_info(&gic_v2_kvm_info); + vgic_set_kvm_info(&gic_v2_kvm_info); } int __init @@ -1618,7 +1618,7 @@ static void __init gic_acpi_setup_kvm_info(void) gic_v2_kvm_info.maint_irq = irq; - gic_set_kvm_info(&gic_v2_kvm_info); + vgic_set_kvm_info(&gic_v2_kvm_info); } static int __init gic_v2_acpi_init(union acpi_subtable_headers *header, diff --git a/drivers/media/cec/platform/s5p/s5p_cec.c b/drivers/media/cec/platform/s5p/s5p_cec.c index 2a3e7ffefe0a..028a09a7531e 100644 --- a/drivers/media/cec/platform/s5p/s5p_cec.c +++ b/drivers/media/cec/platform/s5p/s5p_cec.c @@ -35,10 +35,13 @@ MODULE_PARM_DESC(debug, "debug level (0-2)"); static int s5p_cec_adap_enable(struct cec_adapter *adap, bool enable) { + int ret; struct s5p_cec_dev *cec = cec_get_drvdata(adap); if (enable) { - pm_runtime_get_sync(cec->dev); + ret = pm_runtime_resume_and_get(cec->dev); + if (ret < 0) + return ret; s5p_cec_reset(cec); @@ -51,7 +54,7 @@ static int s5p_cec_adap_enable(struct cec_adapter *adap, bool enable) } else { s5p_cec_mask_tx_interrupts(cec); s5p_cec_mask_rx_interrupts(cec); - pm_runtime_disable(cec->dev); + pm_runtime_put(cec->dev); } return 0; diff --git a/drivers/media/common/Kconfig b/drivers/media/common/Kconfig index 4ea03b7899a8..0f6bde0f793e 100644 --- a/drivers/media/common/Kconfig +++ b/drivers/media/common/Kconfig @@ -13,6 +13,10 @@ config VIDEO_TVEEPROM tristate depends on I2C +config TTPCI_EEPROM + tristate + depends on I2C + config CYPRESS_FIRMWARE tristate depends on USB diff --git a/drivers/media/common/Makefile b/drivers/media/common/Makefile index b71e4b62eea5..55b5a1900124 100644 --- a/drivers/media/common/Makefile +++ b/drivers/media/common/Makefile @@ -3,3 +3,4 @@ obj-y += b2c2/ saa7146/ siano/ v4l2-tpg/ videobuf2/ obj-$(CONFIG_VIDEO_CX2341X) += cx2341x.o obj-$(CONFIG_VIDEO_TVEEPROM) += tveeprom.o obj-$(CONFIG_CYPRESS_FIRMWARE) += cypress_firmware.o +obj-$(CONFIG_TTPCI_EEPROM) += ttpci-eeprom.o diff --git a/drivers/media/common/siano/smscoreapi.c b/drivers/media/common/siano/smscoreapi.c index 410cc3ac6f94..bceaf91faa15 100644 --- a/drivers/media/common/siano/smscoreapi.c +++ b/drivers/media/common/siano/smscoreapi.c @@ -908,7 +908,7 @@ static int smscore_load_firmware_family2(struct smscore_device_t *coredev, void *buffer, size_t size) { struct sms_firmware *firmware = (struct sms_firmware *) buffer; - struct sms_msg_data4 *msg; + struct sms_msg_data5 *msg; u32 mem_address, calc_checksum = 0; u32 i, *ptr; u8 *payload = firmware->payload; @@ -989,24 +989,20 @@ static int smscore_load_firmware_family2(struct smscore_device_t *coredev, goto exit_fw_download; if (coredev->mode == DEVICE_MODE_NONE) { - struct sms_msg_data *trigger_msg = - (struct sms_msg_data *) msg; - pr_debug("sending MSG_SMS_SWDOWNLOAD_TRIGGER_REQ\n"); SMS_INIT_MSG(&msg->x_msg_header, MSG_SMS_SWDOWNLOAD_TRIGGER_REQ, - sizeof(struct sms_msg_hdr) + - sizeof(u32) * 5); + sizeof(*msg)); - trigger_msg->msg_data[0] = firmware->start_address; + msg->msg_data[0] = firmware->start_address; /* Entry point */ - trigger_msg->msg_data[1] = 6; /* Priority */ - trigger_msg->msg_data[2] = 0x200; /* Stack size */ - trigger_msg->msg_data[3] = 0; /* Parameter */ - trigger_msg->msg_data[4] = 4; /* Task ID */ + msg->msg_data[1] = 6; /* Priority */ + msg->msg_data[2] = 0x200; /* Stack size */ + msg->msg_data[3] = 0; /* Parameter */ + msg->msg_data[4] = 4; /* Task ID */ - rc = smscore_sendrequest_and_wait(coredev, trigger_msg, - trigger_msg->x_msg_header.msg_length, + rc = smscore_sendrequest_and_wait(coredev, msg, + msg->x_msg_header.msg_length, &coredev->trigger_done); } else { SMS_INIT_MSG(&msg->x_msg_header, MSG_SW_RELOAD_EXEC_REQ, diff --git a/drivers/media/common/siano/smscoreapi.h b/drivers/media/common/siano/smscoreapi.h index 4a6b9f4c44ac..f8789ee0d554 100644 --- a/drivers/media/common/siano/smscoreapi.h +++ b/drivers/media/common/siano/smscoreapi.h @@ -624,9 +624,9 @@ struct sms_msg_data2 { u32 msg_data[2]; }; -struct sms_msg_data4 { +struct sms_msg_data5 { struct sms_msg_hdr x_msg_header; - u32 msg_data[4]; + u32 msg_data[5]; }; struct sms_data_download { diff --git a/drivers/media/common/siano/smsdvb-main.c b/drivers/media/common/siano/smsdvb-main.c index cd5bafe9a3ac..f80caaa333da 100644 --- a/drivers/media/common/siano/smsdvb-main.c +++ b/drivers/media/common/siano/smsdvb-main.c @@ -26,8 +26,8 @@ Copyright (C) 2006-2008, Uri Shkolnik DVB_DEFINE_MOD_OPT_ADAPTER_NR(adapter_nr); -static struct list_head g_smsdvb_clients; -static struct mutex g_smsdvb_clientslock; +static LIST_HEAD(g_smsdvb_clients); +static DEFINE_MUTEX(g_smsdvb_clientslock); static u32 sms_to_guard_interval_table[] = { [0] = GUARD_INTERVAL_1_32, @@ -1212,6 +1212,10 @@ static int smsdvb_hotplug(struct smscore_device_t *coredev, return 0; media_graph_error: + mutex_lock(&g_smsdvb_clientslock); + list_del(&client->entry); + mutex_unlock(&g_smsdvb_clientslock); + smsdvb_debugfs_release(client); client_error: @@ -1236,9 +1240,6 @@ static int __init smsdvb_module_init(void) { int rc; - INIT_LIST_HEAD(&g_smsdvb_clients); - mutex_init(&g_smsdvb_clientslock); - smsdvb_debugfs_register(); rc = smscore_register_hotplug(smsdvb_hotplug); diff --git a/drivers/media/pci/ttpci/ttpci-eeprom.c b/drivers/media/common/ttpci-eeprom.c similarity index 100% rename from drivers/media/pci/ttpci/ttpci-eeprom.c rename to drivers/media/common/ttpci-eeprom.c diff --git a/drivers/media/pci/ttpci/ttpci-eeprom.h b/drivers/media/common/ttpci-eeprom.h similarity index 100% rename from drivers/media/pci/ttpci/ttpci-eeprom.h rename to drivers/media/common/ttpci-eeprom.h diff --git a/drivers/media/common/videobuf2/videobuf2-v4l2.c b/drivers/media/common/videobuf2/videobuf2-v4l2.c index 7e96f67c60ba..2988bb38ceb1 100644 --- a/drivers/media/common/videobuf2/videobuf2-v4l2.c +++ b/drivers/media/common/videobuf2/videobuf2-v4l2.c @@ -939,6 +939,20 @@ void vb2_queue_release(struct vb2_queue *q) } EXPORT_SYMBOL_GPL(vb2_queue_release); +int vb2_queue_change_type(struct vb2_queue *q, unsigned int type) +{ + if (type == q->type) + return 0; + + if (vb2_is_busy(q)) + return -EBUSY; + + q->type = type; + + return 0; +} +EXPORT_SYMBOL_GPL(vb2_queue_change_type); + __poll_t vb2_poll(struct vb2_queue *q, struct file *file, poll_table *wait) { struct video_device *vfd = video_devdata(file); diff --git a/drivers/media/dvb-core/dmxdev.c b/drivers/media/dvb-core/dmxdev.c index f14a872d1268..5d5a48475a54 100644 --- a/drivers/media/dvb-core/dmxdev.c +++ b/drivers/media/dvb-core/dmxdev.c @@ -720,7 +720,7 @@ static int dvb_dmxdev_filter_start(struct dmxdev_filter *filter) ret = dmxdev->demux->allocate_section_feed(dmxdev->demux, secfeed, dvb_dmxdev_section_callback); - if (ret < 0) { + if (!*secfeed) { pr_err("DVB (%s): could not alloc feed\n", __func__); return ret; diff --git a/drivers/media/dvb-core/dvb_ca_en50221.c b/drivers/media/dvb-core/dvb_ca_en50221.c index b7e4a3371176..15a08d8c69ef 100644 --- a/drivers/media/dvb-core/dvb_ca_en50221.c +++ b/drivers/media/dvb-core/dvb_ca_en50221.c @@ -1386,6 +1386,7 @@ static int dvb_ca_en50221_io_do_ioctl(struct file *file, err = -EINVAL; goto out_unlock; } + slot = array_index_nospec(slot, ca->slot_count); info->type = CA_CI_LINK; info->flags = 0; diff --git a/drivers/media/dvb-core/dvb_frontend.c b/drivers/media/dvb-core/dvb_frontend.c index a6915582d1a6..258637d762d6 100644 --- a/drivers/media/dvb-core/dvb_frontend.c +++ b/drivers/media/dvb-core/dvb_frontend.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -1063,107 +1064,97 @@ static int dvb_frontend_clear_cache(struct dvb_frontend *fe) return 0; } -#define _DTV_CMD(n, s, b) \ -[n] = { \ - .name = #n, \ - .cmd = n, \ - .set = s,\ - .buffer = b \ -} +#define _DTV_CMD(n) \ + [n] = #n -struct dtv_cmds_h { - char *name; /* A display name for debugging purposes */ - - __u32 cmd; /* A unique ID */ - - /* Flags */ - __u32 set:1; /* Either a set or get property */ - __u32 buffer:1; /* Does this property use the buffer? */ - __u32 reserved:30; /* Align */ -}; - -static struct dtv_cmds_h dtv_cmds[DTV_MAX_COMMAND + 1] = { - _DTV_CMD(DTV_TUNE, 1, 0), - _DTV_CMD(DTV_CLEAR, 1, 0), +static char *dtv_cmds[DTV_MAX_COMMAND + 1] = { + _DTV_CMD(DTV_TUNE), + _DTV_CMD(DTV_CLEAR), /* Set */ - _DTV_CMD(DTV_FREQUENCY, 1, 0), - _DTV_CMD(DTV_BANDWIDTH_HZ, 1, 0), - _DTV_CMD(DTV_MODULATION, 1, 0), - _DTV_CMD(DTV_INVERSION, 1, 0), - _DTV_CMD(DTV_DISEQC_MASTER, 1, 1), - _DTV_CMD(DTV_SYMBOL_RATE, 1, 0), - _DTV_CMD(DTV_INNER_FEC, 1, 0), - _DTV_CMD(DTV_VOLTAGE, 1, 0), - _DTV_CMD(DTV_TONE, 1, 0), - _DTV_CMD(DTV_PILOT, 1, 0), - _DTV_CMD(DTV_ROLLOFF, 1, 0), - _DTV_CMD(DTV_DELIVERY_SYSTEM, 1, 0), - _DTV_CMD(DTV_HIERARCHY, 1, 0), - _DTV_CMD(DTV_CODE_RATE_HP, 1, 0), - _DTV_CMD(DTV_CODE_RATE_LP, 1, 0), - _DTV_CMD(DTV_GUARD_INTERVAL, 1, 0), - _DTV_CMD(DTV_TRANSMISSION_MODE, 1, 0), - _DTV_CMD(DTV_INTERLEAVING, 1, 0), + _DTV_CMD(DTV_FREQUENCY), + _DTV_CMD(DTV_BANDWIDTH_HZ), + _DTV_CMD(DTV_MODULATION), + _DTV_CMD(DTV_INVERSION), + _DTV_CMD(DTV_DISEQC_MASTER), + _DTV_CMD(DTV_SYMBOL_RATE), + _DTV_CMD(DTV_INNER_FEC), + _DTV_CMD(DTV_VOLTAGE), + _DTV_CMD(DTV_TONE), + _DTV_CMD(DTV_PILOT), + _DTV_CMD(DTV_ROLLOFF), + _DTV_CMD(DTV_DELIVERY_SYSTEM), + _DTV_CMD(DTV_HIERARCHY), + _DTV_CMD(DTV_CODE_RATE_HP), + _DTV_CMD(DTV_CODE_RATE_LP), + _DTV_CMD(DTV_GUARD_INTERVAL), + _DTV_CMD(DTV_TRANSMISSION_MODE), + _DTV_CMD(DTV_INTERLEAVING), - _DTV_CMD(DTV_ISDBT_PARTIAL_RECEPTION, 1, 0), - _DTV_CMD(DTV_ISDBT_SOUND_BROADCASTING, 1, 0), - _DTV_CMD(DTV_ISDBT_SB_SUBCHANNEL_ID, 1, 0), - _DTV_CMD(DTV_ISDBT_SB_SEGMENT_IDX, 1, 0), - _DTV_CMD(DTV_ISDBT_SB_SEGMENT_COUNT, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYER_ENABLED, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERA_FEC, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERA_MODULATION, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERA_SEGMENT_COUNT, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERA_TIME_INTERLEAVING, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERB_FEC, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERB_MODULATION, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERB_SEGMENT_COUNT, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERB_TIME_INTERLEAVING, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERC_FEC, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERC_MODULATION, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERC_SEGMENT_COUNT, 1, 0), - _DTV_CMD(DTV_ISDBT_LAYERC_TIME_INTERLEAVING, 1, 0), + _DTV_CMD(DTV_ISDBT_PARTIAL_RECEPTION), + _DTV_CMD(DTV_ISDBT_SOUND_BROADCASTING), + _DTV_CMD(DTV_ISDBT_SB_SUBCHANNEL_ID), + _DTV_CMD(DTV_ISDBT_SB_SEGMENT_IDX), + _DTV_CMD(DTV_ISDBT_SB_SEGMENT_COUNT), + _DTV_CMD(DTV_ISDBT_LAYER_ENABLED), + _DTV_CMD(DTV_ISDBT_LAYERA_FEC), + _DTV_CMD(DTV_ISDBT_LAYERA_MODULATION), + _DTV_CMD(DTV_ISDBT_LAYERA_SEGMENT_COUNT), + _DTV_CMD(DTV_ISDBT_LAYERA_TIME_INTERLEAVING), + _DTV_CMD(DTV_ISDBT_LAYERB_FEC), + _DTV_CMD(DTV_ISDBT_LAYERB_MODULATION), + _DTV_CMD(DTV_ISDBT_LAYERB_SEGMENT_COUNT), + _DTV_CMD(DTV_ISDBT_LAYERB_TIME_INTERLEAVING), + _DTV_CMD(DTV_ISDBT_LAYERC_FEC), + _DTV_CMD(DTV_ISDBT_LAYERC_MODULATION), + _DTV_CMD(DTV_ISDBT_LAYERC_SEGMENT_COUNT), + _DTV_CMD(DTV_ISDBT_LAYERC_TIME_INTERLEAVING), - _DTV_CMD(DTV_STREAM_ID, 1, 0), - _DTV_CMD(DTV_DVBT2_PLP_ID_LEGACY, 1, 0), - _DTV_CMD(DTV_SCRAMBLING_SEQUENCE_INDEX, 1, 0), - _DTV_CMD(DTV_LNA, 1, 0), + _DTV_CMD(DTV_STREAM_ID), + _DTV_CMD(DTV_DVBT2_PLP_ID_LEGACY), + _DTV_CMD(DTV_SCRAMBLING_SEQUENCE_INDEX), + _DTV_CMD(DTV_LNA), /* Get */ - _DTV_CMD(DTV_DISEQC_SLAVE_REPLY, 0, 1), - _DTV_CMD(DTV_API_VERSION, 0, 0), + _DTV_CMD(DTV_DISEQC_SLAVE_REPLY), + _DTV_CMD(DTV_API_VERSION), - _DTV_CMD(DTV_ENUM_DELSYS, 0, 0), + _DTV_CMD(DTV_ENUM_DELSYS), - _DTV_CMD(DTV_ATSCMH_PARADE_ID, 1, 0), - _DTV_CMD(DTV_ATSCMH_RS_FRAME_ENSEMBLE, 1, 0), + _DTV_CMD(DTV_ATSCMH_PARADE_ID), + _DTV_CMD(DTV_ATSCMH_RS_FRAME_ENSEMBLE), - _DTV_CMD(DTV_ATSCMH_FIC_VER, 0, 0), - _DTV_CMD(DTV_ATSCMH_NOG, 0, 0), - _DTV_CMD(DTV_ATSCMH_TNOG, 0, 0), - _DTV_CMD(DTV_ATSCMH_SGN, 0, 0), - _DTV_CMD(DTV_ATSCMH_PRC, 0, 0), - _DTV_CMD(DTV_ATSCMH_RS_FRAME_MODE, 0, 0), - _DTV_CMD(DTV_ATSCMH_RS_CODE_MODE_PRI, 0, 0), - _DTV_CMD(DTV_ATSCMH_RS_CODE_MODE_SEC, 0, 0), - _DTV_CMD(DTV_ATSCMH_SCCC_BLOCK_MODE, 0, 0), - _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_A, 0, 0), - _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_B, 0, 0), - _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_C, 0, 0), - _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_D, 0, 0), + _DTV_CMD(DTV_ATSCMH_FIC_VER), + _DTV_CMD(DTV_ATSCMH_NOG), + _DTV_CMD(DTV_ATSCMH_TNOG), + _DTV_CMD(DTV_ATSCMH_SGN), + _DTV_CMD(DTV_ATSCMH_PRC), + _DTV_CMD(DTV_ATSCMH_RS_FRAME_MODE), + _DTV_CMD(DTV_ATSCMH_RS_CODE_MODE_PRI), + _DTV_CMD(DTV_ATSCMH_RS_CODE_MODE_SEC), + _DTV_CMD(DTV_ATSCMH_SCCC_BLOCK_MODE), + _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_A), + _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_B), + _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_C), + _DTV_CMD(DTV_ATSCMH_SCCC_CODE_MODE_D), /* Statistics API */ - _DTV_CMD(DTV_STAT_SIGNAL_STRENGTH, 0, 0), - _DTV_CMD(DTV_STAT_CNR, 0, 0), - _DTV_CMD(DTV_STAT_PRE_ERROR_BIT_COUNT, 0, 0), - _DTV_CMD(DTV_STAT_PRE_TOTAL_BIT_COUNT, 0, 0), - _DTV_CMD(DTV_STAT_POST_ERROR_BIT_COUNT, 0, 0), - _DTV_CMD(DTV_STAT_POST_TOTAL_BIT_COUNT, 0, 0), - _DTV_CMD(DTV_STAT_ERROR_BLOCK_COUNT, 0, 0), - _DTV_CMD(DTV_STAT_TOTAL_BLOCK_COUNT, 0, 0), + _DTV_CMD(DTV_STAT_SIGNAL_STRENGTH), + _DTV_CMD(DTV_STAT_CNR), + _DTV_CMD(DTV_STAT_PRE_ERROR_BIT_COUNT), + _DTV_CMD(DTV_STAT_PRE_TOTAL_BIT_COUNT), + _DTV_CMD(DTV_STAT_POST_ERROR_BIT_COUNT), + _DTV_CMD(DTV_STAT_POST_TOTAL_BIT_COUNT), + _DTV_CMD(DTV_STAT_ERROR_BLOCK_COUNT), + _DTV_CMD(DTV_STAT_TOTAL_BLOCK_COUNT), }; +static char *dtv_cmd_name(u32 cmd) +{ + cmd = array_index_nospec(cmd, DTV_MAX_COMMAND); + return dtv_cmds[cmd]; +} + /* Synchronise the legacy tuning parameters into the cache, so that demodulator * drivers can use a single set_frontend tuning function, regardless of whether * it's being used for the legacy or new API, reducing code and complexity. @@ -1346,6 +1337,7 @@ static int dtv_property_process_get(struct dvb_frontend *fe, struct file *file) { int ncaps; + unsigned int len = 1; switch (tvp->cmd) { case DTV_ENUM_DELSYS: @@ -1355,6 +1347,7 @@ static int dtv_property_process_get(struct dvb_frontend *fe, ncaps++; } tvp->u.buffer.len = ncaps; + len = ncaps; break; case DTV_FREQUENCY: tvp->u.data = c->frequency; @@ -1532,27 +1525,51 @@ static int dtv_property_process_get(struct dvb_frontend *fe, /* Fill quality measures */ case DTV_STAT_SIGNAL_STRENGTH: tvp->u.st = c->strength; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; case DTV_STAT_CNR: tvp->u.st = c->cnr; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; case DTV_STAT_PRE_ERROR_BIT_COUNT: tvp->u.st = c->pre_bit_error; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; case DTV_STAT_PRE_TOTAL_BIT_COUNT: tvp->u.st = c->pre_bit_count; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; case DTV_STAT_POST_ERROR_BIT_COUNT: tvp->u.st = c->post_bit_error; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; case DTV_STAT_POST_TOTAL_BIT_COUNT: tvp->u.st = c->post_bit_count; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; case DTV_STAT_ERROR_BLOCK_COUNT: tvp->u.st = c->block_error; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; case DTV_STAT_TOTAL_BLOCK_COUNT: tvp->u.st = c->block_count; + if (tvp->u.buffer.len > MAX_DTV_STATS * sizeof(u32)) + tvp->u.buffer.len = MAX_DTV_STATS * sizeof(u32); + len = tvp->u.buffer.len; break; default: dev_dbg(fe->dvb->device, @@ -1561,18 +1578,13 @@ static int dtv_property_process_get(struct dvb_frontend *fe, return -EINVAL; } - if (!dtv_cmds[tvp->cmd].buffer) - dev_dbg(fe->dvb->device, - "%s: GET cmd 0x%08x (%s) = 0x%08x\n", - __func__, tvp->cmd, dtv_cmds[tvp->cmd].name, - tvp->u.data); - else - dev_dbg(fe->dvb->device, - "%s: GET cmd 0x%08x (%s) len %d: %*ph\n", - __func__, - tvp->cmd, dtv_cmds[tvp->cmd].name, - tvp->u.buffer.len, - tvp->u.buffer.len, tvp->u.buffer.data); + if (len < 1) + len = 1; + + dev_dbg(fe->dvb->device, + "%s: GET cmd 0x%08x (%s) len %d: %*ph\n", + __func__, tvp->cmd, dtv_cmd_name(tvp->cmd), + tvp->u.buffer.len, tvp->u.buffer.len, tvp->u.buffer.data); return 0; } @@ -1870,7 +1882,7 @@ static int dtv_property_process_set(struct dvb_frontend *fe, else dev_dbg(fe->dvb->device, "%s: SET cmd 0x%08x (%s) to 0x%08x\n", - __func__, cmd, dtv_cmds[cmd].name, data); + __func__, cmd, dtv_cmd_name(cmd), data); switch (cmd) { case DTV_CLEAR: /* diff --git a/drivers/media/dvb-core/dvb_net.c b/drivers/media/dvb-core/dvb_net.c index 89620da983ba..dddebea644bb 100644 --- a/drivers/media/dvb-core/dvb_net.c +++ b/drivers/media/dvb-core/dvb_net.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -1462,14 +1463,20 @@ static int dvb_net_do_ioctl(struct file *file, struct net_device *netdev; struct dvb_net_priv *priv_data; struct dvb_net_if *dvbnetif = parg; + int if_num = dvbnetif->if_num; - if (dvbnetif->if_num >= DVB_NET_DEVICES_MAX || - !dvbnet->state[dvbnetif->if_num]) { + if (if_num >= DVB_NET_DEVICES_MAX) { + ret = -EINVAL; + goto ioctl_error; + } + if_num = array_index_nospec(if_num, DVB_NET_DEVICES_MAX); + + if (!dvbnet->state[if_num]) { ret = -EINVAL; goto ioctl_error; } - netdev = dvbnet->device[dvbnetif->if_num]; + netdev = dvbnet->device[if_num]; priv_data = netdev_priv(netdev); dvbnetif->pid=priv_data->pid; @@ -1522,14 +1529,20 @@ static int dvb_net_do_ioctl(struct file *file, struct net_device *netdev; struct dvb_net_priv *priv_data; struct __dvb_net_if_old *dvbnetif = parg; + int if_num = dvbnetif->if_num; - if (dvbnetif->if_num >= DVB_NET_DEVICES_MAX || - !dvbnet->state[dvbnetif->if_num]) { + if (if_num >= DVB_NET_DEVICES_MAX) { + ret = -EINVAL; + goto ioctl_error; + } + if_num = array_index_nospec(if_num, DVB_NET_DEVICES_MAX); + + if (!dvbnet->state[if_num]) { ret = -EINVAL; goto ioctl_error; } - netdev = dvbnet->device[dvbnetif->if_num]; + netdev = dvbnet->device[if_num]; priv_data = netdev_priv(netdev); dvbnetif->pid=priv_data->pid; diff --git a/drivers/media/dvb-core/dvbdev.c b/drivers/media/dvb-core/dvbdev.c index 3862ddc86ec4..795d9bfaba5c 100644 --- a/drivers/media/dvb-core/dvbdev.c +++ b/drivers/media/dvb-core/dvbdev.c @@ -506,6 +506,7 @@ int dvb_register_device(struct dvb_adapter *adap, struct dvb_device **pdvbdev, break; if (minor == MAX_DVB_MINORS) { + list_del (&dvbdev->list_head); kfree(dvbdevfops); kfree(dvbdev); up_write(&minor_rwsem); @@ -526,6 +527,7 @@ int dvb_register_device(struct dvb_adapter *adap, struct dvb_device **pdvbdev, __func__); dvb_media_device_free(dvbdev); + list_del (&dvbdev->list_head); kfree(dvbdevfops); kfree(dvbdev); mutex_unlock(&dvbdev_register_lock); @@ -541,6 +543,7 @@ int dvb_register_device(struct dvb_adapter *adap, struct dvb_device **pdvbdev, pr_err("%s: failed to create device dvb%d.%s%d (%ld)\n", __func__, adap->num, dnames[type], id, PTR_ERR(clsdev)); dvb_media_device_free(dvbdev); + list_del (&dvbdev->list_head); kfree(dvbdevfops); kfree(dvbdev); return PTR_ERR(clsdev); diff --git a/drivers/media/dvb-frontends/Kconfig b/drivers/media/dvb-frontends/Kconfig index 3468b07b62fe..2c1ed98d43c5 100644 --- a/drivers/media/dvb-frontends/Kconfig +++ b/drivers/media/dvb-frontends/Kconfig @@ -323,18 +323,6 @@ config DVB_TDA10071 comment "DVB-T (terrestrial) frontends" depends on DVB_CORE -config DVB_SP8870 - tristate "Spase sp8870 based" - depends on DVB_CORE && I2C - default m if !MEDIA_SUBDRV_AUTOSELECT - help - A DVB-T tuner module. Say Y when you want to support this frontend. - - This driver needs external firmware. Please use the command - "/scripts/get_dvb_firmware sp8870" to - download/extract it, and then copy it to /usr/lib/hotplug/firmware - or /lib/firmware (depending on configuration of firmware hotplug). - config DVB_SP887X tristate "Spase sp887x based" depends on DVB_CORE && I2C diff --git a/drivers/media/dvb-frontends/Makefile b/drivers/media/dvb-frontends/Makefile index b9f47d68e14e..d32e4c0be576 100644 --- a/drivers/media/dvb-frontends/Makefile +++ b/drivers/media/dvb-frontends/Makefile @@ -20,7 +20,6 @@ obj-$(CONFIG_DVB_PLL) += dvb-pll.o obj-$(CONFIG_DVB_STV0299) += stv0299.o obj-$(CONFIG_DVB_STB0899) += stb0899.o obj-$(CONFIG_DVB_STB6100) += stb6100.o -obj-$(CONFIG_DVB_SP8870) += sp8870.o obj-$(CONFIG_DVB_CX22700) += cx22700.o obj-$(CONFIG_DVB_S5H1432) += s5h1432.o obj-$(CONFIG_DVB_CX24110) += cx24110.o diff --git a/drivers/media/dvb-frontends/drx39xyj/drxj.h b/drivers/media/dvb-frontends/drx39xyj/drxj.h index d62412f71c88..232b3b0d68c8 100644 --- a/drivers/media/dvb-frontends/drx39xyj/drxj.h +++ b/drivers/media/dvb-frontends/drx39xyj/drxj.h @@ -75,9 +75,9 @@ TYPEDEFS u16 result_len; /*< result length in byte */ u16 *parameter; - /*< General purpous param */ + /*< General purpose param */ u16 *result; - /*< General purpous param */}; + /*< General purpose param */}; /*============================================================================*/ /*============================================================================*/ @@ -131,7 +131,7 @@ TYPEDEFS DRXJ_CFG_MAX /* dummy, never to be used */}; /* -* /struct enum drxj_cfg_smart_ant_io * smart antenna i/o. +* /enum drxj_cfg_smart_ant_io * smart antenna i/o. */ enum drxj_cfg_smart_ant_io { DRXJ_SMT_ANT_OUTPUT = 0, @@ -139,7 +139,7 @@ enum drxj_cfg_smart_ant_io { }; /* -* /struct struct drxj_cfg_smart_ant * Set smart antenna. +* /struct drxj_cfg_smart_ant * Set smart antenna. */ struct drxj_cfg_smart_ant { enum drxj_cfg_smart_ant_io io; @@ -159,7 +159,7 @@ struct drxj_agc_status { /* DRXJ_CFG_AGC_RF, DRXJ_CFG_AGC_IF */ /* -* /struct enum drxj_agc_ctrl_mode * Available AGCs modes in the DRXJ. +* /enum drxj_agc_ctrl_mode * Available AGCs modes in the DRXJ. */ enum drxj_agc_ctrl_mode { DRX_AGC_CTRL_AUTO = 0, @@ -167,7 +167,7 @@ struct drxj_agc_status { DRX_AGC_CTRL_OFF}; /* -* /struct struct drxj_cfg_agc * Generic interface for all AGCs present on the DRXJ. +* /struct drxj_cfg_agc * Generic interface for all AGCs present on the DRXJ. */ struct drxj_cfg_agc { enum drx_standard standard; /* standard for which these settings apply */ @@ -183,7 +183,7 @@ struct drxj_agc_status { /* DRXJ_CFG_PRE_SAW */ /* -* /struct struct drxj_cfg_pre_saw * Interface to configure pre SAW sense. +* /struct drxj_cfg_pre_saw * Interface to configure pre SAW sense. */ struct drxj_cfg_pre_saw { enum drx_standard standard; /* standard to which these settings apply */ @@ -193,7 +193,7 @@ struct drxj_agc_status { /* DRXJ_CFG_AFE_GAIN */ /* -* /struct struct drxj_cfg_afe_gain * Interface to configure gain of AFE (LNA + PGA). +* /struct drxj_cfg_afe_gain * Interface to configure gain of AFE (LNA + PGA). */ struct drxj_cfg_afe_gain { enum drx_standard standard; /* standard to which these settings apply */ @@ -220,14 +220,14 @@ struct drxj_agc_status { }; /* -* /struct struct drxj_cfg_vsb_misc * symbol error rate +* /struct drxj_cfg_vsb_misc * symbol error rate */ struct drxj_cfg_vsb_misc { u32 symb_error; /*< symbol error rate sps */}; /* -* /enum enum drxj_mpeg_output_clock_rate * Mpeg output clock rate. +* /enum drxj_mpeg_output_clock_rate * Mpeg output clock rate. * */ enum drxj_mpeg_start_width { @@ -235,7 +235,7 @@ struct drxj_agc_status { DRXJ_MPEG_START_WIDTH_8CLKCYC}; /* -* /enum enum drxj_mpeg_output_clock_rate * Mpeg output clock rate. +* /enum drxj_mpeg_output_clock_rate * Mpeg output clock rate. * */ enum drxj_mpeg_output_clock_rate { @@ -261,7 +261,7 @@ struct drxj_agc_status { enum drxj_mpeg_start_width mpeg_start_width; /*< set MPEG output start width */}; /* -* /enum enum drxj_xtal_freq * Supported external crystal reference frequency. +* /enum drxj_xtal_freq * Supported external crystal reference frequency. */ enum drxj_xtal_freq { DRXJ_XTAL_FREQ_RSVD, @@ -270,14 +270,15 @@ struct drxj_agc_status { DRXJ_XTAL_FREQ_4MHZ}; /* -* /enum enum drxj_xtal_freq * Supported external crystal reference frequency. +* /enum drxj_xtal_freq * Supported external crystal reference frequency. */ enum drxji2c_speed { DRXJ_I2C_SPEED_400KBPS, DRXJ_I2C_SPEED_100KBPS}; /* -* /struct struct drxj_cfg_hw_cfg * Get hw configuration, such as crystal reference frequency, I2C speed, etc... +* /struct drxj_cfg_hw_cfg * Get hw configuration, such as crystal +* reference frequency, I2C speed, etc... */ struct drxj_cfg_hw_cfg { enum drxj_xtal_freq xtal_freq; @@ -364,7 +365,7 @@ struct drxj_cfg_oob_misc { DRXJ_SIF_ATTENUATION_9DB}; /* -* /struct struct drxj_cfg_atv_output * SIF attenuation setting. +* /struct drxj_cfg_atv_output * SIF attenuation setting. * */ struct drxj_cfg_atv_output { @@ -453,10 +454,10 @@ struct drxj_cfg_atv_output { enum drxuio_mode uio_gpio_mode; /*< current mode of ASEL pin */ enum drxuio_mode uio_irqn_mode; /*< current mode of IRQN pin */ - /* IQM fs frequecy shift and inversion */ + /* IQM fs frequency shift and inversion */ u32 iqm_fs_rate_ofs; /*< frequency shifter setting after setchannel */ bool pos_image; /*< True: positive image */ - /* IQM RC frequecy shift */ + /* IQM RC frequency shift */ u32 iqm_rc_rate_ofs; /*< frequency shifter setting after setchannel */ /* ATV configuration */ diff --git a/drivers/media/dvb-frontends/mxl692.c b/drivers/media/dvb-frontends/mxl692.c index 83030643aba7..a246db683cdf 100644 --- a/drivers/media/dvb-frontends/mxl692.c +++ b/drivers/media/dvb-frontends/mxl692.c @@ -224,7 +224,9 @@ static int mxl692_validate_fw_header(struct mxl692_dev *dev, u32 ix, temp; __be32 *local_buf = NULL; u8 temp_cksum = 0; - const u8 fw_hdr[] = { 0x4D, 0x31, 0x10, 0x02, 0x40, 0x00, 0x00, 0x80 }; + static const u8 fw_hdr[] = { + 0x4D, 0x31, 0x10, 0x02, 0x40, 0x00, 0x00, 0x80 + }; if (memcmp(buffer, fw_hdr, 8) != 0) { status = -EINVAL; diff --git a/drivers/media/dvb-frontends/rtl2832_sdr.c b/drivers/media/dvb-frontends/rtl2832_sdr.c index ef6feb299d46..1a2f0d2adadf 100644 --- a/drivers/media/dvb-frontends/rtl2832_sdr.c +++ b/drivers/media/dvb-frontends/rtl2832_sdr.c @@ -1130,8 +1130,6 @@ static int rtl2832_sdr_g_fmt_sdr_cap(struct file *file, void *priv, f->fmt.sdr.pixelformat = dev->pixelformat; f->fmt.sdr.buffersize = dev->buffersize; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); - return 0; } @@ -1149,7 +1147,6 @@ static int rtl2832_sdr_s_fmt_sdr_cap(struct file *file, void *priv, if (vb2_is_busy(q)) return -EBUSY; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < dev->num_formats; i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { dev->pixelformat = formats[i].pixelformat; @@ -1177,7 +1174,6 @@ static int rtl2832_sdr_try_fmt_sdr_cap(struct file *file, void *priv, dev_dbg(&pdev->dev, "pixelformat fourcc %4.4s\n", (char *)&f->fmt.sdr.pixelformat); - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < dev->num_formats; i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { f->fmt.sdr.buffersize = formats[i].buffersize; diff --git a/drivers/media/i2c/Kconfig b/drivers/media/i2c/Kconfig index 462c0e059754..588f8eb95984 100644 --- a/drivers/media/i2c/Kconfig +++ b/drivers/media/i2c/Kconfig @@ -217,6 +217,7 @@ config VIDEO_ADV7180 depends on GPIOLIB && VIDEO_V4L2 && I2C select MEDIA_CONTROLLER select VIDEO_V4L2_SUBDEV_API + select V4L2_ASYNC help Support for the Analog Devices ADV7180 video decoder. @@ -534,6 +535,7 @@ config VIDEO_ADV7175 config VIDEO_ADV7343 tristate "ADV7343 video encoder" depends on I2C + select V4L2_ASYNC help Support for Analog Devices I2C bus based ADV7343 encoder. @@ -652,6 +654,7 @@ config SDR_MAX2175 tristate "Maxim 2175 RF to Bits tuner" depends on VIDEO_V4L2 && MEDIA_SDR_SUPPORT && I2C select REGMAP_I2C + select V4L2_ASYNC help Support for Maxim 2175 tuner. It is an advanced analog/digital radio receiver with RF-to-Bits front-end designed for SDR solutions. @@ -668,6 +671,7 @@ menu "Miscellaneous helper chips" config VIDEO_THS7303 tristate "THS7303/53 Video Amplifier" depends on VIDEO_V4L2 && I2C + select V4L2_ASYNC help Support for TI THS7303/53 video amplifier @@ -738,6 +742,17 @@ config VIDEO_HI556 To compile this driver as a module, choose M here: the module will be called hi556. +config VIDEO_IMX208 + tristate "Sony IMX208 sensor support" + depends on I2C && VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API + depends on MEDIA_CAMERA_SUPPORT + help + This is a Video4Linux2 sensor driver for the Sony + IMX208 camera. + + To compile this driver as a module, choose M here: the + module will be called imx208. + config VIDEO_IMX214 tristate "Sony IMX214 sensor support" depends on GPIOLIB && I2C && VIDEO_V4L2 @@ -1341,6 +1356,7 @@ config VIDEO_AD5820 tristate "AD5820 lens voice coil support" depends on GPIOLIB && I2C && VIDEO_V4L2 select MEDIA_CONTROLLER + select V4L2_ASYNC help This is a driver for the AD5820 camera lens voice coil. It is used for example in Nokia N900 (RX-51). @@ -1350,6 +1366,7 @@ config VIDEO_AK7375 depends on I2C && VIDEO_V4L2 select MEDIA_CONTROLLER select VIDEO_V4L2_SUBDEV_API + select V4L2_ASYNC help This is a driver for the AK7375 camera lens voice coil. AK7375 is a 12 bit DAC with 120mA output current sink @@ -1361,6 +1378,7 @@ config VIDEO_DW9714 depends on I2C && VIDEO_V4L2 select MEDIA_CONTROLLER select VIDEO_V4L2_SUBDEV_API + select V4L2_ASYNC help This is a driver for the DW9714 camera lens voice coil. DW9714 is a 10 bit DAC with 120mA output current sink @@ -1384,6 +1402,7 @@ config VIDEO_DW9807_VCM depends on I2C && VIDEO_V4L2 select MEDIA_CONTROLLER select VIDEO_V4L2_SUBDEV_API + select V4L2_ASYNC help This is a driver for the DW9807 camera lens voice coil. DW9807 is a 10 bit DAC with 100mA output current sink @@ -1399,6 +1418,7 @@ config VIDEO_ADP1653 tristate "ADP1653 flash support" depends on I2C && VIDEO_V4L2 select MEDIA_CONTROLLER + select V4L2_ASYNC help This is a driver for the ADP1653 flash controller. It is used for example in Nokia N900. @@ -1408,6 +1428,7 @@ config VIDEO_LM3560 depends on I2C && VIDEO_V4L2 select MEDIA_CONTROLLER select REGMAP_I2C + select V4L2_ASYNC help This is a driver for the lm3560 dual flash controllers. It controls flash, torch LEDs. @@ -1417,6 +1438,7 @@ config VIDEO_LM3646 depends on I2C && VIDEO_V4L2 select MEDIA_CONTROLLER select REGMAP_I2C + select V4L2_ASYNC help This is a driver for the lm3646 dual flash controllers. It controls flash, torch LEDs. diff --git a/drivers/media/i2c/Makefile b/drivers/media/i2c/Makefile index 0c067beca066..1168fa6b84ed 100644 --- a/drivers/media/i2c/Makefile +++ b/drivers/media/i2c/Makefile @@ -116,6 +116,7 @@ obj-$(CONFIG_VIDEO_ML86V7667) += ml86v7667.o obj-$(CONFIG_VIDEO_OV2659) += ov2659.o obj-$(CONFIG_VIDEO_TC358743) += tc358743.o obj-$(CONFIG_VIDEO_HI556) += hi556.o +obj-$(CONFIG_VIDEO_IMX208) += imx208.o obj-$(CONFIG_VIDEO_IMX214) += imx214.o obj-$(CONFIG_VIDEO_IMX219) += imx219.o obj-$(CONFIG_VIDEO_IMX258) += imx258.o diff --git a/drivers/media/i2c/adv7170.c b/drivers/media/i2c/adv7170.c index e4e8fda51ad8..714e31f993e1 100644 --- a/drivers/media/i2c/adv7170.c +++ b/drivers/media/i2c/adv7170.c @@ -250,7 +250,7 @@ static int adv7170_s_routing(struct v4l2_subdev *sd, } static int adv7170_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(adv7170_codes)) @@ -261,7 +261,7 @@ static int adv7170_enum_mbus_code(struct v4l2_subdev *sd, } static int adv7170_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -284,7 +284,7 @@ static int adv7170_get_fmt(struct v4l2_subdev *sd, } static int adv7170_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; diff --git a/drivers/media/i2c/adv7175.c b/drivers/media/i2c/adv7175.c index 0cdd8e033197..1813f67f0fe1 100644 --- a/drivers/media/i2c/adv7175.c +++ b/drivers/media/i2c/adv7175.c @@ -288,7 +288,7 @@ static int adv7175_s_routing(struct v4l2_subdev *sd, } static int adv7175_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(adv7175_codes)) @@ -299,7 +299,7 @@ static int adv7175_enum_mbus_code(struct v4l2_subdev *sd, } static int adv7175_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -322,7 +322,7 @@ static int adv7175_get_fmt(struct v4l2_subdev *sd, } static int adv7175_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; diff --git a/drivers/media/i2c/adv7180.c b/drivers/media/i2c/adv7180.c index 44bb6fe85644..fa5bc55bc944 100644 --- a/drivers/media/i2c/adv7180.c +++ b/drivers/media/i2c/adv7180.c @@ -633,7 +633,7 @@ static void adv7180_exit_controls(struct adv7180_state *state) } static int adv7180_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index != 0) @@ -699,13 +699,13 @@ static int adv7180_set_field_mode(struct adv7180_state *state) } static int adv7180_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7180_state *state = to_state(sd); if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - format->format = *v4l2_subdev_get_try_format(sd, cfg, 0); + format->format = *v4l2_subdev_get_try_format(sd, sd_state, 0); } else { adv7180_mbus_fmt(sd, &format->format); format->format.field = state->field; @@ -715,7 +715,7 @@ static int adv7180_get_pad_format(struct v4l2_subdev *sd, } static int adv7180_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7180_state *state = to_state(sd); @@ -742,7 +742,7 @@ static int adv7180_set_pad_format(struct v4l2_subdev *sd, adv7180_set_power(state, true); } } else { - framefmt = v4l2_subdev_get_try_format(sd, cfg, 0); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, 0); *framefmt = format->format; } @@ -750,14 +750,14 @@ static int adv7180_set_pad_format(struct v4l2_subdev *sd, } static int adv7180_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format fmt = { - .which = cfg ? V4L2_SUBDEV_FORMAT_TRY - : V4L2_SUBDEV_FORMAT_ACTIVE, + .which = sd_state ? V4L2_SUBDEV_FORMAT_TRY + : V4L2_SUBDEV_FORMAT_ACTIVE, }; - return adv7180_set_pad_format(sd, cfg, &fmt); + return adv7180_set_pad_format(sd, sd_state, &fmt); } static int adv7180_get_mbus_config(struct v4l2_subdev *sd, diff --git a/drivers/media/i2c/adv7183.c b/drivers/media/i2c/adv7183.c index 8bcd632c081a..92cafdea3f1f 100644 --- a/drivers/media/i2c/adv7183.c +++ b/drivers/media/i2c/adv7183.c @@ -409,7 +409,7 @@ static int adv7183_g_input_status(struct v4l2_subdev *sd, u32 *status) } static int adv7183_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index > 0) @@ -420,7 +420,7 @@ static int adv7183_enum_mbus_code(struct v4l2_subdev *sd, } static int adv7183_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7183 *decoder = to_adv7183(sd); @@ -443,12 +443,12 @@ static int adv7183_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) decoder->fmt = *fmt; else - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; return 0; } static int adv7183_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7183 *decoder = to_adv7183(sd); diff --git a/drivers/media/i2c/adv748x/adv748x-afe.c b/drivers/media/i2c/adv748x/adv748x-afe.c index 4052cf67bf16..02eabe10ab97 100644 --- a/drivers/media/i2c/adv748x/adv748x-afe.c +++ b/drivers/media/i2c/adv748x/adv748x-afe.c @@ -331,7 +331,7 @@ static int adv748x_afe_propagate_pixelrate(struct adv748x_afe *afe) } static int adv748x_afe_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index != 0) @@ -343,7 +343,7 @@ static int adv748x_afe_enum_mbus_code(struct v4l2_subdev *sd, } static int adv748x_afe_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct adv748x_afe *afe = adv748x_sd_to_afe(sd); @@ -354,7 +354,8 @@ static int adv748x_afe_get_format(struct v4l2_subdev *sd, return -EINVAL; if (sdformat->which == V4L2_SUBDEV_FORMAT_TRY) { - mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad); + mbusformat = v4l2_subdev_get_try_format(sd, sd_state, + sdformat->pad); sdformat->format = *mbusformat; } else { adv748x_afe_fill_format(afe, &sdformat->format); @@ -365,7 +366,7 @@ static int adv748x_afe_get_format(struct v4l2_subdev *sd, } static int adv748x_afe_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct v4l2_mbus_framefmt *mbusformat; @@ -375,9 +376,9 @@ static int adv748x_afe_set_format(struct v4l2_subdev *sd, return -EINVAL; if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) - return adv748x_afe_get_format(sd, cfg, sdformat); + return adv748x_afe_get_format(sd, sd_state, sdformat); - mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad); + mbusformat = v4l2_subdev_get_try_format(sd, sd_state, sdformat->pad); *mbusformat = sdformat->format; return 0; diff --git a/drivers/media/i2c/adv748x/adv748x-csi2.c b/drivers/media/i2c/adv748x/adv748x-csi2.c index fa9278a08fde..589e9644fcdc 100644 --- a/drivers/media/i2c/adv748x/adv748x-csi2.c +++ b/drivers/media/i2c/adv748x/adv748x-csi2.c @@ -141,26 +141,26 @@ static const struct v4l2_subdev_video_ops adv748x_csi2_video_ops = { static struct v4l2_mbus_framefmt * adv748x_csi2_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd); if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(sd, cfg, pad); + return v4l2_subdev_get_try_format(sd, sd_state, pad); return &tx->format; } static int adv748x_csi2_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd); struct adv748x_state *state = tx->state; struct v4l2_mbus_framefmt *mbusformat; - mbusformat = adv748x_csi2_get_pad_format(sd, cfg, sdformat->pad, + mbusformat = adv748x_csi2_get_pad_format(sd, sd_state, sdformat->pad, sdformat->which); if (!mbusformat) return -EINVAL; @@ -175,7 +175,7 @@ static int adv748x_csi2_get_format(struct v4l2_subdev *sd, } static int adv748x_csi2_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct adv748x_csi2 *tx = adv748x_sd_to_csi2(sd); @@ -183,7 +183,7 @@ static int adv748x_csi2_set_format(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mbusformat; int ret = 0; - mbusformat = adv748x_csi2_get_pad_format(sd, cfg, sdformat->pad, + mbusformat = adv748x_csi2_get_pad_format(sd, sd_state, sdformat->pad, sdformat->which); if (!mbusformat) return -EINVAL; @@ -193,7 +193,7 @@ static int adv748x_csi2_set_format(struct v4l2_subdev *sd, if (sdformat->pad == ADV748X_CSI2_SOURCE) { const struct v4l2_mbus_framefmt *sink_fmt; - sink_fmt = adv748x_csi2_get_pad_format(sd, cfg, + sink_fmt = adv748x_csi2_get_pad_format(sd, sd_state, ADV748X_CSI2_SINK, sdformat->which); diff --git a/drivers/media/i2c/adv748x/adv748x-hdmi.c b/drivers/media/i2c/adv748x/adv748x-hdmi.c index c557f8fdf11a..52fa7bd75660 100644 --- a/drivers/media/i2c/adv748x/adv748x-hdmi.c +++ b/drivers/media/i2c/adv748x/adv748x-hdmi.c @@ -409,7 +409,7 @@ static int adv748x_hdmi_propagate_pixelrate(struct adv748x_hdmi *hdmi) } static int adv748x_hdmi_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index != 0) @@ -421,7 +421,7 @@ static int adv748x_hdmi_enum_mbus_code(struct v4l2_subdev *sd, } static int adv748x_hdmi_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct adv748x_hdmi *hdmi = adv748x_sd_to_hdmi(sd); @@ -431,7 +431,8 @@ static int adv748x_hdmi_get_format(struct v4l2_subdev *sd, return -EINVAL; if (sdformat->which == V4L2_SUBDEV_FORMAT_TRY) { - mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad); + mbusformat = v4l2_subdev_get_try_format(sd, sd_state, + sdformat->pad); sdformat->format = *mbusformat; } else { adv748x_hdmi_fill_format(hdmi, &sdformat->format); @@ -442,7 +443,7 @@ static int adv748x_hdmi_get_format(struct v4l2_subdev *sd, } static int adv748x_hdmi_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct v4l2_mbus_framefmt *mbusformat; @@ -451,9 +452,9 @@ static int adv748x_hdmi_set_format(struct v4l2_subdev *sd, return -EINVAL; if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) - return adv748x_hdmi_get_format(sd, cfg, sdformat); + return adv748x_hdmi_get_format(sd, sd_state, sdformat); - mbusformat = v4l2_subdev_get_try_format(sd, cfg, sdformat->pad); + mbusformat = v4l2_subdev_get_try_format(sd, sd_state, sdformat->pad); *mbusformat = sdformat->format; return 0; diff --git a/drivers/media/i2c/adv7511-v4l2.c b/drivers/media/i2c/adv7511-v4l2.c index 5fc6c06edda1..41f4e749a859 100644 --- a/drivers/media/i2c/adv7511-v4l2.c +++ b/drivers/media/i2c/adv7511-v4l2.c @@ -1216,7 +1216,7 @@ static int adv7511_get_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid) } static int adv7511_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad != 0) @@ -1247,7 +1247,7 @@ static void adv7511_fill_format(struct adv7511_state *state, } static int adv7511_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7511_state *state = get_adv7511_state(sd); @@ -1261,7 +1261,7 @@ static int adv7511_get_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); format->format.code = fmt->code; format->format.colorspace = fmt->colorspace; format->format.ycbcr_enc = fmt->ycbcr_enc; @@ -1279,7 +1279,7 @@ static int adv7511_get_fmt(struct v4l2_subdev *sd, } static int adv7511_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7511_state *state = get_adv7511_state(sd); @@ -1316,7 +1316,7 @@ static int adv7511_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); fmt->code = format->format.code; fmt->colorspace = format->format.colorspace; fmt->ycbcr_enc = format->format.ycbcr_enc; diff --git a/drivers/media/i2c/adv7604.c b/drivers/media/i2c/adv7604.c index 3049aa2fd0f0..122e1fdccd96 100644 --- a/drivers/media/i2c/adv7604.c +++ b/drivers/media/i2c/adv7604.c @@ -1833,7 +1833,7 @@ static int adv76xx_s_routing(struct v4l2_subdev *sd, } static int adv76xx_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct adv76xx_state *state = to_state(sd); @@ -1913,7 +1913,7 @@ static void adv76xx_setup_format(struct adv76xx_state *state) } static int adv76xx_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv76xx_state *state = to_state(sd); @@ -1926,7 +1926,7 @@ static int adv76xx_get_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); format->format.code = fmt->code; } else { format->format.code = state->format->code; @@ -1936,7 +1936,7 @@ static int adv76xx_get_format(struct v4l2_subdev *sd, } static int adv76xx_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct adv76xx_state *state = to_state(sd); @@ -1956,7 +1956,7 @@ static int adv76xx_get_selection(struct v4l2_subdev *sd, } static int adv76xx_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv76xx_state *state = to_state(sd); @@ -1975,7 +1975,7 @@ static int adv76xx_set_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); fmt->code = format->format.code; } else { state->format = info; diff --git a/drivers/media/i2c/adv7842.c b/drivers/media/i2c/adv7842.c index ff10af757b99..7f8acbdf0db4 100644 --- a/drivers/media/i2c/adv7842.c +++ b/drivers/media/i2c/adv7842.c @@ -98,12 +98,12 @@ struct adv7842_state { v4l2_std_id norm; struct { - u8 edid[256]; + u8 edid[512]; u32 blocks; u32 present; } hdmi_edid; struct { - u8 edid[256]; + u8 edid[128]; u32 blocks; u32 present; } vga_edid; @@ -720,6 +720,9 @@ static int edid_write_vga_segment(struct v4l2_subdev *sd) v4l2_dbg(2, debug, sd, "%s: write EDID on VGA port\n", __func__); + if (!state->vga_edid.present) + return 0; + /* HPA disable on port A and B */ io_write_and_or(sd, 0x20, 0xcf, 0x00); @@ -763,7 +766,7 @@ static int edid_write_hdmi_segment(struct v4l2_subdev *sd, u8 port) struct adv7842_state *state = to_state(sd); const u8 *edid = state->hdmi_edid.edid; u32 blocks = state->hdmi_edid.blocks; - int spa_loc; + unsigned int spa_loc; u16 pa, parent_pa; int err = 0; int i; @@ -796,12 +799,14 @@ static int edid_write_hdmi_segment(struct v4l2_subdev *sd, u8 port) pa = (edid[spa_loc] << 8) | edid[spa_loc + 1]; } - /* edid segment pointer '0' for HDMI ports */ - rep_write_and_or(sd, 0x77, 0xef, 0x00); - for (i = 0; !err && i < blocks * 128; i += I2C_SMBUS_BLOCK_MAX) + for (i = 0; !err && i < blocks * 128; i += I2C_SMBUS_BLOCK_MAX) { + /* set edid segment pointer for HDMI ports */ + if (i % 256 == 0) + rep_write_and_or(sd, 0x77, 0xef, i >= 256 ? 0x10 : 0x00); err = i2c_smbus_write_i2c_block_data(state->i2c_edid, i, I2C_SMBUS_BLOCK_MAX, edid + i); + } if (err) return err; @@ -1988,7 +1993,7 @@ static int adv7842_s_routing(struct v4l2_subdev *sd, } static int adv7842_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(adv7842_formats)) @@ -2064,7 +2069,7 @@ static void adv7842_setup_format(struct adv7842_state *state) } static int adv7842_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7842_state *state = to_state(sd); @@ -2092,7 +2097,7 @@ static int adv7842_get_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); format->format.code = fmt->code; } else { format->format.code = state->format->code; @@ -2102,7 +2107,7 @@ static int adv7842_get_format(struct v4l2_subdev *sd, } static int adv7842_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct adv7842_state *state = to_state(sd); @@ -2112,7 +2117,7 @@ static int adv7842_set_format(struct v4l2_subdev *sd, return -EINVAL; if (state->mode == ADV7842_MODE_SDP) - return adv7842_get_format(sd, cfg, format); + return adv7842_get_format(sd, sd_state, format); info = adv7842_format_info(state, format->format.code); if (info == NULL) @@ -2124,7 +2129,7 @@ static int adv7842_set_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); fmt->code = format->format.code; } else { state->format = info; @@ -2491,9 +2496,17 @@ static int adv7842_get_edid(struct v4l2_subdev *sd, struct v4l2_edid *edid) return 0; } +/* + * If the VGA_EDID_ENABLE bit is set (Repeater Map 0x7f, bit 7), then + * the first two blocks of the EDID are for the HDMI, and the first block + * of segment 1 (i.e. the third block of the EDID) is for VGA. + * So if a VGA EDID is installed, then the maximum size of the HDMI EDID + * is 2 blocks. + */ static int adv7842_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *e) { struct adv7842_state *state = to_state(sd); + unsigned int max_blocks = e->pad == ADV7842_EDID_PORT_VGA ? 1 : 4; int err = 0; memset(e->reserved, 0, sizeof(e->reserved)); @@ -2502,8 +2515,12 @@ static int adv7842_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *e) return -EINVAL; if (e->start_block != 0) return -EINVAL; - if (e->blocks > 2) { - e->blocks = 2; + if (e->pad < ADV7842_EDID_PORT_VGA && state->vga_edid.blocks) + max_blocks = 2; + if (e->pad == ADV7842_EDID_PORT_VGA && state->hdmi_edid.blocks > 2) + return -EBUSY; + if (e->blocks > max_blocks) { + e->blocks = max_blocks; return -E2BIG; } @@ -2514,20 +2531,20 @@ static int adv7842_set_edid(struct v4l2_subdev *sd, struct v4l2_edid *e) switch (e->pad) { case ADV7842_EDID_PORT_VGA: - memset(&state->vga_edid.edid, 0, 256); + memset(state->vga_edid.edid, 0, sizeof(state->vga_edid.edid)); state->vga_edid.blocks = e->blocks; state->vga_edid.present = e->blocks ? 0x1 : 0x0; if (e->blocks) - memcpy(&state->vga_edid.edid, e->edid, 128 * e->blocks); + memcpy(state->vga_edid.edid, e->edid, 128); err = edid_write_vga_segment(sd); break; case ADV7842_EDID_PORT_A: case ADV7842_EDID_PORT_B: - memset(&state->hdmi_edid.edid, 0, 256); + memset(state->hdmi_edid.edid, 0, sizeof(state->hdmi_edid.edid)); state->hdmi_edid.blocks = e->blocks; if (e->blocks) { state->hdmi_edid.present |= 0x04 << e->pad; - memcpy(&state->hdmi_edid.edid, e->edid, 128 * e->blocks); + memcpy(state->hdmi_edid.edid, e->edid, 128 * e->blocks); } else { state->hdmi_edid.present &= ~(0x04 << e->pad); adv7842_s_detect_tx_5v_ctrl(sd); diff --git a/drivers/media/i2c/ak7375.c b/drivers/media/i2c/ak7375.c index e1f94ee0f48f..40b1a4aa846c 100644 --- a/drivers/media/i2c/ak7375.c +++ b/drivers/media/i2c/ak7375.c @@ -87,15 +87,7 @@ static const struct v4l2_ctrl_ops ak7375_vcm_ctrl_ops = { static int ak7375_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - int ret; - - ret = pm_runtime_get_sync(sd->dev); - if (ret < 0) { - pm_runtime_put_noidle(sd->dev); - return ret; - } - - return 0; + return pm_runtime_resume_and_get(sd->dev); } static int ak7375_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) diff --git a/drivers/media/i2c/ak881x.c b/drivers/media/i2c/ak881x.c index 1adaf470c75a..dc569d5a4d9d 100644 --- a/drivers/media/i2c/ak881x.c +++ b/drivers/media/i2c/ak881x.c @@ -91,7 +91,7 @@ static int ak881x_s_register(struct v4l2_subdev *sd, #endif static int ak881x_fill_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -111,7 +111,7 @@ static int ak881x_fill_fmt(struct v4l2_subdev *sd, } static int ak881x_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index) @@ -122,7 +122,7 @@ static int ak881x_enum_mbus_code(struct v4l2_subdev *sd, } static int ak881x_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); diff --git a/drivers/media/i2c/ccs/ccs-core.c b/drivers/media/i2c/ccs/ccs-core.c index 9dc3f45da3dc..a9403a227c6b 100644 --- a/drivers/media/i2c/ccs/ccs-core.c +++ b/drivers/media/i2c/ccs/ccs-core.c @@ -1880,21 +1880,33 @@ static int ccs_pm_get_init(struct ccs_sensor *sensor) struct i2c_client *client = v4l2_get_subdevdata(&sensor->src->sd); int rval; + /* + * It can't use pm_runtime_resume_and_get() here, as the driver + * relies at the returned value to detect if the device was already + * active or not. + */ rval = pm_runtime_get_sync(&client->dev); - if (rval < 0) { - pm_runtime_put_noidle(&client->dev); + if (rval < 0) + goto error; - return rval; - } else if (!rval) { - rval = v4l2_ctrl_handler_setup(&sensor->pixel_array-> - ctrl_handler); - if (rval) - return rval; + /* Device was already active, so don't set controls */ + if (rval == 1) + return 0; - return v4l2_ctrl_handler_setup(&sensor->src->ctrl_handler); - } + /* Restore V4L2 controls to the previously suspended device */ + rval = v4l2_ctrl_handler_setup(&sensor->pixel_array->ctrl_handler); + if (rval) + goto error; + rval = v4l2_ctrl_handler_setup(&sensor->src->ctrl_handler); + if (rval) + goto error; + + /* Keep PM runtime usage_count incremented on success */ return 0; +error: + pm_runtime_put(&client->dev); + return rval; } static int ccs_set_stream(struct v4l2_subdev *subdev, int enable) @@ -1932,7 +1944,7 @@ static int ccs_set_stream(struct v4l2_subdev *subdev, int enable) } static int ccs_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct i2c_client *client = v4l2_get_subdevdata(subdev); @@ -1985,13 +1997,13 @@ static u32 __ccs_get_mbus_code(struct v4l2_subdev *subdev, unsigned int pad) } static int __ccs_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ccs_subdev *ssd = to_ccs_subdev(subdev); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - fmt->format = *v4l2_subdev_get_try_format(subdev, cfg, + fmt->format = *v4l2_subdev_get_try_format(subdev, sd_state, fmt->pad); } else { struct v4l2_rect *r; @@ -2011,21 +2023,21 @@ static int __ccs_get_format(struct v4l2_subdev *subdev, } static int ccs_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); int rval; mutex_lock(&sensor->mutex); - rval = __ccs_get_format(subdev, cfg, fmt); + rval = __ccs_get_format(subdev, sd_state, fmt); mutex_unlock(&sensor->mutex); return rval; } static void ccs_get_crop_compose(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_rect **crops, struct v4l2_rect **comps, int which) { @@ -2042,24 +2054,25 @@ static void ccs_get_crop_compose(struct v4l2_subdev *subdev, if (crops) { for (i = 0; i < subdev->entity.num_pads; i++) crops[i] = v4l2_subdev_get_try_crop(subdev, - cfg, i); + sd_state, + i); } if (comps) - *comps = v4l2_subdev_get_try_compose(subdev, cfg, + *comps = v4l2_subdev_get_try_compose(subdev, sd_state, CCS_PAD_SINK); } } /* Changes require propagation only on sink pad. */ static void ccs_propagate(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, int which, + struct v4l2_subdev_state *sd_state, int which, int target) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); struct ccs_subdev *ssd = to_ccs_subdev(subdev); struct v4l2_rect *comp, *crops[CCS_PADS]; - ccs_get_crop_compose(subdev, cfg, crops, &comp, which); + ccs_get_crop_compose(subdev, sd_state, crops, &comp, which); switch (target) { case V4L2_SEL_TGT_CROP: @@ -2099,7 +2112,7 @@ static const struct ccs_csi_data_format } static int ccs_set_format_source(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); @@ -2110,7 +2123,7 @@ static int ccs_set_format_source(struct v4l2_subdev *subdev, unsigned int i; int rval; - rval = __ccs_get_format(subdev, cfg, fmt); + rval = __ccs_get_format(subdev, sd_state, fmt); if (rval) return rval; @@ -2152,7 +2165,7 @@ static int ccs_set_format_source(struct v4l2_subdev *subdev, } static int ccs_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); @@ -2164,7 +2177,7 @@ static int ccs_set_format(struct v4l2_subdev *subdev, if (fmt->pad == ssd->source_pad) { int rval; - rval = ccs_set_format_source(subdev, cfg, fmt); + rval = ccs_set_format_source(subdev, sd_state, fmt); mutex_unlock(&sensor->mutex); @@ -2186,7 +2199,7 @@ static int ccs_set_format(struct v4l2_subdev *subdev, CCS_LIM(sensor, MIN_Y_OUTPUT_SIZE), CCS_LIM(sensor, MAX_Y_OUTPUT_SIZE)); - ccs_get_crop_compose(subdev, cfg, crops, NULL, fmt->which); + ccs_get_crop_compose(subdev, sd_state, crops, NULL, fmt->which); crops[ssd->sink_pad]->left = 0; crops[ssd->sink_pad]->top = 0; @@ -2194,7 +2207,7 @@ static int ccs_set_format(struct v4l2_subdev *subdev, crops[ssd->sink_pad]->height = fmt->format.height; if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) ssd->sink_fmt = *crops[ssd->sink_pad]; - ccs_propagate(subdev, cfg, fmt->which, V4L2_SEL_TGT_CROP); + ccs_propagate(subdev, sd_state, fmt->which, V4L2_SEL_TGT_CROP); mutex_unlock(&sensor->mutex); @@ -2246,7 +2259,7 @@ static int scaling_goodness(struct v4l2_subdev *subdev, int w, int ask_w, } static void ccs_set_compose_binner(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel, struct v4l2_rect **crops, struct v4l2_rect *comp) @@ -2294,7 +2307,7 @@ static void ccs_set_compose_binner(struct v4l2_subdev *subdev, * result. */ static void ccs_set_compose_scaler(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel, struct v4l2_rect **crops, struct v4l2_rect *comp) @@ -2409,25 +2422,25 @@ static void ccs_set_compose_scaler(struct v4l2_subdev *subdev, } /* We're only called on source pads. This function sets scaling. */ static int ccs_set_compose(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); struct ccs_subdev *ssd = to_ccs_subdev(subdev); struct v4l2_rect *comp, *crops[CCS_PADS]; - ccs_get_crop_compose(subdev, cfg, crops, &comp, sel->which); + ccs_get_crop_compose(subdev, sd_state, crops, &comp, sel->which); sel->r.top = 0; sel->r.left = 0; if (ssd == sensor->binner) - ccs_set_compose_binner(subdev, cfg, sel, crops, comp); + ccs_set_compose_binner(subdev, sd_state, sel, crops, comp); else - ccs_set_compose_scaler(subdev, cfg, sel, crops, comp); + ccs_set_compose_scaler(subdev, sd_state, sel, crops, comp); *comp = sel->r; - ccs_propagate(subdev, cfg, sel->which, V4L2_SEL_TGT_COMPOSE); + ccs_propagate(subdev, sd_state, sel->which, V4L2_SEL_TGT_COMPOSE); if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) return ccs_pll_blanking_update(sensor); @@ -2474,7 +2487,7 @@ static int __ccs_sel_supported(struct v4l2_subdev *subdev, } static int ccs_set_crop(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); @@ -2482,7 +2495,7 @@ static int ccs_set_crop(struct v4l2_subdev *subdev, struct v4l2_rect *src_size, *crops[CCS_PADS]; struct v4l2_rect _r; - ccs_get_crop_compose(subdev, cfg, crops, NULL, sel->which); + ccs_get_crop_compose(subdev, sd_state, crops, NULL, sel->which); if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) { if (sel->pad == ssd->sink_pad) @@ -2493,16 +2506,18 @@ static int ccs_set_crop(struct v4l2_subdev *subdev, if (sel->pad == ssd->sink_pad) { _r.left = 0; _r.top = 0; - _r.width = v4l2_subdev_get_try_format(subdev, cfg, + _r.width = v4l2_subdev_get_try_format(subdev, + sd_state, sel->pad) ->width; - _r.height = v4l2_subdev_get_try_format(subdev, cfg, + _r.height = v4l2_subdev_get_try_format(subdev, + sd_state, sel->pad) ->height; src_size = &_r; } else { src_size = v4l2_subdev_get_try_compose( - subdev, cfg, ssd->sink_pad); + subdev, sd_state, ssd->sink_pad); } } @@ -2520,7 +2535,7 @@ static int ccs_set_crop(struct v4l2_subdev *subdev, *crops[sel->pad] = sel->r; if (ssd != sensor->pixel_array && sel->pad == CCS_PAD_SINK) - ccs_propagate(subdev, cfg, sel->which, V4L2_SEL_TGT_CROP); + ccs_propagate(subdev, sd_state, sel->which, V4L2_SEL_TGT_CROP); return 0; } @@ -2534,7 +2549,7 @@ static void ccs_get_native_size(struct ccs_subdev *ssd, struct v4l2_rect *r) } static int __ccs_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); @@ -2547,13 +2562,14 @@ static int __ccs_get_selection(struct v4l2_subdev *subdev, if (ret) return ret; - ccs_get_crop_compose(subdev, cfg, crops, &comp, sel->which); + ccs_get_crop_compose(subdev, sd_state, crops, &comp, sel->which); if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) { sink_fmt = ssd->sink_fmt; } else { struct v4l2_mbus_framefmt *fmt = - v4l2_subdev_get_try_format(subdev, cfg, ssd->sink_pad); + v4l2_subdev_get_try_format(subdev, sd_state, + ssd->sink_pad); sink_fmt.left = 0; sink_fmt.top = 0; @@ -2584,21 +2600,21 @@ static int __ccs_get_selection(struct v4l2_subdev *subdev, } static int ccs_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); int rval; mutex_lock(&sensor->mutex); - rval = __ccs_get_selection(subdev, cfg, sel); + rval = __ccs_get_selection(subdev, sd_state, sel); mutex_unlock(&sensor->mutex); return rval; } static int ccs_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ccs_sensor *sensor = to_ccs_sensor(subdev); @@ -2622,10 +2638,10 @@ static int ccs_set_selection(struct v4l2_subdev *subdev, switch (sel->target) { case V4L2_SEL_TGT_CROP: - ret = ccs_set_crop(subdev, cfg, sel); + ret = ccs_set_crop(subdev, sd_state, sel); break; case V4L2_SEL_TGT_COMPOSE: - ret = ccs_set_compose(subdev, cfg, sel); + ret = ccs_set_compose(subdev, sd_state, sel); break; default: ret = -EINVAL; @@ -3016,9 +3032,9 @@ static int ccs_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) for (i = 0; i < ssd->npads; i++) { struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, i); + v4l2_subdev_get_try_format(sd, fh->state, i); struct v4l2_rect *try_crop = - v4l2_subdev_get_try_crop(sd, fh->pad, i); + v4l2_subdev_get_try_crop(sd, fh->state, i); struct v4l2_rect *try_comp; ccs_get_native_size(ssd, try_crop); @@ -3031,7 +3047,7 @@ static int ccs_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) if (ssd != sensor->pixel_array) continue; - try_comp = v4l2_subdev_get_try_compose(sd, fh->pad, i); + try_comp = v4l2_subdev_get_try_compose(sd, fh->state, i); *try_comp = *try_crop; } @@ -3089,12 +3105,9 @@ static int __maybe_unused ccs_suspend(struct device *dev) bool streaming = sensor->streaming; int rval; - rval = pm_runtime_get_sync(dev); - if (rval < 0) { - pm_runtime_put_noidle(dev); - - return -EAGAIN; - } + rval = pm_runtime_resume_and_get(dev); + if (rval < 0) + return rval; if (sensor->streaming) ccs_stop_streaming(sensor); diff --git a/drivers/media/i2c/ccs/ccs-limits.c b/drivers/media/i2c/ccs/ccs-limits.c index f5511789ac83..4969fa425317 100644 --- a/drivers/media/i2c/ccs/ccs-limits.c +++ b/drivers/media/i2c/ccs/ccs-limits.c @@ -1,5 +1,9 @@ // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause /* Copyright (C) 2019--2020 Intel Corporation */ +/* + * Generated by Documentation/driver-api/media/drivers/ccs/mk-ccs-regs; + * do not modify. + */ #include "ccs-limits.h" #include "ccs-regs.h" diff --git a/drivers/media/i2c/ccs/ccs-limits.h b/drivers/media/i2c/ccs/ccs-limits.h index 1efa43c23a2e..551d3ee9d04e 100644 --- a/drivers/media/i2c/ccs/ccs-limits.h +++ b/drivers/media/i2c/ccs/ccs-limits.h @@ -1,5 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */ /* Copyright (C) 2019--2020 Intel Corporation */ +/* + * Generated by Documentation/driver-api/media/drivers/ccs/mk-ccs-regs; + * do not modify. + */ #ifndef __CCS_LIMITS_H__ #define __CCS_LIMITS_H__ diff --git a/drivers/media/i2c/ccs/ccs-regs.h b/drivers/media/i2c/ccs/ccs-regs.h index 4b3e5df2121f..6ce84c5ecf20 100644 --- a/drivers/media/i2c/ccs/ccs-regs.h +++ b/drivers/media/i2c/ccs/ccs-regs.h @@ -1,5 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause */ /* Copyright (C) 2019--2020 Intel Corporation */ +/* + * Generated by Documentation/driver-api/media/drivers/ccs/mk-ccs-regs; + * do not modify. + */ #ifndef __CCS_REGS_H__ #define __CCS_REGS_H__ @@ -202,7 +206,7 @@ #define CCS_R_OP_PIX_CLK_DIV (0x0308 | CCS_FL_16BIT) #define CCS_R_OP_SYS_CLK_DIV (0x030a | CCS_FL_16BIT) #define CCS_R_OP_PRE_PLL_CLK_DIV (0x030c | CCS_FL_16BIT) -#define CCS_R_OP_PLL_MULTIPLIER (0x031e | CCS_FL_16BIT) +#define CCS_R_OP_PLL_MULTIPLIER (0x030e | CCS_FL_16BIT) #define CCS_R_PLL_MODE 0x0310 #define CCS_PLL_MODE_SHIFT 0U #define CCS_PLL_MODE_MASK 0x1 diff --git a/drivers/media/i2c/cx25840/cx25840-core.c b/drivers/media/i2c/cx25840/cx25840-core.c index e2e935f78986..dc31944c7d5b 100644 --- a/drivers/media/i2c/cx25840/cx25840-core.c +++ b/drivers/media/i2c/cx25840/cx25840-core.c @@ -1746,7 +1746,7 @@ static int cx25840_s_ctrl(struct v4l2_ctrl *ctrl) /* ----------------------------------------------------------------------- */ static int cx25840_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; diff --git a/drivers/media/i2c/dw9714.c b/drivers/media/i2c/dw9714.c index 3f0b082f863f..c8b4292512dc 100644 --- a/drivers/media/i2c/dw9714.c +++ b/drivers/media/i2c/dw9714.c @@ -85,15 +85,7 @@ static const struct v4l2_ctrl_ops dw9714_vcm_ctrl_ops = { static int dw9714_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - int rval; - - rval = pm_runtime_get_sync(sd->dev); - if (rval < 0) { - pm_runtime_put_noidle(sd->dev); - return rval; - } - - return 0; + return pm_runtime_resume_and_get(sd->dev); } static int dw9714_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) diff --git a/drivers/media/i2c/dw9768.c b/drivers/media/i2c/dw9768.c index 8b8cb4b077b5..c086580efac7 100644 --- a/drivers/media/i2c/dw9768.c +++ b/drivers/media/i2c/dw9768.c @@ -374,15 +374,7 @@ static const struct v4l2_ctrl_ops dw9768_ctrl_ops = { static int dw9768_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - int ret; - - ret = pm_runtime_get_sync(sd->dev); - if (ret < 0) { - pm_runtime_put_noidle(sd->dev); - return ret; - } - - return 0; + return pm_runtime_resume_and_get(sd->dev); } static int dw9768_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) diff --git a/drivers/media/i2c/dw9807-vcm.c b/drivers/media/i2c/dw9807-vcm.c index 438a44b76da8..95e06f13bc9e 100644 --- a/drivers/media/i2c/dw9807-vcm.c +++ b/drivers/media/i2c/dw9807-vcm.c @@ -130,15 +130,7 @@ static const struct v4l2_ctrl_ops dw9807_vcm_ctrl_ops = { static int dw9807_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - int rval; - - rval = pm_runtime_get_sync(sd->dev); - if (rval < 0) { - pm_runtime_put_noidle(sd->dev); - return rval; - } - - return 0; + return pm_runtime_resume_and_get(sd->dev); } static int dw9807_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) diff --git a/drivers/media/i2c/et8ek8/et8ek8_driver.c b/drivers/media/i2c/et8ek8/et8ek8_driver.c index bb3eac5e005e..c7b91c0c03b5 100644 --- a/drivers/media/i2c/et8ek8/et8ek8_driver.c +++ b/drivers/media/i2c/et8ek8/et8ek8_driver.c @@ -882,7 +882,7 @@ static int et8ek8_power_on(struct et8ek8_sensor *sensor) */ #define MAX_FMTS 4 static int et8ek8_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct et8ek8_reglist **list = @@ -920,7 +920,7 @@ static int et8ek8_enum_mbus_code(struct v4l2_subdev *subdev, } static int et8ek8_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct et8ek8_reglist **list = @@ -958,7 +958,7 @@ static int et8ek8_enum_frame_size(struct v4l2_subdev *subdev, } static int et8ek8_enum_frame_ival(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct et8ek8_reglist **list = @@ -990,12 +990,13 @@ static int et8ek8_enum_frame_ival(struct v4l2_subdev *subdev, static struct v4l2_mbus_framefmt * __et8ek8_get_pad_format(struct et8ek8_sensor *sensor, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&sensor->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&sensor->subdev, sd_state, + pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &sensor->format; default: @@ -1004,13 +1005,14 @@ __et8ek8_get_pad_format(struct et8ek8_sensor *sensor, } static int et8ek8_get_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct et8ek8_sensor *sensor = to_et8ek8_sensor(subdev); struct v4l2_mbus_framefmt *format; - format = __et8ek8_get_pad_format(sensor, cfg, fmt->pad, fmt->which); + format = __et8ek8_get_pad_format(sensor, sd_state, fmt->pad, + fmt->which); if (!format) return -EINVAL; @@ -1020,14 +1022,15 @@ static int et8ek8_get_pad_format(struct v4l2_subdev *subdev, } static int et8ek8_set_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct et8ek8_sensor *sensor = to_et8ek8_sensor(subdev); struct v4l2_mbus_framefmt *format; struct et8ek8_reglist *reglist; - format = __et8ek8_get_pad_format(sensor, cfg, fmt->pad, fmt->which); + format = __et8ek8_get_pad_format(sensor, sd_state, fmt->pad, + fmt->which); if (!format) return -EINVAL; @@ -1327,7 +1330,7 @@ static int et8ek8_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) struct et8ek8_reglist *reglist; reglist = et8ek8_reglist_find_type(&meta_reglist, ET8EK8_REGLIST_MODE); - format = __et8ek8_get_pad_format(sensor, fh->pad, 0, + format = __et8ek8_get_pad_format(sensor, fh->state, 0, V4L2_SUBDEV_FORMAT_TRY); et8ek8_reglist_to_mbus(reglist, format); diff --git a/drivers/media/i2c/hi556.c b/drivers/media/i2c/hi556.c index 6f05c1138e3b..8db1cbedc1fd 100644 --- a/drivers/media/i2c/hi556.c +++ b/drivers/media/i2c/hi556.c @@ -813,9 +813,8 @@ static int hi556_set_stream(struct v4l2_subdev *sd, int enable) mutex_lock(&hi556->mutex); if (enable) { - ret = pm_runtime_get_sync(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); if (ret < 0) { - pm_runtime_put_noidle(&client->dev); mutex_unlock(&hi556->mutex); return ret; } @@ -876,7 +875,7 @@ static int __maybe_unused hi556_resume(struct device *dev) } static int hi556_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct hi556 *hi556 = to_hi556(sd); @@ -891,7 +890,7 @@ static int hi556_set_format(struct v4l2_subdev *sd, mutex_lock(&hi556->mutex); hi556_assign_pad_format(mode, &fmt->format); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; } else { hi556->cur_mode = mode; __v4l2_ctrl_s_ctrl(hi556->link_freq, mode->link_freq_index); @@ -918,14 +917,15 @@ static int hi556_set_format(struct v4l2_subdev *sd, } static int hi556_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct hi556 *hi556 = to_hi556(sd); mutex_lock(&hi556->mutex); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(&hi556->sd, cfg, + fmt->format = *v4l2_subdev_get_try_format(&hi556->sd, + sd_state, fmt->pad); else hi556_assign_pad_format(hi556->cur_mode, &fmt->format); @@ -936,7 +936,7 @@ static int hi556_get_format(struct v4l2_subdev *sd, } static int hi556_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -948,7 +948,7 @@ static int hi556_enum_mbus_code(struct v4l2_subdev *sd, } static int hi556_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -971,7 +971,7 @@ static int hi556_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) mutex_lock(&hi556->mutex); hi556_assign_pad_format(&supported_modes[0], - v4l2_subdev_get_try_format(sd, fh->pad, 0)); + v4l2_subdev_get_try_format(sd, fh->state, 0)); mutex_unlock(&hi556->mutex); return 0; diff --git a/drivers/media/i2c/imx208.c b/drivers/media/i2c/imx208.c new file mode 100644 index 000000000000..6f3d9c1b5879 --- /dev/null +++ b/drivers/media/i2c/imx208.c @@ -0,0 +1,1088 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2021 Intel Corporation + +#include +#include +#include +#include +#include +#include +#include +#include + +#define IMX208_REG_MODE_SELECT 0x0100 +#define IMX208_MODE_STANDBY 0x00 +#define IMX208_MODE_STREAMING 0x01 + +/* Chip ID */ +#define IMX208_REG_CHIP_ID 0x0000 +#define IMX208_CHIP_ID 0x0208 + +/* V_TIMING internal */ +#define IMX208_REG_VTS 0x0340 +#define IMX208_VTS_60FPS 0x0472 +#define IMX208_VTS_BINNING 0x0239 +#define IMX208_VTS_60FPS_MIN 0x0458 +#define IMX208_VTS_BINNING_MIN 0x0230 +#define IMX208_VTS_MAX 0xffff + +/* HBLANK control - read only */ +#define IMX208_PPL_384MHZ 2248 +#define IMX208_PPL_96MHZ 2248 + +/* Exposure control */ +#define IMX208_REG_EXPOSURE 0x0202 +#define IMX208_EXPOSURE_MIN 4 +#define IMX208_EXPOSURE_STEP 1 +#define IMX208_EXPOSURE_DEFAULT 0x190 +#define IMX208_EXPOSURE_MAX 65535 + +/* Analog gain control */ +#define IMX208_REG_ANALOG_GAIN 0x0204 +#define IMX208_ANA_GAIN_MIN 0 +#define IMX208_ANA_GAIN_MAX 0x00e0 +#define IMX208_ANA_GAIN_STEP 1 +#define IMX208_ANA_GAIN_DEFAULT 0x0 + +/* Digital gain control */ +#define IMX208_REG_GR_DIGITAL_GAIN 0x020e +#define IMX208_REG_R_DIGITAL_GAIN 0x0210 +#define IMX208_REG_B_DIGITAL_GAIN 0x0212 +#define IMX208_REG_GB_DIGITAL_GAIN 0x0214 +#define IMX208_DIGITAL_GAIN_SHIFT 8 + +/* Orientation */ +#define IMX208_REG_ORIENTATION_CONTROL 0x0101 + +/* Test Pattern Control */ +#define IMX208_REG_TEST_PATTERN_MODE 0x0600 +#define IMX208_TEST_PATTERN_DISABLE 0x0 +#define IMX208_TEST_PATTERN_SOLID_COLOR 0x1 +#define IMX208_TEST_PATTERN_COLOR_BARS 0x2 +#define IMX208_TEST_PATTERN_GREY_COLOR 0x3 +#define IMX208_TEST_PATTERN_PN9 0x4 +#define IMX208_TEST_PATTERN_FIX_1 0x100 +#define IMX208_TEST_PATTERN_FIX_2 0x101 +#define IMX208_TEST_PATTERN_FIX_3 0x102 +#define IMX208_TEST_PATTERN_FIX_4 0x103 +#define IMX208_TEST_PATTERN_FIX_5 0x104 +#define IMX208_TEST_PATTERN_FIX_6 0x105 + +/* OTP Access */ +#define IMX208_OTP_BASE 0x3500 +#define IMX208_OTP_SIZE 40 + +struct imx208_reg { + u16 address; + u8 val; +}; + +struct imx208_reg_list { + u32 num_of_regs; + const struct imx208_reg *regs; +}; + +/* Link frequency config */ +struct imx208_link_freq_config { + u32 pixels_per_line; + + /* PLL registers for this link frequency */ + struct imx208_reg_list reg_list; +}; + +/* Mode : resolution and related config&values */ +struct imx208_mode { + /* Frame width */ + u32 width; + /* Frame height */ + u32 height; + + /* V-timing */ + u32 vts_def; + u32 vts_min; + + /* Index of Link frequency config to be used */ + u32 link_freq_index; + /* Default register values */ + struct imx208_reg_list reg_list; +}; + +static const struct imx208_reg pll_ctrl_reg[] = { + {0x0305, 0x02}, + {0x0307, 0x50}, + {0x303C, 0x3C}, +}; + +static const struct imx208_reg mode_1936x1096_60fps_regs[] = { + {0x0340, 0x04}, + {0x0341, 0x72}, + {0x0342, 0x04}, + {0x0343, 0x64}, + {0x034C, 0x07}, + {0x034D, 0x90}, + {0x034E, 0x04}, + {0x034F, 0x48}, + {0x0381, 0x01}, + {0x0383, 0x01}, + {0x0385, 0x01}, + {0x0387, 0x01}, + {0x3048, 0x00}, + {0x3050, 0x01}, + {0x30D5, 0x00}, + {0x3301, 0x00}, + {0x3318, 0x62}, + {0x0202, 0x01}, + {0x0203, 0x90}, + {0x0205, 0x00}, +}; + +static const struct imx208_reg mode_968_548_60fps_regs[] = { + {0x0340, 0x02}, + {0x0341, 0x39}, + {0x0342, 0x08}, + {0x0343, 0xC8}, + {0x034C, 0x03}, + {0x034D, 0xC8}, + {0x034E, 0x02}, + {0x034F, 0x24}, + {0x0381, 0x01}, + {0x0383, 0x03}, + {0x0385, 0x01}, + {0x0387, 0x03}, + {0x3048, 0x01}, + {0x3050, 0x02}, + {0x30D5, 0x03}, + {0x3301, 0x10}, + {0x3318, 0x75}, + {0x0202, 0x01}, + {0x0203, 0x90}, + {0x0205, 0x00}, +}; + +static const s64 imx208_discrete_digital_gain[] = { + 1, 2, 4, 8, 16, +}; + +static const char * const imx208_test_pattern_menu[] = { + "Disabled", + "Solid Color", + "100% Color Bar", + "Fade to Grey Color Bar", + "PN9", + "Fixed Pattern1", + "Fixed Pattern2", + "Fixed Pattern3", + "Fixed Pattern4", + "Fixed Pattern5", + "Fixed Pattern6" +}; + +static const int imx208_test_pattern_val[] = { + IMX208_TEST_PATTERN_DISABLE, + IMX208_TEST_PATTERN_SOLID_COLOR, + IMX208_TEST_PATTERN_COLOR_BARS, + IMX208_TEST_PATTERN_GREY_COLOR, + IMX208_TEST_PATTERN_PN9, + IMX208_TEST_PATTERN_FIX_1, + IMX208_TEST_PATTERN_FIX_2, + IMX208_TEST_PATTERN_FIX_3, + IMX208_TEST_PATTERN_FIX_4, + IMX208_TEST_PATTERN_FIX_5, + IMX208_TEST_PATTERN_FIX_6, +}; + +/* Configurations for supported link frequencies */ +#define IMX208_MHZ (1000 * 1000ULL) +#define IMX208_LINK_FREQ_384MHZ (384ULL * IMX208_MHZ) +#define IMX208_LINK_FREQ_96MHZ (96ULL * IMX208_MHZ) + +#define IMX208_DATA_RATE_DOUBLE 2 +#define IMX208_NUM_OF_LANES 2 +#define IMX208_PIXEL_BITS 10 + +enum { + IMX208_LINK_FREQ_384MHZ_INDEX, + IMX208_LINK_FREQ_96MHZ_INDEX, +}; + +/* + * pixel_rate = link_freq * data-rate * nr_of_lanes / bits_per_sample + * data rate => double data rate; number of lanes => 2; bits per pixel => 10 + */ +static u64 link_freq_to_pixel_rate(u64 f) +{ + f *= IMX208_DATA_RATE_DOUBLE * IMX208_NUM_OF_LANES; + do_div(f, IMX208_PIXEL_BITS); + + return f; +} + +/* Menu items for LINK_FREQ V4L2 control */ +static const s64 link_freq_menu_items[] = { + [IMX208_LINK_FREQ_384MHZ_INDEX] = IMX208_LINK_FREQ_384MHZ, + [IMX208_LINK_FREQ_96MHZ_INDEX] = IMX208_LINK_FREQ_96MHZ, +}; + +/* Link frequency configs */ +static const struct imx208_link_freq_config link_freq_configs[] = { + [IMX208_LINK_FREQ_384MHZ_INDEX] = { + .pixels_per_line = IMX208_PPL_384MHZ, + .reg_list = { + .num_of_regs = ARRAY_SIZE(pll_ctrl_reg), + .regs = pll_ctrl_reg, + } + }, + [IMX208_LINK_FREQ_96MHZ_INDEX] = { + .pixels_per_line = IMX208_PPL_96MHZ, + .reg_list = { + .num_of_regs = ARRAY_SIZE(pll_ctrl_reg), + .regs = pll_ctrl_reg, + } + }, +}; + +/* Mode configs */ +static const struct imx208_mode supported_modes[] = { + { + .width = 1936, + .height = 1096, + .vts_def = IMX208_VTS_60FPS, + .vts_min = IMX208_VTS_60FPS_MIN, + .reg_list = { + .num_of_regs = ARRAY_SIZE(mode_1936x1096_60fps_regs), + .regs = mode_1936x1096_60fps_regs, + }, + .link_freq_index = IMX208_LINK_FREQ_384MHZ_INDEX, + }, + { + .width = 968, + .height = 548, + .vts_def = IMX208_VTS_BINNING, + .vts_min = IMX208_VTS_BINNING_MIN, + .reg_list = { + .num_of_regs = ARRAY_SIZE(mode_968_548_60fps_regs), + .regs = mode_968_548_60fps_regs, + }, + .link_freq_index = IMX208_LINK_FREQ_96MHZ_INDEX, + }, +}; + +struct imx208 { + struct v4l2_subdev sd; + struct media_pad pad; + + struct v4l2_ctrl_handler ctrl_handler; + /* V4L2 Controls */ + struct v4l2_ctrl *link_freq; + struct v4l2_ctrl *pixel_rate; + struct v4l2_ctrl *vblank; + struct v4l2_ctrl *hblank; + struct v4l2_ctrl *vflip; + struct v4l2_ctrl *hflip; + + /* Current mode */ + const struct imx208_mode *cur_mode; + + /* + * Mutex for serialized access: + * Protect sensor set pad format and start/stop streaming safely. + * Protect access to sensor v4l2 controls. + */ + struct mutex imx208_mx; + + /* Streaming on/off */ + bool streaming; + + /* OTP data */ + bool otp_read; + char otp_data[IMX208_OTP_SIZE]; +}; + +static inline struct imx208 *to_imx208(struct v4l2_subdev *_sd) +{ + return container_of(_sd, struct imx208, sd); +} + +/* Get bayer order based on flip setting. */ +static u32 imx208_get_format_code(struct imx208 *imx208) +{ + /* + * Only one bayer order is supported. + * It depends on the flip settings. + */ + static const u32 codes[2][2] = { + { MEDIA_BUS_FMT_SRGGB10_1X10, MEDIA_BUS_FMT_SGRBG10_1X10, }, + { MEDIA_BUS_FMT_SGBRG10_1X10, MEDIA_BUS_FMT_SBGGR10_1X10, }, + }; + + return codes[imx208->vflip->val][imx208->hflip->val]; +} + +/* Read registers up to 4 at a time */ +static int imx208_read_reg(struct imx208 *imx208, u16 reg, u32 len, u32 *val) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + struct i2c_msg msgs[2]; + u8 addr_buf[2] = { reg >> 8, reg & 0xff }; + u8 data_buf[4] = { 0, }; + int ret; + + if (len > 4) + return -EINVAL; + + /* Write register address */ + msgs[0].addr = client->addr; + msgs[0].flags = 0; + msgs[0].len = ARRAY_SIZE(addr_buf); + msgs[0].buf = addr_buf; + + /* Read data from register */ + msgs[1].addr = client->addr; + msgs[1].flags = I2C_M_RD; + msgs[1].len = len; + msgs[1].buf = &data_buf[4 - len]; + + ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs)); + if (ret != ARRAY_SIZE(msgs)) + return -EIO; + + *val = get_unaligned_be32(data_buf); + + return 0; +} + +/* Write registers up to 4 at a time */ +static int imx208_write_reg(struct imx208 *imx208, u16 reg, u32 len, u32 val) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + u8 buf[6]; + + if (len > 4) + return -EINVAL; + + put_unaligned_be16(reg, buf); + put_unaligned_be32(val << (8 * (4 - len)), buf + 2); + if (i2c_master_send(client, buf, len + 2) != len + 2) + return -EIO; + + return 0; +} + +/* Write a list of registers */ +static int imx208_write_regs(struct imx208 *imx208, + const struct imx208_reg *regs, u32 len) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + unsigned int i; + int ret; + + for (i = 0; i < len; i++) { + ret = imx208_write_reg(imx208, regs[i].address, 1, + regs[i].val); + if (ret) { + dev_err_ratelimited(&client->dev, + "Failed to write reg 0x%4.4x. error = %d\n", + regs[i].address, ret); + + return ret; + } + } + + return 0; +} + +/* Open sub-device */ +static int imx208_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) +{ + struct v4l2_mbus_framefmt *try_fmt = + v4l2_subdev_get_try_format(sd, fh->state, 0); + + /* Initialize try_fmt */ + try_fmt->width = supported_modes[0].width; + try_fmt->height = supported_modes[0].height; + try_fmt->code = MEDIA_BUS_FMT_SRGGB10_1X10; + try_fmt->field = V4L2_FIELD_NONE; + + return 0; +} + +static int imx208_update_digital_gain(struct imx208 *imx208, u32 len, u32 val) +{ + int ret; + + val = imx208_discrete_digital_gain[val] << IMX208_DIGITAL_GAIN_SHIFT; + + ret = imx208_write_reg(imx208, IMX208_REG_GR_DIGITAL_GAIN, 2, val); + if (ret) + return ret; + + ret = imx208_write_reg(imx208, IMX208_REG_GB_DIGITAL_GAIN, 2, val); + if (ret) + return ret; + + ret = imx208_write_reg(imx208, IMX208_REG_R_DIGITAL_GAIN, 2, val); + if (ret) + return ret; + + return imx208_write_reg(imx208, IMX208_REG_B_DIGITAL_GAIN, 2, val); +} + +static int imx208_set_ctrl(struct v4l2_ctrl *ctrl) +{ + struct imx208 *imx208 = + container_of(ctrl->handler, struct imx208, ctrl_handler); + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + int ret; + + /* + * Applying V4L2 control value only happens + * when power is up for streaming + */ + if (!pm_runtime_get_if_in_use(&client->dev)) + return 0; + + switch (ctrl->id) { + case V4L2_CID_ANALOGUE_GAIN: + ret = imx208_write_reg(imx208, IMX208_REG_ANALOG_GAIN, + 2, ctrl->val); + break; + case V4L2_CID_EXPOSURE: + ret = imx208_write_reg(imx208, IMX208_REG_EXPOSURE, + 2, ctrl->val); + break; + case V4L2_CID_DIGITAL_GAIN: + ret = imx208_update_digital_gain(imx208, 2, ctrl->val); + break; + case V4L2_CID_VBLANK: + /* Update VTS that meets expected vertical blanking */ + ret = imx208_write_reg(imx208, IMX208_REG_VTS, 2, + imx208->cur_mode->height + ctrl->val); + break; + case V4L2_CID_TEST_PATTERN: + ret = imx208_write_reg(imx208, IMX208_REG_TEST_PATTERN_MODE, + 2, imx208_test_pattern_val[ctrl->val]); + break; + case V4L2_CID_HFLIP: + case V4L2_CID_VFLIP: + ret = imx208_write_reg(imx208, IMX208_REG_ORIENTATION_CONTROL, + 1, + imx208->hflip->val | + imx208->vflip->val << 1); + break; + default: + ret = -EINVAL; + dev_err(&client->dev, + "ctrl(id:0x%x,val:0x%x) is not handled\n", + ctrl->id, ctrl->val); + break; + } + + pm_runtime_put(&client->dev); + + return ret; +} + +static const struct v4l2_ctrl_ops imx208_ctrl_ops = { + .s_ctrl = imx208_set_ctrl, +}; + +static const struct v4l2_ctrl_config imx208_digital_gain_control = { + .ops = &imx208_ctrl_ops, + .id = V4L2_CID_DIGITAL_GAIN, + .name = "Digital Gain", + .type = V4L2_CTRL_TYPE_INTEGER_MENU, + .min = 0, + .max = ARRAY_SIZE(imx208_discrete_digital_gain) - 1, + .step = 0, + .def = 0, + .menu_skip_mask = 0, + .qmenu_int = imx208_discrete_digital_gain, +}; + +static int imx208_enum_mbus_code(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_mbus_code_enum *code) +{ + struct imx208 *imx208 = to_imx208(sd); + + if (code->index > 0) + return -EINVAL; + + code->code = imx208_get_format_code(imx208); + + return 0; +} + +static int imx208_enum_frame_size(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_frame_size_enum *fse) +{ + struct imx208 *imx208 = to_imx208(sd); + + if (fse->index >= ARRAY_SIZE(supported_modes)) + return -EINVAL; + + if (fse->code != imx208_get_format_code(imx208)) + return -EINVAL; + + fse->min_width = supported_modes[fse->index].width; + fse->max_width = fse->min_width; + fse->min_height = supported_modes[fse->index].height; + fse->max_height = fse->min_height; + + return 0; +} + +static void imx208_mode_to_pad_format(struct imx208 *imx208, + const struct imx208_mode *mode, + struct v4l2_subdev_format *fmt) +{ + fmt->format.width = mode->width; + fmt->format.height = mode->height; + fmt->format.code = imx208_get_format_code(imx208); + fmt->format.field = V4L2_FIELD_NONE; +} + +static int __imx208_get_pad_format(struct imx208 *imx208, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) +{ + if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) + fmt->format = *v4l2_subdev_get_try_format(&imx208->sd, + sd_state, + fmt->pad); + else + imx208_mode_to_pad_format(imx208, imx208->cur_mode, fmt); + + return 0; +} + +static int imx208_get_pad_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) +{ + struct imx208 *imx208 = to_imx208(sd); + int ret; + + mutex_lock(&imx208->imx208_mx); + ret = __imx208_get_pad_format(imx208, sd_state, fmt); + mutex_unlock(&imx208->imx208_mx); + + return ret; +} + +static int imx208_set_pad_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) +{ + struct imx208 *imx208 = to_imx208(sd); + const struct imx208_mode *mode; + s32 vblank_def; + s32 vblank_min; + s64 h_blank; + s64 pixel_rate; + s64 link_freq; + + mutex_lock(&imx208->imx208_mx); + + fmt->format.code = imx208_get_format_code(imx208); + mode = v4l2_find_nearest_size(supported_modes, + ARRAY_SIZE(supported_modes), width, height, + fmt->format.width, fmt->format.height); + imx208_mode_to_pad_format(imx208, mode, fmt); + if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; + } else { + imx208->cur_mode = mode; + __v4l2_ctrl_s_ctrl(imx208->link_freq, mode->link_freq_index); + link_freq = link_freq_menu_items[mode->link_freq_index]; + pixel_rate = link_freq_to_pixel_rate(link_freq); + __v4l2_ctrl_s_ctrl_int64(imx208->pixel_rate, pixel_rate); + /* Update limits and set FPS to default */ + vblank_def = imx208->cur_mode->vts_def - + imx208->cur_mode->height; + vblank_min = imx208->cur_mode->vts_min - + imx208->cur_mode->height; + __v4l2_ctrl_modify_range(imx208->vblank, vblank_min, + IMX208_VTS_MAX - imx208->cur_mode->height, + 1, vblank_def); + __v4l2_ctrl_s_ctrl(imx208->vblank, vblank_def); + h_blank = + link_freq_configs[mode->link_freq_index].pixels_per_line + - imx208->cur_mode->width; + __v4l2_ctrl_modify_range(imx208->hblank, h_blank, + h_blank, 1, h_blank); + } + + mutex_unlock(&imx208->imx208_mx); + + return 0; +} + +/* Start streaming */ +static int imx208_start_streaming(struct imx208 *imx208) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + const struct imx208_reg_list *reg_list; + int ret, link_freq_index; + + /* Setup PLL */ + link_freq_index = imx208->cur_mode->link_freq_index; + reg_list = &link_freq_configs[link_freq_index].reg_list; + ret = imx208_write_regs(imx208, reg_list->regs, reg_list->num_of_regs); + if (ret) { + dev_err(&client->dev, "%s failed to set plls\n", __func__); + return ret; + } + + /* Apply default values of current mode */ + reg_list = &imx208->cur_mode->reg_list; + ret = imx208_write_regs(imx208, reg_list->regs, reg_list->num_of_regs); + if (ret) { + dev_err(&client->dev, "%s failed to set mode\n", __func__); + return ret; + } + + /* Apply customized values from user */ + ret = __v4l2_ctrl_handler_setup(imx208->sd.ctrl_handler); + if (ret) + return ret; + + /* set stream on register */ + return imx208_write_reg(imx208, IMX208_REG_MODE_SELECT, + 1, IMX208_MODE_STREAMING); +} + +/* Stop streaming */ +static int imx208_stop_streaming(struct imx208 *imx208) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + int ret; + + /* set stream off register */ + ret = imx208_write_reg(imx208, IMX208_REG_MODE_SELECT, + 1, IMX208_MODE_STANDBY); + if (ret) + dev_err(&client->dev, "%s failed to set stream\n", __func__); + + /* + * Return success even if it was an error, as there is nothing the + * caller can do about it. + */ + return 0; +} + +static int imx208_set_stream(struct v4l2_subdev *sd, int enable) +{ + struct imx208 *imx208 = to_imx208(sd); + struct i2c_client *client = v4l2_get_subdevdata(sd); + int ret = 0; + + mutex_lock(&imx208->imx208_mx); + if (imx208->streaming == enable) { + mutex_unlock(&imx208->imx208_mx); + return 0; + } + + if (enable) { + ret = pm_runtime_get_sync(&client->dev); + if (ret < 0) + goto err_rpm_put; + + /* + * Apply default & customized values + * and then start streaming. + */ + ret = imx208_start_streaming(imx208); + if (ret) + goto err_rpm_put; + } else { + imx208_stop_streaming(imx208); + pm_runtime_put(&client->dev); + } + + imx208->streaming = enable; + mutex_unlock(&imx208->imx208_mx); + + /* vflip and hflip cannot change during streaming */ + v4l2_ctrl_grab(imx208->vflip, enable); + v4l2_ctrl_grab(imx208->hflip, enable); + + return ret; + +err_rpm_put: + pm_runtime_put(&client->dev); + mutex_unlock(&imx208->imx208_mx); + + return ret; +} + +static int __maybe_unused imx208_suspend(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct v4l2_subdev *sd = i2c_get_clientdata(client); + struct imx208 *imx208 = to_imx208(sd); + + if (imx208->streaming) + imx208_stop_streaming(imx208); + + return 0; +} + +static int __maybe_unused imx208_resume(struct device *dev) +{ + struct i2c_client *client = to_i2c_client(dev); + struct v4l2_subdev *sd = i2c_get_clientdata(client); + struct imx208 *imx208 = to_imx208(sd); + int ret; + + if (imx208->streaming) { + ret = imx208_start_streaming(imx208); + if (ret) + goto error; + } + + return 0; + +error: + imx208_stop_streaming(imx208); + imx208->streaming = 0; + + return ret; +} + +/* Verify chip ID */ +static int imx208_identify_module(struct imx208 *imx208) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + int ret; + u32 val; + + ret = imx208_read_reg(imx208, IMX208_REG_CHIP_ID, + 2, &val); + if (ret) { + dev_err(&client->dev, "failed to read chip id %x\n", + IMX208_CHIP_ID); + return ret; + } + + if (val != IMX208_CHIP_ID) { + dev_err(&client->dev, "chip id mismatch: %x!=%x\n", + IMX208_CHIP_ID, val); + return -EIO; + } + + return 0; +} + +static const struct v4l2_subdev_video_ops imx208_video_ops = { + .s_stream = imx208_set_stream, +}; + +static const struct v4l2_subdev_pad_ops imx208_pad_ops = { + .enum_mbus_code = imx208_enum_mbus_code, + .get_fmt = imx208_get_pad_format, + .set_fmt = imx208_set_pad_format, + .enum_frame_size = imx208_enum_frame_size, +}; + +static const struct v4l2_subdev_ops imx208_subdev_ops = { + .video = &imx208_video_ops, + .pad = &imx208_pad_ops, +}; + +static const struct v4l2_subdev_internal_ops imx208_internal_ops = { + .open = imx208_open, +}; + +static int imx208_read_otp(struct imx208 *imx208) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + struct i2c_msg msgs[2]; + u8 addr_buf[2] = { IMX208_OTP_BASE >> 8, IMX208_OTP_BASE & 0xff }; + int ret = 0; + + mutex_lock(&imx208->imx208_mx); + + if (imx208->otp_read) + goto out_unlock; + + ret = pm_runtime_get_sync(&client->dev); + if (ret < 0) { + pm_runtime_put_noidle(&client->dev); + goto out_unlock; + } + + /* Write register address */ + msgs[0].addr = client->addr; + msgs[0].flags = 0; + msgs[0].len = ARRAY_SIZE(addr_buf); + msgs[0].buf = addr_buf; + + /* Read data from registers */ + msgs[1].addr = client->addr; + msgs[1].flags = I2C_M_RD; + msgs[1].len = sizeof(imx208->otp_data); + msgs[1].buf = imx208->otp_data; + + ret = i2c_transfer(client->adapter, msgs, ARRAY_SIZE(msgs)); + if (ret == ARRAY_SIZE(msgs)) { + imx208->otp_read = true; + ret = 0; + } + + pm_runtime_put(&client->dev); + +out_unlock: + mutex_unlock(&imx208->imx208_mx); + + return ret; +} + +static ssize_t otp_read(struct file *filp, struct kobject *kobj, + struct bin_attribute *bin_attr, + char *buf, loff_t off, size_t count) +{ + struct i2c_client *client = to_i2c_client(kobj_to_dev(kobj)); + struct v4l2_subdev *sd = i2c_get_clientdata(client); + struct imx208 *imx208 = to_imx208(sd); + int ret; + + ret = imx208_read_otp(imx208); + if (ret) + return ret; + + memcpy(buf, &imx208->otp_data[off], count); + return count; +} + +static const BIN_ATTR_RO(otp, IMX208_OTP_SIZE); + +/* Initialize control handlers */ +static int imx208_init_controls(struct imx208 *imx208) +{ + struct i2c_client *client = v4l2_get_subdevdata(&imx208->sd); + struct v4l2_ctrl_handler *ctrl_hdlr = &imx208->ctrl_handler; + s64 exposure_max; + s64 vblank_def; + s64 vblank_min; + s64 pixel_rate_min; + s64 pixel_rate_max; + int ret; + + ret = v4l2_ctrl_handler_init(ctrl_hdlr, 8); + if (ret) + return ret; + + mutex_init(&imx208->imx208_mx); + ctrl_hdlr->lock = &imx208->imx208_mx; + imx208->link_freq = + v4l2_ctrl_new_int_menu(ctrl_hdlr, + &imx208_ctrl_ops, + V4L2_CID_LINK_FREQ, + ARRAY_SIZE(link_freq_menu_items) - 1, + 0, link_freq_menu_items); + + if (imx208->link_freq) + imx208->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY; + + pixel_rate_max = link_freq_to_pixel_rate(link_freq_menu_items[0]); + pixel_rate_min = + link_freq_to_pixel_rate(link_freq_menu_items[ARRAY_SIZE(link_freq_menu_items) - 1]); + /* By default, PIXEL_RATE is read only */ + imx208->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &imx208_ctrl_ops, + V4L2_CID_PIXEL_RATE, + pixel_rate_min, pixel_rate_max, + 1, pixel_rate_max); + + vblank_def = imx208->cur_mode->vts_def - imx208->cur_mode->height; + vblank_min = imx208->cur_mode->vts_min - imx208->cur_mode->height; + imx208->vblank = + v4l2_ctrl_new_std(ctrl_hdlr, &imx208_ctrl_ops, V4L2_CID_VBLANK, + vblank_min, + IMX208_VTS_MAX - imx208->cur_mode->height, 1, + vblank_def); + + imx208->hblank = + v4l2_ctrl_new_std(ctrl_hdlr, &imx208_ctrl_ops, V4L2_CID_HBLANK, + IMX208_PPL_384MHZ - imx208->cur_mode->width, + IMX208_PPL_384MHZ - imx208->cur_mode->width, + 1, + IMX208_PPL_384MHZ - imx208->cur_mode->width); + + if (imx208->hblank) + imx208->hblank->flags |= V4L2_CTRL_FLAG_READ_ONLY; + + exposure_max = imx208->cur_mode->vts_def - 8; + v4l2_ctrl_new_std(ctrl_hdlr, &imx208_ctrl_ops, V4L2_CID_EXPOSURE, + IMX208_EXPOSURE_MIN, exposure_max, + IMX208_EXPOSURE_STEP, IMX208_EXPOSURE_DEFAULT); + + imx208->hflip = v4l2_ctrl_new_std(ctrl_hdlr, &imx208_ctrl_ops, + V4L2_CID_HFLIP, 0, 1, 1, 0); + imx208->vflip = v4l2_ctrl_new_std(ctrl_hdlr, &imx208_ctrl_ops, + V4L2_CID_VFLIP, 0, 1, 1, 0); + + v4l2_ctrl_new_std(ctrl_hdlr, &imx208_ctrl_ops, V4L2_CID_ANALOGUE_GAIN, + IMX208_ANA_GAIN_MIN, IMX208_ANA_GAIN_MAX, + IMX208_ANA_GAIN_STEP, IMX208_ANA_GAIN_DEFAULT); + + v4l2_ctrl_new_custom(ctrl_hdlr, &imx208_digital_gain_control, NULL); + + v4l2_ctrl_new_std_menu_items(ctrl_hdlr, &imx208_ctrl_ops, + V4L2_CID_TEST_PATTERN, + ARRAY_SIZE(imx208_test_pattern_menu) - 1, + 0, 0, imx208_test_pattern_menu); + + if (ctrl_hdlr->error) { + ret = ctrl_hdlr->error; + dev_err(&client->dev, "%s control init failed (%d)\n", + __func__, ret); + goto error; + } + + imx208->sd.ctrl_handler = ctrl_hdlr; + + return 0; + +error: + v4l2_ctrl_handler_free(ctrl_hdlr); + mutex_destroy(&imx208->imx208_mx); + + return ret; +} + +static void imx208_free_controls(struct imx208 *imx208) +{ + v4l2_ctrl_handler_free(imx208->sd.ctrl_handler); +} + +static int imx208_probe(struct i2c_client *client) +{ + struct imx208 *imx208; + int ret; + u32 val = 0; + + device_property_read_u32(&client->dev, "clock-frequency", &val); + if (val != 19200000) { + dev_err(&client->dev, + "Unsupported clock-frequency %u. Expected 19200000.\n", + val); + return -EINVAL; + } + + imx208 = devm_kzalloc(&client->dev, sizeof(*imx208), GFP_KERNEL); + if (!imx208) + return -ENOMEM; + + /* Initialize subdev */ + v4l2_i2c_subdev_init(&imx208->sd, client, &imx208_subdev_ops); + + /* Check module identity */ + ret = imx208_identify_module(imx208); + if (ret) { + dev_err(&client->dev, "failed to find sensor: %d", ret); + goto error_probe; + } + + /* Set default mode to max resolution */ + imx208->cur_mode = &supported_modes[0]; + + ret = imx208_init_controls(imx208); + if (ret) { + dev_err(&client->dev, "failed to init controls: %d", ret); + goto error_probe; + } + + /* Initialize subdev */ + imx208->sd.internal_ops = &imx208_internal_ops; + imx208->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; + imx208->sd.entity.function = MEDIA_ENT_F_CAM_SENSOR; + + /* Initialize source pad */ + imx208->pad.flags = MEDIA_PAD_FL_SOURCE; + ret = media_entity_pads_init(&imx208->sd.entity, 1, &imx208->pad); + if (ret) { + dev_err(&client->dev, "%s failed:%d\n", __func__, ret); + goto error_handler_free; + } + + ret = v4l2_async_register_subdev_sensor(&imx208->sd); + if (ret < 0) + goto error_media_entity; + + ret = device_create_bin_file(&client->dev, &bin_attr_otp); + if (ret) { + dev_err(&client->dev, "sysfs otp creation failed\n"); + goto error_async_subdev; + } + + pm_runtime_set_active(&client->dev); + pm_runtime_enable(&client->dev); + pm_runtime_idle(&client->dev); + + return 0; + +error_async_subdev: + v4l2_async_unregister_subdev(&imx208->sd); + +error_media_entity: + media_entity_cleanup(&imx208->sd.entity); + +error_handler_free: + imx208_free_controls(imx208); + +error_probe: + mutex_destroy(&imx208->imx208_mx); + + return ret; +} + +static int imx208_remove(struct i2c_client *client) +{ + struct v4l2_subdev *sd = i2c_get_clientdata(client); + struct imx208 *imx208 = to_imx208(sd); + + device_remove_bin_file(&client->dev, &bin_attr_otp); + v4l2_async_unregister_subdev(sd); + media_entity_cleanup(&sd->entity); + imx208_free_controls(imx208); + + pm_runtime_disable(&client->dev); + pm_runtime_set_suspended(&client->dev); + + mutex_destroy(&imx208->imx208_mx); + + return 0; +} + +static const struct dev_pm_ops imx208_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(imx208_suspend, imx208_resume) +}; + +#ifdef CONFIG_ACPI +static const struct acpi_device_id imx208_acpi_ids[] = { + { "INT3478" }, + { /* sentinel */ } +}; + +MODULE_DEVICE_TABLE(acpi, imx208_acpi_ids); +#endif + +static struct i2c_driver imx208_i2c_driver = { + .driver = { + .name = "imx208", + .pm = &imx208_pm_ops, + .acpi_match_table = ACPI_PTR(imx208_acpi_ids), + }, + .probe_new = imx208_probe, + .remove = imx208_remove, +}; + +module_i2c_driver(imx208_i2c_driver); + +MODULE_AUTHOR("Yeh, Andy "); +MODULE_AUTHOR("Chen, Ping-chung "); +MODULE_AUTHOR("Shawn Tu "); +MODULE_DESCRIPTION("Sony IMX208 sensor driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/i2c/imx214.c b/drivers/media/i2c/imx214.c index e8b281e432e8..83c1737abeec 100644 --- a/drivers/media/i2c/imx214.c +++ b/drivers/media/i2c/imx214.c @@ -474,7 +474,7 @@ static int __maybe_unused imx214_power_off(struct device *dev) } static int imx214_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -486,7 +486,7 @@ static int imx214_enum_mbus_code(struct v4l2_subdev *sd, } static int imx214_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->code != IMX214_MBUS_CODE) @@ -534,13 +534,13 @@ static const struct v4l2_subdev_core_ops imx214_core_ops = { static struct v4l2_mbus_framefmt * __imx214_get_pad_format(struct imx214 *imx214, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&imx214->sd, cfg, pad); + return v4l2_subdev_get_try_format(&imx214->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &imx214->fmt; default: @@ -549,13 +549,14 @@ __imx214_get_pad_format(struct imx214 *imx214, } static int imx214_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct imx214 *imx214 = to_imx214(sd); mutex_lock(&imx214->mutex); - format->format = *__imx214_get_pad_format(imx214, cfg, format->pad, + format->format = *__imx214_get_pad_format(imx214, sd_state, + format->pad, format->which); mutex_unlock(&imx214->mutex); @@ -563,12 +564,13 @@ static int imx214_get_format(struct v4l2_subdev *sd, } static struct v4l2_rect * -__imx214_get_pad_crop(struct imx214 *imx214, struct v4l2_subdev_pad_config *cfg, +__imx214_get_pad_crop(struct imx214 *imx214, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&imx214->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&imx214->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &imx214->crop; default: @@ -577,7 +579,7 @@ __imx214_get_pad_crop(struct imx214 *imx214, struct v4l2_subdev_pad_config *cfg, } static int imx214_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct imx214 *imx214 = to_imx214(sd); @@ -587,7 +589,8 @@ static int imx214_set_format(struct v4l2_subdev *sd, mutex_lock(&imx214->mutex); - __crop = __imx214_get_pad_crop(imx214, cfg, format->pad, format->which); + __crop = __imx214_get_pad_crop(imx214, sd_state, format->pad, + format->which); mode = v4l2_find_nearest_size(imx214_modes, ARRAY_SIZE(imx214_modes), width, height, @@ -597,7 +600,7 @@ static int imx214_set_format(struct v4l2_subdev *sd, __crop->width = mode->width; __crop->height = mode->height; - __format = __imx214_get_pad_format(imx214, cfg, format->pad, + __format = __imx214_get_pad_format(imx214, sd_state, format->pad, format->which); __format->width = __crop->width; __format->height = __crop->height; @@ -617,7 +620,7 @@ static int imx214_set_format(struct v4l2_subdev *sd, } static int imx214_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct imx214 *imx214 = to_imx214(sd); @@ -626,22 +629,22 @@ static int imx214_get_selection(struct v4l2_subdev *sd, return -EINVAL; mutex_lock(&imx214->mutex); - sel->r = *__imx214_get_pad_crop(imx214, cfg, sel->pad, + sel->r = *__imx214_get_pad_crop(imx214, sd_state, sel->pad, sel->which); mutex_unlock(&imx214->mutex); return 0; } static int imx214_entity_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format fmt = { }; - fmt.which = cfg ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; + fmt.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; fmt.format.width = imx214_modes[0].width; fmt.format.height = imx214_modes[0].height; - imx214_set_format(subdev, cfg, &fmt); + imx214_set_format(subdev, sd_state, &fmt); return 0; } @@ -776,11 +779,9 @@ static int imx214_s_stream(struct v4l2_subdev *subdev, int enable) return 0; if (enable) { - ret = pm_runtime_get_sync(imx214->dev); - if (ret < 0) { - pm_runtime_put_noidle(imx214->dev); + ret = pm_runtime_resume_and_get(imx214->dev); + if (ret < 0) return ret; - } ret = imx214_start_streaming(imx214); if (ret < 0) @@ -810,7 +811,7 @@ static int imx214_g_frame_interval(struct v4l2_subdev *subdev, } static int imx214_enum_frame_interval(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { const struct imx214_mode *mode; diff --git a/drivers/media/i2c/imx219.c b/drivers/media/i2c/imx219.c index 1054ffedaefd..e10af3f74b38 100644 --- a/drivers/media/i2c/imx219.c +++ b/drivers/media/i2c/imx219.c @@ -689,7 +689,7 @@ static int imx219_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct imx219 *imx219 = to_imx219(sd); struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); struct v4l2_rect *try_crop; mutex_lock(&imx219->mutex); @@ -702,7 +702,7 @@ static int imx219_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) try_fmt->field = V4L2_FIELD_NONE; /* Initialize try_crop rectangle. */ - try_crop = v4l2_subdev_get_try_crop(sd, fh->pad, 0); + try_crop = v4l2_subdev_get_try_crop(sd, fh->state, 0); try_crop->top = IMX219_PIXEL_ARRAY_TOP; try_crop->left = IMX219_PIXEL_ARRAY_LEFT; try_crop->width = IMX219_PIXEL_ARRAY_WIDTH; @@ -803,7 +803,7 @@ static const struct v4l2_ctrl_ops imx219_ctrl_ops = { }; static int imx219_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct imx219 *imx219 = to_imx219(sd); @@ -819,7 +819,7 @@ static int imx219_enum_mbus_code(struct v4l2_subdev *sd, } static int imx219_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct imx219 *imx219 = to_imx219(sd); @@ -863,12 +863,13 @@ static void imx219_update_pad_format(struct imx219 *imx219, } static int __imx219_get_pad_format(struct imx219 *imx219, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(&imx219->sd, cfg, fmt->pad); + v4l2_subdev_get_try_format(&imx219->sd, sd_state, + fmt->pad); /* update the code which could change due to vflip or hflip: */ try_fmt->code = imx219_get_format_code(imx219, try_fmt->code); fmt->format = *try_fmt; @@ -882,21 +883,21 @@ static int __imx219_get_pad_format(struct imx219 *imx219, } static int imx219_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx219 *imx219 = to_imx219(sd); int ret; mutex_lock(&imx219->mutex); - ret = __imx219_get_pad_format(imx219, cfg, fmt); + ret = __imx219_get_pad_format(imx219, sd_state, fmt); mutex_unlock(&imx219->mutex); return ret; } static int imx219_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx219 *imx219 = to_imx219(sd); @@ -922,7 +923,7 @@ static int imx219_set_pad_format(struct v4l2_subdev *sd, fmt->format.width, fmt->format.height); imx219_update_pad_format(imx219, mode, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *framefmt = fmt->format; } else if (imx219->mode != mode || imx219->fmt.code != fmt->format.code) { @@ -979,12 +980,13 @@ static int imx219_set_framefmt(struct imx219 *imx219) } static const struct v4l2_rect * -__imx219_get_pad_crop(struct imx219 *imx219, struct v4l2_subdev_pad_config *cfg, +__imx219_get_pad_crop(struct imx219 *imx219, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&imx219->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&imx219->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &imx219->mode->crop; } @@ -993,7 +995,7 @@ __imx219_get_pad_crop(struct imx219 *imx219, struct v4l2_subdev_pad_config *cfg, } static int imx219_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { switch (sel->target) { @@ -1001,7 +1003,7 @@ static int imx219_get_selection(struct v4l2_subdev *sd, struct imx219 *imx219 = to_imx219(sd); mutex_lock(&imx219->mutex); - sel->r = *__imx219_get_pad_crop(imx219, cfg, sel->pad, + sel->r = *__imx219_get_pad_crop(imx219, sd_state, sel->pad, sel->which); mutex_unlock(&imx219->mutex); @@ -1035,11 +1037,9 @@ static int imx219_start_streaming(struct imx219 *imx219) const struct imx219_reg_list *reg_list; int ret; - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) return ret; - } /* Apply default values of current mode */ reg_list = &imx219->mode->reg_list; diff --git a/drivers/media/i2c/imx258.c b/drivers/media/i2c/imx258.c index a017ec4e0f50..7ab9e5f9f267 100644 --- a/drivers/media/i2c/imx258.c +++ b/drivers/media/i2c/imx258.c @@ -710,7 +710,7 @@ static int imx258_write_regs(struct imx258 *imx258, static int imx258_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); /* Initialize try_fmt */ try_fmt->width = supported_modes[0].width; @@ -820,7 +820,7 @@ static const struct v4l2_ctrl_ops imx258_ctrl_ops = { }; static int imx258_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { /* Only one bayer order(GRBG) is supported */ @@ -833,7 +833,7 @@ static int imx258_enum_mbus_code(struct v4l2_subdev *sd, } static int imx258_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -860,11 +860,12 @@ static void imx258_update_pad_format(const struct imx258_mode *mode, } static int __imx258_get_pad_format(struct imx258 *imx258, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(&imx258->sd, cfg, + fmt->format = *v4l2_subdev_get_try_format(&imx258->sd, + sd_state, fmt->pad); else imx258_update_pad_format(imx258->cur_mode, fmt); @@ -873,21 +874,21 @@ static int __imx258_get_pad_format(struct imx258 *imx258, } static int imx258_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx258 *imx258 = to_imx258(sd); int ret; mutex_lock(&imx258->mutex); - ret = __imx258_get_pad_format(imx258, cfg, fmt); + ret = __imx258_get_pad_format(imx258, sd_state, fmt); mutex_unlock(&imx258->mutex); return ret; } static int imx258_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx258 *imx258 = to_imx258(sd); @@ -909,7 +910,7 @@ static int imx258_set_pad_format(struct v4l2_subdev *sd, fmt->format.width, fmt->format.height); imx258_update_pad_format(mode, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *framefmt = fmt->format; } else { imx258->cur_mode = mode; @@ -1039,11 +1040,9 @@ static int imx258_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto err_unlock; - } /* * Apply default & customized values diff --git a/drivers/media/i2c/imx274.c b/drivers/media/i2c/imx274.c index cdccaab3043a..0dce92872176 100644 --- a/drivers/media/i2c/imx274.c +++ b/drivers/media/i2c/imx274.c @@ -996,7 +996,7 @@ static int imx274_binning_goodness(struct stimx274 *imx274, * Must be called with imx274->lock locked. * * @imx274: The device object - * @cfg: The pad config we are editing for TRY requests + * @sd_state: The subdev state we are editing for TRY requests * @which: V4L2_SUBDEV_FORMAT_ACTIVE or V4L2_SUBDEV_FORMAT_TRY from the caller * @width: Input-output parameter: set to the desired width before * the call, contains the chosen value after returning successfully @@ -1005,7 +1005,7 @@ static int imx274_binning_goodness(struct stimx274 *imx274, * available (when called from set_fmt) */ static int __imx274_change_compose(struct stimx274 *imx274, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, u32 which, u32 *width, u32 *height, @@ -1019,8 +1019,8 @@ static int __imx274_change_compose(struct stimx274 *imx274, int best_goodness = INT_MIN; if (which == V4L2_SUBDEV_FORMAT_TRY) { - cur_crop = &cfg->try_crop; - tgt_fmt = &cfg->try_fmt; + cur_crop = &sd_state->pads->try_crop; + tgt_fmt = &sd_state->pads->try_fmt; } else { cur_crop = &imx274->crop; tgt_fmt = &imx274->format; @@ -1061,7 +1061,7 @@ static int __imx274_change_compose(struct stimx274 *imx274, /** * imx274_get_fmt - Get the pad format * @sd: Pointer to V4L2 Sub device structure - * @cfg: Pointer to sub device pad information structure + * @sd_state: Pointer to sub device state structure * @fmt: Pointer to pad level media bus format * * This function is used to get the pad format information. @@ -1069,7 +1069,7 @@ static int __imx274_change_compose(struct stimx274 *imx274, * Return: 0 on success */ static int imx274_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct stimx274 *imx274 = to_imx274(sd); @@ -1083,7 +1083,7 @@ static int imx274_get_fmt(struct v4l2_subdev *sd, /** * imx274_set_fmt - This is used to set the pad format * @sd: Pointer to V4L2 Sub device structure - * @cfg: Pointer to sub device pad information structure + * @sd_state: Pointer to sub device state information structure * @format: Pointer to pad level media bus format * * This function is used to set the pad format. @@ -1091,7 +1091,7 @@ static int imx274_get_fmt(struct v4l2_subdev *sd, * Return: 0 on success */ static int imx274_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -1100,7 +1100,7 @@ static int imx274_set_fmt(struct v4l2_subdev *sd, mutex_lock(&imx274->lock); - err = __imx274_change_compose(imx274, cfg, format->which, + err = __imx274_change_compose(imx274, sd_state, format->which, &fmt->width, &fmt->height, 0); if (err) @@ -1113,7 +1113,7 @@ static int imx274_set_fmt(struct v4l2_subdev *sd, */ fmt->field = V4L2_FIELD_NONE; if (format->which == V4L2_SUBDEV_FORMAT_TRY) - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; else imx274->format = *fmt; @@ -1124,7 +1124,7 @@ static int imx274_set_fmt(struct v4l2_subdev *sd, } static int imx274_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct stimx274 *imx274 = to_imx274(sd); @@ -1144,8 +1144,8 @@ static int imx274_get_selection(struct v4l2_subdev *sd, } if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { - src_crop = &cfg->try_crop; - src_fmt = &cfg->try_fmt; + src_crop = &sd_state->pads->try_crop; + src_fmt = &sd_state->pads->try_fmt; } else { src_crop = &imx274->crop; src_fmt = &imx274->format; @@ -1179,7 +1179,7 @@ static int imx274_get_selection(struct v4l2_subdev *sd, } static int imx274_set_selection_crop(struct stimx274 *imx274, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct v4l2_rect *tgt_crop; @@ -1216,7 +1216,7 @@ static int imx274_set_selection_crop(struct stimx274 *imx274, sel->r = new_crop; if (sel->which == V4L2_SUBDEV_FORMAT_TRY) - tgt_crop = &cfg->try_crop; + tgt_crop = &sd_state->pads->try_crop; else tgt_crop = &imx274->crop; @@ -1230,7 +1230,7 @@ static int imx274_set_selection_crop(struct stimx274 *imx274, /* if crop size changed then reset the output image size */ if (size_changed) - __imx274_change_compose(imx274, cfg, sel->which, + __imx274_change_compose(imx274, sd_state, sel->which, &new_crop.width, &new_crop.height, sel->flags); @@ -1240,7 +1240,7 @@ static int imx274_set_selection_crop(struct stimx274 *imx274, } static int imx274_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct stimx274 *imx274 = to_imx274(sd); @@ -1249,13 +1249,13 @@ static int imx274_set_selection(struct v4l2_subdev *sd, return -EINVAL; if (sel->target == V4L2_SEL_TGT_CROP) - return imx274_set_selection_crop(imx274, cfg, sel); + return imx274_set_selection_crop(imx274, sd_state, sel); if (sel->target == V4L2_SEL_TGT_COMPOSE) { int err; mutex_lock(&imx274->lock); - err = __imx274_change_compose(imx274, cfg, sel->which, + err = __imx274_change_compose(imx274, sd_state, sel->which, &sel->r.width, &sel->r.height, sel->flags); mutex_unlock(&imx274->lock); @@ -1441,9 +1441,8 @@ static int imx274_s_stream(struct v4l2_subdev *sd, int on) mutex_lock(&imx274->lock); if (on) { - ret = pm_runtime_get_sync(&imx274->client->dev); + ret = pm_runtime_resume_and_get(&imx274->client->dev); if (ret < 0) { - pm_runtime_put_noidle(&imx274->client->dev); mutex_unlock(&imx274->lock); return ret; } diff --git a/drivers/media/i2c/imx290.c b/drivers/media/i2c/imx290.c index 6319a42057d2..bf7a6c37ca5d 100644 --- a/drivers/media/i2c/imx290.c +++ b/drivers/media/i2c/imx290.c @@ -516,7 +516,7 @@ static const struct v4l2_ctrl_ops imx290_ctrl_ops = { }; static int imx290_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(imx290_formats)) @@ -528,7 +528,7 @@ static int imx290_enum_mbus_code(struct v4l2_subdev *sd, } static int imx290_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { const struct imx290 *imx290 = to_imx290(sd); @@ -550,7 +550,7 @@ static int imx290_enum_frame_size(struct v4l2_subdev *sd, } static int imx290_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx290 *imx290 = to_imx290(sd); @@ -559,7 +559,7 @@ static int imx290_get_fmt(struct v4l2_subdev *sd, mutex_lock(&imx290->lock); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - framefmt = v4l2_subdev_get_try_format(&imx290->sd, cfg, + framefmt = v4l2_subdev_get_try_format(&imx290->sd, sd_state, fmt->pad); else framefmt = &imx290->current_format; @@ -596,8 +596,8 @@ static u64 imx290_calc_pixel_rate(struct imx290 *imx290) } static int imx290_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *fmt) + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) { struct imx290 *imx290 = to_imx290(sd); const struct imx290_mode *mode; @@ -624,7 +624,7 @@ static int imx290_set_fmt(struct v4l2_subdev *sd, fmt->format.field = V4L2_FIELD_NONE; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - format = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + format = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); } else { format = &imx290->current_format; imx290->current_mode = mode; @@ -646,15 +646,15 @@ static int imx290_set_fmt(struct v4l2_subdev *sd, } static int imx290_entity_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format fmt = { 0 }; - fmt.which = cfg ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; + fmt.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; fmt.format.width = 1920; fmt.format.height = 1080; - imx290_set_fmt(subdev, cfg, &fmt); + imx290_set_fmt(subdev, sd_state, &fmt); return 0; } @@ -764,11 +764,9 @@ static int imx290_set_stream(struct v4l2_subdev *sd, int enable) int ret = 0; if (enable) { - ret = pm_runtime_get_sync(imx290->dev); - if (ret < 0) { - pm_runtime_put_noidle(imx290->dev); + ret = pm_runtime_resume_and_get(imx290->dev); + if (ret < 0) goto unlock_and_return; - } ret = imx290_start_streaming(imx290); if (ret) { diff --git a/drivers/media/i2c/imx319.c b/drivers/media/i2c/imx319.c index 38540323a156..dba0854ab5aa 100644 --- a/drivers/media/i2c/imx319.c +++ b/drivers/media/i2c/imx319.c @@ -1860,7 +1860,7 @@ static int imx319_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct imx319 *imx319 = to_imx319(sd); struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); mutex_lock(&imx319->mutex); @@ -1947,7 +1947,7 @@ static const struct v4l2_ctrl_ops imx319_ctrl_ops = { }; static int imx319_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct imx319 *imx319 = to_imx319(sd); @@ -1963,7 +1963,7 @@ static int imx319_enum_mbus_code(struct v4l2_subdev *sd, } static int imx319_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct imx319 *imx319 = to_imx319(sd); @@ -1997,14 +1997,14 @@ static void imx319_update_pad_format(struct imx319 *imx319, } static int imx319_do_get_pad_format(struct imx319 *imx319, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct v4l2_mbus_framefmt *framefmt; struct v4l2_subdev *sd = &imx319->sd; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); fmt->format = *framefmt; } else { imx319_update_pad_format(imx319, imx319->cur_mode, fmt); @@ -2014,14 +2014,14 @@ static int imx319_do_get_pad_format(struct imx319 *imx319, } static int imx319_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx319 *imx319 = to_imx319(sd); int ret; mutex_lock(&imx319->mutex); - ret = imx319_do_get_pad_format(imx319, cfg, fmt); + ret = imx319_do_get_pad_format(imx319, sd_state, fmt); mutex_unlock(&imx319->mutex); return ret; @@ -2029,7 +2029,7 @@ static int imx319_get_pad_format(struct v4l2_subdev *sd, static int imx319_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx319 *imx319 = to_imx319(sd); @@ -2055,7 +2055,7 @@ imx319_set_pad_format(struct v4l2_subdev *sd, fmt->format.width, fmt->format.height); imx319_update_pad_format(imx319, mode, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *framefmt = fmt->format; } else { imx319->cur_mode = mode; @@ -2141,11 +2141,9 @@ static int imx319_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto err_unlock; - } /* * Apply default & customized values diff --git a/drivers/media/i2c/imx334.c b/drivers/media/i2c/imx334.c index 047aa7658d21..062125501788 100644 --- a/drivers/media/i2c/imx334.c +++ b/drivers/media/i2c/imx334.c @@ -497,13 +497,13 @@ static const struct v4l2_ctrl_ops imx334_ctrl_ops = { /** * imx334_enum_mbus_code() - Enumerate V4L2 sub-device mbus codes * @sd: pointer to imx334 V4L2 sub-device structure - * @cfg: V4L2 sub-device pad configuration + * @sd_state: V4L2 sub-device state * @code: V4L2 sub-device code enumeration need to be filled * * Return: 0 if successful, error code otherwise. */ static int imx334_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -517,13 +517,13 @@ static int imx334_enum_mbus_code(struct v4l2_subdev *sd, /** * imx334_enum_frame_size() - Enumerate V4L2 sub-device frame sizes * @sd: pointer to imx334 V4L2 sub-device structure - * @cfg: V4L2 sub-device pad configuration + * @sd_state: V4L2 sub-device state * @fsize: V4L2 sub-device size enumeration need to be filled * * Return: 0 if successful, error code otherwise. */ static int imx334_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fsize) { if (fsize->index > 0) @@ -564,13 +564,13 @@ static void imx334_fill_pad_format(struct imx334 *imx334, /** * imx334_get_pad_format() - Get subdevice pad format * @sd: pointer to imx334 V4L2 sub-device structure - * @cfg: V4L2 sub-device pad configuration + * @sd_state: V4L2 sub-device state * @fmt: V4L2 sub-device format need to be set * * Return: 0 if successful, error code otherwise. */ static int imx334_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx334 *imx334 = to_imx334(sd); @@ -580,7 +580,7 @@ static int imx334_get_pad_format(struct v4l2_subdev *sd, if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *framefmt; - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); fmt->format = *framefmt; } else { imx334_fill_pad_format(imx334, imx334->cur_mode, fmt); @@ -594,13 +594,13 @@ static int imx334_get_pad_format(struct v4l2_subdev *sd, /** * imx334_set_pad_format() - Set subdevice pad format * @sd: pointer to imx334 V4L2 sub-device structure - * @cfg: V4L2 sub-device pad configuration + * @sd_state: V4L2 sub-device state * @fmt: V4L2 sub-device format need to be set * * Return: 0 if successful, error code otherwise. */ static int imx334_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx334 *imx334 = to_imx334(sd); @@ -615,7 +615,7 @@ static int imx334_set_pad_format(struct v4l2_subdev *sd, if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *framefmt; - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *framefmt = fmt->format; } else { ret = imx334_update_controls(imx334, mode); @@ -631,20 +631,20 @@ static int imx334_set_pad_format(struct v4l2_subdev *sd, /** * imx334_init_pad_cfg() - Initialize sub-device pad configuration * @sd: pointer to imx334 V4L2 sub-device structure - * @cfg: V4L2 sub-device pad configuration + * @sd_state: V4L2 sub-device state * * Return: 0 if successful, error code otherwise. */ static int imx334_init_pad_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct imx334 *imx334 = to_imx334(sd); struct v4l2_subdev_format fmt = { 0 }; - fmt.which = cfg ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; + fmt.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; imx334_fill_pad_format(imx334, &supported_mode, &fmt); - return imx334_set_pad_format(sd, cfg, &fmt); + return imx334_set_pad_format(sd, sd_state, &fmt); } /** @@ -717,9 +717,9 @@ static int imx334_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(imx334->dev); - if (ret) - goto error_power_off; + ret = pm_runtime_resume_and_get(imx334->dev); + if (ret < 0) + goto error_unlock; ret = imx334_start_streaming(imx334); if (ret) @@ -737,6 +737,7 @@ static int imx334_set_stream(struct v4l2_subdev *sd, int enable) error_power_off: pm_runtime_put(imx334->dev); +error_unlock: mutex_unlock(&imx334->mutex); return ret; diff --git a/drivers/media/i2c/imx355.c b/drivers/media/i2c/imx355.c index ccedcd4c520a..cb51c81786bd 100644 --- a/drivers/media/i2c/imx355.c +++ b/drivers/media/i2c/imx355.c @@ -1161,7 +1161,7 @@ static int imx355_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct imx355 *imx355 = to_imx355(sd); struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); mutex_lock(&imx355->mutex); @@ -1248,7 +1248,7 @@ static const struct v4l2_ctrl_ops imx355_ctrl_ops = { }; static int imx355_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct imx355 *imx355 = to_imx355(sd); @@ -1264,7 +1264,7 @@ static int imx355_enum_mbus_code(struct v4l2_subdev *sd, } static int imx355_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct imx355 *imx355 = to_imx355(sd); @@ -1298,14 +1298,14 @@ static void imx355_update_pad_format(struct imx355 *imx355, } static int imx355_do_get_pad_format(struct imx355 *imx355, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct v4l2_mbus_framefmt *framefmt; struct v4l2_subdev *sd = &imx355->sd; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); fmt->format = *framefmt; } else { imx355_update_pad_format(imx355, imx355->cur_mode, fmt); @@ -1315,14 +1315,14 @@ static int imx355_do_get_pad_format(struct imx355 *imx355, } static int imx355_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx355 *imx355 = to_imx355(sd); int ret; mutex_lock(&imx355->mutex); - ret = imx355_do_get_pad_format(imx355, cfg, fmt); + ret = imx355_do_get_pad_format(imx355, sd_state, fmt); mutex_unlock(&imx355->mutex); return ret; @@ -1330,7 +1330,7 @@ static int imx355_get_pad_format(struct v4l2_subdev *sd, static int imx355_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imx355 *imx355 = to_imx355(sd); @@ -1356,7 +1356,7 @@ imx355_set_pad_format(struct v4l2_subdev *sd, fmt->format.width, fmt->format.height); imx355_update_pad_format(imx355, mode, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *framefmt = fmt->format; } else { imx355->cur_mode = mode; @@ -1442,11 +1442,9 @@ static int imx355_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto err_unlock; - } /* * Apply default & customized values diff --git a/drivers/media/i2c/ir-kbd-i2c.c b/drivers/media/i2c/ir-kbd-i2c.c index e8119ad0bc71..92376592455e 100644 --- a/drivers/media/i2c/ir-kbd-i2c.c +++ b/drivers/media/i2c/ir-kbd-i2c.c @@ -678,8 +678,8 @@ static int zilog_tx(struct rc_dev *rcdev, unsigned int *txbuf, goto out_unlock; } - i = i2c_master_recv(ir->tx_c, buf, 1); - if (i != 1) { + ret = i2c_master_recv(ir->tx_c, buf, 1); + if (ret != 1) { dev_err(&ir->rc->dev, "i2c_master_recv failed with %d\n", ret); ret = -EIO; goto out_unlock; diff --git a/drivers/media/i2c/m5mols/m5mols_core.c b/drivers/media/i2c/m5mols/m5mols_core.c index 21666d705e37..e29be0242f07 100644 --- a/drivers/media/i2c/m5mols/m5mols_core.c +++ b/drivers/media/i2c/m5mols/m5mols_core.c @@ -539,17 +539,19 @@ static int __find_resolution(struct v4l2_subdev *sd, } static struct v4l2_mbus_framefmt *__find_format(struct m5mols_info *info, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which, enum m5mols_restype type) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return cfg ? v4l2_subdev_get_try_format(&info->sd, cfg, 0) : NULL; + return sd_state ? v4l2_subdev_get_try_format(&info->sd, + sd_state, 0) : NULL; return &info->ffmt[type]; } -static int m5mols_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int m5mols_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct m5mols_info *info = to_m5mols(sd); @@ -558,7 +560,7 @@ static int m5mols_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config mutex_lock(&info->lock); - format = __find_format(info, cfg, fmt->which, info->res_type); + format = __find_format(info, sd_state, fmt->which, info->res_type); if (format) fmt->format = *format; else @@ -568,7 +570,8 @@ static int m5mols_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config return ret; } -static int m5mols_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int m5mols_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct m5mols_info *info = to_m5mols(sd); @@ -582,7 +585,7 @@ static int m5mols_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config if (ret < 0) return ret; - sfmt = __find_format(info, cfg, fmt->which, type); + sfmt = __find_format(info, sd_state, fmt->which, type); if (!sfmt) return 0; @@ -648,7 +651,7 @@ static int m5mols_set_frame_desc(struct v4l2_subdev *sd, unsigned int pad, static int m5mols_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (!code || code->index >= SIZE_DEFAULT_FFMT) @@ -909,7 +912,9 @@ static const struct v4l2_subdev_core_ops m5mols_core_ops = { */ static int m5mols_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd, fh->pad, 0); + struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd, + fh->state, + 0); *format = m5mols_default_ffmt[0]; return 0; diff --git a/drivers/media/i2c/max9271.c b/drivers/media/i2c/max9271.c index c495582dcff6..ff86c8c4ea61 100644 --- a/drivers/media/i2c/max9271.c +++ b/drivers/media/i2c/max9271.c @@ -80,6 +80,18 @@ static int max9271_pclk_detect(struct max9271_device *dev) return -EIO; } +void max9271_wake_up(struct max9271_device *dev) +{ + /* + * Use the chip default address as this function has to be called + * before any other one. + */ + dev->client->addr = MAX9271_DEFAULT_ADDR; + i2c_smbus_read_byte(dev->client); + usleep_range(5000, 8000); +} +EXPORT_SYMBOL_GPL(max9271_wake_up); + int max9271_set_serial_link(struct max9271_device *dev, bool enable) { int ret; @@ -106,7 +118,10 @@ int max9271_set_serial_link(struct max9271_device *dev, bool enable) * Short delays here appear to show bit-errors in the writes following. * Therefore a conservative delay seems best here. */ - max9271_write(dev, 0x04, val); + ret = max9271_write(dev, 0x04, val); + if (ret < 0) + return ret; + usleep_range(5000, 8000); return 0; @@ -118,7 +133,7 @@ int max9271_configure_i2c(struct max9271_device *dev, u8 i2c_config) int ret; ret = max9271_write(dev, 0x0d, i2c_config); - if (ret) + if (ret < 0) return ret; /* The delay required after an I2C bus configuration change is not @@ -143,7 +158,10 @@ int max9271_set_high_threshold(struct max9271_device *dev, bool enable) * Enable or disable reverse channel high threshold to increase * immunity to power supply noise. */ - max9271_write(dev, 0x08, enable ? ret | BIT(0) : ret & ~BIT(0)); + ret = max9271_write(dev, 0x08, enable ? ret | BIT(0) : ret & ~BIT(0)); + if (ret < 0) + return ret; + usleep_range(2000, 2500); return 0; @@ -152,6 +170,8 @@ EXPORT_SYMBOL_GPL(max9271_set_high_threshold); int max9271_configure_gmsl_link(struct max9271_device *dev) { + int ret; + /* * Configure the GMSL link: * @@ -162,16 +182,24 @@ int max9271_configure_gmsl_link(struct max9271_device *dev) * * TODO: Make the GMSL link configuration parametric. */ - max9271_write(dev, 0x07, MAX9271_DBL | MAX9271_HVEN | - MAX9271_EDC_1BIT_PARITY); + ret = max9271_write(dev, 0x07, MAX9271_DBL | MAX9271_HVEN | + MAX9271_EDC_1BIT_PARITY); + if (ret < 0) + return ret; + usleep_range(5000, 8000); /* * Adjust spread spectrum to +4% and auto-detect pixel clock * and serial link rate. */ - max9271_write(dev, 0x02, MAX9271_SPREAD_SPECT_4 | MAX9271_R02_RES | - MAX9271_PCLK_AUTODETECT | MAX9271_SERIAL_AUTODETECT); + ret = max9271_write(dev, 0x02, + MAX9271_SPREAD_SPECT_4 | MAX9271_R02_RES | + MAX9271_PCLK_AUTODETECT | + MAX9271_SERIAL_AUTODETECT); + if (ret < 0) + return ret; + usleep_range(5000, 8000); return 0; diff --git a/drivers/media/i2c/max9271.h b/drivers/media/i2c/max9271.h index d78fb21441e9..dc5e4e70ba6f 100644 --- a/drivers/media/i2c/max9271.h +++ b/drivers/media/i2c/max9271.h @@ -85,6 +85,15 @@ struct max9271_device { struct i2c_client *client; }; +/** + * max9271_wake_up() - Wake up the serializer by issuing an i2c transaction + * @dev: The max9271 device + * + * This function shall be called before any other interaction with the + * serializer. + */ +void max9271_wake_up(struct max9271_device *dev); + /** * max9271_set_serial_link() - Enable/disable serial link * @dev: The max9271 device diff --git a/drivers/media/i2c/max9286.c b/drivers/media/i2c/max9286.c index 6fd4d59fcc72..1aa2c58fd38c 100644 --- a/drivers/media/i2c/max9286.c +++ b/drivers/media/i2c/max9286.c @@ -113,6 +113,7 @@ #define MAX9286_REV_TRF(n) ((n) << 4) #define MAX9286_REV_AMP(n) ((((n) - 30) / 10) << 1) /* in mV */ #define MAX9286_REV_AMP_X BIT(0) +#define MAX9286_REV_AMP_HIGH 170 /* Register 0x3f */ #define MAX9286_EN_REV_CFG BIT(6) #define MAX9286_REV_FLEN(n) ((n) - 20) @@ -163,7 +164,9 @@ struct max9286_priv { unsigned int mux_channel; bool mux_open; - u32 reverse_channel_mv; + /* The initial reverse control channel amplitude. */ + u32 init_rev_chan_mv; + u32 rev_chan_mv; struct v4l2_ctrl_handler ctrls; struct v4l2_ctrl *pixelrate; @@ -287,9 +290,8 @@ static int max9286_i2c_mux_select(struct i2c_mux_core *muxc, u32 chan) priv->mux_channel = chan; - max9286_i2c_mux_configure(priv, - MAX9286_FWDCCEN(chan) | - MAX9286_REVCCEN(chan)); + max9286_i2c_mux_configure(priv, MAX9286_FWDCCEN(chan) | + MAX9286_REVCCEN(chan)); return 0; } @@ -341,8 +343,15 @@ static void max9286_configure_i2c(struct max9286_priv *priv, bool localack) static void max9286_reverse_channel_setup(struct max9286_priv *priv, unsigned int chan_amplitude) { + u8 chan_config; + + if (priv->rev_chan_mv == chan_amplitude) + return; + + priv->rev_chan_mv = chan_amplitude; + /* Reverse channel transmission time: default to 1. */ - u8 chan_config = MAX9286_REV_TRF(1); + chan_config = MAX9286_REV_TRF(1); /* * Reverse channel setup. @@ -547,9 +556,9 @@ static int max9286_notify_bound(struct v4l2_async_notifier *notifier, subdev->name, src_pad, index); /* - * We can only register v4l2_async_notifiers, which do not provide a - * means to register a complete callback. bound_sources allows us to - * identify when all remote serializers have completed their probe. + * As we register a subdev notifiers we won't get a .complete() callback + * here, so we have to use bound_sources to identify when all remote + * serializers have probed. */ if (priv->bound_sources != priv->source_mask) return 0; @@ -559,19 +568,13 @@ static int max9286_notify_bound(struct v4l2_async_notifier *notifier, * channels: * * - Increase the reverse channel amplitude to compensate for the - * remote ends high threshold, if not done already + * remote ends high threshold * - Verify all configuration links are properly detected * - Disable auto-ack as communication on the control channel are now * stable. */ - if (priv->reverse_channel_mv < 170) - max9286_reverse_channel_setup(priv, 170); + max9286_reverse_channel_setup(priv, MAX9286_REV_AMP_HIGH); max9286_check_config_link(priv, priv->source_mask); - - /* - * Re-configure I2C with local acknowledge disabled after cameras have - * probed. - */ max9286_configure_i2c(priv, false); return max9286_set_pixelrate(priv); @@ -712,7 +715,7 @@ static int max9286_s_stream(struct v4l2_subdev *sd, int enable) } static int max9286_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index > 0) @@ -725,12 +728,12 @@ static int max9286_enum_mbus_code(struct v4l2_subdev *sd, static struct v4l2_mbus_framefmt * max9286_get_pad_format(struct max9286_priv *priv, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&priv->sd, cfg, pad); + return v4l2_subdev_get_try_format(&priv->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &priv->fmt[pad]; default: @@ -739,7 +742,7 @@ max9286_get_pad_format(struct max9286_priv *priv, } static int max9286_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct max9286_priv *priv = sd_to_max9286(sd); @@ -760,7 +763,8 @@ static int max9286_set_fmt(struct v4l2_subdev *sd, break; } - cfg_fmt = max9286_get_pad_format(priv, cfg, format->pad, format->which); + cfg_fmt = max9286_get_pad_format(priv, sd_state, format->pad, + format->which); if (!cfg_fmt) return -EINVAL; @@ -772,7 +776,7 @@ static int max9286_set_fmt(struct v4l2_subdev *sd, } static int max9286_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct max9286_priv *priv = sd_to_max9286(sd); @@ -788,7 +792,7 @@ static int max9286_get_fmt(struct v4l2_subdev *sd, if (pad == MAX9286_SRC_PAD) pad = __ffs(priv->bound_sources); - cfg_fmt = max9286_get_pad_format(priv, cfg, pad, format->which); + cfg_fmt = max9286_get_pad_format(priv, sd_state, pad, format->which); if (!cfg_fmt) return -EINVAL; @@ -832,7 +836,7 @@ static int max9286_open(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh) unsigned int i; for (i = 0; i < MAX9286_N_SINKS; i++) { - format = v4l2_subdev_get_try_format(subdev, fh->pad, i); + format = v4l2_subdev_get_try_format(subdev, fh->state, i); max9286_init_format(format); } @@ -972,7 +976,7 @@ static int max9286_setup(struct max9286_priv *priv) * only. This should be disabled after the mux is initialised. */ max9286_configure_i2c(priv, true); - max9286_reverse_channel_setup(priv, priv->reverse_channel_mv); + max9286_reverse_channel_setup(priv, priv->init_rev_chan_mv); /* * Enable GMSL links, mask unused ones and autodetect link @@ -1237,9 +1241,9 @@ static int max9286_parse_dt(struct max9286_priv *priv) if (of_property_read_u32(dev->of_node, "maxim,reverse-channel-microvolt", &reverse_channel_microvolt)) - priv->reverse_channel_mv = 170; + priv->init_rev_chan_mv = 170; else - priv->reverse_channel_mv = reverse_channel_microvolt / 1000U; + priv->init_rev_chan_mv = reverse_channel_microvolt / 1000U; priv->route_mask = priv->source_mask; diff --git a/drivers/media/i2c/ml86v7667.c b/drivers/media/i2c/ml86v7667.c index ff212335326a..4a1410ebb4c8 100644 --- a/drivers/media/i2c/ml86v7667.c +++ b/drivers/media/i2c/ml86v7667.c @@ -188,7 +188,7 @@ static int ml86v7667_g_input_status(struct v4l2_subdev *sd, u32 *status) } static int ml86v7667_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index > 0) @@ -200,7 +200,7 @@ static int ml86v7667_enum_mbus_code(struct v4l2_subdev *sd, } static int ml86v7667_fill_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ml86v7667_priv *priv = to_ml86v7667(sd); diff --git a/drivers/media/i2c/mt9m001.c b/drivers/media/i2c/mt9m001.c index 3b0ba8ed5233..c9f0bd997ea7 100644 --- a/drivers/media/i2c/mt9m001.c +++ b/drivers/media/i2c/mt9m001.c @@ -217,9 +217,9 @@ static int mt9m001_s_stream(struct v4l2_subdev *sd, int enable) goto done; if (enable) { - ret = pm_runtime_get_sync(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); if (ret < 0) - goto put_unlock; + goto unlock; ret = mt9m001_apply_selection(sd); if (ret) @@ -247,13 +247,14 @@ static int mt9m001_s_stream(struct v4l2_subdev *sd, int enable) put_unlock: pm_runtime_put(&client->dev); +unlock: mutex_unlock(&mt9m001->mutex); return ret; } static int mt9m001_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -294,7 +295,7 @@ static int mt9m001_set_selection(struct v4l2_subdev *sd, } static int mt9m001_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -319,7 +320,7 @@ static int mt9m001_get_selection(struct v4l2_subdev *sd, } static int mt9m001_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -330,7 +331,7 @@ static int mt9m001_get_fmt(struct v4l2_subdev *sd, return -EINVAL; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); format->format = *mf; return 0; } @@ -376,7 +377,7 @@ static int mt9m001_s_fmt(struct v4l2_subdev *sd, } static int mt9m001_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -410,7 +411,7 @@ static int mt9m001_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) return mt9m001_s_fmt(sd, fmt, mf); - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } @@ -656,12 +657,12 @@ static const struct v4l2_subdev_core_ops mt9m001_subdev_core_ops = { }; static int mt9m001_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct i2c_client *client = v4l2_get_subdevdata(sd); struct mt9m001 *mt9m001 = to_mt9m001(client); struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, cfg, 0); + v4l2_subdev_get_try_format(sd, sd_state, 0); try_fmt->width = MT9M001_MAX_WIDTH; try_fmt->height = MT9M001_MAX_HEIGHT; @@ -676,7 +677,7 @@ static int mt9m001_init_cfg(struct v4l2_subdev *sd, } static int mt9m001_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -834,6 +835,10 @@ static int mt9m001_remove(struct i2c_client *client) { struct mt9m001 *mt9m001 = to_mt9m001(client); + /* + * As it increments RPM usage_count even on errors, we don't need to + * check the returned code here. + */ pm_runtime_get_sync(&client->dev); v4l2_async_unregister_subdev(&mt9m001->subdev); diff --git a/drivers/media/i2c/mt9m032.c b/drivers/media/i2c/mt9m032.c index 5a4c0f9d1eee..ba0c0ea91c95 100644 --- a/drivers/media/i2c/mt9m032.c +++ b/drivers/media/i2c/mt9m032.c @@ -304,7 +304,7 @@ static int mt9m032_setup_pll(struct mt9m032 *sensor) */ static int mt9m032_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index != 0) @@ -315,7 +315,7 @@ static int mt9m032_enum_mbus_code(struct v4l2_subdev *subdev, } static int mt9m032_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index != 0 || fse->code != MEDIA_BUS_FMT_Y8_1X8) @@ -332,18 +332,19 @@ static int mt9m032_enum_frame_size(struct v4l2_subdev *subdev, /** * __mt9m032_get_pad_crop() - get crop rect * @sensor: pointer to the sensor struct - * @cfg: v4l2_subdev_pad_config for getting the try crop rect from + * @sd_state: v4l2_subdev_state for getting the try crop rect from * @which: select try or active crop rect * * Returns a pointer the current active or fh relative try crop rect */ static struct v4l2_rect * -__mt9m032_get_pad_crop(struct mt9m032 *sensor, struct v4l2_subdev_pad_config *cfg, +__mt9m032_get_pad_crop(struct mt9m032 *sensor, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&sensor->subdev, cfg, 0); + return v4l2_subdev_get_try_crop(&sensor->subdev, sd_state, 0); case V4L2_SUBDEV_FORMAT_ACTIVE: return &sensor->crop; default: @@ -354,18 +355,20 @@ __mt9m032_get_pad_crop(struct mt9m032 *sensor, struct v4l2_subdev_pad_config *cf /** * __mt9m032_get_pad_format() - get format * @sensor: pointer to the sensor struct - * @cfg: v4l2_subdev_pad_config for getting the try format from + * @sd_state: v4l2_subdev_state for getting the try format from * @which: select try or active format * * Returns a pointer the current active or fh relative try format */ static struct v4l2_mbus_framefmt * -__mt9m032_get_pad_format(struct mt9m032 *sensor, struct v4l2_subdev_pad_config *cfg, +__mt9m032_get_pad_format(struct mt9m032 *sensor, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&sensor->subdev, cfg, 0); + return v4l2_subdev_get_try_format(&sensor->subdev, sd_state, + 0); case V4L2_SUBDEV_FORMAT_ACTIVE: return &sensor->format; default: @@ -374,20 +377,20 @@ __mt9m032_get_pad_format(struct mt9m032 *sensor, struct v4l2_subdev_pad_config * } static int mt9m032_get_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct mt9m032 *sensor = to_mt9m032(subdev); mutex_lock(&sensor->lock); - fmt->format = *__mt9m032_get_pad_format(sensor, cfg, fmt->which); + fmt->format = *__mt9m032_get_pad_format(sensor, sd_state, fmt->which); mutex_unlock(&sensor->lock); return 0; } static int mt9m032_set_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct mt9m032 *sensor = to_mt9m032(subdev); @@ -401,7 +404,7 @@ static int mt9m032_set_pad_format(struct v4l2_subdev *subdev, } /* Scaling is not supported, the format is thus fixed. */ - fmt->format = *__mt9m032_get_pad_format(sensor, cfg, fmt->which); + fmt->format = *__mt9m032_get_pad_format(sensor, sd_state, fmt->which); ret = 0; done: @@ -410,7 +413,7 @@ static int mt9m032_set_pad_format(struct v4l2_subdev *subdev, } static int mt9m032_get_pad_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9m032 *sensor = to_mt9m032(subdev); @@ -419,14 +422,14 @@ static int mt9m032_get_pad_selection(struct v4l2_subdev *subdev, return -EINVAL; mutex_lock(&sensor->lock); - sel->r = *__mt9m032_get_pad_crop(sensor, cfg, sel->which); + sel->r = *__mt9m032_get_pad_crop(sensor, sd_state, sel->which); mutex_unlock(&sensor->lock); return 0; } static int mt9m032_set_pad_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9m032 *sensor = to_mt9m032(subdev); @@ -462,13 +465,14 @@ static int mt9m032_set_pad_selection(struct v4l2_subdev *subdev, rect.height = min_t(unsigned int, rect.height, MT9M032_PIXEL_ARRAY_HEIGHT - rect.top); - __crop = __mt9m032_get_pad_crop(sensor, cfg, sel->which); + __crop = __mt9m032_get_pad_crop(sensor, sd_state, sel->which); if (rect.width != __crop->width || rect.height != __crop->height) { /* Reset the output image size if the crop rectangle size has * been modified. */ - format = __mt9m032_get_pad_format(sensor, cfg, sel->which); + format = __mt9m032_get_pad_format(sensor, sd_state, + sel->which); format->width = rect.width; format->height = rect.height; } diff --git a/drivers/media/i2c/mt9m111.c b/drivers/media/i2c/mt9m111.c index 0e11734f75aa..91a44359bcd3 100644 --- a/drivers/media/i2c/mt9m111.c +++ b/drivers/media/i2c/mt9m111.c @@ -449,7 +449,7 @@ static int mt9m111_reset(struct mt9m111 *mt9m111) } static int mt9m111_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -493,7 +493,7 @@ static int mt9m111_set_selection(struct v4l2_subdev *sd, } static int mt9m111_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -518,7 +518,7 @@ static int mt9m111_get_selection(struct v4l2_subdev *sd, } static int mt9m111_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -529,7 +529,7 @@ static int mt9m111_get_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - mf = v4l2_subdev_get_try_format(sd, cfg, format->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, format->pad); format->format = *mf; return 0; #else @@ -624,7 +624,7 @@ static int mt9m111_set_pixfmt(struct mt9m111 *mt9m111, } static int mt9m111_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -678,7 +678,7 @@ static int mt9m111_set_fmt(struct v4l2_subdev *sd, mf->xfer_func = V4L2_XFER_FUNC_DEFAULT; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } @@ -1100,7 +1100,7 @@ static int mt9m111_s_frame_interval(struct v4l2_subdev *sd, } static int mt9m111_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(mt9m111_colour_fmts)) @@ -1119,11 +1119,11 @@ static int mt9m111_s_stream(struct v4l2_subdev *sd, int enable) } static int mt9m111_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API struct v4l2_mbus_framefmt *format = - v4l2_subdev_get_try_format(sd, cfg, 0); + v4l2_subdev_get_try_format(sd, sd_state, 0); format->width = MT9M111_MAX_WIDTH; format->height = MT9M111_MAX_HEIGHT; diff --git a/drivers/media/i2c/mt9p031.c b/drivers/media/i2c/mt9p031.c index a633b934d93e..6eb88ef99783 100644 --- a/drivers/media/i2c/mt9p031.c +++ b/drivers/media/i2c/mt9p031.c @@ -470,7 +470,7 @@ static int mt9p031_s_stream(struct v4l2_subdev *subdev, int enable) } static int mt9p031_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct mt9p031 *mt9p031 = to_mt9p031(subdev); @@ -483,7 +483,7 @@ static int mt9p031_enum_mbus_code(struct v4l2_subdev *subdev, } static int mt9p031_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct mt9p031 *mt9p031 = to_mt9p031(subdev); @@ -501,12 +501,14 @@ static int mt9p031_enum_frame_size(struct v4l2_subdev *subdev, } static struct v4l2_mbus_framefmt * -__mt9p031_get_pad_format(struct mt9p031 *mt9p031, struct v4l2_subdev_pad_config *cfg, +__mt9p031_get_pad_format(struct mt9p031 *mt9p031, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&mt9p031->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&mt9p031->subdev, sd_state, + pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &mt9p031->format; default: @@ -515,12 +517,14 @@ __mt9p031_get_pad_format(struct mt9p031 *mt9p031, struct v4l2_subdev_pad_config } static struct v4l2_rect * -__mt9p031_get_pad_crop(struct mt9p031 *mt9p031, struct v4l2_subdev_pad_config *cfg, - unsigned int pad, u32 which) +__mt9p031_get_pad_crop(struct mt9p031 *mt9p031, + struct v4l2_subdev_state *sd_state, + unsigned int pad, u32 which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&mt9p031->subdev, cfg, pad); + return v4l2_subdev_get_try_crop(&mt9p031->subdev, sd_state, + pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &mt9p031->crop; default: @@ -529,18 +533,18 @@ __mt9p031_get_pad_crop(struct mt9p031 *mt9p031, struct v4l2_subdev_pad_config *c } static int mt9p031_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct mt9p031 *mt9p031 = to_mt9p031(subdev); - fmt->format = *__mt9p031_get_pad_format(mt9p031, cfg, fmt->pad, + fmt->format = *__mt9p031_get_pad_format(mt9p031, sd_state, fmt->pad, fmt->which); return 0; } static int mt9p031_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mt9p031 *mt9p031 = to_mt9p031(subdev); @@ -551,7 +555,7 @@ static int mt9p031_set_format(struct v4l2_subdev *subdev, unsigned int hratio; unsigned int vratio; - __crop = __mt9p031_get_pad_crop(mt9p031, cfg, format->pad, + __crop = __mt9p031_get_pad_crop(mt9p031, sd_state, format->pad, format->which); /* Clamp the width and height to avoid dividing by zero. */ @@ -567,7 +571,7 @@ static int mt9p031_set_format(struct v4l2_subdev *subdev, hratio = DIV_ROUND_CLOSEST(__crop->width, width); vratio = DIV_ROUND_CLOSEST(__crop->height, height); - __format = __mt9p031_get_pad_format(mt9p031, cfg, format->pad, + __format = __mt9p031_get_pad_format(mt9p031, sd_state, format->pad, format->which); __format->width = __crop->width / hratio; __format->height = __crop->height / vratio; @@ -578,7 +582,7 @@ static int mt9p031_set_format(struct v4l2_subdev *subdev, } static int mt9p031_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9p031 *mt9p031 = to_mt9p031(subdev); @@ -586,12 +590,13 @@ static int mt9p031_get_selection(struct v4l2_subdev *subdev, if (sel->target != V4L2_SEL_TGT_CROP) return -EINVAL; - sel->r = *__mt9p031_get_pad_crop(mt9p031, cfg, sel->pad, sel->which); + sel->r = *__mt9p031_get_pad_crop(mt9p031, sd_state, sel->pad, + sel->which); return 0; } static int mt9p031_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9p031 *mt9p031 = to_mt9p031(subdev); @@ -621,13 +626,15 @@ static int mt9p031_set_selection(struct v4l2_subdev *subdev, rect.height = min_t(unsigned int, rect.height, MT9P031_PIXEL_ARRAY_HEIGHT - rect.top); - __crop = __mt9p031_get_pad_crop(mt9p031, cfg, sel->pad, sel->which); + __crop = __mt9p031_get_pad_crop(mt9p031, sd_state, sel->pad, + sel->which); if (rect.width != __crop->width || rect.height != __crop->height) { /* Reset the output image size if the crop rectangle size has * been modified. */ - __format = __mt9p031_get_pad_format(mt9p031, cfg, sel->pad, + __format = __mt9p031_get_pad_format(mt9p031, sd_state, + sel->pad, sel->which); __format->width = rect.width; __format->height = rect.height; @@ -942,13 +949,13 @@ static int mt9p031_open(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh) struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; - crop = v4l2_subdev_get_try_crop(subdev, fh->pad, 0); + crop = v4l2_subdev_get_try_crop(subdev, fh->state, 0); crop->left = MT9P031_COLUMN_START_DEF; crop->top = MT9P031_ROW_START_DEF; crop->width = MT9P031_WINDOW_WIDTH_DEF; crop->height = MT9P031_WINDOW_HEIGHT_DEF; - format = v4l2_subdev_get_try_format(subdev, fh->pad, 0); + format = v4l2_subdev_get_try_format(subdev, fh->state, 0); if (mt9p031->model == MT9P031_MODEL_MONOCHROME) format->code = MEDIA_BUS_FMT_Y12_1X12; diff --git a/drivers/media/i2c/mt9t001.c b/drivers/media/i2c/mt9t001.c index 2e96ff5234b4..b651ee4a26e8 100644 --- a/drivers/media/i2c/mt9t001.c +++ b/drivers/media/i2c/mt9t001.c @@ -252,12 +252,14 @@ static int __mt9t001_set_power(struct mt9t001 *mt9t001, bool on) */ static struct v4l2_mbus_framefmt * -__mt9t001_get_pad_format(struct mt9t001 *mt9t001, struct v4l2_subdev_pad_config *cfg, +__mt9t001_get_pad_format(struct mt9t001 *mt9t001, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&mt9t001->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&mt9t001->subdev, sd_state, + pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &mt9t001->format; default: @@ -266,12 +268,14 @@ __mt9t001_get_pad_format(struct mt9t001 *mt9t001, struct v4l2_subdev_pad_config } static struct v4l2_rect * -__mt9t001_get_pad_crop(struct mt9t001 *mt9t001, struct v4l2_subdev_pad_config *cfg, +__mt9t001_get_pad_crop(struct mt9t001 *mt9t001, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&mt9t001->subdev, cfg, pad); + return v4l2_subdev_get_try_crop(&mt9t001->subdev, sd_state, + pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &mt9t001->crop; default: @@ -335,7 +339,7 @@ static int mt9t001_s_stream(struct v4l2_subdev *subdev, int enable) } static int mt9t001_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -346,7 +350,7 @@ static int mt9t001_enum_mbus_code(struct v4l2_subdev *subdev, } static int mt9t001_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= 8 || fse->code != MEDIA_BUS_FMT_SGRBG10_1X10) @@ -361,18 +365,19 @@ static int mt9t001_enum_frame_size(struct v4l2_subdev *subdev, } static int mt9t001_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mt9t001 *mt9t001 = to_mt9t001(subdev); - format->format = *__mt9t001_get_pad_format(mt9t001, cfg, format->pad, + format->format = *__mt9t001_get_pad_format(mt9t001, sd_state, + format->pad, format->which); return 0; } static int mt9t001_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mt9t001 *mt9t001 = to_mt9t001(subdev); @@ -383,7 +388,7 @@ static int mt9t001_set_format(struct v4l2_subdev *subdev, unsigned int hratio; unsigned int vratio; - __crop = __mt9t001_get_pad_crop(mt9t001, cfg, format->pad, + __crop = __mt9t001_get_pad_crop(mt9t001, sd_state, format->pad, format->which); /* Clamp the width and height to avoid dividing by zero. */ @@ -399,7 +404,7 @@ static int mt9t001_set_format(struct v4l2_subdev *subdev, hratio = DIV_ROUND_CLOSEST(__crop->width, width); vratio = DIV_ROUND_CLOSEST(__crop->height, height); - __format = __mt9t001_get_pad_format(mt9t001, cfg, format->pad, + __format = __mt9t001_get_pad_format(mt9t001, sd_state, format->pad, format->which); __format->width = __crop->width / hratio; __format->height = __crop->height / vratio; @@ -410,7 +415,7 @@ static int mt9t001_set_format(struct v4l2_subdev *subdev, } static int mt9t001_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9t001 *mt9t001 = to_mt9t001(subdev); @@ -418,12 +423,13 @@ static int mt9t001_get_selection(struct v4l2_subdev *subdev, if (sel->target != V4L2_SEL_TGT_CROP) return -EINVAL; - sel->r = *__mt9t001_get_pad_crop(mt9t001, cfg, sel->pad, sel->which); + sel->r = *__mt9t001_get_pad_crop(mt9t001, sd_state, sel->pad, + sel->which); return 0; } static int mt9t001_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9t001 *mt9t001 = to_mt9t001(subdev); @@ -455,13 +461,15 @@ static int mt9t001_set_selection(struct v4l2_subdev *subdev, rect.height = min_t(unsigned int, rect.height, MT9T001_PIXEL_ARRAY_HEIGHT - rect.top); - __crop = __mt9t001_get_pad_crop(mt9t001, cfg, sel->pad, sel->which); + __crop = __mt9t001_get_pad_crop(mt9t001, sd_state, sel->pad, + sel->which); if (rect.width != __crop->width || rect.height != __crop->height) { /* Reset the output image size if the crop rectangle size has * been modified. */ - __format = __mt9t001_get_pad_format(mt9t001, cfg, sel->pad, + __format = __mt9t001_get_pad_format(mt9t001, sd_state, + sel->pad, sel->which); __format->width = rect.width; __format->height = rect.height; @@ -798,13 +806,13 @@ static int mt9t001_open(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh) struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; - crop = v4l2_subdev_get_try_crop(subdev, fh->pad, 0); + crop = v4l2_subdev_get_try_crop(subdev, fh->state, 0); crop->left = MT9T001_COLUMN_START_DEF; crop->top = MT9T001_ROW_START_DEF; crop->width = MT9T001_WINDOW_WIDTH_DEF + 1; crop->height = MT9T001_WINDOW_HEIGHT_DEF + 1; - format = v4l2_subdev_get_try_format(subdev, fh->pad, 0); + format = v4l2_subdev_get_try_format(subdev, fh->state, 0); format->code = MEDIA_BUS_FMT_SGRBG10_1X10; format->width = MT9T001_WINDOW_WIDTH_DEF + 1; format->height = MT9T001_WINDOW_HEIGHT_DEF + 1; diff --git a/drivers/media/i2c/mt9t112.c b/drivers/media/i2c/mt9t112.c index ae3c336eadf5..8d2e3caa9b28 100644 --- a/drivers/media/i2c/mt9t112.c +++ b/drivers/media/i2c/mt9t112.c @@ -872,8 +872,8 @@ static int mt9t112_set_params(struct mt9t112_priv *priv, } static int mt9t112_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_selection *sel) + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); struct mt9t112_priv *priv = to_mt9t112(client); @@ -897,7 +897,7 @@ static int mt9t112_get_selection(struct v4l2_subdev *sd, } static int mt9t112_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -912,7 +912,7 @@ static int mt9t112_set_selection(struct v4l2_subdev *sd, } static int mt9t112_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -953,7 +953,7 @@ static int mt9t112_s_fmt(struct v4l2_subdev *sd, } static int mt9t112_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -982,13 +982,13 @@ static int mt9t112_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) return mt9t112_s_fmt(sd, mf); - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } static int mt9t112_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct i2c_client *client = v4l2_get_subdevdata(sd); diff --git a/drivers/media/i2c/mt9v011.c b/drivers/media/i2c/mt9v011.c index 46ef74a2ca36..7699e64e1127 100644 --- a/drivers/media/i2c/mt9v011.c +++ b/drivers/media/i2c/mt9v011.c @@ -327,7 +327,7 @@ static int mt9v011_reset(struct v4l2_subdev *sd, u32 val) } static int mt9v011_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index > 0) @@ -338,7 +338,7 @@ static int mt9v011_enum_mbus_code(struct v4l2_subdev *sd, } static int mt9v011_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -358,7 +358,7 @@ static int mt9v011_set_fmt(struct v4l2_subdev *sd, set_res(sd); } else { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; } return 0; diff --git a/drivers/media/i2c/mt9v032.c b/drivers/media/i2c/mt9v032.c index 5bd3ae82992f..4cfdd3dfbd42 100644 --- a/drivers/media/i2c/mt9v032.c +++ b/drivers/media/i2c/mt9v032.c @@ -349,12 +349,14 @@ static int __mt9v032_set_power(struct mt9v032 *mt9v032, bool on) */ static struct v4l2_mbus_framefmt * -__mt9v032_get_pad_format(struct mt9v032 *mt9v032, struct v4l2_subdev_pad_config *cfg, +__mt9v032_get_pad_format(struct mt9v032 *mt9v032, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&mt9v032->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&mt9v032->subdev, sd_state, + pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &mt9v032->format; default: @@ -363,12 +365,14 @@ __mt9v032_get_pad_format(struct mt9v032 *mt9v032, struct v4l2_subdev_pad_config } static struct v4l2_rect * -__mt9v032_get_pad_crop(struct mt9v032 *mt9v032, struct v4l2_subdev_pad_config *cfg, +__mt9v032_get_pad_crop(struct mt9v032 *mt9v032, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&mt9v032->subdev, cfg, pad); + return v4l2_subdev_get_try_crop(&mt9v032->subdev, sd_state, + pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &mt9v032->crop; default: @@ -425,7 +429,7 @@ static int mt9v032_s_stream(struct v4l2_subdev *subdev, int enable) } static int mt9v032_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct mt9v032 *mt9v032 = to_mt9v032(subdev); @@ -438,7 +442,7 @@ static int mt9v032_enum_mbus_code(struct v4l2_subdev *subdev, } static int mt9v032_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct mt9v032 *mt9v032 = to_mt9v032(subdev); @@ -457,12 +461,13 @@ static int mt9v032_enum_frame_size(struct v4l2_subdev *subdev, } static int mt9v032_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mt9v032 *mt9v032 = to_mt9v032(subdev); - format->format = *__mt9v032_get_pad_format(mt9v032, cfg, format->pad, + format->format = *__mt9v032_get_pad_format(mt9v032, sd_state, + format->pad, format->which); return 0; } @@ -492,7 +497,7 @@ static unsigned int mt9v032_calc_ratio(unsigned int input, unsigned int output) } static int mt9v032_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mt9v032 *mt9v032 = to_mt9v032(subdev); @@ -503,7 +508,7 @@ static int mt9v032_set_format(struct v4l2_subdev *subdev, unsigned int hratio; unsigned int vratio; - __crop = __mt9v032_get_pad_crop(mt9v032, cfg, format->pad, + __crop = __mt9v032_get_pad_crop(mt9v032, sd_state, format->pad, format->which); /* Clamp the width and height to avoid dividing by zero. */ @@ -519,7 +524,7 @@ static int mt9v032_set_format(struct v4l2_subdev *subdev, hratio = mt9v032_calc_ratio(__crop->width, width); vratio = mt9v032_calc_ratio(__crop->height, height); - __format = __mt9v032_get_pad_format(mt9v032, cfg, format->pad, + __format = __mt9v032_get_pad_format(mt9v032, sd_state, format->pad, format->which); __format->width = __crop->width / hratio; __format->height = __crop->height / vratio; @@ -536,7 +541,7 @@ static int mt9v032_set_format(struct v4l2_subdev *subdev, } static int mt9v032_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9v032 *mt9v032 = to_mt9v032(subdev); @@ -544,12 +549,13 @@ static int mt9v032_get_selection(struct v4l2_subdev *subdev, if (sel->target != V4L2_SEL_TGT_CROP) return -EINVAL; - sel->r = *__mt9v032_get_pad_crop(mt9v032, cfg, sel->pad, sel->which); + sel->r = *__mt9v032_get_pad_crop(mt9v032, sd_state, sel->pad, + sel->which); return 0; } static int mt9v032_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct mt9v032 *mt9v032 = to_mt9v032(subdev); @@ -581,13 +587,15 @@ static int mt9v032_set_selection(struct v4l2_subdev *subdev, rect.height = min_t(unsigned int, rect.height, MT9V032_PIXEL_ARRAY_HEIGHT - rect.top); - __crop = __mt9v032_get_pad_crop(mt9v032, cfg, sel->pad, sel->which); + __crop = __mt9v032_get_pad_crop(mt9v032, sd_state, sel->pad, + sel->which); if (rect.width != __crop->width || rect.height != __crop->height) { /* Reset the output image size if the crop rectangle size has * been modified. */ - __format = __mt9v032_get_pad_format(mt9v032, cfg, sel->pad, + __format = __mt9v032_get_pad_format(mt9v032, sd_state, + sel->pad, sel->which); __format->width = rect.width; __format->height = rect.height; @@ -922,13 +930,13 @@ static int mt9v032_open(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh) struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; - crop = v4l2_subdev_get_try_crop(subdev, fh->pad, 0); + crop = v4l2_subdev_get_try_crop(subdev, fh->state, 0); crop->left = MT9V032_COLUMN_START_DEF; crop->top = MT9V032_ROW_START_DEF; crop->width = MT9V032_WINDOW_WIDTH_DEF; crop->height = MT9V032_WINDOW_HEIGHT_DEF; - format = v4l2_subdev_get_try_format(subdev, fh->pad, 0); + format = v4l2_subdev_get_try_format(subdev, fh->state, 0); if (mt9v032->model->color) format->code = MEDIA_BUS_FMT_SGRBG10_1X10; diff --git a/drivers/media/i2c/mt9v111.c b/drivers/media/i2c/mt9v111.c index 97c7527b74ed..2dc4a0f24ce8 100644 --- a/drivers/media/i2c/mt9v111.c +++ b/drivers/media/i2c/mt9v111.c @@ -791,16 +791,16 @@ static int mt9v111_g_frame_interval(struct v4l2_subdev *sd, static struct v4l2_mbus_framefmt *__mt9v111_get_pad_format( struct mt9v111_dev *mt9v111, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: #if IS_ENABLED(CONFIG_VIDEO_V4L2_SUBDEV_API) - return v4l2_subdev_get_try_format(&mt9v111->sd, cfg, pad); + return v4l2_subdev_get_try_format(&mt9v111->sd, sd_state, pad); #else - return &cfg->try_fmt; + return &sd_state->pads->try_fmt; #endif case V4L2_SUBDEV_FORMAT_ACTIVE: return &mt9v111->fmt; @@ -810,7 +810,7 @@ static struct v4l2_mbus_framefmt *__mt9v111_get_pad_format( } static int mt9v111_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index > ARRAY_SIZE(mt9v111_formats) - 1) @@ -822,7 +822,7 @@ static int mt9v111_enum_mbus_code(struct v4l2_subdev *subdev, } static int mt9v111_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { unsigned int i; @@ -845,7 +845,7 @@ static int mt9v111_enum_frame_interval(struct v4l2_subdev *sd, } static int mt9v111_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->pad || fse->index >= ARRAY_SIZE(mt9v111_frame_sizes)) @@ -860,7 +860,7 @@ static int mt9v111_enum_frame_size(struct v4l2_subdev *subdev, } static int mt9v111_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mt9v111_dev *mt9v111 = sd_to_mt9v111(subdev); @@ -869,7 +869,8 @@ static int mt9v111_get_format(struct v4l2_subdev *subdev, return -EINVAL; mutex_lock(&mt9v111->stream_mutex); - format->format = *__mt9v111_get_pad_format(mt9v111, cfg, format->pad, + format->format = *__mt9v111_get_pad_format(mt9v111, sd_state, + format->pad, format->which); mutex_unlock(&mt9v111->stream_mutex); @@ -877,7 +878,7 @@ static int mt9v111_get_format(struct v4l2_subdev *subdev, } static int mt9v111_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mt9v111_dev *mt9v111 = sd_to_mt9v111(subdev); @@ -925,7 +926,7 @@ static int mt9v111_set_format(struct v4l2_subdev *subdev, new_fmt.height = mt9v111_frame_sizes[idx].height; /* Update the device (or pad) format if it has changed. */ - __fmt = __mt9v111_get_pad_format(mt9v111, cfg, format->pad, + __fmt = __mt9v111_get_pad_format(mt9v111, sd_state, format->pad, format->which); /* Format hasn't changed, stop here. */ @@ -954,9 +955,9 @@ static int mt9v111_set_format(struct v4l2_subdev *subdev, } static int mt9v111_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { - cfg->try_fmt = mt9v111_def_fmt; + sd_state->pads->try_fmt = mt9v111_def_fmt; return 0; } diff --git a/drivers/media/i2c/noon010pc30.c b/drivers/media/i2c/noon010pc30.c index 87d76a7f691a..f3ac379ef34a 100644 --- a/drivers/media/i2c/noon010pc30.c +++ b/drivers/media/i2c/noon010pc30.c @@ -488,7 +488,7 @@ static int noon010_s_ctrl(struct v4l2_ctrl *ctrl) } static int noon010_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(noon010_formats)) @@ -499,15 +499,15 @@ static int noon010_enum_mbus_code(struct v4l2_subdev *sd, } static int noon010_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct noon010_info *info = to_noon010(sd); struct v4l2_mbus_framefmt *mf; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - if (cfg) { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + if (sd_state) { + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); fmt->format = *mf; } return 0; @@ -539,7 +539,8 @@ static const struct noon010_format *noon010_try_fmt(struct v4l2_subdev *sd, return &noon010_formats[i]; } -static int noon010_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int noon010_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct noon010_info *info = to_noon010(sd); @@ -554,8 +555,8 @@ static int noon010_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config fmt->format.field = V4L2_FIELD_NONE; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - if (cfg) { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + if (sd_state) { + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); *mf = fmt->format; } return 0; @@ -637,7 +638,9 @@ static int noon010_log_status(struct v4l2_subdev *sd) static int noon010_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - struct v4l2_mbus_framefmt *mf = v4l2_subdev_get_try_format(sd, fh->pad, 0); + struct v4l2_mbus_framefmt *mf = v4l2_subdev_get_try_format(sd, + fh->state, + 0); mf->width = noon010_sizes[0].width; mf->height = noon010_sizes[0].height; diff --git a/drivers/media/i2c/ov02a10.c b/drivers/media/i2c/ov02a10.c index c47b1d45d8fd..a3ce5500d355 100644 --- a/drivers/media/i2c/ov02a10.c +++ b/drivers/media/i2c/ov02a10.c @@ -295,7 +295,7 @@ static void ov02a10_fill_fmt(const struct ov02a10_mode *mode, } static int ov02a10_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov02a10 *ov02a10 = to_ov02a10(sd); @@ -315,7 +315,7 @@ static int ov02a10_set_fmt(struct v4l2_subdev *sd, ov02a10_fill_fmt(ov02a10->cur_mode, mbus_fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - frame_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + frame_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); else frame_fmt = &ov02a10->fmt; @@ -327,7 +327,7 @@ static int ov02a10_set_fmt(struct v4l2_subdev *sd, } static int ov02a10_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov02a10 *ov02a10 = to_ov02a10(sd); @@ -336,7 +336,8 @@ static int ov02a10_get_fmt(struct v4l2_subdev *sd, mutex_lock(&ov02a10->mutex); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - fmt->format = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + fmt->format = *v4l2_subdev_get_try_format(sd, sd_state, + fmt->pad); } else { fmt->format = ov02a10->fmt; mbus_fmt->code = ov02a10->fmt.code; @@ -349,7 +350,7 @@ static int ov02a10_get_fmt(struct v4l2_subdev *sd, } static int ov02a10_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct ov02a10 *ov02a10 = to_ov02a10(sd); @@ -363,7 +364,7 @@ static int ov02a10_enum_mbus_code(struct v4l2_subdev *sd, } static int ov02a10_enum_frame_sizes(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -511,7 +512,7 @@ static int __ov02a10_stop_stream(struct ov02a10 *ov02a10) } static int ov02a10_entity_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format fmt = { .which = V4L2_SUBDEV_FORMAT_TRY, @@ -521,7 +522,7 @@ static int ov02a10_entity_init_cfg(struct v4l2_subdev *sd, } }; - ov02a10_set_fmt(sd, cfg, &fmt); + ov02a10_set_fmt(sd, sd_state, &fmt); return 0; } @@ -540,11 +541,9 @@ static int ov02a10_s_stream(struct v4l2_subdev *sd, int on) } if (on) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto unlock_and_return; - } ret = __ov02a10_start_stream(ov02a10); if (ret) { diff --git a/drivers/media/i2c/ov13858.c b/drivers/media/i2c/ov13858.c index 4a2885ff0cbe..7fc70af53e45 100644 --- a/drivers/media/i2c/ov13858.c +++ b/drivers/media/i2c/ov13858.c @@ -1150,7 +1150,7 @@ static int ov13858_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct ov13858 *ov13858 = to_ov13858(sd); struct v4l2_mbus_framefmt *try_fmt = v4l2_subdev_get_try_format(sd, - fh->pad, + fh->state, 0); mutex_lock(&ov13858->mutex); @@ -1275,7 +1275,7 @@ static const struct v4l2_ctrl_ops ov13858_ctrl_ops = { }; static int ov13858_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { /* Only one bayer order(GRBG) is supported */ @@ -1288,7 +1288,7 @@ static int ov13858_enum_mbus_code(struct v4l2_subdev *sd, } static int ov13858_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -1315,14 +1315,14 @@ static void ov13858_update_pad_format(const struct ov13858_mode *mode, } static int ov13858_do_get_pad_format(struct ov13858 *ov13858, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct v4l2_mbus_framefmt *framefmt; struct v4l2_subdev *sd = &ov13858->sd; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); fmt->format = *framefmt; } else { ov13858_update_pad_format(ov13858->cur_mode, fmt); @@ -1332,14 +1332,14 @@ static int ov13858_do_get_pad_format(struct ov13858 *ov13858, } static int ov13858_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov13858 *ov13858 = to_ov13858(sd); int ret; mutex_lock(&ov13858->mutex); - ret = ov13858_do_get_pad_format(ov13858, cfg, fmt); + ret = ov13858_do_get_pad_format(ov13858, sd_state, fmt); mutex_unlock(&ov13858->mutex); return ret; @@ -1347,7 +1347,7 @@ static int ov13858_get_pad_format(struct v4l2_subdev *sd, static int ov13858_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov13858 *ov13858 = to_ov13858(sd); @@ -1371,7 +1371,7 @@ ov13858_set_pad_format(struct v4l2_subdev *sd, fmt->format.width, fmt->format.height); ov13858_update_pad_format(mode, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - framefmt = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *framefmt = fmt->format; } else { ov13858->cur_mode = mode; @@ -1472,11 +1472,9 @@ static int ov13858_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto err_unlock; - } /* * Apply default & customized values diff --git a/drivers/media/i2c/ov2640.c b/drivers/media/i2c/ov2640.c index 4a4bd5b665a1..4b75da55b260 100644 --- a/drivers/media/i2c/ov2640.c +++ b/drivers/media/i2c/ov2640.c @@ -913,7 +913,7 @@ static int ov2640_set_params(struct i2c_client *client, } static int ov2640_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -925,7 +925,7 @@ static int ov2640_get_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); format->format = *mf; return 0; #else @@ -946,7 +946,7 @@ static int ov2640_get_fmt(struct v4l2_subdev *sd, } static int ov2640_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -996,7 +996,7 @@ static int ov2640_set_fmt(struct v4l2_subdev *sd, /* select format */ priv->cfmt_code = mf->code; } else { - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; } out: mutex_unlock(&priv->lock); @@ -1005,11 +1005,11 @@ static int ov2640_set_fmt(struct v4l2_subdev *sd, } static int ov2640_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, cfg, 0); + v4l2_subdev_get_try_format(sd, sd_state, 0); const struct ov2640_win_size *win = ov2640_select_win(SVGA_WIDTH, SVGA_HEIGHT); @@ -1026,7 +1026,7 @@ static int ov2640_init_cfg(struct v4l2_subdev *sd, } static int ov2640_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(ov2640_codes)) @@ -1037,7 +1037,7 @@ static int ov2640_enum_mbus_code(struct v4l2_subdev *sd, } static int ov2640_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE) diff --git a/drivers/media/i2c/ov2659.c b/drivers/media/i2c/ov2659.c index 42f64175a6df..13ded5b2aa66 100644 --- a/drivers/media/i2c/ov2659.c +++ b/drivers/media/i2c/ov2659.c @@ -204,6 +204,7 @@ struct ov2659 { struct i2c_client *client; struct v4l2_ctrl_handler ctrls; struct v4l2_ctrl *link_frequency; + struct clk *clk; const struct ov2659_framesize *frame_size; struct sensor_register *format_ctrl_regs; struct ov2659_pll_ctrl pll; @@ -979,7 +980,7 @@ static int ov2659_init(struct v4l2_subdev *sd, u32 val) */ static int ov2659_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -995,7 +996,7 @@ static int ov2659_enum_mbus_code(struct v4l2_subdev *sd, } static int ov2659_enum_frame_sizes(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -1021,7 +1022,7 @@ static int ov2659_enum_frame_sizes(struct v4l2_subdev *sd, } static int ov2659_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -1033,7 +1034,7 @@ static int ov2659_get_fmt(struct v4l2_subdev *sd, #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API struct v4l2_mbus_framefmt *mf; - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); mutex_lock(&ov2659->lock); fmt->format = *mf; mutex_unlock(&ov2659->lock); @@ -1083,7 +1084,7 @@ static void __ov2659_try_frame_size(struct v4l2_mbus_framefmt *mf, } static int ov2659_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -1113,7 +1114,7 @@ static int ov2659_set_fmt(struct v4l2_subdev *sd, if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *mf = fmt->format; #endif } else { @@ -1186,11 +1187,9 @@ static int ov2659_s_stream(struct v4l2_subdev *sd, int on) goto unlock; } - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto unlock; - } ret = ov2659_init(sd, 0); if (!ret) @@ -1270,6 +1269,8 @@ static int ov2659_power_off(struct device *dev) gpiod_set_value(ov2659->pwdn_gpio, 1); + clk_disable_unprepare(ov2659->clk); + return 0; } @@ -1278,9 +1279,17 @@ static int ov2659_power_on(struct device *dev) struct i2c_client *client = to_i2c_client(dev); struct v4l2_subdev *sd = i2c_get_clientdata(client); struct ov2659 *ov2659 = to_ov2659(sd); + int ret; dev_dbg(&client->dev, "%s:\n", __func__); + ret = clk_prepare_enable(ov2659->clk); + if (ret) { + dev_err(&client->dev, "%s: failed to enable clock\n", + __func__); + return ret; + } + gpiod_set_value(ov2659->pwdn_gpio, 0); if (ov2659->resetb_gpio) { @@ -1302,7 +1311,7 @@ static int ov2659_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct i2c_client *client = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); dev_dbg(&client->dev, "%s:\n", __func__); @@ -1368,8 +1377,7 @@ static int ov2659_detect(struct v4l2_subdev *sd) id = OV265X_ID(pid, ver); if (id != OV2659_ID) { dev_err(&client->dev, - "Sensor detection failed (%04X, %d)\n", - id, ret); + "Sensor detection failed (%04X)\n", id); ret = -ENODEV; } else { dev_info(&client->dev, "Found OV%04X sensor\n", id); @@ -1425,7 +1433,6 @@ static int ov2659_probe(struct i2c_client *client) const struct ov2659_platform_data *pdata = ov2659_get_pdata(client); struct v4l2_subdev *sd; struct ov2659 *ov2659; - struct clk *clk; int ret; if (!pdata) { @@ -1440,11 +1447,11 @@ static int ov2659_probe(struct i2c_client *client) ov2659->pdata = pdata; ov2659->client = client; - clk = devm_clk_get(&client->dev, "xvclk"); - if (IS_ERR(clk)) - return PTR_ERR(clk); + ov2659->clk = devm_clk_get(&client->dev, "xvclk"); + if (IS_ERR(ov2659->clk)) + return PTR_ERR(ov2659->clk); - ov2659->xvclk_frequency = clk_get_rate(clk); + ov2659->xvclk_frequency = clk_get_rate(ov2659->clk); if (ov2659->xvclk_frequency < 6000000 || ov2659->xvclk_frequency > 27000000) return -EINVAL; @@ -1506,7 +1513,9 @@ static int ov2659_probe(struct i2c_client *client) ov2659->frame_size = &ov2659_framesizes[2]; ov2659->format_ctrl_regs = ov2659_formats[0].format_ctrl_regs; - ov2659_power_on(&client->dev); + ret = ov2659_power_on(&client->dev); + if (ret < 0) + goto error; ret = ov2659_detect(sd); if (ret < 0) diff --git a/drivers/media/i2c/ov2680.c b/drivers/media/i2c/ov2680.c index 178dfe985a25..906c711f6821 100644 --- a/drivers/media/i2c/ov2680.c +++ b/drivers/media/i2c/ov2680.c @@ -645,7 +645,7 @@ static int ov2680_s_stream(struct v4l2_subdev *sd, int enable) } static int ov2680_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct ov2680_dev *sensor = to_ov2680_dev(sd); @@ -659,7 +659,7 @@ static int ov2680_enum_mbus_code(struct v4l2_subdev *sd, } static int ov2680_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov2680_dev *sensor = to_ov2680_dev(sd); @@ -673,7 +673,8 @@ static int ov2680_get_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - fmt = v4l2_subdev_get_try_format(&sensor->sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(&sensor->sd, sd_state, + format->pad); #else ret = -EINVAL; #endif @@ -690,7 +691,7 @@ static int ov2680_get_fmt(struct v4l2_subdev *sd, } static int ov2680_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov2680_dev *sensor = to_ov2680_dev(sd); @@ -721,7 +722,7 @@ static int ov2680_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - try_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + try_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); format->format = *try_fmt; #endif goto unlock; @@ -743,22 +744,22 @@ static int ov2680_set_fmt(struct v4l2_subdev *sd, } static int ov2680_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format fmt = { - .which = cfg ? V4L2_SUBDEV_FORMAT_TRY - : V4L2_SUBDEV_FORMAT_ACTIVE, + .which = sd_state ? V4L2_SUBDEV_FORMAT_TRY + : V4L2_SUBDEV_FORMAT_ACTIVE, .format = { .width = 800, .height = 600, } }; - return ov2680_set_fmt(sd, cfg, &fmt); + return ov2680_set_fmt(sd, sd_state, &fmt); } static int ov2680_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int index = fse->index; @@ -775,7 +776,7 @@ static int ov2680_enum_frame_size(struct v4l2_subdev *sd, } static int ov2680_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct v4l2_fract tpf; diff --git a/drivers/media/i2c/ov2685.c b/drivers/media/i2c/ov2685.c index 49a2dcedb347..b6e010ea3249 100644 --- a/drivers/media/i2c/ov2685.c +++ b/drivers/media/i2c/ov2685.c @@ -328,7 +328,7 @@ static void ov2685_fill_fmt(const struct ov2685_mode *mode, } static int ov2685_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov2685 *ov2685 = to_ov2685(sd); @@ -341,7 +341,7 @@ static int ov2685_set_fmt(struct v4l2_subdev *sd, } static int ov2685_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov2685 *ov2685 = to_ov2685(sd); @@ -353,7 +353,7 @@ static int ov2685_get_fmt(struct v4l2_subdev *sd, } static int ov2685_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(supported_modes)) @@ -365,7 +365,7 @@ static int ov2685_enum_mbus_code(struct v4l2_subdev *sd, } static int ov2685_enum_frame_sizes(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int index = fse->index; @@ -456,11 +456,10 @@ static int ov2685_s_stream(struct v4l2_subdev *sd, int on) goto unlock_and_return; if (on) { - ret = pm_runtime_get_sync(&ov2685->client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&ov2685->client->dev); + if (ret < 0) goto unlock_and_return; - } + ret = __v4l2_ctrl_handler_setup(&ov2685->ctrl_handler); if (ret) { pm_runtime_put(&client->dev); @@ -494,7 +493,7 @@ static int ov2685_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) mutex_lock(&ov2685->mutex); - try_fmt = v4l2_subdev_get_try_format(sd, fh->pad, 0); + try_fmt = v4l2_subdev_get_try_format(sd, fh->state, 0); /* Initialize try_fmt */ ov2685_fill_fmt(&supported_modes[0], try_fmt); diff --git a/drivers/media/i2c/ov2740.c b/drivers/media/i2c/ov2740.c index 0f3f17f3c426..599369a3d192 100644 --- a/drivers/media/i2c/ov2740.c +++ b/drivers/media/i2c/ov2740.c @@ -751,9 +751,8 @@ static int ov2740_set_stream(struct v4l2_subdev *sd, int enable) mutex_lock(&ov2740->mutex); if (enable) { - ret = pm_runtime_get_sync(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); if (ret < 0) { - pm_runtime_put_noidle(&client->dev); mutex_unlock(&ov2740->mutex); return ret; } @@ -811,7 +810,7 @@ static int __maybe_unused ov2740_resume(struct device *dev) } static int ov2740_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov2740 *ov2740 = to_ov2740(sd); @@ -826,7 +825,7 @@ static int ov2740_set_format(struct v4l2_subdev *sd, mutex_lock(&ov2740->mutex); ov2740_update_pad_format(mode, &fmt->format); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; } else { ov2740->cur_mode = mode; __v4l2_ctrl_s_ctrl(ov2740->link_freq, mode->link_freq_index); @@ -851,14 +850,15 @@ static int ov2740_set_format(struct v4l2_subdev *sd, } static int ov2740_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov2740 *ov2740 = to_ov2740(sd); mutex_lock(&ov2740->mutex); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(&ov2740->sd, cfg, + fmt->format = *v4l2_subdev_get_try_format(&ov2740->sd, + sd_state, fmt->pad); else ov2740_update_pad_format(ov2740->cur_mode, &fmt->format); @@ -869,7 +869,7 @@ static int ov2740_get_format(struct v4l2_subdev *sd, } static int ov2740_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -881,7 +881,7 @@ static int ov2740_enum_mbus_code(struct v4l2_subdev *sd, } static int ov2740_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -904,7 +904,7 @@ static int ov2740_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) mutex_lock(&ov2740->mutex); ov2740_update_pad_format(&supported_modes[0], - v4l2_subdev_get_try_format(sd, fh->pad, 0)); + v4l2_subdev_get_try_format(sd, fh->state, 0)); mutex_unlock(&ov2740->mutex); return 0; @@ -1049,9 +1049,8 @@ static int ov2740_nvmem_read(void *priv, unsigned int off, void *val, goto exit; } - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { - pm_runtime_put_noidle(dev); goto exit; } diff --git a/drivers/media/i2c/ov5640.c b/drivers/media/i2c/ov5640.c index 5b9cc71df473..f6e1e51e0375 100644 --- a/drivers/media/i2c/ov5640.c +++ b/drivers/media/i2c/ov5640.c @@ -2227,7 +2227,7 @@ static int ov5640_try_frame_interval(struct ov5640_dev *sensor, } static int ov5640_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov5640_dev *sensor = to_ov5640_dev(sd); @@ -2239,7 +2239,7 @@ static int ov5640_get_fmt(struct v4l2_subdev *sd, mutex_lock(&sensor->lock); if (format->which == V4L2_SUBDEV_FORMAT_TRY) - fmt = v4l2_subdev_get_try_format(&sensor->sd, cfg, + fmt = v4l2_subdev_get_try_format(&sensor->sd, sd_state, format->pad); else fmt = &sensor->fmt; @@ -2285,7 +2285,7 @@ static int ov5640_try_fmt_internal(struct v4l2_subdev *sd, } static int ov5640_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov5640_dev *sensor = to_ov5640_dev(sd); @@ -2310,7 +2310,7 @@ static int ov5640_set_fmt(struct v4l2_subdev *sd, goto out; if (format->which == V4L2_SUBDEV_FORMAT_TRY) - fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); else fmt = &sensor->fmt; @@ -2818,7 +2818,7 @@ static int ov5640_init_controls(struct ov5640_dev *sensor) } static int ov5640_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->pad != 0) @@ -2838,7 +2838,7 @@ static int ov5640_enum_frame_size(struct v4l2_subdev *sd, static int ov5640_enum_frame_interval( struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct ov5640_dev *sensor = to_ov5640_dev(sd); @@ -2924,7 +2924,7 @@ static int ov5640_s_frame_interval(struct v4l2_subdev *sd, } static int ov5640_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad != 0) diff --git a/drivers/media/i2c/ov5645.c b/drivers/media/i2c/ov5645.c index a6c17d15d754..368fa21e675e 100644 --- a/drivers/media/i2c/ov5645.c +++ b/drivers/media/i2c/ov5645.c @@ -837,7 +837,7 @@ static const struct v4l2_ctrl_ops ov5645_ctrl_ops = { }; static int ov5645_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -849,7 +849,7 @@ static int ov5645_enum_mbus_code(struct v4l2_subdev *sd, } static int ov5645_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->code != MEDIA_BUS_FMT_UYVY8_2X8) @@ -868,13 +868,13 @@ static int ov5645_enum_frame_size(struct v4l2_subdev *subdev, static struct v4l2_mbus_framefmt * __ov5645_get_pad_format(struct ov5645 *ov5645, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&ov5645->sd, cfg, pad); + return v4l2_subdev_get_try_format(&ov5645->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &ov5645->fmt; default: @@ -883,23 +883,25 @@ __ov5645_get_pad_format(struct ov5645 *ov5645, } static int ov5645_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov5645 *ov5645 = to_ov5645(sd); - format->format = *__ov5645_get_pad_format(ov5645, cfg, format->pad, + format->format = *__ov5645_get_pad_format(ov5645, sd_state, + format->pad, format->which); return 0; } static struct v4l2_rect * -__ov5645_get_pad_crop(struct ov5645 *ov5645, struct v4l2_subdev_pad_config *cfg, +__ov5645_get_pad_crop(struct ov5645 *ov5645, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&ov5645->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&ov5645->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &ov5645->crop; default: @@ -908,7 +910,7 @@ __ov5645_get_pad_crop(struct ov5645 *ov5645, struct v4l2_subdev_pad_config *cfg, } static int ov5645_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov5645 *ov5645 = to_ov5645(sd); @@ -917,8 +919,8 @@ static int ov5645_set_format(struct v4l2_subdev *sd, const struct ov5645_mode_info *new_mode; int ret; - __crop = __ov5645_get_pad_crop(ov5645, cfg, format->pad, - format->which); + __crop = __ov5645_get_pad_crop(ov5645, sd_state, format->pad, + format->which); new_mode = v4l2_find_nearest_size(ov5645_mode_info_data, ARRAY_SIZE(ov5645_mode_info_data), @@ -942,8 +944,8 @@ static int ov5645_set_format(struct v4l2_subdev *sd, ov5645->current_mode = new_mode; } - __format = __ov5645_get_pad_format(ov5645, cfg, format->pad, - format->which); + __format = __ov5645_get_pad_format(ov5645, sd_state, format->pad, + format->which); __format->width = __crop->width; __format->height = __crop->height; __format->code = MEDIA_BUS_FMT_UYVY8_2X8; @@ -956,21 +958,21 @@ static int ov5645_set_format(struct v4l2_subdev *sd, } static int ov5645_entity_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format fmt = { 0 }; - fmt.which = cfg ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; + fmt.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; fmt.format.width = 1920; fmt.format.height = 1080; - ov5645_set_format(subdev, cfg, &fmt); + ov5645_set_format(subdev, sd_state, &fmt); return 0; } static int ov5645_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ov5645 *ov5645 = to_ov5645(sd); @@ -978,7 +980,7 @@ static int ov5645_get_selection(struct v4l2_subdev *sd, if (sel->target != V4L2_SEL_TGT_CROP) return -EINVAL; - sel->r = *__ov5645_get_pad_crop(ov5645, cfg, sel->pad, + sel->r = *__ov5645_get_pad_crop(ov5645, sd_state, sel->pad, sel->which); return 0; } diff --git a/drivers/media/i2c/ov5647.c b/drivers/media/i2c/ov5647.c index 1cefa15729ce..d346d18ce629 100644 --- a/drivers/media/i2c/ov5647.c +++ b/drivers/media/i2c/ov5647.c @@ -856,12 +856,13 @@ static const struct v4l2_subdev_core_ops ov5647_subdev_core_ops = { }; static const struct v4l2_rect * -__ov5647_get_pad_crop(struct ov5647 *ov5647, struct v4l2_subdev_pad_config *cfg, +__ov5647_get_pad_crop(struct ov5647 *ov5647, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&ov5647->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&ov5647->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &ov5647->mode->crop; } @@ -882,20 +883,20 @@ static int ov5647_s_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); if (ret < 0) goto error_unlock; ret = ov5647_stream_on(sd); if (ret < 0) { dev_err(&client->dev, "stream start failed: %d\n", ret); - goto error_unlock; + goto error_pm; } } else { ret = ov5647_stream_off(sd); if (ret < 0) { dev_err(&client->dev, "stream stop failed: %d\n", ret); - goto error_unlock; + goto error_pm; } pm_runtime_put(&client->dev); } @@ -905,8 +906,9 @@ static int ov5647_s_stream(struct v4l2_subdev *sd, int enable) return 0; -error_unlock: +error_pm: pm_runtime_put(&client->dev); +error_unlock: mutex_unlock(&sensor->lock); return ret; @@ -917,7 +919,7 @@ static const struct v4l2_subdev_video_ops ov5647_subdev_video_ops = { }; static int ov5647_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -929,7 +931,7 @@ static int ov5647_enum_mbus_code(struct v4l2_subdev *sd, } static int ov5647_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { const struct v4l2_mbus_framefmt *fmt; @@ -948,7 +950,7 @@ static int ov5647_enum_frame_size(struct v4l2_subdev *sd, } static int ov5647_get_pad_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -958,7 +960,8 @@ static int ov5647_get_pad_fmt(struct v4l2_subdev *sd, mutex_lock(&sensor->lock); switch (format->which) { case V4L2_SUBDEV_FORMAT_TRY: - sensor_format = v4l2_subdev_get_try_format(sd, cfg, format->pad); + sensor_format = v4l2_subdev_get_try_format(sd, sd_state, + format->pad); break; default: sensor_format = &sensor->mode->format; @@ -972,7 +975,7 @@ static int ov5647_get_pad_fmt(struct v4l2_subdev *sd, } static int ov5647_set_pad_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -986,7 +989,7 @@ static int ov5647_set_pad_fmt(struct v4l2_subdev *sd, /* Update the sensor mode and apply at it at streamon time. */ mutex_lock(&sensor->lock); if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, format->pad) = mode->format; + *v4l2_subdev_get_try_format(sd, sd_state, format->pad) = mode->format; } else { int exposure_max, exposure_def; int hblank, vblank; @@ -1019,7 +1022,7 @@ static int ov5647_set_pad_fmt(struct v4l2_subdev *sd, } static int ov5647_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { switch (sel->target) { @@ -1027,7 +1030,7 @@ static int ov5647_get_selection(struct v4l2_subdev *sd, struct ov5647 *sensor = to_sensor(sd); mutex_lock(&sensor->lock); - sel->r = *__ov5647_get_pad_crop(sensor, cfg, sel->pad, + sel->r = *__ov5647_get_pad_crop(sensor, sd_state, sel->pad, sel->which); mutex_unlock(&sensor->lock); @@ -1103,8 +1106,8 @@ static int ov5647_detect(struct v4l2_subdev *sd) static int ov5647_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct v4l2_mbus_framefmt *format = - v4l2_subdev_get_try_format(sd, fh->pad, 0); - struct v4l2_rect *crop = v4l2_subdev_get_try_crop(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); + struct v4l2_rect *crop = v4l2_subdev_get_try_crop(sd, fh->state, 0); crop->left = OV5647_PIXEL_ARRAY_LEFT; crop->top = OV5647_PIXEL_ARRAY_TOP; diff --git a/drivers/media/i2c/ov5648.c b/drivers/media/i2c/ov5648.c index 3ecb4a3e8773..947d437ed0ef 100644 --- a/drivers/media/i2c/ov5648.c +++ b/drivers/media/i2c/ov5648.c @@ -2132,11 +2132,9 @@ static int ov5648_s_stream(struct v4l2_subdev *subdev, int enable) int ret; if (enable) { - ret = pm_runtime_get_sync(sensor->dev); - if (ret < 0) { - pm_runtime_put_noidle(sensor->dev); + ret = pm_runtime_resume_and_get(sensor->dev); + if (ret < 0) return ret; - } } mutex_lock(&sensor->mutex); @@ -2190,7 +2188,7 @@ static const struct v4l2_subdev_video_ops ov5648_subdev_video_ops = { /* Subdev Pad Operations */ static int ov5648_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code_enum) { if (code_enum->index >= ARRAY_SIZE(ov5648_mbus_codes)) @@ -2219,7 +2217,7 @@ static void ov5648_mbus_format_fill(struct v4l2_mbus_framefmt *mbus_format, } static int ov5648_get_fmt(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov5648_sensor *sensor = ov5648_subdev_sensor(subdev); @@ -2228,7 +2226,7 @@ static int ov5648_get_fmt(struct v4l2_subdev *subdev, mutex_lock(&sensor->mutex); if (format->which == V4L2_SUBDEV_FORMAT_TRY) - *mbus_format = *v4l2_subdev_get_try_format(subdev, config, + *mbus_format = *v4l2_subdev_get_try_format(subdev, sd_state, format->pad); else ov5648_mbus_format_fill(mbus_format, sensor->state.mbus_code, @@ -2240,7 +2238,7 @@ static int ov5648_get_fmt(struct v4l2_subdev *subdev, } static int ov5648_set_fmt(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov5648_sensor *sensor = ov5648_subdev_sensor(subdev); @@ -2281,7 +2279,7 @@ static int ov5648_set_fmt(struct v4l2_subdev *subdev, ov5648_mbus_format_fill(mbus_format, mbus_code, mode); if (format->which == V4L2_SUBDEV_FORMAT_TRY) - *v4l2_subdev_get_try_format(subdev, config, format->pad) = + *v4l2_subdev_get_try_format(subdev, sd_state, format->pad) = *mbus_format; else if (sensor->state.mode != mode || sensor->state.mbus_code != mbus_code) @@ -2294,7 +2292,7 @@ static int ov5648_set_fmt(struct v4l2_subdev *subdev, } static int ov5648_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *size_enum) { const struct ov5648_mode *mode; @@ -2311,7 +2309,7 @@ static int ov5648_enum_frame_size(struct v4l2_subdev *subdev, } static int ov5648_enum_frame_interval(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *interval_enum) { const struct ov5648_mode *mode = NULL; diff --git a/drivers/media/i2c/ov5670.c b/drivers/media/i2c/ov5670.c index dee7df8dd100..49189926afd6 100644 --- a/drivers/media/i2c/ov5670.c +++ b/drivers/media/i2c/ov5670.c @@ -1937,7 +1937,7 @@ static int ov5670_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct ov5670 *ov5670 = to_ov5670(sd); struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); mutex_lock(&ov5670->mutex); @@ -2153,7 +2153,7 @@ static int ov5670_init_controls(struct ov5670 *ov5670) } static int ov5670_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { /* Only one bayer order GRBG is supported */ @@ -2166,7 +2166,7 @@ static int ov5670_enum_mbus_code(struct v4l2_subdev *sd, } static int ov5670_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -2193,11 +2193,12 @@ static void ov5670_update_pad_format(const struct ov5670_mode *mode, } static int ov5670_do_get_pad_format(struct ov5670 *ov5670, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(&ov5670->sd, cfg, + fmt->format = *v4l2_subdev_get_try_format(&ov5670->sd, + sd_state, fmt->pad); else ov5670_update_pad_format(ov5670->cur_mode, fmt); @@ -2206,21 +2207,21 @@ static int ov5670_do_get_pad_format(struct ov5670 *ov5670, } static int ov5670_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov5670 *ov5670 = to_ov5670(sd); int ret; mutex_lock(&ov5670->mutex); - ret = ov5670_do_get_pad_format(ov5670, cfg, fmt); + ret = ov5670_do_get_pad_format(ov5670, sd_state, fmt); mutex_unlock(&ov5670->mutex); return ret; } static int ov5670_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov5670 *ov5670 = to_ov5670(sd); @@ -2238,7 +2239,7 @@ static int ov5670_set_pad_format(struct v4l2_subdev *sd, fmt->format.width, fmt->format.height); ov5670_update_pad_format(mode, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; } else { ov5670->cur_mode = mode; __v4l2_ctrl_s_ctrl(ov5670->link_freq, mode->link_freq_index); @@ -2347,11 +2348,9 @@ static int ov5670_set_stream(struct v4l2_subdev *sd, int enable) goto unlock_and_return; if (enable) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto unlock_and_return; - } ret = ov5670_start_streaming(ov5670); if (ret) diff --git a/drivers/media/i2c/ov5675.c b/drivers/media/i2c/ov5675.c index dea32859459a..da5850b7ad07 100644 --- a/drivers/media/i2c/ov5675.c +++ b/drivers/media/i2c/ov5675.c @@ -863,9 +863,8 @@ static int ov5675_set_stream(struct v4l2_subdev *sd, int enable) mutex_lock(&ov5675->mutex); if (enable) { - ret = pm_runtime_get_sync(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); if (ret < 0) { - pm_runtime_put_noidle(&client->dev); mutex_unlock(&ov5675->mutex); return ret; } @@ -924,7 +923,7 @@ static int __maybe_unused ov5675_resume(struct device *dev) } static int ov5675_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov5675 *ov5675 = to_ov5675(sd); @@ -939,7 +938,7 @@ static int ov5675_set_format(struct v4l2_subdev *sd, mutex_lock(&ov5675->mutex); ov5675_update_pad_format(mode, &fmt->format); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; } else { ov5675->cur_mode = mode; __v4l2_ctrl_s_ctrl(ov5675->link_freq, mode->link_freq_index); @@ -965,14 +964,15 @@ static int ov5675_set_format(struct v4l2_subdev *sd, } static int ov5675_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov5675 *ov5675 = to_ov5675(sd); mutex_lock(&ov5675->mutex); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(&ov5675->sd, cfg, + fmt->format = *v4l2_subdev_get_try_format(&ov5675->sd, + sd_state, fmt->pad); else ov5675_update_pad_format(ov5675->cur_mode, &fmt->format); @@ -983,7 +983,7 @@ static int ov5675_get_format(struct v4l2_subdev *sd, } static int ov5675_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -995,7 +995,7 @@ static int ov5675_enum_mbus_code(struct v4l2_subdev *sd, } static int ov5675_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -1018,7 +1018,7 @@ static int ov5675_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) mutex_lock(&ov5675->mutex); ov5675_update_pad_format(&supported_modes[0], - v4l2_subdev_get_try_format(sd, fh->pad, 0)); + v4l2_subdev_get_try_format(sd, fh->state, 0)); mutex_unlock(&ov5675->mutex); return 0; diff --git a/drivers/media/i2c/ov5695.c b/drivers/media/i2c/ov5695.c index 09bee57a241d..439385938a51 100644 --- a/drivers/media/i2c/ov5695.c +++ b/drivers/media/i2c/ov5695.c @@ -806,7 +806,7 @@ ov5695_find_best_fit(struct v4l2_subdev_format *fmt) } static int ov5695_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov5695 *ov5695 = to_ov5695(sd); @@ -822,7 +822,7 @@ static int ov5695_set_fmt(struct v4l2_subdev *sd, fmt->format.field = V4L2_FIELD_NONE; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; #endif } else { ov5695->cur_mode = mode; @@ -841,7 +841,7 @@ static int ov5695_set_fmt(struct v4l2_subdev *sd, } static int ov5695_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov5695 *ov5695 = to_ov5695(sd); @@ -850,7 +850,8 @@ static int ov5695_get_fmt(struct v4l2_subdev *sd, mutex_lock(&ov5695->mutex); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - fmt->format = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + fmt->format = *v4l2_subdev_get_try_format(sd, sd_state, + fmt->pad); #else mutex_unlock(&ov5695->mutex); return -EINVAL; @@ -867,7 +868,7 @@ static int ov5695_get_fmt(struct v4l2_subdev *sd, } static int ov5695_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index != 0) @@ -878,7 +879,7 @@ static int ov5695_enum_mbus_code(struct v4l2_subdev *sd, } static int ov5695_enum_frame_sizes(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -946,11 +947,9 @@ static int ov5695_s_stream(struct v4l2_subdev *sd, int on) goto unlock_and_return; if (on) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto unlock_and_return; - } ret = __ov5695_start_stream(ov5695); if (ret) { @@ -1054,7 +1053,7 @@ static int ov5695_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct ov5695 *ov5695 = to_ov5695(sd); struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); const struct ov5695_mode *def_mode = &supported_modes[0]; mutex_lock(&ov5695->mutex); diff --git a/drivers/media/i2c/ov6650.c b/drivers/media/i2c/ov6650.c index 85dd13694bd2..f67412150b16 100644 --- a/drivers/media/i2c/ov6650.c +++ b/drivers/media/i2c/ov6650.c @@ -467,7 +467,7 @@ static int ov6650_s_power(struct v4l2_subdev *sd, int on) } static int ov6650_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -492,7 +492,7 @@ static int ov6650_get_selection(struct v4l2_subdev *sd, } static int ov6650_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -535,7 +535,7 @@ static int ov6650_set_selection(struct v4l2_subdev *sd, } static int ov6650_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -550,9 +550,9 @@ static int ov6650_get_fmt(struct v4l2_subdev *sd, /* update media bus format code and frame size */ if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - mf->width = cfg->try_fmt.width; - mf->height = cfg->try_fmt.height; - mf->code = cfg->try_fmt.code; + mf->width = sd_state->pads->try_fmt.width; + mf->height = sd_state->pads->try_fmt.height; + mf->code = sd_state->pads->try_fmt.code; } else { mf->width = priv->rect.width >> priv->half_scale; @@ -668,7 +668,7 @@ static int ov6650_s_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf) } static int ov6650_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -701,15 +701,15 @@ static int ov6650_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { /* store media bus format code and frame size in pad config */ - cfg->try_fmt.width = mf->width; - cfg->try_fmt.height = mf->height; - cfg->try_fmt.code = mf->code; + sd_state->pads->try_fmt.width = mf->width; + sd_state->pads->try_fmt.height = mf->height; + sd_state->pads->try_fmt.code = mf->code; /* return default mbus frame format updated with pad config */ *mf = ov6650_def_fmt; - mf->width = cfg->try_fmt.width; - mf->height = cfg->try_fmt.height; - mf->code = cfg->try_fmt.code; + mf->width = sd_state->pads->try_fmt.width; + mf->height = sd_state->pads->try_fmt.height; + mf->code = sd_state->pads->try_fmt.code; } else { /* apply new media bus format code and frame size */ @@ -728,7 +728,7 @@ static int ov6650_set_fmt(struct v4l2_subdev *sd, } static int ov6650_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(ov6650_codes)) diff --git a/drivers/media/i2c/ov7251.c b/drivers/media/i2c/ov7251.c index 0c10203f822b..ebb299f207e5 100644 --- a/drivers/media/i2c/ov7251.c +++ b/drivers/media/i2c/ov7251.c @@ -898,7 +898,7 @@ static const struct v4l2_ctrl_ops ov7251_ctrl_ops = { }; static int ov7251_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -910,7 +910,7 @@ static int ov7251_enum_mbus_code(struct v4l2_subdev *sd, } static int ov7251_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->code != MEDIA_BUS_FMT_Y10_1X10) @@ -928,7 +928,7 @@ static int ov7251_enum_frame_size(struct v4l2_subdev *subdev, } static int ov7251_enum_frame_ival(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { unsigned int index = fie->index; @@ -950,13 +950,13 @@ static int ov7251_enum_frame_ival(struct v4l2_subdev *subdev, static struct v4l2_mbus_framefmt * __ov7251_get_pad_format(struct ov7251 *ov7251, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&ov7251->sd, cfg, pad); + return v4l2_subdev_get_try_format(&ov7251->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &ov7251->fmt; default: @@ -965,13 +965,14 @@ __ov7251_get_pad_format(struct ov7251 *ov7251, } static int ov7251_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov7251 *ov7251 = to_ov7251(sd); mutex_lock(&ov7251->lock); - format->format = *__ov7251_get_pad_format(ov7251, cfg, format->pad, + format->format = *__ov7251_get_pad_format(ov7251, sd_state, + format->pad, format->which); mutex_unlock(&ov7251->lock); @@ -979,12 +980,13 @@ static int ov7251_get_format(struct v4l2_subdev *sd, } static struct v4l2_rect * -__ov7251_get_pad_crop(struct ov7251 *ov7251, struct v4l2_subdev_pad_config *cfg, +__ov7251_get_pad_crop(struct ov7251 *ov7251, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_crop(&ov7251->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&ov7251->sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &ov7251->crop; default: @@ -1027,7 +1029,7 @@ ov7251_find_mode_by_ival(struct ov7251 *ov7251, struct v4l2_fract *timeperframe) } static int ov7251_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov7251 *ov7251 = to_ov7251(sd); @@ -1038,7 +1040,8 @@ static int ov7251_set_format(struct v4l2_subdev *sd, mutex_lock(&ov7251->lock); - __crop = __ov7251_get_pad_crop(ov7251, cfg, format->pad, format->which); + __crop = __ov7251_get_pad_crop(ov7251, sd_state, format->pad, + format->which); new_mode = v4l2_find_nearest_size(ov7251_mode_info_data, ARRAY_SIZE(ov7251_mode_info_data), @@ -1077,7 +1080,7 @@ static int ov7251_set_format(struct v4l2_subdev *sd, ov7251->current_mode = new_mode; } - __format = __ov7251_get_pad_format(ov7251, cfg, format->pad, + __format = __ov7251_get_pad_format(ov7251, sd_state, format->pad, format->which); __format->width = __crop->width; __format->height = __crop->height; @@ -1098,24 +1101,24 @@ static int ov7251_set_format(struct v4l2_subdev *sd, } static int ov7251_entity_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format fmt = { - .which = cfg ? V4L2_SUBDEV_FORMAT_TRY - : V4L2_SUBDEV_FORMAT_ACTIVE, + .which = sd_state ? V4L2_SUBDEV_FORMAT_TRY + : V4L2_SUBDEV_FORMAT_ACTIVE, .format = { .width = 640, .height = 480 } }; - ov7251_set_format(subdev, cfg, &fmt); + ov7251_set_format(subdev, sd_state, &fmt); return 0; } static int ov7251_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ov7251 *ov7251 = to_ov7251(sd); @@ -1124,7 +1127,7 @@ static int ov7251_get_selection(struct v4l2_subdev *sd, return -EINVAL; mutex_lock(&ov7251->lock); - sel->r = *__ov7251_get_pad_crop(ov7251, cfg, sel->pad, + sel->r = *__ov7251_get_pad_crop(ov7251, sd_state, sel->pad, sel->which); mutex_unlock(&ov7251->lock); diff --git a/drivers/media/i2c/ov7670.c b/drivers/media/i2c/ov7670.c index d2df811b1a40..196746423116 100644 --- a/drivers/media/i2c/ov7670.c +++ b/drivers/media/i2c/ov7670.c @@ -960,7 +960,7 @@ static int ov7670_set_hw(struct v4l2_subdev *sd, int hstart, int hstop, static int ov7670_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= N_OV7670_FMTS) @@ -1105,7 +1105,7 @@ static int ov7670_apply_fmt(struct v4l2_subdev *sd) * Set a format. */ static int ov7670_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov7670_info *info = to_state(sd); @@ -1122,7 +1122,8 @@ static int ov7670_set_fmt(struct v4l2_subdev *sd, if (ret) return ret; #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - mbus_fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + mbus_fmt = v4l2_subdev_get_try_format(sd, sd_state, + format->pad); *mbus_fmt = format->format; #endif return 0; @@ -1144,7 +1145,7 @@ static int ov7670_set_fmt(struct v4l2_subdev *sd, } static int ov7670_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov7670_info *info = to_state(sd); @@ -1154,7 +1155,7 @@ static int ov7670_get_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - mbus_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + mbus_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); format->format = *mbus_fmt; return 0; #else @@ -1202,7 +1203,7 @@ static int ov7670_s_frame_interval(struct v4l2_subdev *sd, static int ov7670_frame_rates[] = { 30, 15, 10, 5, 1 }; static int ov7670_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct ov7670_info *info = to_state(sd); @@ -1241,7 +1242,7 @@ static int ov7670_enum_frame_interval(struct v4l2_subdev *sd, * Frame size enumeration */ static int ov7670_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct ov7670_info *info = to_state(sd); @@ -1724,7 +1725,7 @@ static void ov7670_get_default_format(struct v4l2_subdev *sd, static int ov7670_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct v4l2_mbus_framefmt *format = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); ov7670_get_default_format(sd, format); diff --git a/drivers/media/i2c/ov772x.c b/drivers/media/i2c/ov772x.c index d94cf2d39c2a..78602a2f70b0 100644 --- a/drivers/media/i2c/ov772x.c +++ b/drivers/media/i2c/ov772x.c @@ -1157,7 +1157,7 @@ static int ov772x_set_params(struct ov772x_priv *priv, } static int ov772x_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct ov772x_priv *priv = to_ov772x(sd); @@ -1179,7 +1179,7 @@ static int ov772x_get_selection(struct v4l2_subdev *sd, } static int ov772x_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -1198,7 +1198,7 @@ static int ov772x_get_fmt(struct v4l2_subdev *sd, } static int ov772x_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov772x_priv *priv = to_ov772x(sd); @@ -1222,7 +1222,7 @@ static int ov772x_set_fmt(struct v4l2_subdev *sd, mf->xfer_func = V4L2_XFER_FUNC_DEFAULT; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } @@ -1320,7 +1320,7 @@ static const struct v4l2_subdev_core_ops ov772x_subdev_core_ops = { }; static int ov772x_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { if (fie->pad || fie->index >= ARRAY_SIZE(ov772x_frame_intervals)) @@ -1338,7 +1338,7 @@ static int ov772x_enum_frame_interval(struct v4l2_subdev *sd, } static int ov772x_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(ov772x_cfmts)) diff --git a/drivers/media/i2c/ov7740.c b/drivers/media/i2c/ov7740.c index 47a9003d29d6..2539cfee85c8 100644 --- a/drivers/media/i2c/ov7740.c +++ b/drivers/media/i2c/ov7740.c @@ -624,11 +624,9 @@ static int ov7740_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); - if (ret < 0) { - pm_runtime_put_noidle(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); + if (ret < 0) goto err_unlock; - } ret = ov7740_start_streaming(ov7740); if (ret) @@ -709,7 +707,7 @@ static const struct ov7740_pixfmt ov7740_formats[] = { #define N_OV7740_FMTS ARRAY_SIZE(ov7740_formats) static int ov7740_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= N_OV7740_FMTS) @@ -721,7 +719,7 @@ static int ov7740_enum_mbus_code(struct v4l2_subdev *sd, } static int ov7740_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { if (fie->pad) @@ -740,7 +738,7 @@ static int ov7740_enum_frame_interval(struct v4l2_subdev *sd, } static int ov7740_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->pad) @@ -803,7 +801,7 @@ static int ov7740_try_fmt_internal(struct v4l2_subdev *sd, } static int ov7740_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev); @@ -825,7 +823,8 @@ static int ov7740_set_fmt(struct v4l2_subdev *sd, if (ret) goto error; #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - mbus_fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + mbus_fmt = v4l2_subdev_get_try_format(sd, sd_state, + format->pad); *mbus_fmt = format->format; #endif mutex_unlock(&ov7740->mutex); @@ -848,7 +847,7 @@ static int ov7740_set_fmt(struct v4l2_subdev *sd, } static int ov7740_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev); @@ -860,7 +859,7 @@ static int ov7740_get_fmt(struct v4l2_subdev *sd, mutex_lock(&ov7740->mutex); if (format->which == V4L2_SUBDEV_FORMAT_TRY) { #ifdef CONFIG_VIDEO_V4L2_SUBDEV_API - mbus_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + mbus_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); format->format = *mbus_fmt; ret = 0; #else @@ -905,7 +904,7 @@ static int ov7740_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct ov7740 *ov7740 = container_of(sd, struct ov7740, subdev); struct v4l2_mbus_framefmt *format = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); mutex_lock(&ov7740->mutex); ov7740_get_default_format(sd, format); diff --git a/drivers/media/i2c/ov8856.c b/drivers/media/i2c/ov8856.c index e3af3ea277af..88e19f30d376 100644 --- a/drivers/media/i2c/ov8856.c +++ b/drivers/media/i2c/ov8856.c @@ -18,8 +18,6 @@ #define OV8856_REG_VALUE_16BIT 2 #define OV8856_REG_VALUE_24BIT 3 -#define OV8856_LINK_FREQ_360MHZ 360000000ULL -#define OV8856_LINK_FREQ_180MHZ 180000000ULL #define OV8856_SCLK 144000000ULL #define OV8856_XVCLK_19_2 19200000 #define OV8856_DATA_LANES 4 @@ -78,6 +76,29 @@ #define OV8856_TEST_PATTERN_ENABLE BIT(7) #define OV8856_TEST_PATTERN_BAR_SHIFT 2 +#define NUM_REGS 7 +#define NUM_MODE_REGS 187 +#define NUM_MODE_REGS_2 200 + +/* Flip Mirror Controls from sensor */ +#define OV8856_REG_FORMAT1 0x3820 +#define OV8856_REG_FORMAT2 0x3821 +#define OV8856_REG_FORMAT1_OP_1 BIT(1) +#define OV8856_REG_FORMAT1_OP_2 BIT(2) +#define OV8856_REG_FORMAT1_OP_3 BIT(6) +#define OV8856_REG_FORMAT2_OP_1 BIT(1) +#define OV8856_REG_FORMAT2_OP_2 BIT(2) +#define OV8856_REG_FORMAT2_OP_3 BIT(6) +#define OV8856_REG_FLIP_OPT_1 0x376b +#define OV8856_REG_FLIP_OPT_2 0x5001 +#define OV8856_REG_FLIP_OPT_3 0x502e +#define OV8856_REG_MIRROR_OPT_1 0x5004 +#define OV8856_REG_FLIP_OP_0 BIT(0) +#define OV8856_REG_FLIP_OP_1 BIT(1) +#define OV8856_REG_FLIP_OP_2 BIT(2) +#define OV8856_REG_MIRROR_OP_1 BIT(1) +#define OV8856_REG_MIRROR_OP_2 BIT(2) + #define to_ov8856(_sd) container_of(_sd, struct ov8856, sd) static const char * const ov8856_supply_names[] = { @@ -86,11 +107,6 @@ static const char * const ov8856_supply_names[] = { "dvdd", /* Digital core power */ }; -enum { - OV8856_LINK_FREQ_720MBPS, - OV8856_LINK_FREQ_360MBPS, -}; - struct ov8856_reg { u16 address; u8 val; @@ -126,812 +142,1234 @@ struct ov8856_mode { /* Sensor register settings for this resolution */ const struct ov8856_reg_list reg_list; + + /* Number of data lanes */ + u8 data_lanes; }; -static const struct ov8856_reg mipi_data_rate_720mbps[] = { - {0x0103, 0x01}, - {0x0100, 0x00}, - {0x0302, 0x4b}, - {0x0303, 0x01}, - {0x030b, 0x02}, - {0x030d, 0x4b}, - {0x031e, 0x0c}, +struct ov8856_mipi_data_rates { + const struct ov8856_reg regs_0[NUM_REGS]; + const struct ov8856_reg regs_1[NUM_REGS]; }; -static const struct ov8856_reg mipi_data_rate_360mbps[] = { - {0x0103, 0x01}, - {0x0100, 0x00}, - {0x0302, 0x4b}, - {0x0303, 0x03}, - {0x030b, 0x02}, - {0x030d, 0x4b}, - {0x031e, 0x0c}, +static const struct ov8856_mipi_data_rates mipi_data_rate_lane_2 = { + //mipi_data_rate_1440mbps + { + {0x0103, 0x01}, + {0x0100, 0x00}, + {0x0302, 0x43}, + {0x0303, 0x00}, + {0x030b, 0x02}, + {0x030d, 0x4b}, + {0x031e, 0x0c} + }, + //mipi_data_rate_720mbps + { + {0x0103, 0x01}, + {0x0100, 0x00}, + {0x0302, 0x4b}, + {0x0303, 0x01}, + {0x030b, 0x02}, + {0x030d, 0x4b}, + {0x031e, 0x0c} + } }; -static const struct ov8856_reg mode_3280x2464_regs[] = { - {0x3000, 0x20}, - {0x3003, 0x08}, - {0x300e, 0x20}, - {0x3010, 0x00}, - {0x3015, 0x84}, - {0x3018, 0x72}, - {0x3021, 0x23}, - {0x3033, 0x24}, - {0x3500, 0x00}, - {0x3501, 0x9a}, - {0x3502, 0x20}, - {0x3503, 0x08}, - {0x3505, 0x83}, - {0x3508, 0x01}, - {0x3509, 0x80}, - {0x350c, 0x00}, - {0x350d, 0x80}, - {0x350e, 0x04}, - {0x350f, 0x00}, - {0x3510, 0x00}, - {0x3511, 0x02}, - {0x3512, 0x00}, - {0x3600, 0x72}, - {0x3601, 0x40}, - {0x3602, 0x30}, - {0x3610, 0xc5}, - {0x3611, 0x58}, - {0x3612, 0x5c}, - {0x3613, 0xca}, - {0x3614, 0x20}, - {0x3628, 0xff}, - {0x3629, 0xff}, - {0x362a, 0xff}, - {0x3633, 0x10}, - {0x3634, 0x10}, - {0x3635, 0x10}, - {0x3636, 0x10}, - {0x3663, 0x08}, - {0x3669, 0x34}, - {0x366e, 0x10}, - {0x3706, 0x86}, - {0x370b, 0x7e}, - {0x3714, 0x23}, - {0x3730, 0x12}, - {0x3733, 0x10}, - {0x3764, 0x00}, - {0x3765, 0x00}, - {0x3769, 0x62}, - {0x376a, 0x2a}, - {0x376b, 0x30}, - {0x3780, 0x00}, - {0x3781, 0x24}, - {0x3782, 0x00}, - {0x3783, 0x23}, - {0x3798, 0x2f}, - {0x37a1, 0x60}, - {0x37a8, 0x6a}, - {0x37ab, 0x3f}, - {0x37c2, 0x04}, - {0x37c3, 0xf1}, - {0x37c9, 0x80}, - {0x37cb, 0x16}, - {0x37cc, 0x16}, - {0x37cd, 0x16}, - {0x37ce, 0x16}, - {0x3800, 0x00}, - {0x3801, 0x00}, - {0x3802, 0x00}, - {0x3803, 0x06}, - {0x3804, 0x0c}, - {0x3805, 0xdf}, - {0x3806, 0x09}, - {0x3807, 0xa7}, - {0x3808, 0x0c}, - {0x3809, 0xd0}, - {0x380a, 0x09}, - {0x380b, 0xa0}, - {0x380c, 0x07}, - {0x380d, 0x88}, - {0x380e, 0x09}, - {0x380f, 0xb8}, - {0x3810, 0x00}, - {0x3811, 0x00}, - {0x3812, 0x00}, - {0x3813, 0x01}, - {0x3814, 0x01}, - {0x3815, 0x01}, - {0x3816, 0x00}, - {0x3817, 0x00}, - {0x3818, 0x00}, - {0x3819, 0x10}, - {0x3820, 0x80}, - {0x3821, 0x46}, - {0x382a, 0x01}, - {0x382b, 0x01}, - {0x3830, 0x06}, - {0x3836, 0x02}, - {0x3862, 0x04}, - {0x3863, 0x08}, - {0x3cc0, 0x33}, - {0x3d85, 0x17}, - {0x3d8c, 0x73}, - {0x3d8d, 0xde}, - {0x4001, 0xe0}, - {0x4003, 0x40}, - {0x4008, 0x00}, - {0x4009, 0x0b}, - {0x400a, 0x00}, - {0x400b, 0x84}, - {0x400f, 0x80}, - {0x4010, 0xf0}, - {0x4011, 0xff}, - {0x4012, 0x02}, - {0x4013, 0x01}, - {0x4014, 0x01}, - {0x4015, 0x01}, - {0x4042, 0x00}, - {0x4043, 0x80}, - {0x4044, 0x00}, - {0x4045, 0x80}, - {0x4046, 0x00}, - {0x4047, 0x80}, - {0x4048, 0x00}, - {0x4049, 0x80}, - {0x4041, 0x03}, - {0x404c, 0x20}, - {0x404d, 0x00}, - {0x404e, 0x20}, - {0x4203, 0x80}, - {0x4307, 0x30}, - {0x4317, 0x00}, - {0x4503, 0x08}, - {0x4601, 0x80}, - {0x4800, 0x44}, - {0x4816, 0x53}, - {0x481b, 0x58}, - {0x481f, 0x27}, - {0x4837, 0x16}, - {0x483c, 0x0f}, - {0x484b, 0x05}, - {0x5000, 0x57}, - {0x5001, 0x0a}, - {0x5004, 0x04}, - {0x502e, 0x03}, - {0x5030, 0x41}, - {0x5780, 0x14}, - {0x5781, 0x0f}, - {0x5782, 0x44}, - {0x5783, 0x02}, - {0x5784, 0x01}, - {0x5785, 0x01}, - {0x5786, 0x00}, - {0x5787, 0x04}, - {0x5788, 0x02}, - {0x5789, 0x0f}, - {0x578a, 0xfd}, - {0x578b, 0xf5}, - {0x578c, 0xf5}, - {0x578d, 0x03}, - {0x578e, 0x08}, - {0x578f, 0x0c}, - {0x5790, 0x08}, - {0x5791, 0x04}, - {0x5792, 0x00}, - {0x5793, 0x52}, - {0x5794, 0xa3}, - {0x5795, 0x02}, - {0x5796, 0x20}, - {0x5797, 0x20}, - {0x5798, 0xd5}, - {0x5799, 0xd5}, - {0x579a, 0x00}, - {0x579b, 0x50}, - {0x579c, 0x00}, - {0x579d, 0x2c}, - {0x579e, 0x0c}, - {0x579f, 0x40}, - {0x57a0, 0x09}, - {0x57a1, 0x40}, - {0x59f8, 0x3d}, - {0x5a08, 0x02}, - {0x5b00, 0x02}, - {0x5b01, 0x10}, - {0x5b02, 0x03}, - {0x5b03, 0xcf}, - {0x5b05, 0x6c}, - {0x5e00, 0x00} +static const struct ov8856_mipi_data_rates mipi_data_rate_lane_4 = { + //mipi_data_rate_720mbps + { + {0x0103, 0x01}, + {0x0100, 0x00}, + {0x0302, 0x4b}, + {0x0303, 0x01}, + {0x030b, 0x02}, + {0x030d, 0x4b}, + {0x031e, 0x0c} + }, + //mipi_data_rate_360mbps + { + {0x0103, 0x01}, + {0x0100, 0x00}, + {0x0302, 0x4b}, + {0x0303, 0x03}, + {0x030b, 0x02}, + {0x030d, 0x4b}, + {0x031e, 0x0c} + } }; -static const struct ov8856_reg mode_3264x2448_regs[] = { - {0x0103, 0x01}, - {0x0302, 0x3c}, - {0x0303, 0x01}, - {0x031e, 0x0c}, - {0x3000, 0x20}, - {0x3003, 0x08}, - {0x300e, 0x20}, - {0x3010, 0x00}, - {0x3015, 0x84}, - {0x3018, 0x72}, - {0x3021, 0x23}, - {0x3033, 0x24}, - {0x3500, 0x00}, - {0x3501, 0x9a}, - {0x3502, 0x20}, - {0x3503, 0x08}, - {0x3505, 0x83}, - {0x3508, 0x01}, - {0x3509, 0x80}, - {0x350c, 0x00}, - {0x350d, 0x80}, - {0x350e, 0x04}, - {0x350f, 0x00}, - {0x3510, 0x00}, - {0x3511, 0x02}, - {0x3512, 0x00}, - {0x3600, 0x72}, - {0x3601, 0x40}, - {0x3602, 0x30}, - {0x3610, 0xc5}, - {0x3611, 0x58}, - {0x3612, 0x5c}, - {0x3613, 0xca}, - {0x3614, 0x60}, - {0x3628, 0xff}, - {0x3629, 0xff}, - {0x362a, 0xff}, - {0x3633, 0x10}, - {0x3634, 0x10}, - {0x3635, 0x10}, - {0x3636, 0x10}, - {0x3663, 0x08}, - {0x3669, 0x34}, - {0x366d, 0x00}, - {0x366e, 0x10}, - {0x3706, 0x86}, - {0x370b, 0x7e}, - {0x3714, 0x23}, - {0x3730, 0x12}, - {0x3733, 0x10}, - {0x3764, 0x00}, - {0x3765, 0x00}, - {0x3769, 0x62}, - {0x376a, 0x2a}, - {0x376b, 0x30}, - {0x3780, 0x00}, - {0x3781, 0x24}, - {0x3782, 0x00}, - {0x3783, 0x23}, - {0x3798, 0x2f}, - {0x37a1, 0x60}, - {0x37a8, 0x6a}, - {0x37ab, 0x3f}, - {0x37c2, 0x04}, - {0x37c3, 0xf1}, - {0x37c9, 0x80}, - {0x37cb, 0x16}, - {0x37cc, 0x16}, - {0x37cd, 0x16}, - {0x37ce, 0x16}, - {0x3800, 0x00}, - {0x3801, 0x00}, - {0x3802, 0x00}, - {0x3803, 0x0c}, - {0x3804, 0x0c}, - {0x3805, 0xdf}, - {0x3806, 0x09}, - {0x3807, 0xa3}, - {0x3808, 0x0c}, - {0x3809, 0xc0}, - {0x380a, 0x09}, - {0x380b, 0x90}, - {0x380c, 0x07}, - {0x380d, 0x8c}, - {0x380e, 0x09}, - {0x380f, 0xb2}, - {0x3810, 0x00}, - {0x3811, 0x04}, - {0x3812, 0x00}, - {0x3813, 0x01}, - {0x3814, 0x01}, - {0x3815, 0x01}, - {0x3816, 0x00}, - {0x3817, 0x00}, - {0x3818, 0x00}, - {0x3819, 0x10}, - {0x3820, 0x80}, - {0x3821, 0x46}, - {0x382a, 0x01}, - {0x382b, 0x01}, - {0x3830, 0x06}, - {0x3836, 0x02}, - {0x3862, 0x04}, - {0x3863, 0x08}, - {0x3cc0, 0x33}, - {0x3d85, 0x17}, - {0x3d8c, 0x73}, - {0x3d8d, 0xde}, - {0x4001, 0xe0}, - {0x4003, 0x40}, - {0x4008, 0x00}, - {0x4009, 0x0b}, - {0x400a, 0x00}, - {0x400b, 0x84}, - {0x400f, 0x80}, - {0x4010, 0xf0}, - {0x4011, 0xff}, - {0x4012, 0x02}, - {0x4013, 0x01}, - {0x4014, 0x01}, - {0x4015, 0x01}, - {0x4042, 0x00}, - {0x4043, 0x80}, - {0x4044, 0x00}, - {0x4045, 0x80}, - {0x4046, 0x00}, - {0x4047, 0x80}, - {0x4048, 0x00}, - {0x4049, 0x80}, - {0x4041, 0x03}, - {0x404c, 0x20}, - {0x404d, 0x00}, - {0x404e, 0x20}, - {0x4203, 0x80}, - {0x4307, 0x30}, - {0x4317, 0x00}, - {0x4502, 0x50}, - {0x4503, 0x08}, - {0x4601, 0x80}, - {0x4800, 0x44}, - {0x4816, 0x53}, - {0x481b, 0x50}, - {0x481f, 0x27}, - {0x4823, 0x3c}, - {0x482b, 0x00}, - {0x4831, 0x66}, - {0x4837, 0x16}, - {0x483c, 0x0f}, - {0x484b, 0x05}, - {0x5000, 0x77}, - {0x5001, 0x0a}, - {0x5003, 0xc8}, - {0x5004, 0x04}, - {0x5006, 0x00}, - {0x5007, 0x00}, - {0x502e, 0x03}, - {0x5030, 0x41}, - {0x5780, 0x14}, - {0x5781, 0x0f}, - {0x5782, 0x44}, - {0x5783, 0x02}, - {0x5784, 0x01}, - {0x5785, 0x01}, - {0x5786, 0x00}, - {0x5787, 0x04}, - {0x5788, 0x02}, - {0x5789, 0x0f}, - {0x578a, 0xfd}, - {0x578b, 0xf5}, - {0x578c, 0xf5}, - {0x578d, 0x03}, - {0x578e, 0x08}, - {0x578f, 0x0c}, - {0x5790, 0x08}, - {0x5791, 0x04}, - {0x5792, 0x00}, - {0x5793, 0x52}, - {0x5794, 0xa3}, - {0x5795, 0x02}, - {0x5796, 0x20}, - {0x5797, 0x20}, - {0x5798, 0xd5}, - {0x5799, 0xd5}, - {0x579a, 0x00}, - {0x579b, 0x50}, - {0x579c, 0x00}, - {0x579d, 0x2c}, - {0x579e, 0x0c}, - {0x579f, 0x40}, - {0x57a0, 0x09}, - {0x57a1, 0x40}, - {0x59f8, 0x3d}, - {0x5a08, 0x02}, - {0x5b00, 0x02}, - {0x5b01, 0x10}, - {0x5b02, 0x03}, - {0x5b03, 0xcf}, - {0x5b05, 0x6c}, - {0x5e00, 0x00}, - {0x5e10, 0xfc} +static const struct ov8856_reg lane_2_mode_3280x2464[] = { + /* 3280x2464 resolution */ + {0x3000, 0x20}, + {0x3003, 0x08}, + {0x300e, 0x20}, + {0x3010, 0x00}, + {0x3015, 0x84}, + {0x3018, 0x32}, + {0x3021, 0x23}, + {0x3033, 0x24}, + {0x3500, 0x00}, + {0x3501, 0x9a}, + {0x3502, 0x20}, + {0x3503, 0x08}, + {0x3505, 0x83}, + {0x3508, 0x01}, + {0x3509, 0x80}, + {0x350c, 0x00}, + {0x350d, 0x80}, + {0x350e, 0x04}, + {0x350f, 0x00}, + {0x3510, 0x00}, + {0x3511, 0x02}, + {0x3512, 0x00}, + {0x3600, 0x72}, + {0x3601, 0x40}, + {0x3602, 0x30}, + {0x3610, 0xc5}, + {0x3611, 0x58}, + {0x3612, 0x5c}, + {0x3613, 0xca}, + {0x3614, 0x50}, + {0x3628, 0xff}, + {0x3629, 0xff}, + {0x362a, 0xff}, + {0x3633, 0x10}, + {0x3634, 0x10}, + {0x3635, 0x10}, + {0x3636, 0x10}, + {0x3663, 0x08}, + {0x3669, 0x34}, + {0x366e, 0x10}, + {0x3706, 0x86}, + {0x370b, 0x7e}, + {0x3714, 0x23}, + {0x3730, 0x12}, + {0x3733, 0x10}, + {0x3764, 0x00}, + {0x3765, 0x00}, + {0x3769, 0x62}, + {0x376a, 0x2a}, + {0x376b, 0x30}, + {0x3780, 0x00}, + {0x3781, 0x24}, + {0x3782, 0x00}, + {0x3783, 0x23}, + {0x3798, 0x2f}, + {0x37a1, 0x60}, + {0x37a8, 0x6a}, + {0x37ab, 0x3f}, + {0x37c2, 0x04}, + {0x37c3, 0xf1}, + {0x37c9, 0x80}, + {0x37cb, 0x16}, + {0x37cc, 0x16}, + {0x37cd, 0x16}, + {0x37ce, 0x16}, + {0x3800, 0x00}, + {0x3801, 0x00}, + {0x3802, 0x00}, + {0x3803, 0x06}, + {0x3804, 0x0c}, + {0x3805, 0xdf}, + {0x3806, 0x09}, + {0x3807, 0xa7}, + {0x3808, 0x0c}, + {0x3809, 0xd0}, + {0x380a, 0x09}, + {0x380b, 0xa0}, + {0x380c, 0x07}, + {0x380d, 0x88}, + {0x380e, 0x09}, + {0x380f, 0xb8}, + {0x3810, 0x00}, + {0x3811, 0x00}, + {0x3812, 0x00}, + {0x3813, 0x01}, + {0x3814, 0x01}, + {0x3815, 0x01}, + {0x3816, 0x00}, + {0x3817, 0x00}, + {0x3818, 0x00}, + {0x3819, 0x00}, + {0x3820, 0x80}, + {0x3821, 0x46}, + {0x382a, 0x01}, + {0x382b, 0x01}, + {0x3830, 0x06}, + {0x3836, 0x02}, + {0x3837, 0x10}, + {0x3862, 0x04}, + {0x3863, 0x08}, + {0x3cc0, 0x33}, + {0x3d85, 0x14}, + {0x3d8c, 0x73}, + {0x3d8d, 0xde}, + {0x4001, 0xe0}, + {0x4003, 0x40}, + {0x4008, 0x00}, + {0x4009, 0x0b}, + {0x400a, 0x00}, + {0x400b, 0x84}, + {0x400f, 0x80}, + {0x4010, 0xf0}, + {0x4011, 0xff}, + {0x4012, 0x02}, + {0x4013, 0x01}, + {0x4014, 0x01}, + {0x4015, 0x01}, + {0x4042, 0x00}, + {0x4043, 0x80}, + {0x4044, 0x00}, + {0x4045, 0x80}, + {0x4046, 0x00}, + {0x4047, 0x80}, + {0x4048, 0x00}, + {0x4049, 0x80}, + {0x4041, 0x03}, + {0x404c, 0x20}, + {0x404d, 0x00}, + {0x404e, 0x20}, + {0x4203, 0x80}, + {0x4307, 0x30}, + {0x4317, 0x00}, + {0x4503, 0x08}, + {0x4601, 0x80}, + {0x4800, 0x44}, + {0x4816, 0x53}, + {0x481b, 0x58}, + {0x481f, 0x27}, + {0x4837, 0x0c}, + {0x483c, 0x0f}, + {0x484b, 0x05}, + {0x5000, 0x57}, + {0x5001, 0x0a}, + {0x5004, 0x04}, + {0x502e, 0x03}, + {0x5030, 0x41}, + {0x5795, 0x02}, + {0x5796, 0x20}, + {0x5797, 0x20}, + {0x5798, 0xd5}, + {0x5799, 0xd5}, + {0x579a, 0x00}, + {0x579b, 0x50}, + {0x579c, 0x00}, + {0x579d, 0x2c}, + {0x579e, 0x0c}, + {0x579f, 0x40}, + {0x57a0, 0x09}, + {0x57a1, 0x40}, + {0x5780, 0x14}, + {0x5781, 0x0f}, + {0x5782, 0x44}, + {0x5783, 0x02}, + {0x5784, 0x01}, + {0x5785, 0x01}, + {0x5786, 0x00}, + {0x5787, 0x04}, + {0x5788, 0x02}, + {0x5789, 0x0f}, + {0x578a, 0xfd}, + {0x578b, 0xf5}, + {0x578c, 0xf5}, + {0x578d, 0x03}, + {0x578e, 0x08}, + {0x578f, 0x0c}, + {0x5790, 0x08}, + {0x5791, 0x04}, + {0x5792, 0x00}, + {0x5793, 0x52}, + {0x5794, 0xa3}, + {0x59f8, 0x3d}, + {0x5a08, 0x02}, + {0x5b00, 0x02}, + {0x5b01, 0x10}, + {0x5b02, 0x03}, + {0x5b03, 0xcf}, + {0x5b05, 0x6c}, + {0x5e00, 0x00} }; -static const struct ov8856_reg mode_1640x1232_regs[] = { - {0x3000, 0x20}, - {0x3003, 0x08}, - {0x300e, 0x20}, - {0x3010, 0x00}, - {0x3015, 0x84}, - {0x3018, 0x72}, - {0x3021, 0x23}, - {0x3033, 0x24}, - {0x3500, 0x00}, - {0x3501, 0x4c}, - {0x3502, 0xe0}, - {0x3503, 0x08}, - {0x3505, 0x83}, - {0x3508, 0x01}, - {0x3509, 0x80}, - {0x350c, 0x00}, - {0x350d, 0x80}, - {0x350e, 0x04}, - {0x350f, 0x00}, - {0x3510, 0x00}, - {0x3511, 0x02}, - {0x3512, 0x00}, - {0x3600, 0x72}, - {0x3601, 0x40}, - {0x3602, 0x30}, - {0x3610, 0xc5}, - {0x3611, 0x58}, - {0x3612, 0x5c}, - {0x3613, 0xca}, - {0x3614, 0x20}, - {0x3628, 0xff}, - {0x3629, 0xff}, - {0x362a, 0xff}, - {0x3633, 0x10}, - {0x3634, 0x10}, - {0x3635, 0x10}, - {0x3636, 0x10}, - {0x3663, 0x08}, - {0x3669, 0x34}, - {0x366e, 0x08}, - {0x3706, 0x86}, - {0x370b, 0x7e}, - {0x3714, 0x27}, - {0x3730, 0x12}, - {0x3733, 0x10}, - {0x3764, 0x00}, - {0x3765, 0x00}, - {0x3769, 0x62}, - {0x376a, 0x2a}, - {0x376b, 0x30}, - {0x3780, 0x00}, - {0x3781, 0x24}, - {0x3782, 0x00}, - {0x3783, 0x23}, - {0x3798, 0x2f}, - {0x37a1, 0x60}, - {0x37a8, 0x6a}, - {0x37ab, 0x3f}, - {0x37c2, 0x14}, - {0x37c3, 0xf1}, - {0x37c9, 0x80}, - {0x37cb, 0x16}, - {0x37cc, 0x16}, - {0x37cd, 0x16}, - {0x37ce, 0x16}, - {0x3800, 0x00}, - {0x3801, 0x00}, - {0x3802, 0x00}, - {0x3803, 0x06}, - {0x3804, 0x0c}, - {0x3805, 0xdf}, - {0x3806, 0x09}, - {0x3807, 0xa7}, - {0x3808, 0x06}, - {0x3809, 0x68}, - {0x380a, 0x04}, - {0x380b, 0xd0}, - {0x380c, 0x0e}, - {0x380d, 0xec}, - {0x380e, 0x04}, - {0x380f, 0xe8}, - {0x3810, 0x00}, - {0x3811, 0x00}, - {0x3812, 0x00}, - {0x3813, 0x01}, - {0x3814, 0x03}, - {0x3815, 0x01}, - {0x3816, 0x00}, - {0x3817, 0x00}, - {0x3818, 0x00}, - {0x3819, 0x10}, - {0x3820, 0x90}, - {0x3821, 0x67}, - {0x382a, 0x03}, - {0x382b, 0x01}, - {0x3830, 0x06}, - {0x3836, 0x02}, - {0x3862, 0x04}, - {0x3863, 0x08}, - {0x3cc0, 0x33}, - {0x3d85, 0x17}, - {0x3d8c, 0x73}, - {0x3d8d, 0xde}, - {0x4001, 0xe0}, - {0x4003, 0x40}, - {0x4008, 0x00}, - {0x4009, 0x05}, - {0x400a, 0x00}, - {0x400b, 0x84}, - {0x400f, 0x80}, - {0x4010, 0xf0}, - {0x4011, 0xff}, - {0x4012, 0x02}, - {0x4013, 0x01}, - {0x4014, 0x01}, - {0x4015, 0x01}, - {0x4042, 0x00}, - {0x4043, 0x80}, - {0x4044, 0x00}, - {0x4045, 0x80}, - {0x4046, 0x00}, - {0x4047, 0x80}, - {0x4048, 0x00}, - {0x4049, 0x80}, - {0x4041, 0x03}, - {0x404c, 0x20}, - {0x404d, 0x00}, - {0x404e, 0x20}, - {0x4203, 0x80}, - {0x4307, 0x30}, - {0x4317, 0x00}, - {0x4503, 0x08}, - {0x4601, 0x80}, - {0x4800, 0x44}, - {0x4816, 0x53}, - {0x481b, 0x58}, - {0x481f, 0x27}, - {0x4837, 0x16}, - {0x483c, 0x0f}, - {0x484b, 0x05}, - {0x5000, 0x57}, - {0x5001, 0x0a}, - {0x5004, 0x04}, - {0x502e, 0x03}, - {0x5030, 0x41}, - {0x5780, 0x14}, - {0x5781, 0x0f}, - {0x5782, 0x44}, - {0x5783, 0x02}, - {0x5784, 0x01}, - {0x5785, 0x01}, - {0x5786, 0x00}, - {0x5787, 0x04}, - {0x5788, 0x02}, - {0x5789, 0x0f}, - {0x578a, 0xfd}, - {0x578b, 0xf5}, - {0x578c, 0xf5}, - {0x578d, 0x03}, - {0x578e, 0x08}, - {0x578f, 0x0c}, - {0x5790, 0x08}, - {0x5791, 0x04}, - {0x5792, 0x00}, - {0x5793, 0x52}, - {0x5794, 0xa3}, - {0x5795, 0x00}, - {0x5796, 0x10}, - {0x5797, 0x10}, - {0x5798, 0x73}, - {0x5799, 0x73}, - {0x579a, 0x00}, - {0x579b, 0x28}, - {0x579c, 0x00}, - {0x579d, 0x16}, - {0x579e, 0x06}, - {0x579f, 0x20}, - {0x57a0, 0x04}, - {0x57a1, 0xa0}, - {0x59f8, 0x3d}, - {0x5a08, 0x02}, - {0x5b00, 0x02}, - {0x5b01, 0x10}, - {0x5b02, 0x03}, - {0x5b03, 0xcf}, - {0x5b05, 0x6c}, - {0x5e00, 0x00} +static const struct ov8856_reg lane_2_mode_1640x1232[] = { + /* 1640x1232 resolution */ + {0x3000, 0x20}, + {0x3003, 0x08}, + {0x300e, 0x20}, + {0x3010, 0x00}, + {0x3015, 0x84}, + {0x3018, 0x32}, + {0x3021, 0x23}, + {0x3033, 0x24}, + {0x3500, 0x00}, + {0x3501, 0x4c}, + {0x3502, 0xe0}, + {0x3503, 0x08}, + {0x3505, 0x83}, + {0x3508, 0x01}, + {0x3509, 0x80}, + {0x350c, 0x00}, + {0x350d, 0x80}, + {0x350e, 0x04}, + {0x350f, 0x00}, + {0x3510, 0x00}, + {0x3511, 0x02}, + {0x3512, 0x00}, + {0x3600, 0x72}, + {0x3601, 0x40}, + {0x3602, 0x30}, + {0x3610, 0xc5}, + {0x3611, 0x58}, + {0x3612, 0x5c}, + {0x3613, 0xca}, + {0x3614, 0x50}, + {0x3628, 0xff}, + {0x3629, 0xff}, + {0x362a, 0xff}, + {0x3633, 0x10}, + {0x3634, 0x10}, + {0x3635, 0x10}, + {0x3636, 0x10}, + {0x3663, 0x08}, + {0x3669, 0x34}, + {0x366e, 0x08}, + {0x3706, 0x86}, + {0x370b, 0x7e}, + {0x3714, 0x27}, + {0x3730, 0x12}, + {0x3733, 0x10}, + {0x3764, 0x00}, + {0x3765, 0x00}, + {0x3769, 0x62}, + {0x376a, 0x2a}, + {0x376b, 0x30}, + {0x3780, 0x00}, + {0x3781, 0x24}, + {0x3782, 0x00}, + {0x3783, 0x23}, + {0x3798, 0x2f}, + {0x37a1, 0x60}, + {0x37a8, 0x6a}, + {0x37ab, 0x3f}, + {0x37c2, 0x14}, + {0x37c3, 0xf1}, + {0x37c9, 0x80}, + {0x37cb, 0x16}, + {0x37cc, 0x16}, + {0x37cd, 0x16}, + {0x37ce, 0x16}, + {0x3800, 0x00}, + {0x3801, 0x00}, + {0x3802, 0x00}, + {0x3803, 0x00}, + {0x3804, 0x0c}, + {0x3805, 0xdf}, + {0x3806, 0x09}, + {0x3807, 0xaf}, + {0x3808, 0x06}, + {0x3809, 0x68}, + {0x380a, 0x04}, + {0x380b, 0xd0}, + {0x380c, 0x0c}, + {0x380d, 0x60}, + {0x380e, 0x05}, + {0x380f, 0xea}, + {0x3810, 0x00}, + {0x3811, 0x04}, + {0x3812, 0x00}, + {0x3813, 0x05}, + {0x3814, 0x03}, + {0x3815, 0x01}, + {0x3816, 0x00}, + {0x3817, 0x00}, + {0x3818, 0x00}, + {0x3819, 0x00}, + {0x3820, 0x90}, + {0x3821, 0x67}, + {0x382a, 0x03}, + {0x382b, 0x01}, + {0x3830, 0x06}, + {0x3836, 0x02}, + {0x3837, 0x10}, + {0x3862, 0x04}, + {0x3863, 0x08}, + {0x3cc0, 0x33}, + {0x3d85, 0x14}, + {0x3d8c, 0x73}, + {0x3d8d, 0xde}, + {0x4001, 0xe0}, + {0x4003, 0x40}, + {0x4008, 0x00}, + {0x4009, 0x05}, + {0x400a, 0x00}, + {0x400b, 0x84}, + {0x400f, 0x80}, + {0x4010, 0xf0}, + {0x4011, 0xff}, + {0x4012, 0x02}, + {0x4013, 0x01}, + {0x4014, 0x01}, + {0x4015, 0x01}, + {0x4042, 0x00}, + {0x4043, 0x80}, + {0x4044, 0x00}, + {0x4045, 0x80}, + {0x4046, 0x00}, + {0x4047, 0x80}, + {0x4048, 0x00}, + {0x4049, 0x80}, + {0x4041, 0x03}, + {0x404c, 0x20}, + {0x404d, 0x00}, + {0x404e, 0x20}, + {0x4203, 0x80}, + {0x4307, 0x30}, + {0x4317, 0x00}, + {0x4503, 0x08}, + {0x4601, 0x80}, + {0x4800, 0x44}, + {0x4816, 0x53}, + {0x481b, 0x58}, + {0x481f, 0x27}, + {0x4837, 0x16}, + {0x483c, 0x0f}, + {0x484b, 0x05}, + {0x5000, 0x57}, + {0x5001, 0x0a}, + {0x5004, 0x04}, + {0x502e, 0x03}, + {0x5030, 0x41}, + {0x5795, 0x00}, + {0x5796, 0x10}, + {0x5797, 0x10}, + {0x5798, 0x73}, + {0x5799, 0x73}, + {0x579a, 0x00}, + {0x579b, 0x28}, + {0x579c, 0x00}, + {0x579d, 0x16}, + {0x579e, 0x06}, + {0x579f, 0x20}, + {0x57a0, 0x04}, + {0x57a1, 0xa0}, + {0x5780, 0x14}, + {0x5781, 0x0f}, + {0x5782, 0x44}, + {0x5783, 0x02}, + {0x5784, 0x01}, + {0x5785, 0x01}, + {0x5786, 0x00}, + {0x5787, 0x04}, + {0x5788, 0x02}, + {0x5789, 0x0f}, + {0x578a, 0xfd}, + {0x578b, 0xf5}, + {0x578c, 0xf5}, + {0x578d, 0x03}, + {0x578e, 0x08}, + {0x578f, 0x0c}, + {0x5790, 0x08}, + {0x5791, 0x04}, + {0x5792, 0x00}, + {0x5793, 0x52}, + {0x5794, 0xa3}, + {0x59f8, 0x3d}, + {0x5a08, 0x02}, + {0x5b00, 0x02}, + {0x5b01, 0x10}, + {0x5b02, 0x03}, + {0x5b03, 0xcf}, + {0x5b05, 0x6c}, + {0x5e00, 0x00} }; -static const struct ov8856_reg mode_1632x1224_regs[] = { - {0x0103, 0x01}, - {0x0302, 0x3c}, - {0x0303, 0x01}, - {0x031e, 0x0c}, - {0x3000, 0x20}, - {0x3003, 0x08}, - {0x300e, 0x20}, - {0x3010, 0x00}, - {0x3015, 0x84}, - {0x3018, 0x72}, - {0x3021, 0x23}, - {0x3033, 0x24}, - {0x3500, 0x00}, - {0x3501, 0x4c}, - {0x3502, 0xe0}, - {0x3503, 0x08}, - {0x3505, 0x83}, - {0x3508, 0x01}, - {0x3509, 0x80}, - {0x350c, 0x00}, - {0x350d, 0x80}, - {0x350e, 0x04}, - {0x350f, 0x00}, - {0x3510, 0x00}, - {0x3511, 0x02}, - {0x3512, 0x00}, - {0x3600, 0x72}, - {0x3601, 0x40}, - {0x3602, 0x30}, - {0x3610, 0xc5}, - {0x3611, 0x58}, - {0x3612, 0x5c}, - {0x3613, 0xca}, - {0x3614, 0x60}, - {0x3628, 0xff}, - {0x3629, 0xff}, - {0x362a, 0xff}, - {0x3633, 0x10}, - {0x3634, 0x10}, - {0x3635, 0x10}, - {0x3636, 0x10}, - {0x3663, 0x08}, - {0x3669, 0x34}, - {0x366d, 0x00}, - {0x366e, 0x08}, - {0x3706, 0x86}, - {0x370b, 0x7e}, - {0x3714, 0x27}, - {0x3730, 0x12}, - {0x3733, 0x10}, - {0x3764, 0x00}, - {0x3765, 0x00}, - {0x3769, 0x62}, - {0x376a, 0x2a}, - {0x376b, 0x30}, - {0x3780, 0x00}, - {0x3781, 0x24}, - {0x3782, 0x00}, - {0x3783, 0x23}, - {0x3798, 0x2f}, - {0x37a1, 0x60}, - {0x37a8, 0x6a}, - {0x37ab, 0x3f}, - {0x37c2, 0x14}, - {0x37c3, 0xf1}, - {0x37c9, 0x80}, - {0x37cb, 0x16}, - {0x37cc, 0x16}, - {0x37cd, 0x16}, - {0x37ce, 0x16}, - {0x3800, 0x00}, - {0x3801, 0x00}, - {0x3802, 0x00}, - {0x3803, 0x0c}, - {0x3804, 0x0c}, - {0x3805, 0xdf}, - {0x3806, 0x09}, - {0x3807, 0xa3}, - {0x3808, 0x06}, - {0x3809, 0x60}, - {0x380a, 0x04}, - {0x380b, 0xc8}, - {0x380c, 0x07}, - {0x380d, 0x8c}, - {0x380e, 0x09}, - {0x380f, 0xb2}, - {0x3810, 0x00}, - {0x3811, 0x02}, - {0x3812, 0x00}, - {0x3813, 0x01}, - {0x3814, 0x03}, - {0x3815, 0x01}, - {0x3816, 0x00}, - {0x3817, 0x00}, - {0x3818, 0x00}, - {0x3819, 0x10}, - {0x3820, 0x80}, - {0x3821, 0x47}, - {0x382a, 0x03}, - {0x382b, 0x01}, - {0x3830, 0x06}, - {0x3836, 0x02}, - {0x3862, 0x04}, - {0x3863, 0x08}, - {0x3cc0, 0x33}, - {0x3d85, 0x17}, - {0x3d8c, 0x73}, - {0x3d8d, 0xde}, - {0x4001, 0xe0}, - {0x4003, 0x40}, - {0x4008, 0x00}, - {0x4009, 0x05}, - {0x400a, 0x00}, - {0x400b, 0x84}, - {0x400f, 0x80}, - {0x4010, 0xf0}, - {0x4011, 0xff}, - {0x4012, 0x02}, - {0x4013, 0x01}, - {0x4014, 0x01}, - {0x4015, 0x01}, - {0x4042, 0x00}, - {0x4043, 0x80}, - {0x4044, 0x00}, - {0x4045, 0x80}, - {0x4046, 0x00}, - {0x4047, 0x80}, - {0x4048, 0x00}, - {0x4049, 0x80}, - {0x4041, 0x03}, - {0x404c, 0x20}, - {0x404d, 0x00}, - {0x404e, 0x20}, - {0x4203, 0x80}, - {0x4307, 0x30}, - {0x4317, 0x00}, - {0x4502, 0x50}, - {0x4503, 0x08}, - {0x4601, 0x80}, - {0x4800, 0x44}, - {0x4816, 0x53}, - {0x481b, 0x50}, - {0x481f, 0x27}, - {0x4823, 0x3c}, - {0x482b, 0x00}, - {0x4831, 0x66}, - {0x4837, 0x16}, - {0x483c, 0x0f}, - {0x484b, 0x05}, - {0x5000, 0x77}, - {0x5001, 0x0a}, - {0x5003, 0xc8}, - {0x5004, 0x04}, - {0x5006, 0x00}, - {0x5007, 0x00}, - {0x502e, 0x03}, - {0x5030, 0x41}, - {0x5795, 0x00}, - {0x5796, 0x10}, - {0x5797, 0x10}, - {0x5798, 0x73}, - {0x5799, 0x73}, - {0x579a, 0x00}, - {0x579b, 0x28}, - {0x579c, 0x00}, - {0x579d, 0x16}, - {0x579e, 0x06}, - {0x579f, 0x20}, - {0x57a0, 0x04}, - {0x57a1, 0xa0}, - {0x5780, 0x14}, - {0x5781, 0x0f}, - {0x5782, 0x44}, - {0x5783, 0x02}, - {0x5784, 0x01}, - {0x5785, 0x01}, - {0x5786, 0x00}, - {0x5787, 0x04}, - {0x5788, 0x02}, - {0x5789, 0x0f}, - {0x578a, 0xfd}, - {0x578b, 0xf5}, - {0x578c, 0xf5}, - {0x578d, 0x03}, - {0x578e, 0x08}, - {0x578f, 0x0c}, - {0x5790, 0x08}, - {0x5791, 0x04}, - {0x5792, 0x00}, - {0x5793, 0x52}, - {0x5794, 0xa3}, - {0x59f8, 0x3d}, - {0x5a08, 0x02}, - {0x5b00, 0x02}, - {0x5b01, 0x10}, - {0x5b02, 0x03}, - {0x5b03, 0xcf}, - {0x5b05, 0x6c}, - {0x5e00, 0x00}, - {0x5e10, 0xfc} +static const struct ov8856_reg lane_4_mode_3280x2464[] = { + /* 3280x2464 resolution */ + {0x3000, 0x20}, + {0x3003, 0x08}, + {0x300e, 0x20}, + {0x3010, 0x00}, + {0x3015, 0x84}, + {0x3018, 0x72}, + {0x3021, 0x23}, + {0x3033, 0x24}, + {0x3500, 0x00}, + {0x3501, 0x9a}, + {0x3502, 0x20}, + {0x3503, 0x08}, + {0x3505, 0x83}, + {0x3508, 0x01}, + {0x3509, 0x80}, + {0x350c, 0x00}, + {0x350d, 0x80}, + {0x350e, 0x04}, + {0x350f, 0x00}, + {0x3510, 0x00}, + {0x3511, 0x02}, + {0x3512, 0x00}, + {0x3600, 0x72}, + {0x3601, 0x40}, + {0x3602, 0x30}, + {0x3610, 0xc5}, + {0x3611, 0x58}, + {0x3612, 0x5c}, + {0x3613, 0xca}, + {0x3614, 0x20}, + {0x3628, 0xff}, + {0x3629, 0xff}, + {0x362a, 0xff}, + {0x3633, 0x10}, + {0x3634, 0x10}, + {0x3635, 0x10}, + {0x3636, 0x10}, + {0x3663, 0x08}, + {0x3669, 0x34}, + {0x366e, 0x10}, + {0x3706, 0x86}, + {0x370b, 0x7e}, + {0x3714, 0x23}, + {0x3730, 0x12}, + {0x3733, 0x10}, + {0x3764, 0x00}, + {0x3765, 0x00}, + {0x3769, 0x62}, + {0x376a, 0x2a}, + {0x376b, 0x30}, + {0x3780, 0x00}, + {0x3781, 0x24}, + {0x3782, 0x00}, + {0x3783, 0x23}, + {0x3798, 0x2f}, + {0x37a1, 0x60}, + {0x37a8, 0x6a}, + {0x37ab, 0x3f}, + {0x37c2, 0x04}, + {0x37c3, 0xf1}, + {0x37c9, 0x80}, + {0x37cb, 0x16}, + {0x37cc, 0x16}, + {0x37cd, 0x16}, + {0x37ce, 0x16}, + {0x3800, 0x00}, + {0x3801, 0x00}, + {0x3802, 0x00}, + {0x3803, 0x06}, + {0x3804, 0x0c}, + {0x3805, 0xdf}, + {0x3806, 0x09}, + {0x3807, 0xa7}, + {0x3808, 0x0c}, + {0x3809, 0xd0}, + {0x380a, 0x09}, + {0x380b, 0xa0}, + {0x380c, 0x07}, + {0x380d, 0x88}, + {0x380e, 0x09}, + {0x380f, 0xb8}, + {0x3810, 0x00}, + {0x3811, 0x00}, + {0x3812, 0x00}, + {0x3813, 0x01}, + {0x3814, 0x01}, + {0x3815, 0x01}, + {0x3816, 0x00}, + {0x3817, 0x00}, + {0x3818, 0x00}, + {0x3819, 0x10}, + {0x3820, 0x80}, + {0x3821, 0x46}, + {0x382a, 0x01}, + {0x382b, 0x01}, + {0x3830, 0x06}, + {0x3836, 0x02}, + {0x3862, 0x04}, + {0x3863, 0x08}, + {0x3cc0, 0x33}, + {0x3d85, 0x17}, + {0x3d8c, 0x73}, + {0x3d8d, 0xde}, + {0x4001, 0xe0}, + {0x4003, 0x40}, + {0x4008, 0x00}, + {0x4009, 0x0b}, + {0x400a, 0x00}, + {0x400b, 0x84}, + {0x400f, 0x80}, + {0x4010, 0xf0}, + {0x4011, 0xff}, + {0x4012, 0x02}, + {0x4013, 0x01}, + {0x4014, 0x01}, + {0x4015, 0x01}, + {0x4042, 0x00}, + {0x4043, 0x80}, + {0x4044, 0x00}, + {0x4045, 0x80}, + {0x4046, 0x00}, + {0x4047, 0x80}, + {0x4048, 0x00}, + {0x4049, 0x80}, + {0x4041, 0x03}, + {0x404c, 0x20}, + {0x404d, 0x00}, + {0x404e, 0x20}, + {0x4203, 0x80}, + {0x4307, 0x30}, + {0x4317, 0x00}, + {0x4503, 0x08}, + {0x4601, 0x80}, + {0x4800, 0x44}, + {0x4816, 0x53}, + {0x481b, 0x58}, + {0x481f, 0x27}, + {0x4837, 0x16}, + {0x483c, 0x0f}, + {0x484b, 0x05}, + {0x5000, 0x57}, + {0x5001, 0x0a}, + {0x5004, 0x04}, + {0x502e, 0x03}, + {0x5030, 0x41}, + {0x5780, 0x14}, + {0x5781, 0x0f}, + {0x5782, 0x44}, + {0x5783, 0x02}, + {0x5784, 0x01}, + {0x5785, 0x01}, + {0x5786, 0x00}, + {0x5787, 0x04}, + {0x5788, 0x02}, + {0x5789, 0x0f}, + {0x578a, 0xfd}, + {0x578b, 0xf5}, + {0x578c, 0xf5}, + {0x578d, 0x03}, + {0x578e, 0x08}, + {0x578f, 0x0c}, + {0x5790, 0x08}, + {0x5791, 0x04}, + {0x5792, 0x00}, + {0x5793, 0x52}, + {0x5794, 0xa3}, + {0x5795, 0x02}, + {0x5796, 0x20}, + {0x5797, 0x20}, + {0x5798, 0xd5}, + {0x5799, 0xd5}, + {0x579a, 0x00}, + {0x579b, 0x50}, + {0x579c, 0x00}, + {0x579d, 0x2c}, + {0x579e, 0x0c}, + {0x579f, 0x40}, + {0x57a0, 0x09}, + {0x57a1, 0x40}, + {0x59f8, 0x3d}, + {0x5a08, 0x02}, + {0x5b00, 0x02}, + {0x5b01, 0x10}, + {0x5b02, 0x03}, + {0x5b03, 0xcf}, + {0x5b05, 0x6c}, + {0x5e00, 0x00} +}; + +static const struct ov8856_reg lane_4_mode_1640x1232[] = { + /* 1640x1232 resolution */ + {0x3000, 0x20}, + {0x3003, 0x08}, + {0x300e, 0x20}, + {0x3010, 0x00}, + {0x3015, 0x84}, + {0x3018, 0x72}, + {0x3021, 0x23}, + {0x3033, 0x24}, + {0x3500, 0x00}, + {0x3501, 0x4c}, + {0x3502, 0xe0}, + {0x3503, 0x08}, + {0x3505, 0x83}, + {0x3508, 0x01}, + {0x3509, 0x80}, + {0x350c, 0x00}, + {0x350d, 0x80}, + {0x350e, 0x04}, + {0x350f, 0x00}, + {0x3510, 0x00}, + {0x3511, 0x02}, + {0x3512, 0x00}, + {0x3600, 0x72}, + {0x3601, 0x40}, + {0x3602, 0x30}, + {0x3610, 0xc5}, + {0x3611, 0x58}, + {0x3612, 0x5c}, + {0x3613, 0xca}, + {0x3614, 0x20}, + {0x3628, 0xff}, + {0x3629, 0xff}, + {0x362a, 0xff}, + {0x3633, 0x10}, + {0x3634, 0x10}, + {0x3635, 0x10}, + {0x3636, 0x10}, + {0x3663, 0x08}, + {0x3669, 0x34}, + {0x366e, 0x08}, + {0x3706, 0x86}, + {0x370b, 0x7e}, + {0x3714, 0x27}, + {0x3730, 0x12}, + {0x3733, 0x10}, + {0x3764, 0x00}, + {0x3765, 0x00}, + {0x3769, 0x62}, + {0x376a, 0x2a}, + {0x376b, 0x30}, + {0x3780, 0x00}, + {0x3781, 0x24}, + {0x3782, 0x00}, + {0x3783, 0x23}, + {0x3798, 0x2f}, + {0x37a1, 0x60}, + {0x37a8, 0x6a}, + {0x37ab, 0x3f}, + {0x37c2, 0x14}, + {0x37c3, 0xf1}, + {0x37c9, 0x80}, + {0x37cb, 0x16}, + {0x37cc, 0x16}, + {0x37cd, 0x16}, + {0x37ce, 0x16}, + {0x3800, 0x00}, + {0x3801, 0x00}, + {0x3802, 0x00}, + {0x3803, 0x00}, + {0x3804, 0x0c}, + {0x3805, 0xdf}, + {0x3806, 0x09}, + {0x3807, 0xaf}, + {0x3808, 0x06}, + {0x3809, 0x68}, + {0x380a, 0x04}, + {0x380b, 0xd0}, + {0x380c, 0x0e}, + {0x380d, 0xec}, + {0x380e, 0x04}, + {0x380f, 0xe8}, + {0x3810, 0x00}, + {0x3811, 0x04}, + {0x3812, 0x00}, + {0x3813, 0x05}, + {0x3814, 0x03}, + {0x3815, 0x01}, + {0x3816, 0x00}, + {0x3817, 0x00}, + {0x3818, 0x00}, + {0x3819, 0x10}, + {0x3820, 0x90}, + {0x3821, 0x67}, + {0x382a, 0x03}, + {0x382b, 0x01}, + {0x3830, 0x06}, + {0x3836, 0x02}, + {0x3862, 0x04}, + {0x3863, 0x08}, + {0x3cc0, 0x33}, + {0x3d85, 0x17}, + {0x3d8c, 0x73}, + {0x3d8d, 0xde}, + {0x4001, 0xe0}, + {0x4003, 0x40}, + {0x4008, 0x00}, + {0x4009, 0x05}, + {0x400a, 0x00}, + {0x400b, 0x84}, + {0x400f, 0x80}, + {0x4010, 0xf0}, + {0x4011, 0xff}, + {0x4012, 0x02}, + {0x4013, 0x01}, + {0x4014, 0x01}, + {0x4015, 0x01}, + {0x4042, 0x00}, + {0x4043, 0x80}, + {0x4044, 0x00}, + {0x4045, 0x80}, + {0x4046, 0x00}, + {0x4047, 0x80}, + {0x4048, 0x00}, + {0x4049, 0x80}, + {0x4041, 0x03}, + {0x404c, 0x20}, + {0x404d, 0x00}, + {0x404e, 0x20}, + {0x4203, 0x80}, + {0x4307, 0x30}, + {0x4317, 0x00}, + {0x4503, 0x08}, + {0x4601, 0x80}, + {0x4800, 0x44}, + {0x4816, 0x53}, + {0x481b, 0x58}, + {0x481f, 0x27}, + {0x4837, 0x16}, + {0x483c, 0x0f}, + {0x484b, 0x05}, + {0x5000, 0x57}, + {0x5001, 0x0a}, + {0x5004, 0x04}, + {0x502e, 0x03}, + {0x5030, 0x41}, + {0x5780, 0x14}, + {0x5781, 0x0f}, + {0x5782, 0x44}, + {0x5783, 0x02}, + {0x5784, 0x01}, + {0x5785, 0x01}, + {0x5786, 0x00}, + {0x5787, 0x04}, + {0x5788, 0x02}, + {0x5789, 0x0f}, + {0x578a, 0xfd}, + {0x578b, 0xf5}, + {0x578c, 0xf5}, + {0x578d, 0x03}, + {0x578e, 0x08}, + {0x578f, 0x0c}, + {0x5790, 0x08}, + {0x5791, 0x04}, + {0x5792, 0x00}, + {0x5793, 0x52}, + {0x5794, 0xa3}, + {0x5795, 0x00}, + {0x5796, 0x10}, + {0x5797, 0x10}, + {0x5798, 0x73}, + {0x5799, 0x73}, + {0x579a, 0x00}, + {0x579b, 0x28}, + {0x579c, 0x00}, + {0x579d, 0x16}, + {0x579e, 0x06}, + {0x579f, 0x20}, + {0x57a0, 0x04}, + {0x57a1, 0xa0}, + {0x59f8, 0x3d}, + {0x5a08, 0x02}, + {0x5b00, 0x02}, + {0x5b01, 0x10}, + {0x5b02, 0x03}, + {0x5b03, 0xcf}, + {0x5b05, 0x6c}, + {0x5e00, 0x00} +}; + +static const struct ov8856_reg lane_4_mode_3264x2448[] = { + /* 3264x2448 resolution */ + {0x0103, 0x01}, + {0x0302, 0x3c}, + {0x0303, 0x01}, + {0x031e, 0x0c}, + {0x3000, 0x20}, + {0x3003, 0x08}, + {0x300e, 0x20}, + {0x3010, 0x00}, + {0x3015, 0x84}, + {0x3018, 0x72}, + {0x3021, 0x23}, + {0x3033, 0x24}, + {0x3500, 0x00}, + {0x3501, 0x9a}, + {0x3502, 0x20}, + {0x3503, 0x08}, + {0x3505, 0x83}, + {0x3508, 0x01}, + {0x3509, 0x80}, + {0x350c, 0x00}, + {0x350d, 0x80}, + {0x350e, 0x04}, + {0x350f, 0x00}, + {0x3510, 0x00}, + {0x3511, 0x02}, + {0x3512, 0x00}, + {0x3600, 0x72}, + {0x3601, 0x40}, + {0x3602, 0x30}, + {0x3610, 0xc5}, + {0x3611, 0x58}, + {0x3612, 0x5c}, + {0x3613, 0xca}, + {0x3614, 0x60}, + {0x3628, 0xff}, + {0x3629, 0xff}, + {0x362a, 0xff}, + {0x3633, 0x10}, + {0x3634, 0x10}, + {0x3635, 0x10}, + {0x3636, 0x10}, + {0x3663, 0x08}, + {0x3669, 0x34}, + {0x366d, 0x00}, + {0x366e, 0x10}, + {0x3706, 0x86}, + {0x370b, 0x7e}, + {0x3714, 0x23}, + {0x3730, 0x12}, + {0x3733, 0x10}, + {0x3764, 0x00}, + {0x3765, 0x00}, + {0x3769, 0x62}, + {0x376a, 0x2a}, + {0x376b, 0x30}, + {0x3780, 0x00}, + {0x3781, 0x24}, + {0x3782, 0x00}, + {0x3783, 0x23}, + {0x3798, 0x2f}, + {0x37a1, 0x60}, + {0x37a8, 0x6a}, + {0x37ab, 0x3f}, + {0x37c2, 0x04}, + {0x37c3, 0xf1}, + {0x37c9, 0x80}, + {0x37cb, 0x16}, + {0x37cc, 0x16}, + {0x37cd, 0x16}, + {0x37ce, 0x16}, + {0x3800, 0x00}, + {0x3801, 0x00}, + {0x3802, 0x00}, + {0x3803, 0x0c}, + {0x3804, 0x0c}, + {0x3805, 0xdf}, + {0x3806, 0x09}, + {0x3807, 0xa3}, + {0x3808, 0x0c}, + {0x3809, 0xc0}, + {0x380a, 0x09}, + {0x380b, 0x90}, + {0x380c, 0x07}, + {0x380d, 0x8c}, + {0x380e, 0x09}, + {0x380f, 0xb2}, + {0x3810, 0x00}, + {0x3811, 0x04}, + {0x3812, 0x00}, + {0x3813, 0x01}, + {0x3814, 0x01}, + {0x3815, 0x01}, + {0x3816, 0x00}, + {0x3817, 0x00}, + {0x3818, 0x00}, + {0x3819, 0x10}, + {0x3820, 0x80}, + {0x3821, 0x46}, + {0x382a, 0x01}, + {0x382b, 0x01}, + {0x3830, 0x06}, + {0x3836, 0x02}, + {0x3862, 0x04}, + {0x3863, 0x08}, + {0x3cc0, 0x33}, + {0x3d85, 0x17}, + {0x3d8c, 0x73}, + {0x3d8d, 0xde}, + {0x4001, 0xe0}, + {0x4003, 0x40}, + {0x4008, 0x00}, + {0x4009, 0x0b}, + {0x400a, 0x00}, + {0x400b, 0x84}, + {0x400f, 0x80}, + {0x4010, 0xf0}, + {0x4011, 0xff}, + {0x4012, 0x02}, + {0x4013, 0x01}, + {0x4014, 0x01}, + {0x4015, 0x01}, + {0x4042, 0x00}, + {0x4043, 0x80}, + {0x4044, 0x00}, + {0x4045, 0x80}, + {0x4046, 0x00}, + {0x4047, 0x80}, + {0x4048, 0x00}, + {0x4049, 0x80}, + {0x4041, 0x03}, + {0x404c, 0x20}, + {0x404d, 0x00}, + {0x404e, 0x20}, + {0x4203, 0x80}, + {0x4307, 0x30}, + {0x4317, 0x00}, + {0x4502, 0x50}, + {0x4503, 0x08}, + {0x4601, 0x80}, + {0x4800, 0x44}, + {0x4816, 0x53}, + {0x481b, 0x50}, + {0x481f, 0x27}, + {0x4823, 0x3c}, + {0x482b, 0x00}, + {0x4831, 0x66}, + {0x4837, 0x16}, + {0x483c, 0x0f}, + {0x484b, 0x05}, + {0x5000, 0x77}, + {0x5001, 0x0a}, + {0x5003, 0xc8}, + {0x5004, 0x04}, + {0x5006, 0x00}, + {0x5007, 0x00}, + {0x502e, 0x03}, + {0x5030, 0x41}, + {0x5780, 0x14}, + {0x5781, 0x0f}, + {0x5782, 0x44}, + {0x5783, 0x02}, + {0x5784, 0x01}, + {0x5785, 0x01}, + {0x5786, 0x00}, + {0x5787, 0x04}, + {0x5788, 0x02}, + {0x5789, 0x0f}, + {0x578a, 0xfd}, + {0x578b, 0xf5}, + {0x578c, 0xf5}, + {0x578d, 0x03}, + {0x578e, 0x08}, + {0x578f, 0x0c}, + {0x5790, 0x08}, + {0x5791, 0x04}, + {0x5792, 0x00}, + {0x5793, 0x52}, + {0x5794, 0xa3}, + {0x5795, 0x02}, + {0x5796, 0x20}, + {0x5797, 0x20}, + {0x5798, 0xd5}, + {0x5799, 0xd5}, + {0x579a, 0x00}, + {0x579b, 0x50}, + {0x579c, 0x00}, + {0x579d, 0x2c}, + {0x579e, 0x0c}, + {0x579f, 0x40}, + {0x57a0, 0x09}, + {0x57a1, 0x40}, + {0x59f8, 0x3d}, + {0x5a08, 0x02}, + {0x5b00, 0x02}, + {0x5b01, 0x10}, + {0x5b02, 0x03}, + {0x5b03, 0xcf}, + {0x5b05, 0x6c}, + {0x5e00, 0x00}, + {0x5e10, 0xfc} +}; + +static const struct ov8856_reg lane_4_mode_1632x1224[] = { + /* 1632x1224 resolution */ + {0x0103, 0x01}, + {0x0302, 0x3c}, + {0x0303, 0x01}, + {0x031e, 0x0c}, + {0x3000, 0x20}, + {0x3003, 0x08}, + {0x300e, 0x20}, + {0x3010, 0x00}, + {0x3015, 0x84}, + {0x3018, 0x72}, + {0x3021, 0x23}, + {0x3033, 0x24}, + {0x3500, 0x00}, + {0x3501, 0x4c}, + {0x3502, 0xe0}, + {0x3503, 0x08}, + {0x3505, 0x83}, + {0x3508, 0x01}, + {0x3509, 0x80}, + {0x350c, 0x00}, + {0x350d, 0x80}, + {0x350e, 0x04}, + {0x350f, 0x00}, + {0x3510, 0x00}, + {0x3511, 0x02}, + {0x3512, 0x00}, + {0x3600, 0x72}, + {0x3601, 0x40}, + {0x3602, 0x30}, + {0x3610, 0xc5}, + {0x3611, 0x58}, + {0x3612, 0x5c}, + {0x3613, 0xca}, + {0x3614, 0x60}, + {0x3628, 0xff}, + {0x3629, 0xff}, + {0x362a, 0xff}, + {0x3633, 0x10}, + {0x3634, 0x10}, + {0x3635, 0x10}, + {0x3636, 0x10}, + {0x3663, 0x08}, + {0x3669, 0x34}, + {0x366d, 0x00}, + {0x366e, 0x08}, + {0x3706, 0x86}, + {0x370b, 0x7e}, + {0x3714, 0x27}, + {0x3730, 0x12}, + {0x3733, 0x10}, + {0x3764, 0x00}, + {0x3765, 0x00}, + {0x3769, 0x62}, + {0x376a, 0x2a}, + {0x376b, 0x30}, + {0x3780, 0x00}, + {0x3781, 0x24}, + {0x3782, 0x00}, + {0x3783, 0x23}, + {0x3798, 0x2f}, + {0x37a1, 0x60}, + {0x37a8, 0x6a}, + {0x37ab, 0x3f}, + {0x37c2, 0x14}, + {0x37c3, 0xf1}, + {0x37c9, 0x80}, + {0x37cb, 0x16}, + {0x37cc, 0x16}, + {0x37cd, 0x16}, + {0x37ce, 0x16}, + {0x3800, 0x00}, + {0x3801, 0x00}, + {0x3802, 0x00}, + {0x3803, 0x0c}, + {0x3804, 0x0c}, + {0x3805, 0xdf}, + {0x3806, 0x09}, + {0x3807, 0xa3}, + {0x3808, 0x06}, + {0x3809, 0x60}, + {0x380a, 0x04}, + {0x380b, 0xc8}, + {0x380c, 0x07}, + {0x380d, 0x8c}, + {0x380e, 0x09}, + {0x380f, 0xb2}, + {0x3810, 0x00}, + {0x3811, 0x02}, + {0x3812, 0x00}, + {0x3813, 0x01}, + {0x3814, 0x03}, + {0x3815, 0x01}, + {0x3816, 0x00}, + {0x3817, 0x00}, + {0x3818, 0x00}, + {0x3819, 0x10}, + {0x3820, 0x80}, + {0x3821, 0x47}, + {0x382a, 0x03}, + {0x382b, 0x01}, + {0x3830, 0x06}, + {0x3836, 0x02}, + {0x3862, 0x04}, + {0x3863, 0x08}, + {0x3cc0, 0x33}, + {0x3d85, 0x17}, + {0x3d8c, 0x73}, + {0x3d8d, 0xde}, + {0x4001, 0xe0}, + {0x4003, 0x40}, + {0x4008, 0x00}, + {0x4009, 0x05}, + {0x400a, 0x00}, + {0x400b, 0x84}, + {0x400f, 0x80}, + {0x4010, 0xf0}, + {0x4011, 0xff}, + {0x4012, 0x02}, + {0x4013, 0x01}, + {0x4014, 0x01}, + {0x4015, 0x01}, + {0x4042, 0x00}, + {0x4043, 0x80}, + {0x4044, 0x00}, + {0x4045, 0x80}, + {0x4046, 0x00}, + {0x4047, 0x80}, + {0x4048, 0x00}, + {0x4049, 0x80}, + {0x4041, 0x03}, + {0x404c, 0x20}, + {0x404d, 0x00}, + {0x404e, 0x20}, + {0x4203, 0x80}, + {0x4307, 0x30}, + {0x4317, 0x00}, + {0x4502, 0x50}, + {0x4503, 0x08}, + {0x4601, 0x80}, + {0x4800, 0x44}, + {0x4816, 0x53}, + {0x481b, 0x50}, + {0x481f, 0x27}, + {0x4823, 0x3c}, + {0x482b, 0x00}, + {0x4831, 0x66}, + {0x4837, 0x16}, + {0x483c, 0x0f}, + {0x484b, 0x05}, + {0x5000, 0x77}, + {0x5001, 0x0a}, + {0x5003, 0xc8}, + {0x5004, 0x04}, + {0x5006, 0x00}, + {0x5007, 0x00}, + {0x502e, 0x03}, + {0x5030, 0x41}, + {0x5795, 0x00}, + {0x5796, 0x10}, + {0x5797, 0x10}, + {0x5798, 0x73}, + {0x5799, 0x73}, + {0x579a, 0x00}, + {0x579b, 0x28}, + {0x579c, 0x00}, + {0x579d, 0x16}, + {0x579e, 0x06}, + {0x579f, 0x20}, + {0x57a0, 0x04}, + {0x57a1, 0xa0}, + {0x5780, 0x14}, + {0x5781, 0x0f}, + {0x5782, 0x44}, + {0x5783, 0x02}, + {0x5784, 0x01}, + {0x5785, 0x01}, + {0x5786, 0x00}, + {0x5787, 0x04}, + {0x5788, 0x02}, + {0x5789, 0x0f}, + {0x578a, 0xfd}, + {0x578b, 0xf5}, + {0x578c, 0xf5}, + {0x578d, 0x03}, + {0x578e, 0x08}, + {0x578f, 0x0c}, + {0x5790, 0x08}, + {0x5791, 0x04}, + {0x5792, 0x00}, + {0x5793, 0x52}, + {0x5794, 0xa3}, + {0x59f8, 0x3d}, + {0x5a08, 0x02}, + {0x5b00, 0x02}, + {0x5b01, 0x10}, + {0x5b02, 0x03}, + {0x5b03, 0xcf}, + {0x5b05, 0x6c}, + {0x5e00, 0x00}, + {0x5e10, 0xfc} }; static const char * const ov8856_test_pattern_menu[] = { @@ -942,77 +1380,6 @@ static const char * const ov8856_test_pattern_menu[] = { "Bottom-Top Darker Color Bar" }; -static const s64 link_freq_menu_items[] = { - OV8856_LINK_FREQ_360MHZ, - OV8856_LINK_FREQ_180MHZ -}; - -static const struct ov8856_link_freq_config link_freq_configs[] = { - [OV8856_LINK_FREQ_720MBPS] = { - .reg_list = { - .num_of_regs = ARRAY_SIZE(mipi_data_rate_720mbps), - .regs = mipi_data_rate_720mbps, - } - }, - [OV8856_LINK_FREQ_360MBPS] = { - .reg_list = { - .num_of_regs = ARRAY_SIZE(mipi_data_rate_360mbps), - .regs = mipi_data_rate_360mbps, - } - } -}; - -static const struct ov8856_mode supported_modes[] = { - { - .width = 3280, - .height = 2464, - .hts = 1928, - .vts_def = 2488, - .vts_min = 2488, - .reg_list = { - .num_of_regs = ARRAY_SIZE(mode_3280x2464_regs), - .regs = mode_3280x2464_regs, - }, - .link_freq_index = OV8856_LINK_FREQ_720MBPS, - }, - { - .width = 3264, - .height = 2448, - .hts = 1932, - .vts_def = 2482, - .vts_min = 2482, - .reg_list = { - .num_of_regs = ARRAY_SIZE(mode_3264x2448_regs), - .regs = mode_3264x2448_regs, - }, - .link_freq_index = OV8856_LINK_FREQ_720MBPS, - }, - { - .width = 1640, - .height = 1232, - .hts = 3820, - .vts_def = 1256, - .vts_min = 1256, - .reg_list = { - .num_of_regs = ARRAY_SIZE(mode_1640x1232_regs), - .regs = mode_1640x1232_regs, - }, - .link_freq_index = OV8856_LINK_FREQ_360MBPS, - }, - { - .width = 1632, - .height = 1224, - .hts = 1932, - .vts_def = 2482, - .vts_min = 2482, - .reg_list = { - .num_of_regs = ARRAY_SIZE(mode_1632x1224_regs), - .regs = mode_1632x1224_regs, - }, - .link_freq_index = OV8856_LINK_FREQ_360MBPS, - } -}; - struct ov8856 { struct v4l2_subdev sd; struct media_pad pad; @@ -1037,20 +1404,173 @@ struct ov8856 { /* Streaming on/off */ bool streaming; + + /* lanes index */ + u8 nlanes; + + const struct ov8856_lane_cfg *priv_lane; + u8 modes_size; }; -static u64 to_pixel_rate(u32 f_index) +struct ov8856_lane_cfg { + const s64 link_freq_menu_items[2]; + const struct ov8856_link_freq_config link_freq_configs[2]; + const struct ov8856_mode supported_modes[4]; +}; + +static const struct ov8856_lane_cfg lane_cfg_2 = { + { + 720000000, + 360000000, + }, + {{ + .reg_list = { + .num_of_regs = + ARRAY_SIZE(mipi_data_rate_lane_2.regs_0), + .regs = mipi_data_rate_lane_2.regs_0, + } + }, + { + .reg_list = { + .num_of_regs = + ARRAY_SIZE(mipi_data_rate_lane_2.regs_1), + .regs = mipi_data_rate_lane_2.regs_1, + } + }}, + {{ + .width = 3280, + .height = 2464, + .hts = 1928, + .vts_def = 2488, + .vts_min = 2488, + .reg_list = { + .num_of_regs = + ARRAY_SIZE(lane_2_mode_3280x2464), + .regs = lane_2_mode_3280x2464, + }, + .link_freq_index = 0, + .data_lanes = 2, + }, + { + .width = 1640, + .height = 1232, + .hts = 3168, + .vts_def = 1514, + .vts_min = 1514, + .reg_list = { + .num_of_regs = + ARRAY_SIZE(lane_2_mode_1640x1232), + .regs = lane_2_mode_1640x1232, + }, + .link_freq_index = 1, + .data_lanes = 2, + }} +}; + +static const struct ov8856_lane_cfg lane_cfg_4 = { + { + 360000000, + 180000000, + }, + {{ + .reg_list = { + .num_of_regs = + ARRAY_SIZE(mipi_data_rate_lane_4.regs_0), + .regs = mipi_data_rate_lane_4.regs_0, + } + }, + { + .reg_list = { + .num_of_regs = + ARRAY_SIZE(mipi_data_rate_lane_4.regs_1), + .regs = mipi_data_rate_lane_4.regs_1, + } + }}, + {{ + .width = 3280, + .height = 2464, + .hts = 1928, + .vts_def = 2488, + .vts_min = 2488, + .reg_list = { + .num_of_regs = + ARRAY_SIZE(lane_4_mode_3280x2464), + .regs = lane_4_mode_3280x2464, + }, + .link_freq_index = 0, + .data_lanes = 4, + }, + { + .width = 1640, + .height = 1232, + .hts = 3820, + .vts_def = 1256, + .vts_min = 1256, + .reg_list = { + .num_of_regs = + ARRAY_SIZE(lane_4_mode_1640x1232), + .regs = lane_4_mode_1640x1232, + }, + .link_freq_index = 1, + .data_lanes = 4, + }, + { + .width = 3264, + .height = 2448, + .hts = 1932, + .vts_def = 2482, + .vts_min = 2482, + .reg_list = { + .num_of_regs = + ARRAY_SIZE(lane_4_mode_3264x2448), + .regs = lane_4_mode_3264x2448, + }, + .link_freq_index = 0, + .data_lanes = 4, + }, + { + .width = 1632, + .height = 1224, + .hts = 1932, + .vts_def = 2482, + .vts_min = 2482, + .reg_list = { + .num_of_regs = + ARRAY_SIZE(lane_4_mode_1632x1224), + .regs = lane_4_mode_1632x1224, + }, + .link_freq_index = 1, + .data_lanes = 4, + }} +}; + +static unsigned int ov8856_modes_num(const struct ov8856 *ov8856) { - u64 pixel_rate = link_freq_menu_items[f_index] * 2 * OV8856_DATA_LANES; + unsigned int i, count = 0; + + for (i = 0; i < ARRAY_SIZE(ov8856->priv_lane->supported_modes); i++) { + if (ov8856->priv_lane->supported_modes[i].width == 0) + break; + count++; + } + + return count; +} + +static u64 to_rate(const s64 *link_freq_menu_items, + u32 f_index, u8 nlanes) +{ + u64 pixel_rate = link_freq_menu_items[f_index] * 2 * nlanes; do_div(pixel_rate, OV8856_RGB_DEPTH); return pixel_rate; } -static u64 to_pixels_per_line(u32 hts, u32 f_index) +static u64 to_pixels_per_line(const s64 *link_freq_menu_items, u32 hts, + u32 f_index, u8 nlanes) { - u64 ppl = hts * to_pixel_rate(f_index); + u64 ppl = hts * to_rate(link_freq_menu_items, f_index, nlanes); do_div(ppl, OV8856_SCLK); @@ -1152,6 +1672,93 @@ static int ov8856_test_pattern(struct ov8856 *ov8856, u32 pattern) OV8856_REG_VALUE_08BIT, pattern); } +static int ov8856_set_ctrl_hflip(struct ov8856 *ov8856, u32 ctrl_val) +{ + int ret; + u32 val; + + ret = ov8856_read_reg(ov8856, OV8856_REG_MIRROR_OPT_1, + OV8856_REG_VALUE_08BIT, &val); + if (ret) + return ret; + + ret = ov8856_write_reg(ov8856, OV8856_REG_MIRROR_OPT_1, + OV8856_REG_VALUE_08BIT, + ctrl_val ? val & ~OV8856_REG_MIRROR_OP_2 : + val | OV8856_REG_MIRROR_OP_2); + + if (ret) + return ret; + + ret = ov8856_read_reg(ov8856, OV8856_REG_FORMAT2, + OV8856_REG_VALUE_08BIT, &val); + if (ret) + return ret; + + return ov8856_write_reg(ov8856, OV8856_REG_FORMAT2, + OV8856_REG_VALUE_08BIT, + ctrl_val ? val & ~OV8856_REG_FORMAT2_OP_1 & + ~OV8856_REG_FORMAT2_OP_2 & + ~OV8856_REG_FORMAT2_OP_3 : + val | OV8856_REG_FORMAT2_OP_1 | + OV8856_REG_FORMAT2_OP_2 | + OV8856_REG_FORMAT2_OP_3); +} + +static int ov8856_set_ctrl_vflip(struct ov8856 *ov8856, u8 ctrl_val) +{ + int ret; + u32 val; + + ret = ov8856_read_reg(ov8856, OV8856_REG_FLIP_OPT_1, + OV8856_REG_VALUE_08BIT, &val); + if (ret) + return ret; + + ret = ov8856_write_reg(ov8856, OV8856_REG_FLIP_OPT_1, + OV8856_REG_VALUE_08BIT, + ctrl_val ? val | OV8856_REG_FLIP_OP_1 | + OV8856_REG_FLIP_OP_2 : + val & ~OV8856_REG_FLIP_OP_1 & + ~OV8856_REG_FLIP_OP_2); + + ret = ov8856_read_reg(ov8856, OV8856_REG_FLIP_OPT_2, + OV8856_REG_VALUE_08BIT, &val); + if (ret) + return ret; + + ret = ov8856_write_reg(ov8856, OV8856_REG_FLIP_OPT_2, + OV8856_REG_VALUE_08BIT, + ctrl_val ? val | OV8856_REG_FLIP_OP_2 : + val & ~OV8856_REG_FLIP_OP_2); + + ret = ov8856_read_reg(ov8856, OV8856_REG_FLIP_OPT_3, + OV8856_REG_VALUE_08BIT, &val); + if (ret) + return ret; + + ret = ov8856_write_reg(ov8856, OV8856_REG_FLIP_OPT_3, + OV8856_REG_VALUE_08BIT, + ctrl_val ? val & ~OV8856_REG_FLIP_OP_0 & + ~OV8856_REG_FLIP_OP_1 : + val | OV8856_REG_FLIP_OP_0 | + OV8856_REG_FLIP_OP_1); + + ret = ov8856_read_reg(ov8856, OV8856_REG_FORMAT1, + OV8856_REG_VALUE_08BIT, &val); + if (ret) + return ret; + + return ov8856_write_reg(ov8856, OV8856_REG_FORMAT1, + OV8856_REG_VALUE_08BIT, + ctrl_val ? val | OV8856_REG_FORMAT1_OP_1 | + OV8856_REG_FORMAT1_OP_3 | + OV8856_REG_FORMAT1_OP_2 : + val & ~OV8856_REG_FORMAT1_OP_1 & + ~OV8856_REG_FORMAT1_OP_3 & + ~OV8856_REG_FORMAT1_OP_2); +} + static int ov8856_set_ctrl(struct v4l2_ctrl *ctrl) { struct ov8856 *ov8856 = container_of(ctrl->handler, @@ -1201,6 +1808,14 @@ static int ov8856_set_ctrl(struct v4l2_ctrl *ctrl) ret = ov8856_test_pattern(ov8856, ctrl->val); break; + case V4L2_CID_HFLIP: + ret = ov8856_set_ctrl_hflip(ov8856, ctrl->val); + break; + + case V4L2_CID_VFLIP: + ret = ov8856_set_ctrl_vflip(ov8856, ctrl->val); + break; + default: ret = -EINVAL; break; @@ -1229,23 +1844,32 @@ static int ov8856_init_controls(struct ov8856 *ov8856) ctrl_hdlr->lock = &ov8856->mutex; ov8856->link_freq = v4l2_ctrl_new_int_menu(ctrl_hdlr, &ov8856_ctrl_ops, V4L2_CID_LINK_FREQ, - ARRAY_SIZE(link_freq_menu_items) - 1, - 0, link_freq_menu_items); + ARRAY_SIZE + (ov8856->priv_lane->link_freq_menu_items) + - 1, + 0, ov8856->priv_lane->link_freq_menu_items); if (ov8856->link_freq) ov8856->link_freq->flags |= V4L2_CTRL_FLAG_READ_ONLY; ov8856->pixel_rate = v4l2_ctrl_new_std(ctrl_hdlr, &ov8856_ctrl_ops, V4L2_CID_PIXEL_RATE, 0, - to_pixel_rate(OV8856_LINK_FREQ_720MBPS), - 1, - to_pixel_rate(OV8856_LINK_FREQ_720MBPS)); + to_rate(ov8856->priv_lane->link_freq_menu_items, + 0, + ov8856->cur_mode->data_lanes), 1, + to_rate(ov8856->priv_lane->link_freq_menu_items, + 0, + ov8856->cur_mode->data_lanes)); ov8856->vblank = v4l2_ctrl_new_std(ctrl_hdlr, &ov8856_ctrl_ops, V4L2_CID_VBLANK, ov8856->cur_mode->vts_min - ov8856->cur_mode->height, OV8856_VTS_MAX - ov8856->cur_mode->height, 1, - ov8856->cur_mode->vts_def - ov8856->cur_mode->height); - h_blank = to_pixels_per_line(ov8856->cur_mode->hts, - ov8856->cur_mode->link_freq_index) - ov8856->cur_mode->width; + ov8856->cur_mode->vts_def - + ov8856->cur_mode->height); + h_blank = to_pixels_per_line(ov8856->priv_lane->link_freq_menu_items, + ov8856->cur_mode->hts, + ov8856->cur_mode->link_freq_index, + ov8856->cur_mode->data_lanes) - + ov8856->cur_mode->width; ov8856->hblank = v4l2_ctrl_new_std(ctrl_hdlr, &ov8856_ctrl_ops, V4L2_CID_HBLANK, h_blank, h_blank, 1, h_blank); @@ -1268,6 +1892,10 @@ static int ov8856_init_controls(struct ov8856 *ov8856) V4L2_CID_TEST_PATTERN, ARRAY_SIZE(ov8856_test_pattern_menu) - 1, 0, 0, ov8856_test_pattern_menu); + v4l2_ctrl_new_std(ctrl_hdlr, &ov8856_ctrl_ops, + V4L2_CID_HFLIP, 0, 1, 1, 0); + v4l2_ctrl_new_std(ctrl_hdlr, &ov8856_ctrl_ops, + V4L2_CID_VFLIP, 0, 1, 1, 0); if (ctrl_hdlr->error) return ctrl_hdlr->error; @@ -1292,7 +1920,8 @@ static int ov8856_start_streaming(struct ov8856 *ov8856) int link_freq_index, ret; link_freq_index = ov8856->cur_mode->link_freq_index; - reg_list = &link_freq_configs[link_freq_index].reg_list; + reg_list = &ov8856->priv_lane->link_freq_configs[link_freq_index].reg_list; + ret = ov8856_write_reg_list(ov8856, reg_list); if (ret) { dev_err(&client->dev, "failed to set plls"); @@ -1340,9 +1969,8 @@ static int ov8856_set_stream(struct v4l2_subdev *sd, int enable) mutex_lock(&ov8856->mutex); if (enable) { - ret = pm_runtime_get_sync(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); if (ret < 0) { - pm_runtime_put_noidle(&client->dev); mutex_unlock(&ov8856->mutex); return ret; } @@ -1455,27 +2083,29 @@ static int __maybe_unused ov8856_resume(struct device *dev) } static int ov8856_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov8856 *ov8856 = to_ov8856(sd); const struct ov8856_mode *mode; s32 vblank_def, h_blank; - mode = v4l2_find_nearest_size(supported_modes, - ARRAY_SIZE(supported_modes), width, - height, fmt->format.width, + mode = v4l2_find_nearest_size(ov8856->priv_lane->supported_modes, + ov8856->modes_size, + width, height, fmt->format.width, fmt->format.height); mutex_lock(&ov8856->mutex); ov8856_update_pad_format(mode, &fmt->format); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; } else { ov8856->cur_mode = mode; __v4l2_ctrl_s_ctrl(ov8856->link_freq, mode->link_freq_index); __v4l2_ctrl_s_ctrl_int64(ov8856->pixel_rate, - to_pixel_rate(mode->link_freq_index)); + to_rate(ov8856->priv_lane->link_freq_menu_items, + mode->link_freq_index, + ov8856->cur_mode->data_lanes)); /* Update limits and set FPS to default */ vblank_def = mode->vts_def - mode->height; @@ -1484,8 +2114,11 @@ static int ov8856_set_format(struct v4l2_subdev *sd, OV8856_VTS_MAX - mode->height, 1, vblank_def); __v4l2_ctrl_s_ctrl(ov8856->vblank, vblank_def); - h_blank = to_pixels_per_line(mode->hts, mode->link_freq_index) - - mode->width; + h_blank = to_pixels_per_line(ov8856->priv_lane->link_freq_menu_items, + mode->hts, + mode->link_freq_index, + ov8856->cur_mode->data_lanes) + - mode->width; __v4l2_ctrl_modify_range(ov8856->hblank, h_blank, h_blank, 1, h_blank); } @@ -1496,14 +2129,15 @@ static int ov8856_set_format(struct v4l2_subdev *sd, } static int ov8856_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov8856 *ov8856 = to_ov8856(sd); mutex_lock(&ov8856->mutex); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(&ov8856->sd, cfg, + fmt->format = *v4l2_subdev_get_try_format(&ov8856->sd, + sd_state, fmt->pad); else ov8856_update_pad_format(ov8856->cur_mode, &fmt->format); @@ -1514,7 +2148,7 @@ static int ov8856_get_format(struct v4l2_subdev *sd, } static int ov8856_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { /* Only one bayer order GRBG is supported */ @@ -1527,18 +2161,20 @@ static int ov8856_enum_mbus_code(struct v4l2_subdev *sd, } static int ov8856_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { - if (fse->index >= ARRAY_SIZE(supported_modes)) + struct ov8856 *ov8856 = to_ov8856(sd); + + if (fse->index >= ov8856->modes_size) return -EINVAL; if (fse->code != MEDIA_BUS_FMT_SGRBG10_1X10) return -EINVAL; - fse->min_width = supported_modes[fse->index].width; + fse->min_width = ov8856->priv_lane->supported_modes[fse->index].width; fse->max_width = fse->min_width; - fse->min_height = supported_modes[fse->index].height; + fse->min_height = ov8856->priv_lane->supported_modes[fse->index].height; fse->max_height = fse->min_height; return 0; @@ -1549,8 +2185,8 @@ static int ov8856_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) struct ov8856 *ov8856 = to_ov8856(sd); mutex_lock(&ov8856->mutex); - ov8856_update_pad_format(&supported_modes[0], - v4l2_subdev_get_try_format(sd, fh->pad, 0)); + ov8856_update_pad_format(&ov8856->priv_lane->supported_modes[0], + v4l2_subdev_get_try_format(sd, fh->state, 0)); mutex_unlock(&ov8856->mutex); return 0; @@ -1696,29 +2332,40 @@ static int ov8856_get_hwcfg(struct ov8856 *ov8856, struct device *dev) if (ret) return ret; - if (bus_cfg.bus.mipi_csi2.num_data_lanes != OV8856_DATA_LANES) { + /* Get number of data lanes */ + if (bus_cfg.bus.mipi_csi2.num_data_lanes != 2 && + bus_cfg.bus.mipi_csi2.num_data_lanes != 4) { dev_err(dev, "number of CSI2 data lanes %d is not supported", bus_cfg.bus.mipi_csi2.num_data_lanes); ret = -EINVAL; goto check_hwcfg_error; } + dev_dbg(dev, "Using %u data lanes\n", ov8856->cur_mode->data_lanes); + + if (bus_cfg.bus.mipi_csi2.num_data_lanes == 2) + ov8856->priv_lane = &lane_cfg_2; + else + ov8856->priv_lane = &lane_cfg_4; + + ov8856->modes_size = ov8856_modes_num(ov8856); + if (!bus_cfg.nr_of_link_frequencies) { dev_err(dev, "no link frequencies defined"); ret = -EINVAL; goto check_hwcfg_error; } - for (i = 0; i < ARRAY_SIZE(link_freq_menu_items); i++) { + for (i = 0; i < ARRAY_SIZE(ov8856->priv_lane->link_freq_menu_items); i++) { for (j = 0; j < bus_cfg.nr_of_link_frequencies; j++) { - if (link_freq_menu_items[i] == - bus_cfg.link_frequencies[j]) + if (ov8856->priv_lane->link_freq_menu_items[i] == + bus_cfg.link_frequencies[j]) break; } if (j == bus_cfg.nr_of_link_frequencies) { dev_err(dev, "no link frequency %lld supported", - link_freq_menu_items[i]); + ov8856->priv_lane->link_freq_menu_items[i]); ret = -EINVAL; goto check_hwcfg_error; } @@ -1777,7 +2424,7 @@ static int ov8856_probe(struct i2c_client *client) } mutex_init(&ov8856->mutex); - ov8856->cur_mode = &supported_modes[0]; + ov8856->cur_mode = &ov8856->priv_lane->supported_modes[0]; ret = ov8856_init_controls(ov8856); if (ret) { dev_err(&client->dev, "failed to init controls: %d", ret); diff --git a/drivers/media/i2c/ov8865.c b/drivers/media/i2c/ov8865.c index 9ecf180635ee..ce50f3ea87b8 100644 --- a/drivers/media/i2c/ov8865.c +++ b/drivers/media/i2c/ov8865.c @@ -2497,11 +2497,9 @@ static int ov8865_s_stream(struct v4l2_subdev *subdev, int enable) int ret; if (enable) { - ret = pm_runtime_get_sync(sensor->dev); - if (ret < 0) { - pm_runtime_put_noidle(sensor->dev); + ret = pm_runtime_resume_and_get(sensor->dev); + if (ret < 0) return ret; - } } mutex_lock(&sensor->mutex); @@ -2544,7 +2542,7 @@ static const struct v4l2_subdev_video_ops ov8865_subdev_video_ops = { /* Subdev Pad Operations */ static int ov8865_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code_enum) { if (code_enum->index >= ARRAY_SIZE(ov8865_mbus_codes)) @@ -2573,7 +2571,7 @@ static void ov8865_mbus_format_fill(struct v4l2_mbus_framefmt *mbus_format, } static int ov8865_get_fmt(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov8865_sensor *sensor = ov8865_subdev_sensor(subdev); @@ -2582,7 +2580,7 @@ static int ov8865_get_fmt(struct v4l2_subdev *subdev, mutex_lock(&sensor->mutex); if (format->which == V4L2_SUBDEV_FORMAT_TRY) - *mbus_format = *v4l2_subdev_get_try_format(subdev, config, + *mbus_format = *v4l2_subdev_get_try_format(subdev, sd_state, format->pad); else ov8865_mbus_format_fill(mbus_format, sensor->state.mbus_code, @@ -2594,7 +2592,7 @@ static int ov8865_get_fmt(struct v4l2_subdev *subdev, } static int ov8865_set_fmt(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct ov8865_sensor *sensor = ov8865_subdev_sensor(subdev); @@ -2635,7 +2633,7 @@ static int ov8865_set_fmt(struct v4l2_subdev *subdev, ov8865_mbus_format_fill(mbus_format, mbus_code, mode); if (format->which == V4L2_SUBDEV_FORMAT_TRY) - *v4l2_subdev_get_try_format(subdev, config, format->pad) = + *v4l2_subdev_get_try_format(subdev, sd_state, format->pad) = *mbus_format; else if (sensor->state.mode != mode || sensor->state.mbus_code != mbus_code) @@ -2648,7 +2646,7 @@ static int ov8865_set_fmt(struct v4l2_subdev *subdev, } static int ov8865_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *size_enum) { const struct ov8865_mode *mode; @@ -2665,7 +2663,7 @@ static int ov8865_enum_frame_size(struct v4l2_subdev *subdev, } static int ov8865_enum_frame_interval(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *interval_enum) { const struct ov8865_mode *mode = NULL; @@ -2691,7 +2689,7 @@ static int ov8865_enum_frame_interval(struct v4l2_subdev *subdev, } } - if (mode_index == ARRAY_SIZE(ov8865_modes) || !mode) + if (mode_index == ARRAY_SIZE(ov8865_modes)) return -EINVAL; interval_enum->interval = mode->frame_interval; diff --git a/drivers/media/i2c/ov9640.c b/drivers/media/i2c/ov9640.c index d36b04c49628..0bab8c2cf160 100644 --- a/drivers/media/i2c/ov9640.c +++ b/drivers/media/i2c/ov9640.c @@ -519,7 +519,7 @@ static int ov9640_s_fmt(struct v4l2_subdev *sd, } static int ov9640_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -547,13 +547,13 @@ static int ov9640_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) return ov9640_s_fmt(sd, mf); - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } static int ov9640_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(ov9640_codes)) @@ -565,7 +565,7 @@ static int ov9640_enum_mbus_code(struct v4l2_subdev *sd, } static int ov9640_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { if (sel->which != V4L2_SUBDEV_FORMAT_ACTIVE) diff --git a/drivers/media/i2c/ov9650.c b/drivers/media/i2c/ov9650.c index 4fe68aa55789..c313e11a9754 100644 --- a/drivers/media/i2c/ov9650.c +++ b/drivers/media/i2c/ov9650.c @@ -1070,7 +1070,7 @@ static void ov965x_get_default_format(struct v4l2_mbus_framefmt *mf) } static int ov965x_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(ov965x_formats)) @@ -1081,7 +1081,7 @@ static int ov965x_enum_mbus_code(struct v4l2_subdev *sd, } static int ov965x_enum_frame_sizes(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int i = ARRAY_SIZE(ov965x_formats); @@ -1167,14 +1167,14 @@ static int ov965x_s_frame_interval(struct v4l2_subdev *sd, } static int ov965x_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov965x *ov965x = to_ov965x(sd); struct v4l2_mbus_framefmt *mf; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); fmt->format = *mf; return 0; } @@ -1212,7 +1212,7 @@ static void __ov965x_try_frame_size(struct v4l2_mbus_framefmt *mf, } static int ov965x_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { unsigned int index = ARRAY_SIZE(ov965x_formats); @@ -1234,8 +1234,9 @@ static int ov965x_set_fmt(struct v4l2_subdev *sd, mutex_lock(&ov965x->lock); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - if (cfg) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + if (sd_state) { + mf = v4l2_subdev_get_try_format(sd, sd_state, + fmt->pad); *mf = fmt->format; } } else { @@ -1364,7 +1365,7 @@ static int ov965x_s_stream(struct v4l2_subdev *sd, int on) static int ov965x_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct v4l2_mbus_framefmt *mf = - v4l2_subdev_get_try_format(sd, fh->pad, 0); + v4l2_subdev_get_try_format(sd, fh->state, 0); ov965x_get_default_format(mf); return 0; @@ -1479,8 +1480,8 @@ static int ov965x_detect_sensor(struct v4l2_subdev *sd) if (ov965x->id == OV9650_ID || ov965x->id == OV9652_ID) { v4l2_info(sd, "Found OV%04X sensor\n", ov965x->id); } else { - v4l2_err(sd, "Sensor detection failed (%04X, %d)\n", - ov965x->id, ret); + v4l2_err(sd, "Sensor detection failed (%04X)\n", + ov965x->id); ret = -ENODEV; } } diff --git a/drivers/media/i2c/ov9734.c b/drivers/media/i2c/ov9734.c index b7309a551cae..af50c66cf5ce 100644 --- a/drivers/media/i2c/ov9734.c +++ b/drivers/media/i2c/ov9734.c @@ -644,9 +644,8 @@ static int ov9734_set_stream(struct v4l2_subdev *sd, int enable) } if (enable) { - ret = pm_runtime_get_sync(&client->dev); + ret = pm_runtime_resume_and_get(&client->dev); if (ret < 0) { - pm_runtime_put_noidle(&client->dev); mutex_unlock(&ov9734->mutex); return ret; } @@ -706,7 +705,7 @@ static int __maybe_unused ov9734_resume(struct device *dev) } static int ov9734_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov9734 *ov9734 = to_ov9734(sd); @@ -721,7 +720,7 @@ static int ov9734_set_format(struct v4l2_subdev *sd, mutex_lock(&ov9734->mutex); ov9734_update_pad_format(mode, &fmt->format); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = fmt->format; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = fmt->format; } else { ov9734->cur_mode = mode; __v4l2_ctrl_s_ctrl(ov9734->link_freq, mode->link_freq_index); @@ -747,14 +746,15 @@ static int ov9734_set_format(struct v4l2_subdev *sd, } static int ov9734_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ov9734 *ov9734 = to_ov9734(sd); mutex_lock(&ov9734->mutex); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(&ov9734->sd, cfg, + fmt->format = *v4l2_subdev_get_try_format(&ov9734->sd, + sd_state, fmt->pad); else ov9734_update_pad_format(ov9734->cur_mode, &fmt->format); @@ -765,7 +765,7 @@ static int ov9734_get_format(struct v4l2_subdev *sd, } static int ov9734_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index > 0) @@ -777,7 +777,7 @@ static int ov9734_enum_mbus_code(struct v4l2_subdev *sd, } static int ov9734_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index >= ARRAY_SIZE(supported_modes)) @@ -800,7 +800,7 @@ static int ov9734_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) mutex_lock(&ov9734->mutex); ov9734_update_pad_format(&supported_modes[0], - v4l2_subdev_get_try_format(sd, fh->pad, 0)); + v4l2_subdev_get_try_format(sd, fh->state, 0)); mutex_unlock(&ov9734->mutex); return 0; diff --git a/drivers/media/i2c/rdacm20.c b/drivers/media/i2c/rdacm20.c index 90eb73f0e6e9..025a610de893 100644 --- a/drivers/media/i2c/rdacm20.c +++ b/drivers/media/i2c/rdacm20.c @@ -312,7 +312,7 @@ static const struct ov10635_reg { struct rdacm20_device { struct device *dev; - struct max9271_device *serializer; + struct max9271_device serializer; struct i2c_client *sensor; struct v4l2_subdev sd; struct media_pad pad; @@ -399,11 +399,11 @@ static int rdacm20_s_stream(struct v4l2_subdev *sd, int enable) { struct rdacm20_device *dev = sd_to_rdacm20(sd); - return max9271_set_serial_link(dev->serializer, enable); + return max9271_set_serial_link(&dev->serializer, enable); } static int rdacm20_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index > 0) @@ -415,7 +415,7 @@ static int rdacm20_enum_mbus_code(struct v4l2_subdev *sd, } static int rdacm20_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -455,12 +455,10 @@ static int rdacm20_initialize(struct rdacm20_device *dev) unsigned int retry = 3; int ret; - /* Verify communication with the MAX9271: ping to wakeup. */ - dev->serializer->client->addr = MAX9271_DEFAULT_ADDR; - i2c_smbus_read_byte(dev->serializer->client); + max9271_wake_up(&dev->serializer); /* Serial link disabled during config as it needs a valid pixel clock. */ - ret = max9271_set_serial_link(dev->serializer, false); + ret = max9271_set_serial_link(&dev->serializer, false); if (ret) return ret; @@ -468,38 +466,48 @@ static int rdacm20_initialize(struct rdacm20_device *dev) * Ensure that we have a good link configuration before attempting to * identify the device. */ - max9271_configure_i2c(dev->serializer, MAX9271_I2CSLVSH_469NS_234NS | - MAX9271_I2CSLVTO_1024US | - MAX9271_I2CMSTBT_105KBPS); - - max9271_configure_gmsl_link(dev->serializer); - - ret = max9271_verify_id(dev->serializer); - if (ret < 0) + ret = max9271_configure_i2c(&dev->serializer, + MAX9271_I2CSLVSH_469NS_234NS | + MAX9271_I2CSLVTO_1024US | + MAX9271_I2CMSTBT_105KBPS); + if (ret) return ret; - ret = max9271_set_address(dev->serializer, dev->addrs[0]); - if (ret < 0) - return ret; - dev->serializer->client->addr = dev->addrs[0]; - /* - * Reset the sensor by cycling the OV10635 reset signal connected to the - * MAX9271 GPIO1 and verify communication with the OV10635. + * Hold OV10635 in reset during max9271 configuration. The reset signal + * has to be asserted for at least 200 microseconds. */ - ret = max9271_enable_gpios(dev->serializer, MAX9271_GPIO1OUT); + ret = max9271_enable_gpios(&dev->serializer, MAX9271_GPIO1OUT); if (ret) return ret; - ret = max9271_clear_gpios(dev->serializer, MAX9271_GPIO1OUT); + ret = max9271_clear_gpios(&dev->serializer, MAX9271_GPIO1OUT); if (ret) return ret; - usleep_range(10000, 15000); + usleep_range(200, 500); - ret = max9271_set_gpios(dev->serializer, MAX9271_GPIO1OUT); + ret = max9271_configure_gmsl_link(&dev->serializer); if (ret) return ret; - usleep_range(10000, 15000); + + ret = max9271_verify_id(&dev->serializer); + if (ret < 0) + return ret; + + ret = max9271_set_address(&dev->serializer, dev->addrs[0]); + if (ret < 0) + return ret; + dev->serializer.client->addr = dev->addrs[0]; + + /* + * Release ov10635 from reset and initialize it. The image sensor + * requires at least 2048 XVCLK cycles (85 micro-seconds at 24MHz) + * before being available. Stay safe and wait up to 500 micro-seconds. + */ + ret = max9271_set_gpios(&dev->serializer, MAX9271_GPIO1OUT); + if (ret) + return ret; + usleep_range(100, 500); again: ret = ov10635_read16(dev, OV10635_PID); @@ -539,9 +547,21 @@ static int rdacm20_initialize(struct rdacm20_device *dev) if (ret) return ret; - dev_info(dev->dev, "Identified MAX9271 + OV10635 device\n"); + dev_info(dev->dev, "Identified RDACM20 camera module\n"); - return 0; + /* + * Set reverse channel high threshold to increase noise immunity. + * + * This should be compensated by increasing the reverse channel + * amplitude on the remote deserializer side. + * + * TODO Inspect the embedded MCU programming sequence to make sure + * there are no conflicts with the configuration applied here. + * + * TODO Clarify the embedded MCU startup delay to avoid write + * collisions on the I2C bus. + */ + return max9271_set_high_threshold(&dev->serializer, true); } static int rdacm20_probe(struct i2c_client *client) @@ -554,13 +574,7 @@ static int rdacm20_probe(struct i2c_client *client) if (!dev) return -ENOMEM; dev->dev = &client->dev; - - dev->serializer = devm_kzalloc(&client->dev, sizeof(*dev->serializer), - GFP_KERNEL); - if (!dev->serializer) - return -ENOMEM; - - dev->serializer->client = client; + dev->serializer.client = client; ret = of_property_read_u32_array(client->dev.of_node, "reg", dev->addrs, 2); diff --git a/drivers/media/i2c/rdacm21.c b/drivers/media/i2c/rdacm21.c index 179d107f494c..12ec5467ed1e 100644 --- a/drivers/media/i2c/rdacm21.c +++ b/drivers/media/i2c/rdacm21.c @@ -69,6 +69,7 @@ #define OV490_ISP_VSIZE_LOW 0x80820062 #define OV490_ISP_VSIZE_HIGH 0x80820063 +#define OV10640_PID_TIMEOUT 20 #define OV10640_ID_HIGH 0xa6 #define OV10640_CHIP_ID 0x300a #define OV10640_PIXEL_RATE 55000000 @@ -281,7 +282,7 @@ static int rdacm21_s_stream(struct v4l2_subdev *sd, int enable) } static int rdacm21_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index > 0) @@ -293,7 +294,7 @@ static int rdacm21_enum_mbus_code(struct v4l2_subdev *sd, } static int rdacm21_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -329,30 +330,51 @@ static const struct v4l2_subdev_ops rdacm21_subdev_ops = { .pad = &rdacm21_subdev_pad_ops, }; -static int ov10640_initialize(struct rdacm21_device *dev) +static void ov10640_power_up(struct rdacm21_device *dev) { - u8 val; - - /* Power-up OV10640 by setting RESETB and PWDNB pins high. */ + /* Enable GPIO0#0 (reset) and GPIO1#0 (pwdn) as output lines. */ ov490_write_reg(dev, OV490_GPIO_SEL0, OV490_GPIO0); ov490_write_reg(dev, OV490_GPIO_SEL1, OV490_SPWDN0); ov490_write_reg(dev, OV490_GPIO_DIRECTION0, OV490_GPIO0); ov490_write_reg(dev, OV490_GPIO_DIRECTION1, OV490_SPWDN0); + + /* Power up OV10640 and then reset it. */ + ov490_write_reg(dev, OV490_GPIO_OUTPUT_VALUE1, OV490_SPWDN0); + usleep_range(1500, 3000); + + ov490_write_reg(dev, OV490_GPIO_OUTPUT_VALUE0, 0x00); + usleep_range(1500, 3000); ov490_write_reg(dev, OV490_GPIO_OUTPUT_VALUE0, OV490_GPIO0); - ov490_write_reg(dev, OV490_GPIO_OUTPUT_VALUE0, OV490_SPWDN0); usleep_range(3000, 5000); +} + +static int ov10640_check_id(struct rdacm21_device *dev) +{ + unsigned int i; + u8 val; /* Read OV10640 ID to test communications. */ - ov490_write_reg(dev, OV490_SCCB_SLAVE0_DIR, OV490_SCCB_SLAVE_READ); - ov490_write_reg(dev, OV490_SCCB_SLAVE0_ADDR_HIGH, OV10640_CHIP_ID >> 8); - ov490_write_reg(dev, OV490_SCCB_SLAVE0_ADDR_LOW, OV10640_CHIP_ID & 0xff); + for (i = 0; i < OV10640_PID_TIMEOUT; ++i) { + ov490_write_reg(dev, OV490_SCCB_SLAVE0_DIR, + OV490_SCCB_SLAVE_READ); + ov490_write_reg(dev, OV490_SCCB_SLAVE0_ADDR_HIGH, + OV10640_CHIP_ID >> 8); + ov490_write_reg(dev, OV490_SCCB_SLAVE0_ADDR_LOW, + OV10640_CHIP_ID & 0xff); - /* Trigger SCCB slave transaction and give it some time to complete. */ - ov490_write_reg(dev, OV490_HOST_CMD, OV490_HOST_CMD_TRIGGER); - usleep_range(1000, 1500); + /* + * Trigger SCCB slave transaction and give it some time + * to complete. + */ + ov490_write_reg(dev, OV490_HOST_CMD, OV490_HOST_CMD_TRIGGER); + usleep_range(1000, 1500); - ov490_read_reg(dev, OV490_SCCB_SLAVE0_DIR, &val); - if (val != OV10640_ID_HIGH) { + ov490_read_reg(dev, OV490_SCCB_SLAVE0_DIR, &val); + if (val == OV10640_ID_HIGH) + break; + usleep_range(1000, 1500); + } + if (i == OV10640_PID_TIMEOUT) { dev_err(dev->dev, "OV10640 ID mismatch: (0x%02x)\n", val); return -ENODEV; } @@ -368,6 +390,8 @@ static int ov490_initialize(struct rdacm21_device *dev) unsigned int i; int ret; + ov10640_power_up(dev); + /* * Read OV490 Id to test communications. Give it up to 40msec to * exit from reset. @@ -405,7 +429,7 @@ static int ov490_initialize(struct rdacm21_device *dev) return -ENODEV; } - ret = ov10640_initialize(dev); + ret = ov10640_check_id(dev); if (ret) return ret; @@ -450,10 +474,7 @@ static int rdacm21_initialize(struct rdacm21_device *dev) { int ret; - /* Verify communication with the MAX9271: ping to wakeup. */ - dev->serializer.client->addr = MAX9271_DEFAULT_ADDR; - i2c_smbus_read_byte(dev->serializer.client); - usleep_range(3000, 5000); + max9271_wake_up(&dev->serializer); /* Enable reverse channel and disable the serial link. */ ret = max9271_set_serial_link(&dev->serializer, false); @@ -472,7 +493,10 @@ static int rdacm21_initialize(struct rdacm21_device *dev) if (ret) return ret; - /* Enable GPIO1 and hold OV490 in reset during max9271 configuration. */ + /* + * Enable GPIO1 and hold OV490 in reset during max9271 configuration. + * The reset signal has to be asserted for at least 250 useconds. + */ ret = max9271_enable_gpios(&dev->serializer, MAX9271_GPIO1OUT); if (ret) return ret; @@ -480,6 +504,7 @@ static int rdacm21_initialize(struct rdacm21_device *dev) ret = max9271_clear_gpios(&dev->serializer, MAX9271_GPIO1OUT); if (ret) return ret; + usleep_range(250, 500); ret = max9271_configure_gmsl_link(&dev->serializer); if (ret) diff --git a/drivers/media/i2c/rj54n1cb0c.c b/drivers/media/i2c/rj54n1cb0c.c index 4cc51e001874..2e4018c26912 100644 --- a/drivers/media/i2c/rj54n1cb0c.c +++ b/drivers/media/i2c/rj54n1cb0c.c @@ -488,7 +488,7 @@ static int reg_write_multiple(struct i2c_client *client, } static int rj54n1_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(rj54n1_colour_fmts)) @@ -541,7 +541,7 @@ static int rj54n1_sensor_scale(struct v4l2_subdev *sd, s32 *in_w, s32 *in_h, s32 *out_w, s32 *out_h); static int rj54n1_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -578,7 +578,7 @@ static int rj54n1_set_selection(struct v4l2_subdev *sd, } static int rj54n1_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -603,7 +603,7 @@ static int rj54n1_get_selection(struct v4l2_subdev *sd, } static int rj54n1_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -973,7 +973,7 @@ static int rj54n1_reg_init(struct i2c_client *client) } static int rj54n1_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -1009,7 +1009,7 @@ static int rj54n1_set_fmt(struct v4l2_subdev *sd, &mf->height, 84, RJ54N1_MAX_HEIGHT, align, 0); if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } diff --git a/drivers/media/i2c/s5c73m3/s5c73m3-core.c b/drivers/media/i2c/s5c73m3/s5c73m3-core.c index 5b4c4a3547c9..e2b88c5e4f98 100644 --- a/drivers/media/i2c/s5c73m3/s5c73m3-core.c +++ b/drivers/media/i2c/s5c73m3/s5c73m3-core.c @@ -817,7 +817,7 @@ static const struct s5c73m3_frame_size *s5c73m3_find_frame_size( } static void s5c73m3_oif_try_format(struct s5c73m3 *state, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt, const struct s5c73m3_frame_size **fs) { @@ -844,8 +844,8 @@ static void s5c73m3_oif_try_format(struct s5c73m3 *state, *fs = state->oif_pix_size[RES_ISP]; else *fs = s5c73m3_find_frame_size( - v4l2_subdev_get_try_format(sd, cfg, - OIF_ISP_PAD), + v4l2_subdev_get_try_format(sd, sd_state, + OIF_ISP_PAD), RES_ISP); break; } @@ -854,7 +854,7 @@ static void s5c73m3_oif_try_format(struct s5c73m3 *state, } static void s5c73m3_try_format(struct s5c73m3 *state, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt, const struct s5c73m3_frame_size **fs) { @@ -946,7 +946,7 @@ static int s5c73m3_oif_s_frame_interval(struct v4l2_subdev *sd, } static int s5c73m3_oif_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct s5c73m3 *state = oif_sd_to_s5c73m3(sd); @@ -984,7 +984,7 @@ static int s5c73m3_oif_get_pad_code(int pad, int index) } static int s5c73m3_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct s5c73m3 *state = sensor_sd_to_s5c73m3(sd); @@ -992,7 +992,8 @@ static int s5c73m3_get_fmt(struct v4l2_subdev *sd, u32 code; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - fmt->format = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + fmt->format = *v4l2_subdev_get_try_format(sd, sd_state, + fmt->pad); return 0; } @@ -1018,7 +1019,7 @@ static int s5c73m3_get_fmt(struct v4l2_subdev *sd, } static int s5c73m3_oif_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct s5c73m3 *state = oif_sd_to_s5c73m3(sd); @@ -1026,7 +1027,8 @@ static int s5c73m3_oif_get_fmt(struct v4l2_subdev *sd, u32 code; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - fmt->format = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + fmt->format = *v4l2_subdev_get_try_format(sd, sd_state, + fmt->pad); return 0; } @@ -1056,7 +1058,7 @@ static int s5c73m3_oif_get_fmt(struct v4l2_subdev *sd, } static int s5c73m3_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { const struct s5c73m3_frame_size *frame_size = NULL; @@ -1066,10 +1068,10 @@ static int s5c73m3_set_fmt(struct v4l2_subdev *sd, mutex_lock(&state->lock); - s5c73m3_try_format(state, cfg, fmt, &frame_size); + s5c73m3_try_format(state, sd_state, fmt, &frame_size); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *mf = fmt->format; } else { switch (fmt->pad) { @@ -1095,7 +1097,7 @@ static int s5c73m3_set_fmt(struct v4l2_subdev *sd, } static int s5c73m3_oif_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { const struct s5c73m3_frame_size *frame_size = NULL; @@ -1105,13 +1107,14 @@ static int s5c73m3_oif_set_fmt(struct v4l2_subdev *sd, mutex_lock(&state->lock); - s5c73m3_oif_try_format(state, cfg, fmt, &frame_size); + s5c73m3_oif_try_format(state, sd_state, fmt, &frame_size); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *mf = fmt->format; if (fmt->pad == OIF_ISP_PAD) { - mf = v4l2_subdev_get_try_format(sd, cfg, OIF_SOURCE_PAD); + mf = v4l2_subdev_get_try_format(sd, sd_state, + OIF_SOURCE_PAD); mf->width = fmt->format.width; mf->height = fmt->format.height; } @@ -1183,7 +1186,7 @@ static int s5c73m3_oif_set_frame_desc(struct v4l2_subdev *sd, unsigned int pad, } static int s5c73m3_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { static const int codes[] = { @@ -1199,7 +1202,7 @@ static int s5c73m3_enum_mbus_code(struct v4l2_subdev *sd, } static int s5c73m3_oif_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { int ret; @@ -1214,7 +1217,7 @@ static int s5c73m3_oif_enum_mbus_code(struct v4l2_subdev *sd, } static int s5c73m3_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int idx; @@ -1241,7 +1244,7 @@ static int s5c73m3_enum_frame_size(struct v4l2_subdev *sd, } static int s5c73m3_oif_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct s5c73m3 *state = oif_sd_to_s5c73m3(sd); @@ -1259,7 +1262,7 @@ static int s5c73m3_oif_enum_frame_size(struct v4l2_subdev *sd, if (fse->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *mf; - mf = v4l2_subdev_get_try_format(sd, cfg, + mf = v4l2_subdev_get_try_format(sd, sd_state, OIF_ISP_PAD); w = mf->width; @@ -1315,11 +1318,11 @@ static int s5c73m3_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct v4l2_mbus_framefmt *mf; - mf = v4l2_subdev_get_try_format(sd, fh->pad, S5C73M3_ISP_PAD); + mf = v4l2_subdev_get_try_format(sd, fh->state, S5C73M3_ISP_PAD); s5c73m3_fill_mbus_fmt(mf, &s5c73m3_isp_resolutions[1], S5C73M3_ISP_FMT); - mf = v4l2_subdev_get_try_format(sd, fh->pad, S5C73M3_JPEG_PAD); + mf = v4l2_subdev_get_try_format(sd, fh->state, S5C73M3_JPEG_PAD); s5c73m3_fill_mbus_fmt(mf, &s5c73m3_jpeg_resolutions[1], S5C73M3_JPEG_FMT); @@ -1330,15 +1333,15 @@ static int s5c73m3_oif_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct v4l2_mbus_framefmt *mf; - mf = v4l2_subdev_get_try_format(sd, fh->pad, OIF_ISP_PAD); + mf = v4l2_subdev_get_try_format(sd, fh->state, OIF_ISP_PAD); s5c73m3_fill_mbus_fmt(mf, &s5c73m3_isp_resolutions[1], S5C73M3_ISP_FMT); - mf = v4l2_subdev_get_try_format(sd, fh->pad, OIF_JPEG_PAD); + mf = v4l2_subdev_get_try_format(sd, fh->state, OIF_JPEG_PAD); s5c73m3_fill_mbus_fmt(mf, &s5c73m3_jpeg_resolutions[1], S5C73M3_JPEG_FMT); - mf = v4l2_subdev_get_try_format(sd, fh->pad, OIF_SOURCE_PAD); + mf = v4l2_subdev_get_try_format(sd, fh->state, OIF_SOURCE_PAD); s5c73m3_fill_mbus_fmt(mf, &s5c73m3_isp_resolutions[1], S5C73M3_ISP_FMT); return 0; @@ -1386,7 +1389,7 @@ static int __s5c73m3_power_on(struct s5c73m3 *state) s5c73m3_gpio_deassert(state, STBY); usleep_range(100, 200); - s5c73m3_gpio_deassert(state, RST); + s5c73m3_gpio_deassert(state, RSET); usleep_range(50, 100); return 0; @@ -1401,7 +1404,7 @@ static int __s5c73m3_power_off(struct s5c73m3 *state) { int i, ret; - if (s5c73m3_gpio_assert(state, RST)) + if (s5c73m3_gpio_assert(state, RSET)) usleep_range(10, 50); if (s5c73m3_gpio_assert(state, STBY)) @@ -1606,7 +1609,7 @@ static int s5c73m3_get_platform_data(struct s5c73m3 *state) state->mclk_frequency = pdata->mclk_frequency; state->gpio[STBY] = pdata->gpio_stby; - state->gpio[RST] = pdata->gpio_reset; + state->gpio[RSET] = pdata->gpio_reset; return 0; } diff --git a/drivers/media/i2c/s5c73m3/s5c73m3.h b/drivers/media/i2c/s5c73m3/s5c73m3.h index ef7e85b34263..c3fcfdd3ea66 100644 --- a/drivers/media/i2c/s5c73m3/s5c73m3.h +++ b/drivers/media/i2c/s5c73m3/s5c73m3.h @@ -353,7 +353,7 @@ struct s5c73m3_ctrls { enum s5c73m3_gpio_id { STBY, - RST, + RSET, GPIO_NUM, }; diff --git a/drivers/media/i2c/s5k4ecgx.c b/drivers/media/i2c/s5k4ecgx.c index b2d53417badf..af9a305242cd 100644 --- a/drivers/media/i2c/s5k4ecgx.c +++ b/drivers/media/i2c/s5k4ecgx.c @@ -173,7 +173,7 @@ static const char * const s5k4ecgx_supply_names[] = { enum s5k4ecgx_gpio_id { STBY, - RST, + RSET, GPIO_NUM, }; @@ -476,7 +476,7 @@ static int __s5k4ecgx_power_on(struct s5k4ecgx *priv) if (s5k4ecgx_gpio_set_value(priv, STBY, priv->gpio[STBY].level)) usleep_range(30, 50); - if (s5k4ecgx_gpio_set_value(priv, RST, priv->gpio[RST].level)) + if (s5k4ecgx_gpio_set_value(priv, RSET, priv->gpio[RSET].level)) usleep_range(30, 50); return 0; @@ -484,7 +484,7 @@ static int __s5k4ecgx_power_on(struct s5k4ecgx *priv) static int __s5k4ecgx_power_off(struct s5k4ecgx *priv) { - if (s5k4ecgx_gpio_set_value(priv, RST, !priv->gpio[RST].level)) + if (s5k4ecgx_gpio_set_value(priv, RSET, !priv->gpio[RSET].level)) usleep_range(30, 50); if (s5k4ecgx_gpio_set_value(priv, STBY, !priv->gpio[STBY].level)) @@ -525,7 +525,7 @@ static int s5k4ecgx_try_frame_size(struct v4l2_mbus_framefmt *mf, } static int s5k4ecgx_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(s5k4ecgx_formats)) @@ -535,15 +535,16 @@ static int s5k4ecgx_enum_mbus_code(struct v4l2_subdev *sd, return 0; } -static int s5k4ecgx_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *fmt) +static int s5k4ecgx_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) { struct s5k4ecgx *priv = to_s5k4ecgx(sd); struct v4l2_mbus_framefmt *mf; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - if (cfg) { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + if (sd_state) { + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); fmt->format = *mf; } return 0; @@ -575,7 +576,8 @@ static const struct s5k4ecgx_pixfmt *s5k4ecgx_try_fmt(struct v4l2_subdev *sd, return &s5k4ecgx_formats[i]; } -static int s5k4ecgx_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int s5k4ecgx_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct s5k4ecgx *priv = to_s5k4ecgx(sd); @@ -590,8 +592,8 @@ static int s5k4ecgx_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_confi fmt->format.field = V4L2_FIELD_NONE; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - if (cfg) { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + if (sd_state) { + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); *mf = fmt->format; } return 0; @@ -686,7 +688,9 @@ static int s5k4ecgx_registered(struct v4l2_subdev *sd) */ static int s5k4ecgx_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - struct v4l2_mbus_framefmt *mf = v4l2_subdev_get_try_format(sd, fh->pad, 0); + struct v4l2_mbus_framefmt *mf = v4l2_subdev_get_try_format(sd, + fh->state, + 0); mf->width = s5k4ecgx_prev_sizes[0].size.width; mf->height = s5k4ecgx_prev_sizes[0].size.height; @@ -872,7 +876,7 @@ static int s5k4ecgx_config_gpios(struct s5k4ecgx *priv, int ret; priv->gpio[STBY].gpio = -EINVAL; - priv->gpio[RST].gpio = -EINVAL; + priv->gpio[RSET].gpio = -EINVAL; ret = s5k4ecgx_config_gpio(gpio->gpio, gpio->level, "S5K4ECGX_STBY"); @@ -891,7 +895,7 @@ static int s5k4ecgx_config_gpios(struct s5k4ecgx *priv, s5k4ecgx_free_gpios(priv); return ret; } - priv->gpio[RST] = *gpio; + priv->gpio[RSET] = *gpio; if (gpio_is_valid(gpio->gpio)) gpio_set_value(gpio->gpio, 0); diff --git a/drivers/media/i2c/s5k5baf.c b/drivers/media/i2c/s5k5baf.c index 6e702b57c37d..6a5dceb699a8 100644 --- a/drivers/media/i2c/s5k5baf.c +++ b/drivers/media/i2c/s5k5baf.c @@ -235,7 +235,7 @@ struct s5k5baf_gpio { enum s5k5baf_gpio_id { STBY, - RST, + RSET, NUM_GPIOS, }; @@ -969,7 +969,7 @@ static int s5k5baf_power_on(struct s5k5baf *state) s5k5baf_gpio_deassert(state, STBY); usleep_range(50, 100); - s5k5baf_gpio_deassert(state, RST); + s5k5baf_gpio_deassert(state, RSET); return 0; err_reg_dis: @@ -987,7 +987,7 @@ static int s5k5baf_power_off(struct s5k5baf *state) state->apply_cfg = 0; state->apply_crop = 0; - s5k5baf_gpio_assert(state, RST); + s5k5baf_gpio_assert(state, RSET); s5k5baf_gpio_assert(state, STBY); if (!IS_ERR(state->clock)) @@ -1180,7 +1180,7 @@ static int s5k5baf_s_frame_interval(struct v4l2_subdev *sd, * V4L2 subdev pad level and video operations */ static int s5k5baf_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { if (fie->index > S5K5BAF_MAX_FR_TIME - S5K5BAF_MIN_FR_TIME || @@ -1199,7 +1199,7 @@ static int s5k5baf_enum_frame_interval(struct v4l2_subdev *sd, } static int s5k5baf_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad == PAD_CIS) { @@ -1217,7 +1217,7 @@ static int s5k5baf_enum_mbus_code(struct v4l2_subdev *sd, } static int s5k5baf_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int i; @@ -1274,15 +1274,16 @@ static int s5k5baf_try_isp_format(struct v4l2_mbus_framefmt *mf) return pixfmt; } -static int s5k5baf_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *fmt) +static int s5k5baf_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) { struct s5k5baf *state = to_s5k5baf(sd); const struct s5k5baf_pixfmt *pixfmt; struct v4l2_mbus_framefmt *mf; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); fmt->format = *mf; return 0; } @@ -1304,8 +1305,9 @@ static int s5k5baf_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config return 0; } -static int s5k5baf_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *fmt) +static int s5k5baf_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) { struct v4l2_mbus_framefmt *mf = &fmt->format; struct s5k5baf *state = to_s5k5baf(sd); @@ -1315,7 +1317,7 @@ static int s5k5baf_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config mf->field = V4L2_FIELD_NONE; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) = *mf; + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) = *mf; return 0; } @@ -1367,7 +1369,7 @@ static int s5k5baf_is_bound_target(u32 target) } static int s5k5baf_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { enum selection_rect rtype; @@ -1387,9 +1389,11 @@ static int s5k5baf_get_selection(struct v4l2_subdev *sd, if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { if (rtype == R_COMPOSE) - sel->r = *v4l2_subdev_get_try_compose(sd, cfg, sel->pad); + sel->r = *v4l2_subdev_get_try_compose(sd, sd_state, + sel->pad); else - sel->r = *v4l2_subdev_get_try_crop(sd, cfg, sel->pad); + sel->r = *v4l2_subdev_get_try_crop(sd, sd_state, + sel->pad); return 0; } @@ -1458,7 +1462,7 @@ static bool s5k5baf_cmp_rect(const struct v4l2_rect *r1, } static int s5k5baf_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { static enum selection_rect rtype; @@ -1479,9 +1483,12 @@ static int s5k5baf_set_selection(struct v4l2_subdev *sd, if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { rects = (struct v4l2_rect * []) { &s5k5baf_cis_rect, - v4l2_subdev_get_try_crop(sd, cfg, PAD_CIS), - v4l2_subdev_get_try_compose(sd, cfg, PAD_CIS), - v4l2_subdev_get_try_crop(sd, cfg, PAD_OUT) + v4l2_subdev_get_try_crop(sd, sd_state, + PAD_CIS), + v4l2_subdev_get_try_compose(sd, sd_state, + PAD_CIS), + v4l2_subdev_get_try_crop(sd, sd_state, + PAD_OUT) }; s5k5baf_set_rect_and_adjust(rects, rtype, &sel->r); return 0; @@ -1699,22 +1706,22 @@ static int s5k5baf_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { struct v4l2_mbus_framefmt *mf; - mf = v4l2_subdev_get_try_format(sd, fh->pad, PAD_CIS); + mf = v4l2_subdev_get_try_format(sd, fh->state, PAD_CIS); s5k5baf_try_cis_format(mf); if (s5k5baf_is_cis_subdev(sd)) return 0; - mf = v4l2_subdev_get_try_format(sd, fh->pad, PAD_OUT); + mf = v4l2_subdev_get_try_format(sd, fh->state, PAD_OUT); mf->colorspace = s5k5baf_formats[0].colorspace; mf->code = s5k5baf_formats[0].code; mf->width = s5k5baf_cis_rect.width; mf->height = s5k5baf_cis_rect.height; mf->field = V4L2_FIELD_NONE; - *v4l2_subdev_get_try_crop(sd, fh->pad, PAD_CIS) = s5k5baf_cis_rect; - *v4l2_subdev_get_try_compose(sd, fh->pad, PAD_CIS) = s5k5baf_cis_rect; - *v4l2_subdev_get_try_crop(sd, fh->pad, PAD_OUT) = s5k5baf_cis_rect; + *v4l2_subdev_get_try_crop(sd, fh->state, PAD_CIS) = s5k5baf_cis_rect; + *v4l2_subdev_get_try_compose(sd, fh->state, PAD_CIS) = s5k5baf_cis_rect; + *v4l2_subdev_get_try_crop(sd, fh->state, PAD_OUT) = s5k5baf_cis_rect; return 0; } diff --git a/drivers/media/i2c/s5k6a3.c b/drivers/media/i2c/s5k6a3.c index f26c168ef942..b97dd6149e90 100644 --- a/drivers/media/i2c/s5k6a3.c +++ b/drivers/media/i2c/s5k6a3.c @@ -99,7 +99,7 @@ static const struct v4l2_mbus_framefmt *find_sensor_format( } static int s5k6a3_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(s5k6a3_formats)) @@ -123,17 +123,18 @@ static void s5k6a3_try_format(struct v4l2_mbus_framefmt *mf) } static struct v4l2_mbus_framefmt *__s5k6a3_get_format( - struct s5k6a3 *sensor, struct v4l2_subdev_pad_config *cfg, + struct s5k6a3 *sensor, struct v4l2_subdev_state *sd_state, u32 pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return cfg ? v4l2_subdev_get_try_format(&sensor->subdev, cfg, pad) : NULL; + return sd_state ? v4l2_subdev_get_try_format(&sensor->subdev, + sd_state, pad) : NULL; return &sensor->format; } static int s5k6a3_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct s5k6a3 *sensor = sd_to_s5k6a3(sd); @@ -141,7 +142,7 @@ static int s5k6a3_set_fmt(struct v4l2_subdev *sd, s5k6a3_try_format(&fmt->format); - mf = __s5k6a3_get_format(sensor, cfg, fmt->pad, fmt->which); + mf = __s5k6a3_get_format(sensor, sd_state, fmt->pad, fmt->which); if (mf) { mutex_lock(&sensor->lock); *mf = fmt->format; @@ -151,13 +152,13 @@ static int s5k6a3_set_fmt(struct v4l2_subdev *sd, } static int s5k6a3_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct s5k6a3 *sensor = sd_to_s5k6a3(sd); struct v4l2_mbus_framefmt *mf; - mf = __s5k6a3_get_format(sensor, cfg, fmt->pad, fmt->which); + mf = __s5k6a3_get_format(sensor, sd_state, fmt->pad, fmt->which); mutex_lock(&sensor->lock); fmt->format = *mf; @@ -173,7 +174,9 @@ static const struct v4l2_subdev_pad_ops s5k6a3_pad_ops = { static int s5k6a3_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd, fh->pad, 0); + struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd, + fh->state, + 0); *format = s5k6a3_formats[0]; format->width = S5K6A3_DEFAULT_WIDTH; diff --git a/drivers/media/i2c/s5k6aa.c b/drivers/media/i2c/s5k6aa.c index 038e38500760..105a4b7d8354 100644 --- a/drivers/media/i2c/s5k6aa.c +++ b/drivers/media/i2c/s5k6aa.c @@ -177,7 +177,7 @@ static const char * const s5k6aa_supply_names[] = { enum s5k6aa_gpio_id { STBY, - RST, + RSET, GPIO_NUM, }; @@ -841,7 +841,7 @@ static int __s5k6aa_power_on(struct s5k6aa *s5k6aa) ret = s5k6aa->s_power(1); usleep_range(4000, 5000); - if (s5k6aa_gpio_deassert(s5k6aa, RST)) + if (s5k6aa_gpio_deassert(s5k6aa, RSET)) msleep(20); return ret; @@ -851,7 +851,7 @@ static int __s5k6aa_power_off(struct s5k6aa *s5k6aa) { int ret; - if (s5k6aa_gpio_assert(s5k6aa, RST)) + if (s5k6aa_gpio_assert(s5k6aa, RSET)) usleep_range(100, 150); if (s5k6aa->s_power) { @@ -997,7 +997,7 @@ static int s5k6aa_s_frame_interval(struct v4l2_subdev *sd, * V4L2 subdev pad level and video operations */ static int s5k6aa_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct s5k6aa *s5k6aa = to_s5k6aa(sd); @@ -1024,7 +1024,7 @@ static int s5k6aa_enum_frame_interval(struct v4l2_subdev *sd, } static int s5k6aa_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(s5k6aa_formats)) @@ -1035,7 +1035,7 @@ static int s5k6aa_enum_mbus_code(struct v4l2_subdev *sd, } static int s5k6aa_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int i = ARRAY_SIZE(s5k6aa_formats); @@ -1057,14 +1057,15 @@ static int s5k6aa_enum_frame_size(struct v4l2_subdev *sd, } static struct v4l2_rect * -__s5k6aa_get_crop_rect(struct s5k6aa *s5k6aa, struct v4l2_subdev_pad_config *cfg, +__s5k6aa_get_crop_rect(struct s5k6aa *s5k6aa, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_ACTIVE) return &s5k6aa->ccd_rect; WARN_ON(which != V4L2_SUBDEV_FORMAT_TRY); - return v4l2_subdev_get_try_crop(&s5k6aa->sd, cfg, 0); + return v4l2_subdev_get_try_crop(&s5k6aa->sd, sd_state, 0); } static void s5k6aa_try_format(struct s5k6aa *s5k6aa, @@ -1088,7 +1089,8 @@ static void s5k6aa_try_format(struct s5k6aa *s5k6aa, mf->field = V4L2_FIELD_NONE; } -static int s5k6aa_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int s5k6aa_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct s5k6aa *s5k6aa = to_s5k6aa(sd); @@ -1097,7 +1099,7 @@ static int s5k6aa_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config memset(fmt->reserved, 0, sizeof(fmt->reserved)); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); fmt->format = *mf; return 0; } @@ -1109,7 +1111,8 @@ static int s5k6aa_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config return 0; } -static int s5k6aa_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int s5k6aa_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct s5k6aa *s5k6aa = to_s5k6aa(sd); @@ -1122,8 +1125,8 @@ static int s5k6aa_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config s5k6aa_try_format(s5k6aa, &fmt->format); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); - crop = v4l2_subdev_get_try_crop(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); + crop = v4l2_subdev_get_try_crop(sd, sd_state, 0); } else { if (s5k6aa->streaming) { ret = -EBUSY; @@ -1163,7 +1166,7 @@ static int s5k6aa_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config } static int s5k6aa_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct s5k6aa *s5k6aa = to_s5k6aa(sd); @@ -1175,7 +1178,7 @@ static int s5k6aa_get_selection(struct v4l2_subdev *sd, memset(sel->reserved, 0, sizeof(sel->reserved)); mutex_lock(&s5k6aa->lock); - rect = __s5k6aa_get_crop_rect(s5k6aa, cfg, sel->which); + rect = __s5k6aa_get_crop_rect(s5k6aa, sd_state, sel->which); sel->r = *rect; mutex_unlock(&s5k6aa->lock); @@ -1186,7 +1189,7 @@ static int s5k6aa_get_selection(struct v4l2_subdev *sd, } static int s5k6aa_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct s5k6aa *s5k6aa = to_s5k6aa(sd); @@ -1198,13 +1201,13 @@ static int s5k6aa_set_selection(struct v4l2_subdev *sd, return -EINVAL; mutex_lock(&s5k6aa->lock); - crop_r = __s5k6aa_get_crop_rect(s5k6aa, cfg, sel->which); + crop_r = __s5k6aa_get_crop_rect(s5k6aa, sd_state, sel->which); if (sel->which == V4L2_SUBDEV_FORMAT_ACTIVE) { mf = &s5k6aa->preset->mbus_fmt; s5k6aa->apply_crop = 1; } else { - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); } v4l_bound_align_image(&sel->r.width, mf->width, S5K6AA_WIN_WIDTH_MAX, 1, @@ -1425,8 +1428,10 @@ static int s5k6aa_initialize_ctrls(struct s5k6aa *s5k6aa) */ static int s5k6aa_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd, fh->pad, 0); - struct v4l2_rect *crop = v4l2_subdev_get_try_crop(sd, fh->pad, 0); + struct v4l2_mbus_framefmt *format = v4l2_subdev_get_try_format(sd, + fh->state, + 0); + struct v4l2_rect *crop = v4l2_subdev_get_try_crop(sd, fh->state, 0); format->colorspace = s5k6aa_formats[0].colorspace; format->code = s5k6aa_formats[0].code; @@ -1510,7 +1515,7 @@ static int s5k6aa_configure_gpios(struct s5k6aa *s5k6aa, int ret; s5k6aa->gpio[STBY].gpio = -EINVAL; - s5k6aa->gpio[RST].gpio = -EINVAL; + s5k6aa->gpio[RSET].gpio = -EINVAL; gpio = &pdata->gpio_stby; if (gpio_is_valid(gpio->gpio)) { @@ -1533,7 +1538,7 @@ static int s5k6aa_configure_gpios(struct s5k6aa *s5k6aa, if (ret < 0) return ret; - s5k6aa->gpio[RST] = *gpio; + s5k6aa->gpio[RSET] = *gpio; } return 0; diff --git a/drivers/media/i2c/saa6588.c b/drivers/media/i2c/saa6588.c index ecb491d5f2ab..d1e0716bdfff 100644 --- a/drivers/media/i2c/saa6588.c +++ b/drivers/media/i2c/saa6588.c @@ -380,7 +380,7 @@ static void saa6588_configure(struct saa6588 *s) /* ---------------------------------------------------------------------- */ -static long saa6588_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg) +static long saa6588_command(struct v4l2_subdev *sd, unsigned int cmd, void *arg) { struct saa6588 *s = to_saa6588(sd); struct saa6588_command *a = arg; @@ -433,7 +433,7 @@ static int saa6588_s_tuner(struct v4l2_subdev *sd, const struct v4l2_tuner *vt) /* ----------------------------------------------------------------------- */ static const struct v4l2_subdev_core_ops saa6588_core_ops = { - .ioctl = saa6588_ioctl, + .command = saa6588_command, }; static const struct v4l2_subdev_tuner_ops saa6588_tuner_ops = { diff --git a/drivers/media/i2c/saa6752hs.c b/drivers/media/i2c/saa6752hs.c index 6171ced809bb..a7f043cad149 100644 --- a/drivers/media/i2c/saa6752hs.c +++ b/drivers/media/i2c/saa6752hs.c @@ -543,7 +543,7 @@ static int saa6752hs_init(struct v4l2_subdev *sd, u32 leading_null_bytes) } static int saa6752hs_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *f = &format->format; @@ -563,7 +563,7 @@ static int saa6752hs_get_fmt(struct v4l2_subdev *sd, } static int saa6752hs_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *f = &format->format; @@ -595,7 +595,7 @@ static int saa6752hs_set_fmt(struct v4l2_subdev *sd, f->colorspace = V4L2_COLORSPACE_SMPTE170M; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *f; + sd_state->pads->try_fmt = *f; return 0; } diff --git a/drivers/media/i2c/saa7115.c b/drivers/media/i2c/saa7115.c index 88dc6baac639..a958bbc2c33d 100644 --- a/drivers/media/i2c/saa7115.c +++ b/drivers/media/i2c/saa7115.c @@ -1167,7 +1167,7 @@ static int saa711x_s_sliced_fmt(struct v4l2_subdev *sd, struct v4l2_sliced_vbi_f } static int saa711x_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; diff --git a/drivers/media/i2c/saa717x.c b/drivers/media/i2c/saa717x.c index ba103a6a1875..adf905360171 100644 --- a/drivers/media/i2c/saa717x.c +++ b/drivers/media/i2c/saa717x.c @@ -980,7 +980,7 @@ static int saa717x_s_register(struct v4l2_subdev *sd, const struct v4l2_dbg_regi #endif static int saa717x_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; diff --git a/drivers/media/i2c/sr030pc30.c b/drivers/media/i2c/sr030pc30.c index 46924024faa8..19c0252df2f1 100644 --- a/drivers/media/i2c/sr030pc30.c +++ b/drivers/media/i2c/sr030pc30.c @@ -468,7 +468,7 @@ static int sr030pc30_s_ctrl(struct v4l2_ctrl *ctrl) } static int sr030pc30_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (!code || code->pad || @@ -480,7 +480,7 @@ static int sr030pc30_enum_mbus_code(struct v4l2_subdev *sd, } static int sr030pc30_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf; @@ -525,7 +525,7 @@ static const struct sr030pc30_format *try_fmt(struct v4l2_subdev *sd, /* Return nearest media bus frame format. */ static int sr030pc30_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct sr030pc30_info *info = sd ? to_sr030pc30(sd) : NULL; @@ -541,7 +541,7 @@ static int sr030pc30_set_fmt(struct v4l2_subdev *sd, fmt = try_fmt(sd, mf); if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } diff --git a/drivers/media/i2c/st-mipid02.c b/drivers/media/i2c/st-mipid02.c index 7f07ef56fbbd..f630b88cbfaa 100644 --- a/drivers/media/i2c/st-mipid02.c +++ b/drivers/media/i2c/st-mipid02.c @@ -643,7 +643,7 @@ static int mipid02_s_stream(struct v4l2_subdev *sd, int enable) } static int mipid02_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct mipid02_dev *bridge = to_mipid02_dev(sd); @@ -670,7 +670,7 @@ static int mipid02_enum_mbus_code(struct v4l2_subdev *sd, } static int mipid02_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mbus_fmt = &format->format; @@ -687,7 +687,8 @@ static int mipid02_get_fmt(struct v4l2_subdev *sd, return -EINVAL; if (format->which == V4L2_SUBDEV_FORMAT_TRY) - fmt = v4l2_subdev_get_try_format(&bridge->sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(&bridge->sd, sd_state, + format->pad); else fmt = &bridge->fmt; @@ -704,7 +705,7 @@ static int mipid02_get_fmt(struct v4l2_subdev *sd, } static void mipid02_set_fmt_source(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mipid02_dev *bridge = to_mipid02_dev(sd); @@ -718,11 +719,11 @@ static void mipid02_set_fmt_source(struct v4l2_subdev *sd, if (format->which != V4L2_SUBDEV_FORMAT_TRY) return; - *v4l2_subdev_get_try_format(sd, cfg, format->pad) = format->format; + *v4l2_subdev_get_try_format(sd, sd_state, format->pad) = format->format; } static void mipid02_set_fmt_sink(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mipid02_dev *bridge = to_mipid02_dev(sd); @@ -731,7 +732,7 @@ static void mipid02_set_fmt_sink(struct v4l2_subdev *sd, format->format.code = get_fmt_code(format->format.code); if (format->which == V4L2_SUBDEV_FORMAT_TRY) - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); else fmt = &bridge->fmt; @@ -739,7 +740,7 @@ static void mipid02_set_fmt_sink(struct v4l2_subdev *sd, } static int mipid02_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct mipid02_dev *bridge = to_mipid02_dev(sd); @@ -762,9 +763,9 @@ static int mipid02_set_fmt(struct v4l2_subdev *sd, } if (format->pad == MIPID02_SOURCE) - mipid02_set_fmt_source(sd, cfg, format); + mipid02_set_fmt_source(sd, sd_state, format); else - mipid02_set_fmt_sink(sd, cfg, format); + mipid02_set_fmt_sink(sd, sd_state, format); error: mutex_unlock(&bridge->lock); diff --git a/drivers/media/i2c/tc358743.c b/drivers/media/i2c/tc358743.c index 1b309bb743c7..3205cd8298dd 100644 --- a/drivers/media/i2c/tc358743.c +++ b/drivers/media/i2c/tc358743.c @@ -1649,7 +1649,7 @@ static int tc358743_s_stream(struct v4l2_subdev *sd, int enable) /* --------------- PAD OPS --------------- */ static int tc358743_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { switch (code->index) { @@ -1666,7 +1666,7 @@ static int tc358743_enum_mbus_code(struct v4l2_subdev *sd, } static int tc358743_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct tc358743_state *state = to_state(sd); @@ -1702,13 +1702,13 @@ static int tc358743_get_fmt(struct v4l2_subdev *sd, } static int tc358743_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct tc358743_state *state = to_state(sd); u32 code = format->format.code; /* is overwritten by get_fmt */ - int ret = tc358743_get_fmt(sd, cfg, format); + int ret = tc358743_get_fmt(sd, sd_state, format); format->format.code = code; @@ -1974,6 +1974,7 @@ static int tc358743_probe_of(struct tc358743_state *state) bps_pr_lane = 2 * endpoint.link_frequencies[0]; if (bps_pr_lane < 62500000U || bps_pr_lane > 1000000000U) { dev_err(dev, "unsupported bps per lane: %u bps\n", bps_pr_lane); + ret = -EINVAL; goto disable_clk; } diff --git a/drivers/media/i2c/tda1997x.c b/drivers/media/i2c/tda1997x.c index 89bb7e6dc7a4..91e6db847bb5 100644 --- a/drivers/media/i2c/tda1997x.c +++ b/drivers/media/i2c/tda1997x.c @@ -1718,19 +1718,19 @@ static const struct v4l2_subdev_video_ops tda1997x_video_ops = { */ static int tda1997x_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct tda1997x_state *state = to_state(sd); struct v4l2_mbus_framefmt *mf; - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); mf->code = state->mbus_codes[0]; return 0; } static int tda1997x_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct tda1997x_state *state = to_state(sd); @@ -1762,7 +1762,7 @@ static void tda1997x_fill_format(struct tda1997x_state *state, } static int tda1997x_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct tda1997x_state *state = to_state(sd); @@ -1775,7 +1775,7 @@ static int tda1997x_get_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); format->format.code = fmt->code; } else format->format.code = state->mbus_code; @@ -1784,7 +1784,7 @@ static int tda1997x_get_format(struct v4l2_subdev *sd, } static int tda1997x_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct tda1997x_state *state = to_state(sd); @@ -1809,7 +1809,7 @@ static int tda1997x_set_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(sd, cfg, format->pad); + fmt = v4l2_subdev_get_try_format(sd, sd_state, format->pad); *fmt = format->format; } else { int ret = tda1997x_setup_format(state, format->format.code); diff --git a/drivers/media/i2c/tvp514x.c b/drivers/media/i2c/tvp514x.c index a7fbe5b400c2..cee60f945036 100644 --- a/drivers/media/i2c/tvp514x.c +++ b/drivers/media/i2c/tvp514x.c @@ -853,13 +853,13 @@ static const struct v4l2_ctrl_ops tvp514x_ctrl_ops = { /** * tvp514x_enum_mbus_code() - V4L2 decoder interface handler for enum_mbus_code * @sd: pointer to standard V4L2 sub-device structure - * @cfg: pad configuration + * @sd_state: subdev state * @code: pointer to v4l2_subdev_mbus_code_enum structure * * Enumertaes mbus codes supported */ static int tvp514x_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { u32 pad = code->pad; @@ -880,13 +880,13 @@ static int tvp514x_enum_mbus_code(struct v4l2_subdev *sd, /** * tvp514x_get_pad_format() - V4L2 decoder interface handler for get pad format * @sd: pointer to standard V4L2 sub-device structure - * @cfg: pad configuration + * @sd_state: subdev state * @format: pointer to v4l2_subdev_format structure * * Retrieves pad format which is active or tried based on requirement */ static int tvp514x_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct tvp514x_decoder *decoder = to_decoder(sd); @@ -912,13 +912,13 @@ static int tvp514x_get_pad_format(struct v4l2_subdev *sd, /** * tvp514x_set_pad_format() - V4L2 decoder interface handler for set pad format * @sd: pointer to standard V4L2 sub-device structure - * @cfg: pad configuration + * @sd_state: subdev state * @fmt: pointer to v4l2_subdev_format structure * * Set pad format for the output pad */ static int tvp514x_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct tvp514x_decoder *decoder = to_decoder(sd); diff --git a/drivers/media/i2c/tvp5150.c b/drivers/media/i2c/tvp5150.c index e26e3f544054..30c63552556d 100644 --- a/drivers/media/i2c/tvp5150.c +++ b/drivers/media/i2c/tvp5150.c @@ -1027,7 +1027,7 @@ static void tvp5150_set_default(v4l2_std_id std, struct v4l2_rect *crop) static struct v4l2_rect * tvp5150_get_pad_crop(struct tvp5150 *decoder, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { switch (which) { @@ -1035,7 +1035,7 @@ tvp5150_get_pad_crop(struct tvp5150 *decoder, return &decoder->rect; case V4L2_SUBDEV_FORMAT_TRY: #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API) - return v4l2_subdev_get_try_crop(&decoder->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&decoder->sd, sd_state, pad); #else return ERR_PTR(-EINVAL); #endif @@ -1045,7 +1045,7 @@ tvp5150_get_pad_crop(struct tvp5150 *decoder, } static int tvp5150_fill_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *f; @@ -1104,7 +1104,7 @@ static void tvp5150_set_hw_selection(struct v4l2_subdev *sd, } static int tvp5150_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct tvp5150 *decoder = to_tvp5150(sd); @@ -1138,7 +1138,7 @@ static int tvp5150_set_selection(struct v4l2_subdev *sd, sel->which == V4L2_SUBDEV_FORMAT_TRY) return 0; - crop = tvp5150_get_pad_crop(decoder, cfg, sel->pad, sel->which); + crop = tvp5150_get_pad_crop(decoder, sd_state, sel->pad, sel->which); if (IS_ERR(crop)) return PTR_ERR(crop); @@ -1156,7 +1156,7 @@ static int tvp5150_set_selection(struct v4l2_subdev *sd, } static int tvp5150_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct tvp5150 *decoder = container_of(sd, struct tvp5150, sd); @@ -1180,7 +1180,7 @@ static int tvp5150_get_selection(struct v4l2_subdev *sd, sel->r.height = TVP5150_V_MAX_OTHERS; return 0; case V4L2_SEL_TGT_CROP: - crop = tvp5150_get_pad_crop(decoder, cfg, sel->pad, + crop = tvp5150_get_pad_crop(decoder, sd_state, sel->pad, sel->which); if (IS_ERR(crop)) return PTR_ERR(crop); @@ -1208,7 +1208,7 @@ static int tvp5150_get_mbus_config(struct v4l2_subdev *sd, V4L2 subdev pad ops ****************************************************************************/ static int tvp5150_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct tvp5150 *decoder = to_tvp5150(sd); v4l2_std_id std; @@ -1229,7 +1229,7 @@ static int tvp5150_init_cfg(struct v4l2_subdev *sd, } static int tvp5150_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index) @@ -1240,7 +1240,7 @@ static int tvp5150_enum_mbus_code(struct v4l2_subdev *sd, } static int tvp5150_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct tvp5150 *decoder = to_tvp5150(sd); @@ -1448,11 +1448,9 @@ static int tvp5150_s_stream(struct v4l2_subdev *sd, int enable) TVP5150_MISC_CTL_CLOCK_OE; if (enable) { - ret = pm_runtime_get_sync(sd->dev); - if (ret < 0) { - pm_runtime_put_noidle(sd->dev); + ret = pm_runtime_resume_and_get(sd->dev); + if (ret < 0) return ret; - } tvp5150_enable(sd); @@ -1675,15 +1673,7 @@ static int tvp5150_registered(struct v4l2_subdev *sd) static int tvp5150_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) { - int ret; - - ret = pm_runtime_get_sync(sd->dev); - if (ret < 0) { - pm_runtime_put_noidle(sd->dev); - return ret; - } - - return 0; + return pm_runtime_resume_and_get(sd->dev); } static int tvp5150_close(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) diff --git a/drivers/media/i2c/tvp7002.c b/drivers/media/i2c/tvp7002.c index ada4ec5ef782..2de18833b07b 100644 --- a/drivers/media/i2c/tvp7002.c +++ b/drivers/media/i2c/tvp7002.c @@ -797,7 +797,8 @@ static const struct v4l2_ctrl_ops tvp7002_ctrl_ops = { * Enumerate supported digital video formats for pad. */ static int -tvp7002_enum_mbus_code(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +tvp7002_enum_mbus_code(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { /* Check requested format index is within range */ @@ -818,7 +819,8 @@ tvp7002_enum_mbus_code(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cf * get video format for pad. */ static int -tvp7002_get_pad_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +tvp7002_get_pad_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct tvp7002 *tvp7002 = to_tvp7002(sd); @@ -841,10 +843,11 @@ tvp7002_get_pad_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cf * set video format for pad. */ static int -tvp7002_set_pad_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +tvp7002_set_pad_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - return tvp7002_get_pad_format(sd, cfg, fmt); + return tvp7002_get_pad_format(sd, sd_state, fmt); } /* V4L2 core operation handlers */ diff --git a/drivers/media/i2c/tw9910.c b/drivers/media/i2c/tw9910.c index a25a350b0ddc..09f5b3986928 100644 --- a/drivers/media/i2c/tw9910.c +++ b/drivers/media/i2c/tw9910.c @@ -720,7 +720,7 @@ static int tw9910_set_frame(struct v4l2_subdev *sd, u32 *width, u32 *height) } static int tw9910_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -746,7 +746,7 @@ static int tw9910_get_selection(struct v4l2_subdev *sd, } static int tw9910_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -797,7 +797,7 @@ static int tw9910_s_fmt(struct v4l2_subdev *sd, } static int tw9910_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *mf = &format->format; @@ -829,7 +829,7 @@ static int tw9910_set_fmt(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) return tw9910_s_fmt(sd, mf); - cfg->try_fmt = *mf; + sd_state->pads->try_fmt = *mf; return 0; } @@ -886,7 +886,7 @@ static const struct v4l2_subdev_core_ops tw9910_subdev_core_ops = { }; static int tw9910_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index) diff --git a/drivers/media/i2c/video-i2c.c b/drivers/media/i2c/video-i2c.c index 0465832a4090..de12f38f347c 100644 --- a/drivers/media/i2c/video-i2c.c +++ b/drivers/media/i2c/video-i2c.c @@ -286,11 +286,9 @@ static int amg88xx_read(struct device *dev, enum hwmon_sensor_types type, __le16 buf; int tmp; - tmp = pm_runtime_get_sync(regmap_get_device(data->regmap)); - if (tmp < 0) { - pm_runtime_put_noidle(regmap_get_device(data->regmap)); + tmp = pm_runtime_resume_and_get(regmap_get_device(data->regmap)); + if (tmp < 0) return tmp; - } tmp = regmap_bulk_read(data->regmap, AMG88XX_REG_TTHL, &buf, 2); pm_runtime_mark_last_busy(regmap_get_device(data->regmap)); @@ -512,11 +510,9 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count) if (data->kthread_vid_cap) return 0; - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put_noidle(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) goto error_del_list; - } ret = data->chip->setup(data); if (ret) diff --git a/drivers/media/i2c/vs6624.c b/drivers/media/i2c/vs6624.c index c292c92e37b9..29003dec6f2d 100644 --- a/drivers/media/i2c/vs6624.c +++ b/drivers/media/i2c/vs6624.c @@ -546,7 +546,7 @@ static int vs6624_s_ctrl(struct v4l2_ctrl *ctrl) } static int vs6624_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(vs6624_formats)) @@ -557,7 +557,7 @@ static int vs6624_enum_mbus_code(struct v4l2_subdev *sd, } static int vs6624_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -587,7 +587,7 @@ static int vs6624_set_fmt(struct v4l2_subdev *sd, fmt->colorspace = vs6624_formats[index].colorspace; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; return 0; } @@ -637,7 +637,7 @@ static int vs6624_set_fmt(struct v4l2_subdev *sd, } static int vs6624_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct vs6624 *sensor = to_vs6624(sd); diff --git a/drivers/media/mc/Makefile b/drivers/media/mc/Makefile index 119037f0e686..2b7af42ba59c 100644 --- a/drivers/media/mc/Makefile +++ b/drivers/media/mc/Makefile @@ -3,7 +3,7 @@ mc-objs := mc-device.o mc-devnode.o mc-entity.o \ mc-request.o -ifeq ($(CONFIG_USB),y) +ifneq ($(CONFIG_USB),) mc-objs += mc-dev-allocator.o endif diff --git a/drivers/media/mc/mc-entity.c b/drivers/media/mc/mc-entity.c index 678b99771cfa..f40f41977142 100644 --- a/drivers/media/mc/mc-entity.c +++ b/drivers/media/mc/mc-entity.c @@ -323,7 +323,7 @@ static void media_graph_walk_iter(struct media_graph *graph) return; } - /* Get the entity in the other end of the link . */ + /* Get the entity at the other end of the link. */ next = media_entity_other(entity, link); /* Has the entity already been visited? */ diff --git a/drivers/media/mc/mc-request.c b/drivers/media/mc/mc-request.c index c0782fd96c59..addb8f2d8939 100644 --- a/drivers/media/mc/mc-request.c +++ b/drivers/media/mc/mc-request.c @@ -414,7 +414,8 @@ int media_request_object_bind(struct media_request *req, spin_lock_irqsave(&req->lock, flags); - if (WARN_ON(req->state != MEDIA_REQUEST_STATE_UPDATING)) + if (WARN_ON(req->state != MEDIA_REQUEST_STATE_UPDATING && + req->state != MEDIA_REQUEST_STATE_QUEUED)) goto unlock; obj->req = req; diff --git a/drivers/media/pci/bt8xx/bt878.c b/drivers/media/pci/bt8xx/bt878.c index 78dd35c9b65d..90972d6952f1 100644 --- a/drivers/media/pci/bt8xx/bt878.c +++ b/drivers/media/pci/bt8xx/bt878.c @@ -300,7 +300,8 @@ static irqreturn_t bt878_irq(int irq, void *dev_id) } if (astat & BT878_ARISCI) { bt->finished_block = (stat & BT878_ARISCS) >> 28; - tasklet_schedule(&bt->tasklet); + if (bt->tasklet.callback) + tasklet_schedule(&bt->tasklet); break; } count++; @@ -477,6 +478,9 @@ static int bt878_probe(struct pci_dev *dev, const struct pci_device_id *pci_id) btwrite(0, BT878_AINT_MASK); bt878_num++; + if (!bt->tasklet.func) + tasklet_disable(&bt->tasklet); + return 0; fail2: diff --git a/drivers/media/pci/bt8xx/bttv-driver.c b/drivers/media/pci/bt8xx/bttv-driver.c index 1f62a9d8ea1d..0e9df8b35ac6 100644 --- a/drivers/media/pci/bt8xx/bttv-driver.c +++ b/drivers/media/pci/bt8xx/bttv-driver.c @@ -3179,7 +3179,7 @@ static int radio_release(struct file *file) btv->radio_user--; - bttv_call_all(btv, core, ioctl, SAA6588_CMD_CLOSE, &cmd); + bttv_call_all(btv, core, command, SAA6588_CMD_CLOSE, &cmd); if (btv->radio_user == 0) btv->has_radio_tuner = 0; @@ -3260,7 +3260,7 @@ static ssize_t radio_read(struct file *file, char __user *data, cmd.result = -ENODEV; radio_enable(btv); - bttv_call_all(btv, core, ioctl, SAA6588_CMD_READ, &cmd); + bttv_call_all(btv, core, command, SAA6588_CMD_READ, &cmd); return cmd.result; } @@ -3281,7 +3281,7 @@ static __poll_t radio_poll(struct file *file, poll_table *wait) cmd.instance = file; cmd.event_list = wait; cmd.poll_mask = res; - bttv_call_all(btv, core, ioctl, SAA6588_CMD_POLL, &cmd); + bttv_call_all(btv, core, command, SAA6588_CMD_POLL, &cmd); return cmd.poll_mask; } diff --git a/drivers/media/pci/cobalt/cobalt-driver.c b/drivers/media/pci/cobalt/cobalt-driver.c index 839503e654f4..16af58f2f93c 100644 --- a/drivers/media/pci/cobalt/cobalt-driver.c +++ b/drivers/media/pci/cobalt/cobalt-driver.c @@ -667,6 +667,7 @@ static int cobalt_probe(struct pci_dev *pci_dev, return -ENOMEM; cobalt->pci_dev = pci_dev; cobalt->instance = i; + mutex_init(&cobalt->pci_lock); retval = v4l2_device_register(&pci_dev->dev, &cobalt->v4l2_dev); if (retval) { diff --git a/drivers/media/pci/cobalt/cobalt-driver.h b/drivers/media/pci/cobalt/cobalt-driver.h index bca68572b324..12c33e035904 100644 --- a/drivers/media/pci/cobalt/cobalt-driver.h +++ b/drivers/media/pci/cobalt/cobalt-driver.h @@ -251,6 +251,8 @@ struct cobalt { int instance; struct pci_dev *pci_dev; struct v4l2_device v4l2_dev; + /* serialize PCI access in cobalt_s_bit_sysctrl() */ + struct mutex pci_lock; void __iomem *bar0, *bar1; @@ -320,10 +322,13 @@ static inline u32 cobalt_g_sysctrl(struct cobalt *cobalt) static inline void cobalt_s_bit_sysctrl(struct cobalt *cobalt, int bit, int val) { - u32 ctrl = cobalt_read_bar1(cobalt, COBALT_SYS_CTRL_BASE); + u32 ctrl; + mutex_lock(&cobalt->pci_lock); + ctrl = cobalt_read_bar1(cobalt, COBALT_SYS_CTRL_BASE); cobalt_write_bar1(cobalt, COBALT_SYS_CTRL_BASE, (ctrl & ~(1UL << bit)) | (val << bit)); + mutex_unlock(&cobalt->pci_lock); } static inline u32 cobalt_g_sysstat(struct cobalt *cobalt) diff --git a/drivers/media/pci/cx18/cx18-av-core.c b/drivers/media/pci/cx18/cx18-av-core.c index 11cfe35fd730..76e5a504df8c 100644 --- a/drivers/media/pci/cx18/cx18-av-core.c +++ b/drivers/media/pci/cx18/cx18-av-core.c @@ -930,7 +930,7 @@ static int cx18_av_s_ctrl(struct v4l2_ctrl *ctrl) } static int cx18_av_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; diff --git a/drivers/media/pci/cx88/cx88-alsa.c b/drivers/media/pci/cx88/cx88-alsa.c index c83814c052d3..29fb1311e443 100644 --- a/drivers/media/pci/cx88/cx88-alsa.c +++ b/drivers/media/pci/cx88/cx88-alsa.c @@ -357,8 +357,8 @@ static int dsp_buffer_free(struct cx88_audio_dev *chip) cx88_alsa_dma_unmap(chip); cx88_alsa_dma_free(chip->buf); if (risc->cpu) - pci_free_consistent(chip->pci, risc->size, - risc->cpu, risc->dma); + dma_free_coherent(&chip->pci->dev, risc->size, risc->cpu, + risc->dma); kfree(chip->buf); chip->buf = NULL; @@ -868,7 +868,7 @@ static int snd_cx88_create(struct snd_card *card, struct pci_dev *pci, return err; } - err = pci_set_dma_mask(pci, DMA_BIT_MASK(32)); + err = dma_set_mask(&pci->dev, DMA_BIT_MASK(32)); if (err) { dprintk(0, "%s/1: Oops: no 32bit PCI DMA ???\n", core->name); cx88_core_put(core, pci); diff --git a/drivers/media/pci/cx88/cx88-blackbird.c b/drivers/media/pci/cx88/cx88-blackbird.c index fa4ca002ed19..d5da3bd5695d 100644 --- a/drivers/media/pci/cx88/cx88-blackbird.c +++ b/drivers/media/pci/cx88/cx88-blackbird.c @@ -685,7 +685,8 @@ static void buffer_finish(struct vb2_buffer *vb) struct cx88_riscmem *risc = &buf->risc; if (risc->cpu) - pci_free_consistent(dev->pci, risc->size, risc->cpu, risc->dma); + dma_free_coherent(&dev->pci->dev, risc->size, risc->cpu, + risc->dma); memset(risc, 0, sizeof(*risc)); } diff --git a/drivers/media/pci/cx88/cx88-core.c b/drivers/media/pci/cx88/cx88-core.c index 48c8a3429542..89d4d5a3ba34 100644 --- a/drivers/media/pci/cx88/cx88-core.c +++ b/drivers/media/pci/cx88/cx88-core.c @@ -152,7 +152,8 @@ int cx88_risc_buffer(struct pci_dev *pci, struct cx88_riscmem *risc, instructions += 4; risc->size = instructions * 8; risc->dma = 0; - risc->cpu = pci_zalloc_consistent(pci, risc->size, &risc->dma); + risc->cpu = dma_alloc_coherent(&pci->dev, risc->size, &risc->dma, + GFP_KERNEL); if (!risc->cpu) return -ENOMEM; @@ -190,7 +191,8 @@ int cx88_risc_databuffer(struct pci_dev *pci, struct cx88_riscmem *risc, instructions += 3; risc->size = instructions * 8; risc->dma = 0; - risc->cpu = pci_zalloc_consistent(pci, risc->size, &risc->dma); + risc->cpu = dma_alloc_coherent(&pci->dev, risc->size, &risc->dma, + GFP_KERNEL); if (!risc->cpu) return -ENOMEM; diff --git a/drivers/media/pci/cx88/cx88-dvb.c b/drivers/media/pci/cx88/cx88-dvb.c index 202ff9e8c257..2087f2491c42 100644 --- a/drivers/media/pci/cx88/cx88-dvb.c +++ b/drivers/media/pci/cx88/cx88-dvb.c @@ -103,7 +103,8 @@ static void buffer_finish(struct vb2_buffer *vb) struct cx88_riscmem *risc = &buf->risc; if (risc->cpu) - pci_free_consistent(dev->pci, risc->size, risc->cpu, risc->dma); + dma_free_coherent(&dev->pci->dev, risc->size, risc->cpu, + risc->dma); memset(risc, 0, sizeof(*risc)); } diff --git a/drivers/media/pci/cx88/cx88-mpeg.c b/drivers/media/pci/cx88/cx88-mpeg.c index a3edb548afde..680e1e3fe89b 100644 --- a/drivers/media/pci/cx88/cx88-mpeg.c +++ b/drivers/media/pci/cx88/cx88-mpeg.c @@ -226,8 +226,8 @@ int cx8802_buf_prepare(struct vb2_queue *q, struct cx8802_dev *dev, dev->ts_packet_size, dev->ts_packet_count, 0); if (rc) { if (risc->cpu) - pci_free_consistent(dev->pci, risc->size, - risc->cpu, risc->dma); + dma_free_coherent(&dev->pci->dev, risc->size, + risc->cpu, risc->dma); memset(risc, 0, sizeof(*risc)); return rc; } @@ -386,7 +386,7 @@ static int cx8802_init_common(struct cx8802_dev *dev) if (pci_enable_device(dev->pci)) return -EIO; pci_set_master(dev->pci); - err = pci_set_dma_mask(dev->pci, DMA_BIT_MASK(32)); + err = dma_set_mask(&dev->pci->dev, DMA_BIT_MASK(32)); if (err) { pr_err("Oops: no 32bit PCI DMA ???\n"); return -EIO; diff --git a/drivers/media/pci/cx88/cx88-vbi.c b/drivers/media/pci/cx88/cx88-vbi.c index 58489ea0c1da..a075788c64d4 100644 --- a/drivers/media/pci/cx88/cx88-vbi.c +++ b/drivers/media/pci/cx88/cx88-vbi.c @@ -159,7 +159,8 @@ static void buffer_finish(struct vb2_buffer *vb) struct cx88_riscmem *risc = &buf->risc; if (risc->cpu) - pci_free_consistent(dev->pci, risc->size, risc->cpu, risc->dma); + dma_free_coherent(&dev->pci->dev, risc->size, risc->cpu, + risc->dma); memset(risc, 0, sizeof(*risc)); } diff --git a/drivers/media/pci/cx88/cx88-video.c b/drivers/media/pci/cx88/cx88-video.c index 8cffdacf6007..c17ad9f7d822 100644 --- a/drivers/media/pci/cx88/cx88-video.c +++ b/drivers/media/pci/cx88/cx88-video.c @@ -492,7 +492,8 @@ static void buffer_finish(struct vb2_buffer *vb) struct cx88_riscmem *risc = &buf->risc; if (risc->cpu) - pci_free_consistent(dev->pci, risc->size, risc->cpu, risc->dma); + dma_free_coherent(&dev->pci->dev, risc->size, risc->cpu, + risc->dma); memset(risc, 0, sizeof(*risc)); } @@ -1288,7 +1289,7 @@ static int cx8800_initdev(struct pci_dev *pci_dev, (unsigned long long)pci_resource_start(pci_dev, 0)); pci_set_master(pci_dev); - err = pci_set_dma_mask(pci_dev, DMA_BIT_MASK(32)); + err = dma_set_mask(&pci_dev->dev, DMA_BIT_MASK(32)); if (err) { pr_err("Oops: no 32bit PCI DMA ???\n"); goto fail_core; diff --git a/drivers/media/pci/intel/ipu3/cio2-bridge.c b/drivers/media/pci/intel/ipu3/cio2-bridge.c index e8511787c1e4..4657e99df033 100644 --- a/drivers/media/pci/intel/ipu3/cio2-bridge.c +++ b/drivers/media/pci/intel/ipu3/cio2-bridge.c @@ -173,14 +173,15 @@ static int cio2_bridge_connect_sensor(const struct cio2_sensor_config *cfg, int ret; for_each_acpi_dev_match(adev, cfg->hid, NULL, -1) { - if (!adev->status.enabled) + if (!adev->status.enabled) { + acpi_dev_put(adev); continue; + } if (bridge->n_sensors >= CIO2_NUM_PORTS) { + acpi_dev_put(adev); dev_err(&cio2->dev, "Exceeded available CIO2 ports\n"); - cio2_bridge_unregister_sensors(bridge); - ret = -EINVAL; - goto err_out; + return -EINVAL; } sensor = &bridge->sensors[bridge->n_sensors]; @@ -228,7 +229,6 @@ static int cio2_bridge_connect_sensor(const struct cio2_sensor_config *cfg, software_node_unregister_nodes(sensor->swnodes); err_put_adev: acpi_dev_put(sensor->adev); -err_out: return ret; } diff --git a/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c b/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c index fecef85bd62e..47db0ee0fcbf 100644 --- a/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c +++ b/drivers/media/pci/intel/ipu3/ipu3-cio2-main.c @@ -975,10 +975,9 @@ static int cio2_vb2_start_streaming(struct vb2_queue *vq, unsigned int count) cio2->cur_queue = q; atomic_set(&q->frame_sequence, 0); - r = pm_runtime_get_sync(&cio2->pci_dev->dev); + r = pm_runtime_resume_and_get(&cio2->pci_dev->dev); if (r < 0) { dev_info(&cio2->pci_dev->dev, "failed to set power %d\n", r); - pm_runtime_put_noidle(&cio2->pci_dev->dev); return r; } @@ -1200,11 +1199,11 @@ static int cio2_subdev_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) }; /* Initialize try_fmt */ - format = v4l2_subdev_get_try_format(sd, fh->pad, CIO2_PAD_SINK); + format = v4l2_subdev_get_try_format(sd, fh->state, CIO2_PAD_SINK); *format = fmt_default; /* same as sink */ - format = v4l2_subdev_get_try_format(sd, fh->pad, CIO2_PAD_SOURCE); + format = v4l2_subdev_get_try_format(sd, fh->state, CIO2_PAD_SOURCE); *format = fmt_default; return 0; @@ -1218,7 +1217,7 @@ static int cio2_subdev_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) * return -EINVAL or zero on success */ static int cio2_subdev_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct cio2_queue *q = container_of(sd, struct cio2_queue, subdev); @@ -1226,7 +1225,8 @@ static int cio2_subdev_get_fmt(struct v4l2_subdev *sd, mutex_lock(&q->subdev_lock); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - fmt->format = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + fmt->format = *v4l2_subdev_get_try_format(sd, sd_state, + fmt->pad); else fmt->format = q->subdev_fmt; @@ -1243,7 +1243,7 @@ static int cio2_subdev_get_fmt(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int cio2_subdev_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct cio2_queue *q = container_of(sd, struct cio2_queue, subdev); @@ -1256,10 +1256,10 @@ static int cio2_subdev_set_fmt(struct v4l2_subdev *sd, * source always propagates from sink */ if (fmt->pad == CIO2_PAD_SOURCE) - return cio2_subdev_get_fmt(sd, cfg, fmt); + return cio2_subdev_get_fmt(sd, sd_state, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - mbus = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mbus = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); else mbus = &q->subdev_fmt; @@ -1284,7 +1284,7 @@ static int cio2_subdev_set_fmt(struct v4l2_subdev *sd, } static int cio2_subdev_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(formats)) diff --git a/drivers/media/pci/ivtv/Kconfig b/drivers/media/pci/ivtv/Kconfig index c729e54692c4..e70502902b73 100644 --- a/drivers/media/pci/ivtv/Kconfig +++ b/drivers/media/pci/ivtv/Kconfig @@ -29,18 +29,6 @@ config VIDEO_IVTV To compile this driver as a module, choose M here: the module will be called ivtv. -config VIDEO_IVTV_DEPRECATED_IOCTLS - bool "enable the DVB ioctls abuse on ivtv driver" - depends on VIDEO_IVTV - help - Enable the usage of the a DVB set of ioctls that were abused by - IVTV driver for a while. - - Those ioctls were not needed for a long time, as IVTV implements - the proper V4L2 ioctls since kernel 3.3. - - If unsure, say N. - config VIDEO_IVTV_ALSA tristate "Conexant cx23415/cx23416 ALSA interface for PCM audio capture" depends on VIDEO_IVTV && SND diff --git a/drivers/media/pci/ivtv/ivtv-driver.h b/drivers/media/pci/ivtv/ivtv-driver.h index e5efe525ad7b..4cf92dee6527 100644 --- a/drivers/media/pci/ivtv/ivtv-driver.h +++ b/drivers/media/pci/ivtv/ivtv-driver.h @@ -57,8 +57,6 @@ #include #include -#include -#include #include #include #include diff --git a/drivers/media/pci/ivtv/ivtv-ioctl.c b/drivers/media/pci/ivtv/ivtv-ioctl.c index 35dccb31174c..da19b2e95e6c 100644 --- a/drivers/media/pci/ivtv/ivtv-ioctl.c +++ b/drivers/media/pci/ivtv/ivtv-ioctl.c @@ -23,11 +23,6 @@ #include #include #include -#ifdef CONFIG_VIDEO_IVTV_DEPRECATED_IOCTLS -#include -#include -#include -#endif u16 ivtv_service2vbi(int type) { @@ -1606,38 +1601,11 @@ static int ivtv_try_decoder_cmd(struct file *file, void *fh, struct v4l2_decoder return ivtv_video_command(itv, id, dec, true); } -#ifdef CONFIG_VIDEO_IVTV_DEPRECATED_IOCTLS -static __inline__ void warn_deprecated_ioctl(const char *name) -{ - pr_warn_once("warning: the %s ioctl is deprecated. Don't use it, as it will be removed soon\n", - name); -} - -#ifdef CONFIG_COMPAT -struct compat_video_event { - __s32 type; - /* unused, make sure to use atomic time for y2038 if it ever gets used */ - compat_long_t timestamp; - union { - video_size_t size; - unsigned int frame_rate; /* in frames per 1000sec */ - unsigned char vsync_field; /* unknown/odd/even/progressive */ - } u; -}; -#define VIDEO_GET_EVENT32 _IOR('o', 28, struct compat_video_event) -#endif - -#endif - static int ivtv_decoder_ioctls(struct file *filp, unsigned int cmd, void *arg) { struct ivtv_open_id *id = fh2id(filp->private_data); struct ivtv *itv = id->itv; struct ivtv_stream *s = &itv->streams[id->type]; -#ifdef CONFIG_VIDEO_IVTV_DEPRECATED_IOCTLS - int nonblocking = filp->f_flags & O_NONBLOCK; - unsigned long iarg = (unsigned long)arg; -#endif switch (cmd) { case IVTV_IOC_DMA_FRAME: { @@ -1669,169 +1637,6 @@ static int ivtv_decoder_ioctls(struct file *filp, unsigned int cmd, void *arg) if (!(itv->v4l2_cap & V4L2_CAP_VIDEO_OUTPUT)) return -EINVAL; return ivtv_passthrough_mode(itv, *(int *)arg != 0); -#ifdef CONFIG_VIDEO_IVTV_DEPRECATED_IOCTLS - case VIDEO_GET_PTS: { - s64 *pts = arg; - s64 frame; - - warn_deprecated_ioctl("VIDEO_GET_PTS"); - if (s->type < IVTV_DEC_STREAM_TYPE_MPG) { - *pts = s->dma_pts; - break; - } - if (!(itv->v4l2_cap & V4L2_CAP_VIDEO_OUTPUT)) - return -EINVAL; - return ivtv_g_pts_frame(itv, pts, &frame); - } - - case VIDEO_GET_FRAME_COUNT: { - s64 *frame = arg; - s64 pts; - - warn_deprecated_ioctl("VIDEO_GET_FRAME_COUNT"); - if (s->type < IVTV_DEC_STREAM_TYPE_MPG) { - *frame = 0; - break; - } - if (!(itv->v4l2_cap & V4L2_CAP_VIDEO_OUTPUT)) - return -EINVAL; - return ivtv_g_pts_frame(itv, &pts, frame); - } - - case VIDEO_PLAY: { - struct v4l2_decoder_cmd dc; - - warn_deprecated_ioctl("VIDEO_PLAY"); - memset(&dc, 0, sizeof(dc)); - dc.cmd = V4L2_DEC_CMD_START; - return ivtv_video_command(itv, id, &dc, 0); - } - - case VIDEO_STOP: { - struct v4l2_decoder_cmd dc; - - warn_deprecated_ioctl("VIDEO_STOP"); - memset(&dc, 0, sizeof(dc)); - dc.cmd = V4L2_DEC_CMD_STOP; - dc.flags = V4L2_DEC_CMD_STOP_TO_BLACK | V4L2_DEC_CMD_STOP_IMMEDIATELY; - return ivtv_video_command(itv, id, &dc, 0); - } - - case VIDEO_FREEZE: { - struct v4l2_decoder_cmd dc; - - warn_deprecated_ioctl("VIDEO_FREEZE"); - memset(&dc, 0, sizeof(dc)); - dc.cmd = V4L2_DEC_CMD_PAUSE; - return ivtv_video_command(itv, id, &dc, 0); - } - - case VIDEO_CONTINUE: { - struct v4l2_decoder_cmd dc; - - warn_deprecated_ioctl("VIDEO_CONTINUE"); - memset(&dc, 0, sizeof(dc)); - dc.cmd = V4L2_DEC_CMD_RESUME; - return ivtv_video_command(itv, id, &dc, 0); - } - - case VIDEO_COMMAND: - case VIDEO_TRY_COMMAND: { - /* Note: struct v4l2_decoder_cmd has the same layout as - struct video_command */ - struct v4l2_decoder_cmd *dc = arg; - int try = (cmd == VIDEO_TRY_COMMAND); - - if (try) - warn_deprecated_ioctl("VIDEO_TRY_COMMAND"); - else - warn_deprecated_ioctl("VIDEO_COMMAND"); - return ivtv_video_command(itv, id, dc, try); - } - -#ifdef CONFIG_COMPAT - case VIDEO_GET_EVENT32: -#endif - case VIDEO_GET_EVENT: { -#ifdef CONFIG_COMPAT - struct compat_video_event *ev32 = arg; -#endif - struct video_event *ev = arg; - DEFINE_WAIT(wait); - - warn_deprecated_ioctl("VIDEO_GET_EVENT"); - if (!(itv->v4l2_cap & V4L2_CAP_VIDEO_OUTPUT)) - return -EINVAL; - memset(ev, 0, sizeof(*ev)); - set_bit(IVTV_F_I_EV_VSYNC_ENABLED, &itv->i_flags); - - while (1) { - if (test_and_clear_bit(IVTV_F_I_EV_DEC_STOPPED, &itv->i_flags)) - ev->type = VIDEO_EVENT_DECODER_STOPPED; - else if (test_and_clear_bit(IVTV_F_I_EV_VSYNC, &itv->i_flags)) { - unsigned char vsync_field; - - ev->type = VIDEO_EVENT_VSYNC; - vsync_field = test_bit(IVTV_F_I_EV_VSYNC_FIELD, &itv->i_flags) ? - VIDEO_VSYNC_FIELD_ODD : VIDEO_VSYNC_FIELD_EVEN; - if (itv->output_mode == OUT_UDMA_YUV && - (itv->yuv_info.lace_mode & IVTV_YUV_MODE_MASK) == - IVTV_YUV_MODE_PROGRESSIVE) { - vsync_field = VIDEO_VSYNC_FIELD_PROGRESSIVE; - } -#ifdef CONFIG_COMPAT - if (cmd == VIDEO_GET_EVENT32) - ev32->u.vsync_field = vsync_field; - else -#endif - ev->u.vsync_field = vsync_field; - } - if (ev->type) - return 0; - if (nonblocking) - return -EAGAIN; - /* Wait for event. Note that serialize_lock is locked, - so to allow other processes to access the driver while - we are waiting unlock first and later lock again. */ - mutex_unlock(&itv->serialize_lock); - prepare_to_wait(&itv->event_waitq, &wait, TASK_INTERRUPTIBLE); - if (!test_bit(IVTV_F_I_EV_DEC_STOPPED, &itv->i_flags) && - !test_bit(IVTV_F_I_EV_VSYNC, &itv->i_flags)) - schedule(); - finish_wait(&itv->event_waitq, &wait); - mutex_lock(&itv->serialize_lock); - if (signal_pending(current)) { - /* return if a signal was received */ - IVTV_DEBUG_INFO("User stopped wait for event\n"); - return -EINTR; - } - } - break; - } - - case VIDEO_SELECT_SOURCE: - warn_deprecated_ioctl("VIDEO_SELECT_SOURCE"); - if (!(itv->v4l2_cap & V4L2_CAP_VIDEO_OUTPUT)) - return -EINVAL; - return ivtv_passthrough_mode(itv, iarg == VIDEO_SOURCE_DEMUX); - - case AUDIO_SET_MUTE: - warn_deprecated_ioctl("AUDIO_SET_MUTE"); - itv->speed_mute_audio = iarg; - return 0; - - case AUDIO_CHANNEL_SELECT: - warn_deprecated_ioctl("AUDIO_CHANNEL_SELECT"); - if (iarg > AUDIO_STEREO_SWAPPED) - return -EINVAL; - return v4l2_ctrl_s_ctrl(itv->ctrl_audio_playback, iarg + 1); - - case AUDIO_BILINGUAL_CHANNEL_SELECT: - warn_deprecated_ioctl("AUDIO_BILINGUAL_CHANNEL_SELECT"); - if (iarg > AUDIO_STEREO_SWAPPED) - return -EINVAL; - return v4l2_ctrl_s_ctrl(itv->ctrl_audio_multilingual_playback, iarg + 1); -#endif default: return -EINVAL; } @@ -1846,17 +1651,6 @@ static long ivtv_default(struct file *file, void *fh, bool valid_prio, if (!valid_prio) { switch (cmd) { case IVTV_IOC_PASSTHROUGH_MODE: -#ifdef CONFIG_VIDEO_IVTV_DEPRECATED_IOCTLS - case VIDEO_PLAY: - case VIDEO_STOP: - case VIDEO_FREEZE: - case VIDEO_CONTINUE: - case VIDEO_COMMAND: - case VIDEO_SELECT_SOURCE: - case AUDIO_SET_MUTE: - case AUDIO_CHANNEL_SELECT: - case AUDIO_BILINGUAL_CHANNEL_SELECT: -#endif return -EBUSY; } } @@ -1874,21 +1668,6 @@ static long ivtv_default(struct file *file, void *fh, bool valid_prio, case IVTV_IOC_DMA_FRAME: case IVTV_IOC_PASSTHROUGH_MODE: -#ifdef CONFIG_VIDEO_IVTV_DEPRECATED_IOCTLS - case VIDEO_GET_PTS: - case VIDEO_GET_FRAME_COUNT: - case VIDEO_GET_EVENT: - case VIDEO_PLAY: - case VIDEO_STOP: - case VIDEO_FREEZE: - case VIDEO_CONTINUE: - case VIDEO_COMMAND: - case VIDEO_TRY_COMMAND: - case VIDEO_SELECT_SOURCE: - case AUDIO_SET_MUTE: - case AUDIO_CHANNEL_SELECT: - case AUDIO_BILINGUAL_CHANNEL_SELECT: -#endif return ivtv_decoder_ioctls(file, cmd, (void *)arg); default: diff --git a/drivers/media/pci/saa7134/saa7134-core.c b/drivers/media/pci/saa7134/saa7134-core.c index efb757d5168a..47158ab3956b 100644 --- a/drivers/media/pci/saa7134/saa7134-core.c +++ b/drivers/media/pci/saa7134/saa7134-core.c @@ -1031,7 +1031,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, dev->media_dev = kzalloc(sizeof(*dev->media_dev), GFP_KERNEL); if (!dev->media_dev) { err = -ENOMEM; - goto fail0; + goto err_free_dev; } media_device_pci_init(dev->media_dev, pci_dev, dev->name); dev->v4l2_dev.mdev = dev->media_dev; @@ -1039,13 +1039,13 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = v4l2_device_register(&pci_dev->dev, &dev->v4l2_dev); if (err) - goto fail0; + goto err_free_dev; /* pci init */ dev->pci = pci_dev; if (pci_enable_device(pci_dev)) { err = -EIO; - goto fail1; + goto err_v4l2_unregister; } /* pci quirks */ @@ -1095,7 +1095,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = pci_set_dma_mask(pci_dev, DMA_BIT_MASK(32)); if (err) { pr_warn("%s: Oops: no 32bit PCI DMA ???\n", dev->name); - goto fail1; + goto err_v4l2_unregister; } /* board config */ @@ -1129,7 +1129,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = -EBUSY; pr_err("%s: can't get MMIO memory @ 0x%llx\n", dev->name,(unsigned long long)pci_resource_start(pci_dev,0)); - goto fail1; + goto err_v4l2_unregister; } dev->lmmio = ioremap(pci_resource_start(pci_dev, 0), pci_resource_len(pci_dev, 0)); @@ -1138,7 +1138,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = -EIO; pr_err("%s: can't ioremap() MMIO memory\n", dev->name); - goto fail2; + goto err_release_mem_reg; } /* initialize hardware #1 */ @@ -1151,7 +1151,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, if (err < 0) { pr_err("%s: can't get IRQ %d\n", dev->name,pci_dev->irq); - goto fail3; + goto err_iounmap; } /* wait a bit, register i2c bus */ @@ -1217,7 +1217,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, if (err < 0) { pr_info("%s: can't register video device\n", dev->name); - goto fail4; + goto err_unregister_video; } pr_info("%s: registered device %s [v4l2]\n", dev->name, video_device_node_name(dev->video_dev)); @@ -1234,7 +1234,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = video_register_device(dev->vbi_dev,VFL_TYPE_VBI, vbi_nr[dev->nr]); if (err < 0) - goto fail4; + goto err_unregister_video; pr_info("%s: registered device %s\n", dev->name, video_device_node_name(dev->vbi_dev)); @@ -1248,7 +1248,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = video_register_device(dev->radio_dev,VFL_TYPE_RADIO, radio_nr[dev->nr]); if (err < 0) - goto fail4; + goto err_unregister_video; pr_info("%s: registered device %s\n", dev->name, video_device_node_name(dev->radio_dev)); } @@ -1259,7 +1259,7 @@ static int saa7134_initdev(struct pci_dev *pci_dev, err = v4l2_mc_create_media_graph(dev->media_dev); if (err) { pr_err("failed to create media graph\n"); - goto fail4; + goto err_unregister_video; } #endif /* everything worked */ @@ -1277,25 +1277,28 @@ static int saa7134_initdev(struct pci_dev *pci_dev, */ #ifdef CONFIG_MEDIA_CONTROLLER err = media_device_register(dev->media_dev); - if (err) - goto fail4; + if (err) { + media_device_cleanup(dev->media_dev); + goto err_unregister_video; + } #endif return 0; - fail4: +err_unregister_video: saa7134_unregister_video(dev); + list_del(&dev->devlist); saa7134_i2c_unregister(dev); free_irq(pci_dev->irq, dev); - fail3: +err_iounmap: saa7134_hwfini(dev); iounmap(dev->lmmio); - fail2: +err_release_mem_reg: release_mem_region(pci_resource_start(pci_dev,0), pci_resource_len(pci_dev,0)); - fail1: +err_v4l2_unregister: v4l2_device_unregister(&dev->v4l2_dev); - fail0: +err_free_dev: #ifdef CONFIG_MEDIA_CONTROLLER kfree(dev->media_dev); #endif @@ -1524,7 +1527,6 @@ static struct pci_driver saa7134_pci_driver = { static int __init saa7134_init(void) { - INIT_LIST_HEAD(&saa7134_devlist); pr_info("saa7130/34: v4l2 driver version %s loaded\n", SAA7134_VERSION); return pci_register_driver(&saa7134_pci_driver); diff --git a/drivers/media/pci/saa7134/saa7134-empress.c b/drivers/media/pci/saa7134/saa7134-empress.c index 76a37fbd8458..aafbb34765b0 100644 --- a/drivers/media/pci/saa7134/saa7134-empress.c +++ b/drivers/media/pci/saa7134/saa7134-empress.c @@ -138,12 +138,15 @@ static int empress_try_fmt_vid_cap(struct file *file, void *priv, { struct saa7134_dev *dev = video_drvdata(file); struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; v4l2_fill_mbus_format(&format.format, &f->fmt.pix, MEDIA_BUS_FMT_FIXED); - saa_call_all(dev, pad, set_fmt, &pad_cfg, &format); + saa_call_all(dev, pad, set_fmt, &pad_state, &format); v4l2_fill_pix_format(&f->fmt.pix, &format.format); f->fmt.pix.pixelformat = V4L2_PIX_FMT_MPEG; diff --git a/drivers/media/pci/saa7134/saa7134-tvaudio.c b/drivers/media/pci/saa7134/saa7134-tvaudio.c index aa0895d2d735..9e0c442abc76 100644 --- a/drivers/media/pci/saa7134/saa7134-tvaudio.c +++ b/drivers/media/pci/saa7134/saa7134-tvaudio.c @@ -871,7 +871,7 @@ void saa7134_enable_i2s(struct saa7134_dev *dev) switch (dev->pci->device) { case PCI_DEVICE_ID_PHILIPS_SAA7133: case PCI_DEVICE_ID_PHILIPS_SAA7135: - /* Set I2S format (SONY)  */ + /* Set I2S format (SONY) */ saa_writeb(SAA7133_I2S_AUDIO_CONTROL, 0x00); /* Start I2S */ saa_writeb(SAA7134_I2S_AUDIO_OUTPUT, 0x11); diff --git a/drivers/media/pci/saa7134/saa7134-video.c b/drivers/media/pci/saa7134/saa7134-video.c index 0f9d6b9edb90..374c8e1087de 100644 --- a/drivers/media/pci/saa7134/saa7134-video.c +++ b/drivers/media/pci/saa7134/saa7134-video.c @@ -1181,7 +1181,7 @@ static int video_release(struct file *file) saa_call_all(dev, tuner, standby); if (vdev->vfl_type == VFL_TYPE_RADIO) - saa_call_all(dev, core, ioctl, SAA6588_CMD_CLOSE, &cmd); + saa_call_all(dev, core, command, SAA6588_CMD_CLOSE, &cmd); mutex_unlock(&dev->lock); return 0; @@ -1200,7 +1200,7 @@ static ssize_t radio_read(struct file *file, char __user *data, cmd.result = -ENODEV; mutex_lock(&dev->lock); - saa_call_all(dev, core, ioctl, SAA6588_CMD_READ, &cmd); + saa_call_all(dev, core, command, SAA6588_CMD_READ, &cmd); mutex_unlock(&dev->lock); return cmd.result; @@ -1216,7 +1216,7 @@ static __poll_t radio_poll(struct file *file, poll_table *wait) cmd.event_list = wait; cmd.poll_mask = 0; mutex_lock(&dev->lock); - saa_call_all(dev, core, ioctl, SAA6588_CMD_POLL, &cmd); + saa_call_all(dev, core, command, SAA6588_CMD_POLL, &cmd); mutex_unlock(&dev->lock); return rc | cmd.poll_mask; diff --git a/drivers/media/pci/ttpci/Kconfig b/drivers/media/pci/ttpci/Kconfig index 8a362ee9105f..65a6832a6b96 100644 --- a/drivers/media/pci/ttpci/Kconfig +++ b/drivers/media/pci/ttpci/Kconfig @@ -1,56 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -config DVB_AV7110_IR - bool - depends on RC_CORE=y || RC_CORE = DVB_AV7110 - default DVB_AV7110 - -config DVB_AV7110 - tristate "AV7110 cards" - depends on DVB_CORE && PCI && I2C - select TTPCI_EEPROM - select VIDEO_SAA7146_VV - depends on VIDEO_DEV # dependencies of VIDEO_SAA7146_VV - select DVB_VES1820 if MEDIA_SUBDRV_AUTOSELECT - select DVB_VES1X93 if MEDIA_SUBDRV_AUTOSELECT - select DVB_STV0299 if MEDIA_SUBDRV_AUTOSELECT - select DVB_TDA8083 if MEDIA_SUBDRV_AUTOSELECT - select DVB_SP8870 if MEDIA_SUBDRV_AUTOSELECT - select DVB_STV0297 if MEDIA_SUBDRV_AUTOSELECT - select DVB_L64781 if MEDIA_SUBDRV_AUTOSELECT - select DVB_LNBP21 if MEDIA_SUBDRV_AUTOSELECT - help - Support for SAA7146 and AV7110 based DVB cards as produced - by Fujitsu-Siemens, Technotrend, Hauppauge and others. - - This driver only supports the fullfeatured cards with - onboard MPEG2 decoder. - - This driver needs an external firmware. Please use the script - "/scripts/get_dvb_firmware av7110" to - download/extract it, and then copy it to /usr/lib/hotplug/firmware - or /lib/firmware (depending on configuration of firmware hotplug). - - Alternatively, you can download the file and use the kernel's - EXTRA_FIRMWARE configuration option to build it into your - kernel image by adding the filename to the EXTRA_FIRMWARE - configuration option string. - - Say Y if you own such a card and want to use it. - -config DVB_AV7110_OSD - bool "AV7110 OSD support" - depends on DVB_AV7110 - default y if DVB_AV7110=y || DVB_AV7110=m - help - The AV7110 firmware provides some code to generate an OnScreenDisplay - on the video output. This is kind of nonstandard and not guaranteed to - be maintained. - - Anyway, some popular DVB software like VDR uses this OSD to render - its menus, so say Y if you want to use this software. - - All other people say N. - config DVB_BUDGET_CORE tristate "SAA7146 DVB cards (aka Budget, Nova-PCI)" depends on DVB_CORE && PCI && I2C @@ -136,25 +84,3 @@ config DVB_BUDGET_AV To compile this driver as a module, choose M here: the module will be called budget-av. - -config DVB_BUDGET_PATCH - tristate "AV7110 cards with Budget Patch" - depends on DVB_BUDGET_CORE && I2C - depends on DVB_AV7110 - select DVB_STV0299 if MEDIA_SUBDRV_AUTOSELECT - select DVB_VES1X93 if MEDIA_SUBDRV_AUTOSELECT - select DVB_TDA8083 if MEDIA_SUBDRV_AUTOSELECT - help - Support for Budget Patch (full TS) modification on - SAA7146+AV7110 based cards (DVB-S cards). This - driver doesn't use onboard MPEG2 decoder. The - card is driven in Budget-only mode. Card is - required to have loaded firmware to tune properly. - Firmware can be loaded by insertion and removal of - standard AV7110 driver prior to loading this - driver. - - Say Y if you own such a card and want to use it. - - To compile this driver as a module, choose M here: the - module will be called budget-patch. diff --git a/drivers/media/pci/ttpci/Makefile b/drivers/media/pci/ttpci/Makefile index 9b44c479fcdd..b0708f6e40cc 100644 --- a/drivers/media/pci/ttpci/Makefile +++ b/drivers/media/pci/ttpci/Makefile @@ -1,22 +1,13 @@ # SPDX-License-Identifier: GPL-2.0 # # Makefile for the kernel SAA7146 FULL TS DVB device driver -# and the AV7110 DVB device driver # -dvb-ttpci-objs := av7110_hw.o av7110_v4l.o av7110_av.o av7110_ca.o av7110.o av7110_ipack.o dvb_filter.o - -ifdef CONFIG_DVB_AV7110_IR -dvb-ttpci-objs += av7110_ir.o -endif - -obj-$(CONFIG_TTPCI_EEPROM) += ttpci-eeprom.o obj-$(CONFIG_DVB_BUDGET_CORE) += budget-core.o obj-$(CONFIG_DVB_BUDGET) += budget.o obj-$(CONFIG_DVB_BUDGET_AV) += budget-av.o obj-$(CONFIG_DVB_BUDGET_CI) += budget-ci.o -obj-$(CONFIG_DVB_BUDGET_PATCH) += budget-patch.o -obj-$(CONFIG_DVB_AV7110) += dvb-ttpci.o ccflags-y += -I $(srctree)/drivers/media/dvb-frontends/ ccflags-y += -I $(srctree)/drivers/media/tuners +ccflags-y += -I $(srctree)/drivers/media/common diff --git a/drivers/media/pci/ttpci/budget-core.c b/drivers/media/pci/ttpci/budget-core.c index d405eea5c37f..5d5796f24469 100644 --- a/drivers/media/pci/ttpci/budget-core.c +++ b/drivers/media/pci/ttpci/budget-core.c @@ -180,7 +180,8 @@ static void vpeirq(struct tasklet_struct *t) u32 count; /* Ensure streamed PCI data is synced to CPU */ - pci_dma_sync_sg_for_cpu(budget->dev->pci, budget->pt.slist, budget->pt.nents, PCI_DMA_FROMDEVICE); + dma_sync_sg_for_cpu(&budget->dev->pci->dev, budget->pt.slist, + budget->pt.nents, DMA_FROM_DEVICE); /* nearest lower position divisible by 188 */ newdma -= newdma % 188; diff --git a/drivers/media/pci/ttpci/budget.h b/drivers/media/pci/ttpci/budget.h index a7463daf39f1..bd87432e6cde 100644 --- a/drivers/media/pci/ttpci/budget.h +++ b/drivers/media/pci/ttpci/budget.h @@ -8,7 +8,6 @@ #include #include #include -#include "dvb_filter.h" #include #include @@ -28,6 +27,7 @@ extern int budget_debug; __func__, ##arg); \ } while (0) +#define TS_SIZE 188 struct budget_info { char *name; diff --git a/drivers/media/pci/tw5864/tw5864-reg.h b/drivers/media/pci/tw5864/tw5864-reg.h index a74f30f2f78e..a26a439c4dc0 100644 --- a/drivers/media/pci/tw5864/tw5864-reg.h +++ b/drivers/media/pci/tw5864/tw5864-reg.h @@ -289,13 +289,13 @@ /* OSD enable bit for each channel */ #define TW5864_DSP_OSD_ENABLE 0x0228 -/* 0x0280 ~ 0x029c – Motion Vector for 1st 4x4 Block, e.g., 80 (X), 84 (Y) */ +/* 0x0280 ~ 0x029c - Motion Vector for 1st 4x4 Block, e.g., 80 (X), 84 (Y) */ #define TW5864_ME_MV_VEC1 0x0280 -/* 0x02a0 ~ 0x02bc – Motion Vector for 2nd 4x4 Block, e.g., A0 (X), A4 (Y) */ +/* 0x02a0 ~ 0x02bc - Motion Vector for 2nd 4x4 Block, e.g., A0 (X), A4 (Y) */ #define TW5864_ME_MV_VEC2 0x02a0 -/* 0x02c0 ~ 0x02dc – Motion Vector for 3rd 4x4 Block, e.g., C0 (X), C4 (Y) */ +/* 0x02c0 ~ 0x02dc - Motion Vector for 3rd 4x4 Block, e.g., C0 (X), C4 (Y) */ #define TW5864_ME_MV_VEC3 0x02c0 -/* 0x02e0 ~ 0x02fc – Motion Vector for 4th 4x4 Block, e.g., E0 (X), E4 (Y) */ +/* 0x02e0 ~ 0x02fc - Motion Vector for 4th 4x4 Block, e.g., E0 (X), E4 (Y) */ #define TW5864_ME_MV_VEC4 0x02e0 /* @@ -462,13 +462,13 @@ #define TW5864_VLC_BUF 0x100c /* Define controls in register TW5864_VLC_BUF */ -/* VLC BK0 full status, write ‘1’ to clear */ +/* VLC BK0 full status, write '1' to clear */ #define TW5864_VLC_BK0_FULL BIT(0) -/* VLC BK1 full status, write ‘1’ to clear */ +/* VLC BK1 full status, write '1' to clear */ #define TW5864_VLC_BK1_FULL BIT(1) -/* VLC end slice status, write ‘1’ to clear */ +/* VLC end slice status, write '1' to clear */ #define TW5864_VLC_END_SLICE BIT(2) -/* VLC Buffer overflow status, write ‘1’ to clear */ +/* VLC Buffer overflow status, write '1' to clear */ #define TW5864_DSP_RD_OF BIT(3) /* VLC string length in either buffer 0 or 1 at end of frame */ #define TW5864_VLC_STREAM_LEN_SHIFT 4 @@ -476,7 +476,7 @@ /* [15:0] Total coefficient number in a frame */ #define TW5864_TOTAL_COEF_NO 0x1010 -/* [0] VLC Encoder Interrupt. Write ‘1’ to clear */ +/* [0] VLC Encoder Interrupt. Write '1' to clear */ #define TW5864_VLC_DSP_INTR 0x1014 /* [31:0] VLC stream CRC checksum */ #define TW5864_VLC_STREAM_CRC 0x1018 @@ -494,7 +494,7 @@ */ #define TW5864_VLC_RD_BRST BIT(1) -/* 0x2000 ~ 0x2ffc -- H264 Stream Memory Map */ +/* 0x2000 ~ 0x2ffc - H264 Stream Memory Map */ /* * A word is 4 bytes. I.e., * VLC_STREAM_MEM[0] address: 0x2000 @@ -506,7 +506,7 @@ #define TW5864_VLC_STREAM_MEM_MAX_OFFSET 0x3ff #define TW5864_VLC_STREAM_MEM(offset) (TW5864_VLC_STREAM_MEM_START + 4 * offset) -/* 0x4000 ~ 0x4ffc -- Audio Register Map */ +/* 0x4000 ~ 0x4ffc - Audio Register Map */ /* [31:0] config 1ms cnt = Realtime clk/1000 */ #define TW5864_CFG_1MS_CNT 0x4000 @@ -688,10 +688,10 @@ /* * [1:0] - * 2’b00 phase set to 180 degree - * 2’b01 phase set to 270 degree - * 2’b10 phase set to 0 degree - * 2’b11 phase set to 90 degree + * 2'b00 phase set to 180 degree + * 2'b01 phase set to 270 degree + * 2'b10 phase set to 0 degree + * 2'b11 phase set to 90 degree */ #define TW5864_I2C_PHASE_CFG 0x800c @@ -826,7 +826,7 @@ /* SPLL_IREF, SPLL_LPX4, SPLL_CPX4, SPLL_PD, SPLL_DBG */ #define TW5864_SPLL 0x8028 -/* 0x8800 ~ 0x88fc -- Interrupt Register Map */ +/* 0x8800 ~ 0x88fc - Interrupt Register Map */ /* * Trigger mode of interrupt source 0 ~ 15 * 1 Edge trigger mode @@ -909,7 +909,7 @@ #define TW5864_INTR_I2C_DONE BIT(25) #define TW5864_INTR_AD BIT(26) -/* 0x9000 ~ 0x920c -- Video Capture (VIF) Register Map */ +/* 0x9000 ~ 0x920c - Video Capture (VIF) Register Map */ /* * H264EN_CH_STATUS[n] Status of Vsync synchronized H264EN_CH_EN (Read Only) * 1 Channel Enabled @@ -1009,7 +1009,7 @@ /* GPIO Output Enable of Group n */ #define TW5864_GPIO_OEN (0xff << 8) -/* 0xa000 ~ 0xa8ff – DDR Controller Register Map */ +/* 0xa000 ~ 0xa8ff - DDR Controller Register Map */ /* DDR Controller A */ /* * [2:0] Data valid counter after read command to DDR. This is the delay value @@ -1111,7 +1111,7 @@ */ #define TW5864_DDR_B_OFFSET 0x0800 -/* 0xb004 ~ 0xb018 – HW version/ARB12 Register Map */ +/* 0xb004 ~ 0xb018 - HW version/ARB12 Register Map */ /* [15:0] Default is C013 */ #define TW5864_HW_VERSION 0xb004 @@ -1145,7 +1145,7 @@ /* ARB12 maximum value of time out counter (default 15"h1FF) */ #define TW5864_ARB12_TIME_OUT_CNT 0x7fff -/* 0xb800 ~ 0xb80c -- Indirect Access Register Map */ +/* 0xb800 ~ 0xb80c - Indirect Access Register Map */ /* * Spec says: * In order to access the indirect register space, the following procedure is @@ -1177,7 +1177,7 @@ /* [31:0] Data used to read/write indirect register space */ #define TW5864_IND_DATA 0xb804 -/* 0xc000 ~ 0xc7fc -- Preview Register Map */ +/* 0xc000 ~ 0xc7fc - Preview Register Map */ /* Mostly skipped this section. */ /* * [15:0] Status of Vsync Synchronized PCI_PV_CH_EN (Read Only) @@ -1192,12 +1192,12 @@ */ #define TW5864_PCI_PV_CH_EN 0xc004 -/* 0xc800 ~ 0xc804 -- JPEG Capture Register Map */ +/* 0xc800 ~ 0xc804 - JPEG Capture Register Map */ /* Skipped. */ -/* 0xd000 ~ 0xd0fc -- JPEG Control Register Map */ +/* 0xd000 ~ 0xd0fc - JPEG Control Register Map */ /* Skipped. */ -/* 0xe000 ~ 0xfc04 – Motion Vector Register Map */ +/* 0xe000 ~ 0xfc04 - Motion Vector Register Map */ /* ME Motion Vector data (Four Byte Each) 0xe000 ~ 0xe7fc */ #define TW5864_ME_MV_VEC_START 0xe000 @@ -1231,7 +1231,7 @@ */ #define TW5864_MPI_DDR_SEL2 BIT(15) -/* 0x18000 ~ 0x181fc – PCI Master/Slave Control Map */ +/* 0x18000 ~ 0x181fc - PCI Master/Slave Control Map */ #define TW5864_PCI_INTR_STATUS 0x18000 /* Define controls in register TW5864_PCI_INTR_STATUS */ /* vlc done */ @@ -1400,11 +1400,11 @@ #define TW5864_VLC_STREAM_BASE_ADDR 0x18080 /* MV stream base address */ #define TW5864_MV_STREAM_BASE_ADDR 0x18084 -/* 0x180a0 – 0x180bc: audio burst base address. Skipped. */ -/* 0x180c0 ~ 0x180dc – JPEG Push Mode Buffer Base Address. Skipped. */ -/* 0x18100 – 0x1817c: preview burst base address. Skipped. */ +/* 0x180a0 ~ 0x180bc: audio burst base address. Skipped. */ +/* 0x180c0 ~ 0x180dc: JPEG Push Mode Buffer Base Address. Skipped. */ +/* 0x18100 ~ 0x1817c: preview burst base address. Skipped. */ -/* 0x80000 ~ 0x87fff -- DDR Burst RW Register Map */ +/* 0x80000 ~ 0x87fff - DDR Burst RW Register Map */ #define TW5864_DDR_CTL 0x80000 /* Define controls in register TW5864_DDR_CTL */ #define TW5864_BRST_LENGTH_SHIFT 2 @@ -1516,7 +1516,7 @@ * Vertical Sharpness Control. Writable. * 0 = None (default) * 7 = Highest - * **Note: VSHP must be set to ‘0’ if COMB = 0 + * **Note: VSHP must be set to '0' if COMB = 0 */ #define TW5864_INDIR_VIN_1_VSHP 0x07 @@ -1595,7 +1595,7 @@ #define TW5864_INDIR_VIN_9_CNTRST(channel) (0x009 + channel * 0x010) /* - * These bits control the brightness. They have value of –128 to 127 in 2's + * These bits control the brightness. They have value of -128 to 127 in 2's * complement form. Positive value increases brightness. A value 0 has no * effect on the data. The default is 00h. */ diff --git a/drivers/media/platform/Makefile b/drivers/media/platform/Makefile index eedc14aafb32..73ce083c2fc6 100644 --- a/drivers/media/platform/Makefile +++ b/drivers/media/platform/Makefile @@ -67,6 +67,7 @@ obj-$(CONFIG_VIDEO_RCAR_VIN) += rcar-vin/ obj-$(CONFIG_VIDEO_ATMEL_ISC) += atmel/ obj-$(CONFIG_VIDEO_ATMEL_ISI) += atmel/ +obj-$(CONFIG_VIDEO_ATMEL_XISC) += atmel/ obj-$(CONFIG_VIDEO_STM32_DCMI) += stm32/ diff --git a/drivers/media/platform/allegro-dvt/nal-h264.c b/drivers/media/platform/allegro-dvt/nal-h264.c index 94dd9266d850..0ab2fcbee1b9 100644 --- a/drivers/media/platform/allegro-dvt/nal-h264.c +++ b/drivers/media/platform/allegro-dvt/nal-h264.c @@ -25,7 +25,7 @@ #include "nal-rbsp.h" /* - * See Rec. ITU-T H.264 (04/2017) Table 7-1 – NAL unit type codes, syntax + * See Rec. ITU-T H.264 (04/2017) Table 7-1 - NAL unit type codes, syntax * element categories, and NAL unit type classes */ enum nal_unit_type { diff --git a/drivers/media/platform/allegro-dvt/nal-hevc.c b/drivers/media/platform/allegro-dvt/nal-hevc.c index 5db540c69bfe..15a352e45831 100644 --- a/drivers/media/platform/allegro-dvt/nal-hevc.c +++ b/drivers/media/platform/allegro-dvt/nal-hevc.c @@ -25,7 +25,7 @@ #include "nal-rbsp.h" /* - * See Rec. ITU-T H.265 (02/2018) Table 7-1 – NAL unit type codes and NAL unit + * See Rec. ITU-T H.265 (02/2018) Table 7-1 - NAL unit type codes and NAL unit * type classes */ enum nal_unit_type { diff --git a/drivers/media/platform/am437x/am437x-vpfe.c b/drivers/media/platform/am437x/am437x-vpfe.c index 6cdc77dda0e4..1c9cb9e05fdf 100644 --- a/drivers/media/platform/am437x/am437x-vpfe.c +++ b/drivers/media/platform/am437x/am437x-vpfe.c @@ -1021,7 +1021,9 @@ static int vpfe_initialize_device(struct vpfe_device *vpfe) if (ret) return ret; - pm_runtime_get_sync(vpfe->pdev); + ret = pm_runtime_resume_and_get(vpfe->pdev); + if (ret < 0) + return ret; vpfe_config_enable(&vpfe->ccdc, 1); @@ -2443,7 +2445,11 @@ static int vpfe_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); /* for now just enable it here instead of waiting for the open */ - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + vpfe_err(vpfe, "Unable to resume device.\n"); + goto probe_out_v4l2_unregister; + } vpfe_ccdc_config_defaults(ccdc); @@ -2530,6 +2536,11 @@ static int vpfe_suspend(struct device *dev) /* only do full suspend if streaming has started */ if (vb2_start_streaming_called(&vpfe->buffer_queue)) { + /* + * ignore RPM resume errors here, as it is already too late. + * A check like that should happen earlier, either at + * open() or just before start streaming. + */ pm_runtime_get_sync(dev); vpfe_config_enable(ccdc, 1); diff --git a/drivers/media/platform/atmel/Kconfig b/drivers/media/platform/atmel/Kconfig index 1850fe7f9360..99b51213f871 100644 --- a/drivers/media/platform/atmel/Kconfig +++ b/drivers/media/platform/atmel/Kconfig @@ -12,6 +12,17 @@ config VIDEO_ATMEL_ISC This module makes the ATMEL Image Sensor Controller available as a v4l2 device. +config VIDEO_ATMEL_XISC + tristate "ATMEL eXtended Image Sensor Controller (XISC) support" + depends on VIDEO_V4L2 && COMMON_CLK && VIDEO_V4L2_SUBDEV_API + depends on ARCH_AT91 || COMPILE_TEST + select VIDEOBUF2_DMA_CONTIG + select REGMAP_MMIO + select V4L2_FWNODE + help + This module makes the ATMEL eXtended Image Sensor Controller + available as a v4l2 device. + config VIDEO_ATMEL_ISI tristate "ATMEL Image Sensor Interface (ISI) support" depends on VIDEO_V4L2 && OF diff --git a/drivers/media/platform/atmel/Makefile b/drivers/media/platform/atmel/Makefile index 2dba38994a70..c5c01556c653 100644 --- a/drivers/media/platform/atmel/Makefile +++ b/drivers/media/platform/atmel/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only atmel-isc-objs = atmel-sama5d2-isc.o atmel-isc-base.o +atmel-xisc-objs = atmel-sama7g5-isc.o atmel-isc-base.o obj-$(CONFIG_VIDEO_ATMEL_ISI) += atmel-isi.o obj-$(CONFIG_VIDEO_ATMEL_ISC) += atmel-isc.o +obj-$(CONFIG_VIDEO_ATMEL_XISC) += atmel-xisc.o diff --git a/drivers/media/platform/atmel/atmel-isc-base.c b/drivers/media/platform/atmel/atmel-isc-base.c index fe3ec8d0eaee..19daa49bf604 100644 --- a/drivers/media/platform/atmel/atmel-isc-base.c +++ b/drivers/media/platform/atmel/atmel-isc-base.c @@ -45,179 +45,6 @@ module_param(sensor_preferred, uint, 0644); MODULE_PARM_DESC(sensor_preferred, "Sensor is preferred to output the specified format (1-on 0-off), default 1"); -/* This is a list of the formats that the ISC can *output* */ -const struct isc_format controller_formats[] = { - { - .fourcc = V4L2_PIX_FMT_ARGB444, - }, - { - .fourcc = V4L2_PIX_FMT_ARGB555, - }, - { - .fourcc = V4L2_PIX_FMT_RGB565, - }, - { - .fourcc = V4L2_PIX_FMT_ABGR32, - }, - { - .fourcc = V4L2_PIX_FMT_XBGR32, - }, - { - .fourcc = V4L2_PIX_FMT_YUV420, - }, - { - .fourcc = V4L2_PIX_FMT_YUYV, - }, - { - .fourcc = V4L2_PIX_FMT_YUV422P, - }, - { - .fourcc = V4L2_PIX_FMT_GREY, - }, - { - .fourcc = V4L2_PIX_FMT_Y10, - }, -}; - -/* This is a list of formats that the ISC can receive as *input* */ -struct isc_format formats_list[] = { - { - .fourcc = V4L2_PIX_FMT_SBGGR8, - .mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8, - .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, - .cfa_baycfg = ISC_BAY_CFG_BGBG, - }, - { - .fourcc = V4L2_PIX_FMT_SGBRG8, - .mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8, - .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, - .cfa_baycfg = ISC_BAY_CFG_GBGB, - }, - { - .fourcc = V4L2_PIX_FMT_SGRBG8, - .mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8, - .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, - .cfa_baycfg = ISC_BAY_CFG_GRGR, - }, - { - .fourcc = V4L2_PIX_FMT_SRGGB8, - .mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8, - .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, - .cfa_baycfg = ISC_BAY_CFG_RGRG, - }, - { - .fourcc = V4L2_PIX_FMT_SBGGR10, - .mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, - .cfa_baycfg = ISC_BAY_CFG_RGRG, - }, - { - .fourcc = V4L2_PIX_FMT_SGBRG10, - .mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, - .cfa_baycfg = ISC_BAY_CFG_GBGB, - }, - { - .fourcc = V4L2_PIX_FMT_SGRBG10, - .mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, - .cfa_baycfg = ISC_BAY_CFG_GRGR, - }, - { - .fourcc = V4L2_PIX_FMT_SRGGB10, - .mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, - .cfa_baycfg = ISC_BAY_CFG_RGRG, - }, - { - .fourcc = V4L2_PIX_FMT_SBGGR12, - .mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, - .cfa_baycfg = ISC_BAY_CFG_BGBG, - }, - { - .fourcc = V4L2_PIX_FMT_SGBRG12, - .mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, - .cfa_baycfg = ISC_BAY_CFG_GBGB, - }, - { - .fourcc = V4L2_PIX_FMT_SGRBG12, - .mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, - .cfa_baycfg = ISC_BAY_CFG_GRGR, - }, - { - .fourcc = V4L2_PIX_FMT_SRGGB12, - .mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, - .cfa_baycfg = ISC_BAY_CFG_RGRG, - }, - { - .fourcc = V4L2_PIX_FMT_GREY, - .mbus_code = MEDIA_BUS_FMT_Y8_1X8, - .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, - }, - { - .fourcc = V4L2_PIX_FMT_YUYV, - .mbus_code = MEDIA_BUS_FMT_YUYV8_2X8, - .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, - }, - { - .fourcc = V4L2_PIX_FMT_RGB565, - .mbus_code = MEDIA_BUS_FMT_RGB565_2X8_LE, - .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, - }, - { - .fourcc = V4L2_PIX_FMT_Y10, - .mbus_code = MEDIA_BUS_FMT_Y10_1X10, - .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, - }, - -}; - -/* Gamma table with gamma 1/2.2 */ -const u32 isc_gamma_table[GAMMA_MAX + 1][GAMMA_ENTRIES] = { - /* 0 --> gamma 1/1.8 */ - { 0x65, 0x66002F, 0x950025, 0xBB0020, 0xDB001D, 0xF8001A, - 0x1130018, 0x12B0017, 0x1420016, 0x1580014, 0x16D0013, 0x1810012, - 0x1940012, 0x1A60012, 0x1B80011, 0x1C90010, 0x1DA0010, 0x1EA000F, - 0x1FA000F, 0x209000F, 0x218000F, 0x227000E, 0x235000E, 0x243000E, - 0x251000E, 0x25F000D, 0x26C000D, 0x279000D, 0x286000D, 0x293000C, - 0x2A0000C, 0x2AC000C, 0x2B8000C, 0x2C4000C, 0x2D0000B, 0x2DC000B, - 0x2E7000B, 0x2F3000B, 0x2FE000B, 0x309000B, 0x314000B, 0x31F000A, - 0x32A000A, 0x334000B, 0x33F000A, 0x349000A, 0x354000A, 0x35E000A, - 0x368000A, 0x372000A, 0x37C000A, 0x386000A, 0x3900009, 0x399000A, - 0x3A30009, 0x3AD0009, 0x3B60009, 0x3BF000A, 0x3C90009, 0x3D20009, - 0x3DB0009, 0x3E40009, 0x3ED0009, 0x3F60009 }, - - /* 1 --> gamma 1/2 */ - { 0x7F, 0x800034, 0xB50028, 0xDE0021, 0x100001E, 0x11E001B, - 0x1390019, 0x1520017, 0x16A0015, 0x1800014, 0x1940014, 0x1A80013, - 0x1BB0012, 0x1CD0011, 0x1DF0010, 0x1EF0010, 0x200000F, 0x20F000F, - 0x21F000E, 0x22D000F, 0x23C000E, 0x24A000E, 0x258000D, 0x265000D, - 0x273000C, 0x27F000D, 0x28C000C, 0x299000C, 0x2A5000C, 0x2B1000B, - 0x2BC000C, 0x2C8000B, 0x2D3000C, 0x2DF000B, 0x2EA000A, 0x2F5000A, - 0x2FF000B, 0x30A000A, 0x314000B, 0x31F000A, 0x329000A, 0x333000A, - 0x33D0009, 0x3470009, 0x350000A, 0x35A0009, 0x363000A, 0x36D0009, - 0x3760009, 0x37F0009, 0x3880009, 0x3910009, 0x39A0009, 0x3A30009, - 0x3AC0008, 0x3B40009, 0x3BD0008, 0x3C60008, 0x3CE0008, 0x3D60009, - 0x3DF0008, 0x3E70008, 0x3EF0008, 0x3F70008 }, - - /* 2 --> gamma 1/2.2 */ - { 0x99, 0x9B0038, 0xD4002A, 0xFF0023, 0x122001F, 0x141001B, - 0x15D0019, 0x1760017, 0x18E0015, 0x1A30015, 0x1B80013, 0x1CC0012, - 0x1DE0011, 0x1F00010, 0x2010010, 0x2110010, 0x221000F, 0x230000F, - 0x23F000E, 0x24D000E, 0x25B000D, 0x269000C, 0x276000C, 0x283000C, - 0x28F000C, 0x29B000C, 0x2A7000C, 0x2B3000B, 0x2BF000B, 0x2CA000B, - 0x2D5000B, 0x2E0000A, 0x2EB000A, 0x2F5000A, 0x2FF000A, 0x30A000A, - 0x3140009, 0x31E0009, 0x327000A, 0x3310009, 0x33A0009, 0x3440009, - 0x34D0009, 0x3560009, 0x35F0009, 0x3680008, 0x3710008, 0x3790009, - 0x3820008, 0x38A0008, 0x3930008, 0x39B0008, 0x3A30008, 0x3AB0008, - 0x3B30008, 0x3BB0008, 0x3C30008, 0x3CB0007, 0x3D20008, 0x3DA0007, - 0x3E20007, 0x3E90007, 0x3F00008, 0x3F80007 }, -}; - #define ISC_IS_FORMAT_RAW(mbus_code) \ (((mbus_code) & 0xf000) == 0x3000) @@ -294,9 +121,13 @@ static int isc_wait_clk_stable(struct clk_hw *hw) static int isc_clk_prepare(struct clk_hw *hw) { struct isc_clk *isc_clk = to_isc_clk(hw); + int ret; - if (isc_clk->id == ISC_ISPCK) - pm_runtime_get_sync(isc_clk->dev); + if (isc_clk->id == ISC_ISPCK) { + ret = pm_runtime_resume_and_get(isc_clk->dev); + if (ret < 0) + return ret; + } return isc_wait_clk_stable(hw); } @@ -319,8 +150,8 @@ static int isc_clk_enable(struct clk_hw *hw) unsigned long flags; unsigned int status; - dev_dbg(isc_clk->dev, "ISC CLK: %s, div = %d, parent id = %d\n", - __func__, isc_clk->div, isc_clk->parent_id); + dev_dbg(isc_clk->dev, "ISC CLK: %s, id = %d, div = %d, parent id = %d\n", + __func__, id, isc_clk->div, isc_clk->parent_id); spin_lock_irqsave(&isc_clk->lock, flags); regmap_update_bits(regmap, ISC_CLKCFG, @@ -353,9 +184,13 @@ static int isc_clk_is_enabled(struct clk_hw *hw) { struct isc_clk *isc_clk = to_isc_clk(hw); u32 status; + int ret; - if (isc_clk->id == ISC_ISPCK) - pm_runtime_get_sync(isc_clk->dev); + if (isc_clk->id == ISC_ISPCK) { + ret = pm_runtime_resume_and_get(isc_clk->dev); + if (ret < 0) + return 0; + } regmap_read(isc_clk->regmap, ISC_CLKSR, &status); @@ -635,16 +470,20 @@ static void isc_start_dma(struct isc_device *isc) ISC_PFE_CFG0_COLEN | ISC_PFE_CFG0_ROWEN); addr0 = vb2_dma_contig_plane_dma_addr(&isc->cur_frm->vb.vb2_buf, 0); - regmap_write(regmap, ISC_DAD0, addr0); + regmap_write(regmap, ISC_DAD0 + isc->offsets.dma, addr0); switch (isc->config.fourcc) { case V4L2_PIX_FMT_YUV420: - regmap_write(regmap, ISC_DAD1, addr0 + (sizeimage * 2) / 3); - regmap_write(regmap, ISC_DAD2, addr0 + (sizeimage * 5) / 6); + regmap_write(regmap, ISC_DAD1 + isc->offsets.dma, + addr0 + (sizeimage * 2) / 3); + regmap_write(regmap, ISC_DAD2 + isc->offsets.dma, + addr0 + (sizeimage * 5) / 6); break; case V4L2_PIX_FMT_YUV422P: - regmap_write(regmap, ISC_DAD1, addr0 + sizeimage / 2); - regmap_write(regmap, ISC_DAD2, addr0 + (sizeimage * 3) / 4); + regmap_write(regmap, ISC_DAD1 + isc->offsets.dma, + addr0 + sizeimage / 2); + regmap_write(regmap, ISC_DAD2 + isc->offsets.dma, + addr0 + (sizeimage * 3) / 4); break; default: break; @@ -652,7 +491,8 @@ static void isc_start_dma(struct isc_device *isc) dctrl_dview = isc->config.dctrl_dview; - regmap_write(regmap, ISC_DCTRL, dctrl_dview | ISC_DCTRL_IE_IS); + regmap_write(regmap, ISC_DCTRL + isc->offsets.dma, + dctrl_dview | ISC_DCTRL_IE_IS); spin_lock(&isc->awb_lock); regmap_write(regmap, ISC_CTRLEN, ISC_CTRL_CAPTURE); spin_unlock(&isc->awb_lock); @@ -683,21 +523,16 @@ static void isc_set_pipeline(struct isc_device *isc, u32 pipeline) regmap_write(regmap, ISC_CFA_CFG, bay_cfg | ISC_CFA_CFG_EITPOL); - gamma = &isc_gamma_table[ctrls->gamma_index][0]; + gamma = &isc->gamma_table[ctrls->gamma_index][0]; regmap_bulk_write(regmap, ISC_GAM_BENTRY, gamma, GAMMA_ENTRIES); regmap_bulk_write(regmap, ISC_GAM_GENTRY, gamma, GAMMA_ENTRIES); regmap_bulk_write(regmap, ISC_GAM_RENTRY, gamma, GAMMA_ENTRIES); - /* Convert RGB to YUV */ - regmap_write(regmap, ISC_CSC_YR_YG, 0x42 | (0x81 << 16)); - regmap_write(regmap, ISC_CSC_YB_OY, 0x19 | (0x10 << 16)); - regmap_write(regmap, ISC_CSC_CBR_CBG, 0xFDA | (0xFB6 << 16)); - regmap_write(regmap, ISC_CSC_CBB_OCB, 0x70 | (0x80 << 16)); - regmap_write(regmap, ISC_CSC_CRR_CRG, 0x70 | (0xFA2 << 16)); - regmap_write(regmap, ISC_CSC_CRB_OCR, 0xFEE | (0x80 << 16)); - - regmap_write(regmap, ISC_CBC_BRIGHT, ctrls->brightness); - regmap_write(regmap, ISC_CBC_CONTRAST, ctrls->contrast); + isc->config_dpc(isc); + isc->config_csc(isc); + isc->config_cbc(isc); + isc->config_cc(isc); + isc->config_gam(isc); } static int isc_update_profile(struct isc_device *isc) @@ -728,12 +563,13 @@ static void isc_set_histogram(struct isc_device *isc, bool enable) struct isc_ctrls *ctrls = &isc->ctrls; if (enable) { - regmap_write(regmap, ISC_HIS_CFG, + regmap_write(regmap, ISC_HIS_CFG + isc->offsets.his, ISC_HIS_CFG_MODE_GR | (isc->config.sd_format->cfa_baycfg << ISC_HIS_CFG_BAYSEL_SHIFT) | ISC_HIS_CFG_RAR); - regmap_write(regmap, ISC_HIS_CTRL, ISC_HIS_CTRL_EN); + regmap_write(regmap, ISC_HIS_CTRL + isc->offsets.his, + ISC_HIS_CTRL_EN); regmap_write(regmap, ISC_INTEN, ISC_INT_HISDONE); ctrls->hist_id = ISC_HIS_CFG_MODE_GR; isc_update_profile(isc); @@ -742,7 +578,8 @@ static void isc_set_histogram(struct isc_device *isc, bool enable) ctrls->hist_stat = HIST_ENABLED; } else { regmap_write(regmap, ISC_INTDIS, ISC_INT_HISDONE); - regmap_write(regmap, ISC_HIS_CTRL, ISC_HIS_CTRL_DIS); + regmap_write(regmap, ISC_HIS_CTRL + isc->offsets.his, + ISC_HIS_CTRL_DIS); ctrls->hist_stat = HIST_DISABLED; } @@ -751,28 +588,25 @@ static void isc_set_histogram(struct isc_device *isc, bool enable) static int isc_configure(struct isc_device *isc) { struct regmap *regmap = isc->regmap; - u32 pfe_cfg0, rlp_mode, dcfg, mask, pipeline; + u32 pfe_cfg0, dcfg, mask, pipeline; struct isc_subdev_entity *subdev = isc->current_subdev; pfe_cfg0 = isc->config.sd_format->pfe_cfg0_bps; - rlp_mode = isc->config.rlp_cfg_mode; pipeline = isc->config.bits_pipeline; - dcfg = isc->config.dcfg_imode | - ISC_DCFG_YMBSIZE_BEATS8 | ISC_DCFG_CMBSIZE_BEATS8; + dcfg = isc->config.dcfg_imode | isc->dcfg; pfe_cfg0 |= subdev->pfe_cfg0 | ISC_PFE_CFG0_MODE_PROGRESSIVE; mask = ISC_PFE_CFG0_BPS_MASK | ISC_PFE_CFG0_HPOL_LOW | ISC_PFE_CFG0_VPOL_LOW | ISC_PFE_CFG0_PPOL_LOW | ISC_PFE_CFG0_MODE_MASK | ISC_PFE_CFG0_CCIR_CRC | - ISC_PFE_CFG0_CCIR656; + ISC_PFE_CFG0_CCIR656 | ISC_PFE_CFG0_MIPI; regmap_update_bits(regmap, ISC_PFE_CFG0, mask, pfe_cfg0); - regmap_update_bits(regmap, ISC_RLP_CFG, ISC_RLP_CFG_MODE_MASK, - rlp_mode); + isc->config_rlp(isc); - regmap_write(regmap, ISC_DCFG, dcfg); + regmap_write(regmap, ISC_DCFG + isc->offsets.dma, dcfg); /* Set the pipeline */ isc_set_pipeline(isc, pipeline); @@ -807,7 +641,12 @@ static int isc_start_streaming(struct vb2_queue *vq, unsigned int count) goto err_start_stream; } - pm_runtime_get_sync(isc->dev); + ret = pm_runtime_resume_and_get(isc->dev); + if (ret < 0) { + v4l2_err(&isc->v4l2_dev, "RPM resume failed in subdev %d\n", + ret); + goto err_pm_get; + } ret = isc_configure(isc); if (unlikely(ret)) @@ -838,7 +677,7 @@ static int isc_start_streaming(struct vb2_queue *vq, unsigned int count) err_configure: pm_runtime_put_sync(isc->dev); - +err_pm_get: v4l2_subdev_call(isc->current_subdev->sd, video, s_stream, 0); err_start_stream: @@ -938,7 +777,7 @@ static int isc_querycap(struct file *file, void *priv, { struct isc_device *isc = video_drvdata(file); - strscpy(cap->driver, ATMEL_ISC_NAME, sizeof(cap->driver)); + strscpy(cap->driver, "microchip-isc", sizeof(cap->driver)); strscpy(cap->card, "Atmel Image Sensor Controller", sizeof(cap->card)); snprintf(cap->bus_info, sizeof(cap->bus_info), "platform:%s", isc->v4l2_dev.name); @@ -949,25 +788,25 @@ static int isc_querycap(struct file *file, void *priv, static int isc_enum_fmt_vid_cap(struct file *file, void *priv, struct v4l2_fmtdesc *f) { + struct isc_device *isc = video_drvdata(file); u32 index = f->index; u32 i, supported_index; - if (index < ARRAY_SIZE(controller_formats)) { - f->pixelformat = controller_formats[index].fourcc; + if (index < isc->controller_formats_size) { + f->pixelformat = isc->controller_formats[index].fourcc; return 0; } - index -= ARRAY_SIZE(controller_formats); + index -= isc->controller_formats_size; - i = 0; supported_index = 0; - for (i = 0; i < ARRAY_SIZE(formats_list); i++) { - if (!ISC_IS_FORMAT_RAW(formats_list[i].mbus_code) || - !formats_list[i].sd_support) + for (i = 0; i < isc->formats_list_size; i++) { + if (!ISC_IS_FORMAT_RAW(isc->formats_list[i].mbus_code) || + !isc->formats_list[i].sd_support) continue; if (supported_index == index) { - f->pixelformat = formats_list[i].fourcc; + f->pixelformat = isc->formats_list[i].fourcc; return 0; } supported_index++; @@ -1016,6 +855,8 @@ static int isc_try_validate_formats(struct isc_device *isc) case V4L2_PIX_FMT_YUV420: case V4L2_PIX_FMT_YUV422P: case V4L2_PIX_FMT_YUYV: + case V4L2_PIX_FMT_UYVY: + case V4L2_PIX_FMT_VYUY: ret = 0; yuv = true; break; @@ -1030,6 +871,7 @@ static int isc_try_validate_formats(struct isc_device *isc) break; case V4L2_PIX_FMT_GREY: case V4L2_PIX_FMT_Y10: + case V4L2_PIX_FMT_Y16: ret = 0; grey = true; break; @@ -1060,6 +902,8 @@ static int isc_try_validate_formats(struct isc_device *isc) */ static int isc_try_configure_rlp_dma(struct isc_device *isc, bool direct_dump) { + isc->try_config.rlp_cfg_mode = 0; + switch (isc->try_config.fourcc) { case V4L2_PIX_FMT_SBGGR8: case V4L2_PIX_FMT_SGBRG8: @@ -1126,7 +970,19 @@ static int isc_try_configure_rlp_dma(struct isc_device *isc, bool direct_dump) isc->try_config.bpp = 16; break; case V4L2_PIX_FMT_YUYV: - isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YYCC; + isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YCYC | ISC_RLP_CFG_YMODE_YUYV; + isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32; + isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; + isc->try_config.bpp = 16; + break; + case V4L2_PIX_FMT_UYVY: + isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YCYC | ISC_RLP_CFG_YMODE_UYVY; + isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32; + isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; + isc->try_config.bpp = 16; + break; + case V4L2_PIX_FMT_VYUY: + isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_YCYC | ISC_RLP_CFG_YMODE_VYUY; isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED32; isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; isc->try_config.bpp = 16; @@ -1137,8 +993,11 @@ static int isc_try_configure_rlp_dma(struct isc_device *isc, bool direct_dump) isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; isc->try_config.bpp = 8; break; + case V4L2_PIX_FMT_Y16: + isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DATY10 | ISC_RLP_CFG_LSH; + fallthrough; case V4L2_PIX_FMT_Y10: - isc->try_config.rlp_cfg_mode = ISC_RLP_CFG_MODE_DATY10; + isc->try_config.rlp_cfg_mode |= ISC_RLP_CFG_MODE_DATY10; isc->try_config.dcfg_imode = ISC_DCFG_IMODE_PACKED16; isc->try_config.dctrl_dview = ISC_DCTRL_DVIEW_PACKED; isc->try_config.bpp = 16; @@ -1172,7 +1031,8 @@ static int isc_try_configure_pipeline(struct isc_device *isc) /* if sensor format is RAW, we convert inside ISC */ if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) { isc->try_config.bits_pipeline = CFA_ENABLE | - WB_ENABLE | GAM_ENABLES; + WB_ENABLE | GAM_ENABLES | DPC_BLCENABLE | + CC_ENABLE; } else { isc->try_config.bits_pipeline = 0x0; } @@ -1181,8 +1041,9 @@ static int isc_try_configure_pipeline(struct isc_device *isc) /* if sensor format is RAW, we convert inside ISC */ if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) { isc->try_config.bits_pipeline = CFA_ENABLE | - CSC_ENABLE | WB_ENABLE | GAM_ENABLES | - SUB420_ENABLE | SUB422_ENABLE | CBC_ENABLE; + CSC_ENABLE | GAM_ENABLES | WB_ENABLE | + SUB420_ENABLE | SUB422_ENABLE | CBC_ENABLE | + DPC_BLCENABLE; } else { isc->try_config.bits_pipeline = 0x0; } @@ -1192,39 +1053,49 @@ static int isc_try_configure_pipeline(struct isc_device *isc) if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) { isc->try_config.bits_pipeline = CFA_ENABLE | CSC_ENABLE | WB_ENABLE | GAM_ENABLES | - SUB422_ENABLE | CBC_ENABLE; + SUB422_ENABLE | CBC_ENABLE | DPC_BLCENABLE; } else { isc->try_config.bits_pipeline = 0x0; } break; case V4L2_PIX_FMT_YUYV: + case V4L2_PIX_FMT_UYVY: + case V4L2_PIX_FMT_VYUY: /* if sensor format is RAW, we convert inside ISC */ if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) { isc->try_config.bits_pipeline = CFA_ENABLE | CSC_ENABLE | WB_ENABLE | GAM_ENABLES | - SUB422_ENABLE | CBC_ENABLE; + SUB422_ENABLE | CBC_ENABLE | DPC_BLCENABLE; } else { isc->try_config.bits_pipeline = 0x0; } break; case V4L2_PIX_FMT_GREY: - if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) { + case V4L2_PIX_FMT_Y16: /* if sensor format is RAW, we convert inside ISC */ + if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) { isc->try_config.bits_pipeline = CFA_ENABLE | CSC_ENABLE | WB_ENABLE | GAM_ENABLES | - CBC_ENABLE; + CBC_ENABLE | DPC_BLCENABLE; } else { isc->try_config.bits_pipeline = 0x0; } break; default: - isc->try_config.bits_pipeline = 0x0; + if (ISC_IS_FORMAT_RAW(isc->try_config.sd_format->mbus_code)) + isc->try_config.bits_pipeline = WB_ENABLE | DPC_BLCENABLE; + else + isc->try_config.bits_pipeline = 0x0; } + + /* Tune the pipeline to product specific */ + isc->adapt_pipeline(isc); + return 0; } static void isc_try_fse(struct isc_device *isc, - struct v4l2_subdev_pad_config *pad_cfg) + struct v4l2_subdev_state *sd_state) { int ret; struct v4l2_subdev_frame_size_enum fse = {}; @@ -1240,17 +1111,17 @@ static void isc_try_fse(struct isc_device *isc, fse.which = V4L2_SUBDEV_FORMAT_TRY; ret = v4l2_subdev_call(isc->current_subdev->sd, pad, enum_frame_size, - pad_cfg, &fse); + sd_state, &fse); /* * Attempt to obtain format size from subdev. If not available, * just use the maximum ISC can receive. */ if (ret) { - pad_cfg->try_crop.width = ISC_MAX_SUPPORT_WIDTH; - pad_cfg->try_crop.height = ISC_MAX_SUPPORT_HEIGHT; + sd_state->pads->try_crop.width = isc->max_width; + sd_state->pads->try_crop.height = isc->max_height; } else { - pad_cfg->try_crop.width = fse.max_width; - pad_cfg->try_crop.height = fse.max_height; + sd_state->pads->try_crop.width = fse.max_width; + sd_state->pads->try_crop.height = fse.max_height; } } @@ -1261,6 +1132,9 @@ static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f, struct isc_format *sd_fmt = NULL, *direct_fmt = NULL; struct v4l2_pix_format *pixfmt = &f->fmt.pix; struct v4l2_subdev_pad_config pad_cfg = {}; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -1324,10 +1198,10 @@ static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f, isc->try_config.sd_format = sd_fmt; /* Limit to Atmel ISC hardware capabilities */ - if (pixfmt->width > ISC_MAX_SUPPORT_WIDTH) - pixfmt->width = ISC_MAX_SUPPORT_WIDTH; - if (pixfmt->height > ISC_MAX_SUPPORT_HEIGHT) - pixfmt->height = ISC_MAX_SUPPORT_HEIGHT; + if (pixfmt->width > isc->max_width) + pixfmt->width = isc->max_width; + if (pixfmt->height > isc->max_height) + pixfmt->height = isc->max_height; /* * The mbus format is the one the subdev outputs. @@ -1358,16 +1232,22 @@ static int isc_try_fmt(struct isc_device *isc, struct v4l2_format *f, goto isc_try_fmt_err; /* Obtain frame sizes if possible to have crop requirements ready */ - isc_try_fse(isc, &pad_cfg); + isc_try_fse(isc, &pad_state); v4l2_fill_mbus_format(&format.format, pixfmt, mbus_code); ret = v4l2_subdev_call(isc->current_subdev->sd, pad, set_fmt, - &pad_cfg, &format); + &pad_state, &format); if (ret < 0) goto isc_try_fmt_subdev_err; v4l2_fill_pix_format(pixfmt, &format.format); + /* Limit to Atmel ISC hardware capabilities */ + if (pixfmt->width > isc->max_width) + pixfmt->width = isc->max_width; + if (pixfmt->height > isc->max_height) + pixfmt->height = isc->max_height; + pixfmt->field = V4L2_FIELD_NONE; pixfmt->bytesperline = (pixfmt->width * isc->try_config.bpp) >> 3; pixfmt->sizeimage = pixfmt->bytesperline * pixfmt->height; @@ -1403,6 +1283,12 @@ static int isc_set_fmt(struct isc_device *isc, struct v4l2_format *f) if (ret < 0) return ret; + /* Limit to Atmel ISC hardware capabilities */ + if (f->fmt.pix.width > isc->max_width) + f->fmt.pix.width = isc->max_width; + if (f->fmt.pix.height > isc->max_height) + f->fmt.pix.height = isc->max_height; + isc->fmt = *f; if (isc->try_config.sd_format && isc->config.sd_format && @@ -1496,8 +1382,8 @@ static int isc_enum_framesizes(struct file *file, void *fh, if (isc->user_formats[i]->fourcc == fsize->pixel_format) ret = 0; - for (i = 0; i < ARRAY_SIZE(controller_formats); i++) - if (controller_formats[i].fourcc == fsize->pixel_format) + for (i = 0; i < isc->controller_formats_size; i++) + if (isc->controller_formats[i].fourcc == fsize->pixel_format) ret = 0; if (ret) @@ -1533,8 +1419,8 @@ static int isc_enum_frameintervals(struct file *file, void *fh, if (isc->user_formats[i]->fourcc == fival->pixel_format) ret = 0; - for (i = 0; i < ARRAY_SIZE(controller_formats); i++) - if (controller_formats[i].fourcc == fival->pixel_format) + for (i = 0; i < isc->controller_formats_size; i++) + if (isc->controller_formats[i].fourcc == fival->pixel_format) ret = 0; if (ret) @@ -1704,7 +1590,8 @@ static void isc_hist_count(struct isc_device *isc, u32 *min, u32 *max) *min = 0; *max = HIST_ENTRIES; - regmap_bulk_read(regmap, ISC_HIS_ENTRY, hist_entry, HIST_ENTRIES); + regmap_bulk_read(regmap, ISC_HIS_ENTRY + isc->offsets.his_entry, + hist_entry, HIST_ENTRIES); *hist_count = 0; /* @@ -1809,6 +1696,7 @@ static void isc_awb_work(struct work_struct *w) u32 baysel; unsigned long flags; u32 min, max; + int ret; /* streaming is not active anymore */ if (isc->stop) @@ -1831,7 +1719,9 @@ static void isc_awb_work(struct work_struct *w) ctrls->hist_id = hist_id; baysel = isc->config.sd_format->cfa_baycfg << ISC_HIS_CFG_BAYSEL_SHIFT; - pm_runtime_get_sync(isc->dev); + ret = pm_runtime_resume_and_get(isc->dev); + if (ret < 0) + return; /* * only update if we have all the required histograms and controls @@ -1860,7 +1750,8 @@ static void isc_awb_work(struct work_struct *w) ctrls->awb = ISC_WB_NONE; } } - regmap_write(regmap, ISC_HIS_CFG, hist_id | baysel | ISC_HIS_CFG_RAR); + regmap_write(regmap, ISC_HIS_CFG + isc->offsets.his, + hist_id | baysel | ISC_HIS_CFG_RAR); isc_update_profile(isc); /* if awb has been disabled, we don't need to start another histogram */ if (ctrls->awb) @@ -2065,12 +1956,14 @@ static int isc_ctrl_init(struct isc_device *isc) if (ret < 0) return ret; + /* Initialize product specific controls. For example, contrast */ + isc->config_ctrls(isc, ops); + ctrls->brightness = 0; - ctrls->contrast = 256; v4l2_ctrl_new_std(hdl, ops, V4L2_CID_BRIGHTNESS, -1024, 1023, 1, 0); - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_CONTRAST, -2048, 2047, 1, 256); - v4l2_ctrl_new_std(hdl, ops, V4L2_CID_GAMMA, 0, GAMMA_MAX, 1, 2); + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_GAMMA, 0, isc->gamma_max, 1, + isc->gamma_max); isc->awb_ctrl = v4l2_ctrl_new_std(hdl, &isc_awb_ops, V4L2_CID_AUTO_WHITE_BALANCE, 0, 1, 1, 1); @@ -2138,12 +2031,13 @@ static void isc_async_unbind(struct v4l2_async_notifier *notifier, v4l2_ctrl_handler_free(&isc->ctrls.handler); } -static struct isc_format *find_format_by_code(unsigned int code, int *index) +static struct isc_format *find_format_by_code(struct isc_device *isc, + unsigned int code, int *index) { - struct isc_format *fmt = &formats_list[0]; + struct isc_format *fmt = &isc->formats_list[0]; unsigned int i; - for (i = 0; i < ARRAY_SIZE(formats_list); i++) { + for (i = 0; i < isc->formats_list_size; i++) { if (fmt->mbus_code == code) { *index = i; return fmt; @@ -2160,7 +2054,7 @@ static int isc_formats_init(struct isc_device *isc) struct isc_format *fmt; struct v4l2_subdev *subdev = isc->current_subdev->sd; unsigned int num_fmts, i, j; - u32 list_size = ARRAY_SIZE(formats_list); + u32 list_size = isc->formats_list_size; struct v4l2_subdev_mbus_code_enum mbus_code = { .which = V4L2_SUBDEV_FORMAT_ACTIVE, }; @@ -2170,7 +2064,7 @@ static int isc_formats_init(struct isc_device *isc) NULL, &mbus_code)) { mbus_code.index++; - fmt = find_format_by_code(mbus_code.code, &i); + fmt = find_format_by_code(isc, mbus_code.code, &i); if (!fmt) { v4l2_warn(&isc->v4l2_dev, "Mbus code %x not supported\n", mbus_code.code); @@ -2191,7 +2085,7 @@ static int isc_formats_init(struct isc_device *isc) if (!isc->user_formats) return -ENOMEM; - fmt = &formats_list[0]; + fmt = &isc->formats_list[0]; for (i = 0, j = 0; i < list_size; i++) { if (fmt->sd_support) isc->user_formats[j++] = fmt; @@ -2287,7 +2181,7 @@ static int isc_async_complete(struct v4l2_async_notifier *notifier) } /* Register video device */ - strscpy(vdev->name, ATMEL_ISC_NAME, sizeof(vdev->name)); + strscpy(vdev->name, "microchip-isc", sizeof(vdev->name)); vdev->release = video_device_release_empty; vdev->fops = &isc_fops; vdev->ioctl_ops = &isc_ioctl_ops; @@ -2338,8 +2232,14 @@ int isc_pipeline_init(struct isc_device *isc) struct regmap_field *regs; unsigned int i; - /* WB-->CFA-->CC-->GAM-->CSC-->CBC-->SUB422-->SUB420 */ + /* + * DPCEN-->GDCEN-->BLCEN-->WB-->CFA-->CC--> + * GAM-->VHXS-->CSC-->CBC-->SUB422-->SUB420 + */ const struct reg_field regfields[ISC_PIPE_LINE_NODE_NUM] = { + REG_FIELD(ISC_DPC_CTRL, 0, 0), + REG_FIELD(ISC_DPC_CTRL, 1, 1), + REG_FIELD(ISC_DPC_CTRL, 2, 2), REG_FIELD(ISC_WB_CTRL, 0, 0), REG_FIELD(ISC_CFA_CTRL, 0, 0), REG_FIELD(ISC_CC_CTRL, 0, 0), @@ -2347,10 +2247,11 @@ int isc_pipeline_init(struct isc_device *isc) REG_FIELD(ISC_GAM_CTRL, 1, 1), REG_FIELD(ISC_GAM_CTRL, 2, 2), REG_FIELD(ISC_GAM_CTRL, 3, 3), - REG_FIELD(ISC_CSC_CTRL, 0, 0), - REG_FIELD(ISC_CBC_CTRL, 0, 0), - REG_FIELD(ISC_SUB422_CTRL, 0, 0), - REG_FIELD(ISC_SUB420_CTRL, 0, 0), + REG_FIELD(ISC_VHXS_CTRL, 0, 0), + REG_FIELD(ISC_CSC_CTRL + isc->offsets.csc, 0, 0), + REG_FIELD(ISC_CBC_CTRL + isc->offsets.cbc, 0, 0), + REG_FIELD(ISC_SUB422_CTRL + isc->offsets.sub422, 0, 0), + REG_FIELD(ISC_SUB420_CTRL + isc->offsets.sub420, 0, 0), }; for (i = 0; i < ISC_PIPE_LINE_NODE_NUM; i++) { @@ -2365,7 +2266,7 @@ int isc_pipeline_init(struct isc_device *isc) } /* regmap configuration */ -#define ATMEL_ISC_REG_MAX 0xbfc +#define ATMEL_ISC_REG_MAX 0xd5c const struct regmap_config isc_regmap_config = { .reg_bits = 32, .reg_stride = 4, diff --git a/drivers/media/platform/atmel/atmel-isc-regs.h b/drivers/media/platform/atmel/atmel-isc-regs.h index f1e160ed4351..d06b72228d4f 100644 --- a/drivers/media/platform/atmel/atmel-isc-regs.h +++ b/drivers/media/platform/atmel/atmel-isc-regs.h @@ -26,6 +26,7 @@ #define ISC_PFE_CFG0_PPOL_LOW BIT(2) #define ISC_PFE_CFG0_CCIR656 BIT(9) #define ISC_PFE_CFG0_CCIR_CRC BIT(10) +#define ISC_PFE_CFG0_MIPI BIT(14) #define ISC_PFE_CFG0_MODE_PROGRESSIVE (0x0 << 4) #define ISC_PFE_CFG0_MODE_MASK GENMASK(6, 4) @@ -90,6 +91,46 @@ #define ISC_INT_DDONE BIT(8) #define ISC_INT_HISDONE BIT(12) +/* ISC DPC Control Register */ +#define ISC_DPC_CTRL 0x40 + +#define ISC_DPC_CTRL_DPCEN BIT(0) +#define ISC_DPC_CTRL_GDCEN BIT(1) +#define ISC_DPC_CTRL_BLCEN BIT(2) + +/* ISC DPC Config Register */ +#define ISC_DPC_CFG 0x44 + +#define ISC_DPC_CFG_BAYSEL_SHIFT 0 + +#define ISC_DPC_CFG_EITPOL BIT(4) + +#define ISC_DPC_CFG_TA_ENABLE BIT(14) +#define ISC_DPC_CFG_TC_ENABLE BIT(13) +#define ISC_DPC_CFG_TM_ENABLE BIT(12) + +#define ISC_DPC_CFG_RE_MODE BIT(17) + +#define ISC_DPC_CFG_GDCCLP_SHIFT 20 +#define ISC_DPC_CFG_GDCCLP_MASK GENMASK(22, 20) + +#define ISC_DPC_CFG_BLOFF_SHIFT 24 +#define ISC_DPC_CFG_BLOFF_MASK GENMASK(31, 24) + +#define ISC_DPC_CFG_BAYCFG_SHIFT 0 +#define ISC_DPC_CFG_BAYCFG_MASK GENMASK(1, 0) +/* ISC DPC Threshold Median Register */ +#define ISC_DPC_THRESHM 0x48 + +/* ISC DPC Threshold Closest Register */ +#define ISC_DPC_THRESHC 0x4C + +/* ISC DPC Threshold Average Register */ +#define ISC_DPC_THRESHA 0x50 + +/* ISC DPC STatus Register */ +#define ISC_DPC_SR 0x54 + /* ISC White Balance Control Register */ #define ISC_WB_CTRL 0x00000058 @@ -144,6 +185,8 @@ /* ISC Gamma Correction Control Register */ #define ISC_GAM_CTRL 0x00000094 +#define ISC_GAM_CTRL_BIPART BIT(4) + /* ISC_Gamma Correction Blue Entry Register */ #define ISC_GAM_BENTRY 0x00000098 @@ -153,6 +196,38 @@ /* ISC_Gamma Correction Green Entry Register */ #define ISC_GAM_RENTRY 0x00000298 +/* ISC VHXS Control Register */ +#define ISC_VHXS_CTRL 0x398 + +/* ISC VHXS Source Size Register */ +#define ISC_VHXS_SS 0x39C + +/* ISC VHXS Destination Size Register */ +#define ISC_VHXS_DS 0x3A0 + +/* ISC Vertical Factor Register */ +#define ISC_VXS_FACT 0x3a4 + +/* ISC Horizontal Factor Register */ +#define ISC_HXS_FACT 0x3a8 + +/* ISC Vertical Config Register */ +#define ISC_VXS_CFG 0x3ac + +/* ISC Horizontal Config Register */ +#define ISC_HXS_CFG 0x3b0 + +/* ISC Vertical Tap Register */ +#define ISC_VXS_TAP 0x3b4 + +/* ISC Horizontal Tap Register */ +#define ISC_HXS_TAP 0x434 + +/* Offset for CSC register specific to sama5d2 product */ +#define ISC_SAMA5D2_CSC_OFFSET 0 +/* Offset for CSC register specific to sama7g5 product */ +#define ISC_SAMA7G5_CSC_OFFSET 0x11c + /* Color Space Conversion Control Register */ #define ISC_CSC_CTRL 0x00000398 @@ -174,6 +249,11 @@ /* Color Space Conversion CRB OCR Register */ #define ISC_CSC_CRB_OCR 0x000003b0 +/* Offset for CBC register specific to sama5d2 product */ +#define ISC_SAMA5D2_CBC_OFFSET 0 +/* Offset for CBC register specific to sama7g5 product */ +#define ISC_SAMA7G5_CBC_OFFSET 0x11c + /* Contrast And Brightness Control Register */ #define ISC_CBC_CTRL 0x000003b4 @@ -188,12 +268,30 @@ #define ISC_CBC_CONTRAST 0x000003c0 #define ISC_CBC_CONTRAST_MASK GENMASK(11, 0) +/* Hue Register */ +#define ISC_CBCHS_HUE 0x4e0 +/* Saturation Register */ +#define ISC_CBCHS_SAT 0x4e4 + +/* Offset for SUB422 register specific to sama5d2 product */ +#define ISC_SAMA5D2_SUB422_OFFSET 0 +/* Offset for SUB422 register specific to sama7g5 product */ +#define ISC_SAMA7G5_SUB422_OFFSET 0x124 + /* Subsampling 4:4:4 to 4:2:2 Control Register */ #define ISC_SUB422_CTRL 0x000003c4 +/* Offset for SUB420 register specific to sama5d2 product */ +#define ISC_SAMA5D2_SUB420_OFFSET 0 +/* Offset for SUB420 register specific to sama7g5 product */ +#define ISC_SAMA7G5_SUB420_OFFSET 0x124 /* Subsampling 4:2:2 to 4:2:0 Control Register */ #define ISC_SUB420_CTRL 0x000003cc +/* Offset for RLP register specific to sama5d2 product */ +#define ISC_SAMA5D2_RLP_OFFSET 0 +/* Offset for RLP register specific to sama7g5 product */ +#define ISC_SAMA7G5_RLP_OFFSET 0x124 /* Rounding, Limiting and Packing Configuration Register */ #define ISC_RLP_CFG 0x000003d0 @@ -210,8 +308,22 @@ #define ISC_RLP_CFG_MODE_ARGB32 0xa #define ISC_RLP_CFG_MODE_YYCC 0xb #define ISC_RLP_CFG_MODE_YYCC_LIMITED 0xc +#define ISC_RLP_CFG_MODE_YCYC 0xd #define ISC_RLP_CFG_MODE_MASK GENMASK(3, 0) +#define ISC_RLP_CFG_LSH BIT(5) + +#define ISC_RLP_CFG_YMODE_YUYV (3 << 6) +#define ISC_RLP_CFG_YMODE_YVYU (2 << 6) +#define ISC_RLP_CFG_YMODE_VYUY (0 << 6) +#define ISC_RLP_CFG_YMODE_UYVY (1 << 6) + +#define ISC_RLP_CFG_YMODE_MASK GENMASK(7, 6) + +/* Offset for HIS register specific to sama5d2 product */ +#define ISC_SAMA5D2_HIS_OFFSET 0 +/* Offset for HIS register specific to sama7g5 product */ +#define ISC_SAMA7G5_HIS_OFFSET 0x124 /* Histogram Control Register */ #define ISC_HIS_CTRL 0x000003d4 @@ -233,6 +345,11 @@ #define ISC_HIS_CFG_RAR BIT(8) +/* Offset for DMA register specific to sama5d2 product */ +#define ISC_SAMA5D2_DMA_OFFSET 0 +/* Offset for DMA register specific to sama7g5 product */ +#define ISC_SAMA7G5_DMA_OFFSET 0x13c + /* DMA Configuration Register */ #define ISC_DCFG 0x000003e0 #define ISC_DCFG_IMODE_PACKED8 0x0 @@ -248,13 +365,15 @@ #define ISC_DCFG_YMBSIZE_BEATS4 (0x1 << 4) #define ISC_DCFG_YMBSIZE_BEATS8 (0x2 << 4) #define ISC_DCFG_YMBSIZE_BEATS16 (0x3 << 4) -#define ISC_DCFG_YMBSIZE_MASK GENMASK(5, 4) +#define ISC_DCFG_YMBSIZE_BEATS32 (0x4 << 4) +#define ISC_DCFG_YMBSIZE_MASK GENMASK(6, 4) #define ISC_DCFG_CMBSIZE_SINGLE (0x0 << 8) #define ISC_DCFG_CMBSIZE_BEATS4 (0x1 << 8) #define ISC_DCFG_CMBSIZE_BEATS8 (0x2 << 8) #define ISC_DCFG_CMBSIZE_BEATS16 (0x3 << 8) -#define ISC_DCFG_CMBSIZE_MASK GENMASK(9, 8) +#define ISC_DCFG_CMBSIZE_BEATS32 (0x4 << 8) +#define ISC_DCFG_CMBSIZE_MASK GENMASK(10, 8) /* DMA Control Register */ #define ISC_DCTRL 0x000003e4 @@ -278,6 +397,16 @@ /* DMA Address 2 Register */ #define ISC_DAD2 0x000003fc +/* Offset for version register specific to sama5d2 product */ +#define ISC_SAMA5D2_VERSION_OFFSET 0 +#define ISC_SAMA7G5_VERSION_OFFSET 0x13c +/* Version Register */ +#define ISC_VERSION 0x0000040c + +/* Offset for version register specific to sama5d2 product */ +#define ISC_SAMA5D2_HIS_ENTRY_OFFSET 0 +/* Offset for version register specific to sama7g5 product */ +#define ISC_SAMA7G5_HIS_ENTRY_OFFSET 0x14c /* Histogram Entry */ #define ISC_HIS_ENTRY 0x00000410 diff --git a/drivers/media/platform/atmel/atmel-isc.h b/drivers/media/platform/atmel/atmel-isc.h index fab8eca58d93..19cc60dfcbe0 100644 --- a/drivers/media/platform/atmel/atmel-isc.h +++ b/drivers/media/platform/atmel/atmel-isc.h @@ -10,9 +10,6 @@ */ #ifndef _ATMEL_ISC_H_ -#define ISC_MAX_SUPPORT_WIDTH 2592 -#define ISC_MAX_SUPPORT_HEIGHT 1944 - #define ISC_CLK_MAX_DIV 255 enum isc_clk_id { @@ -71,17 +68,21 @@ struct isc_format { }; /* Pipeline bitmap */ -#define WB_ENABLE BIT(0) -#define CFA_ENABLE BIT(1) -#define CC_ENABLE BIT(2) -#define GAM_ENABLE BIT(3) -#define GAM_BENABLE BIT(4) -#define GAM_GENABLE BIT(5) -#define GAM_RENABLE BIT(6) -#define CSC_ENABLE BIT(7) -#define CBC_ENABLE BIT(8) -#define SUB422_ENABLE BIT(9) -#define SUB420_ENABLE BIT(10) +#define DPC_DPCENABLE BIT(0) +#define DPC_GDCENABLE BIT(1) +#define DPC_BLCENABLE BIT(2) +#define WB_ENABLE BIT(3) +#define CFA_ENABLE BIT(4) +#define CC_ENABLE BIT(5) +#define GAM_ENABLE BIT(6) +#define GAM_BENABLE BIT(7) +#define GAM_GENABLE BIT(8) +#define GAM_RENABLE BIT(9) +#define VHXS_ENABLE BIT(10) +#define CSC_ENABLE BIT(11) +#define CBC_ENABLE BIT(12) +#define SUB422_ENABLE BIT(13) +#define SUB420_ENABLE BIT(14) #define GAM_ENABLES (GAM_RENABLE | GAM_GENABLE | GAM_BENABLE | GAM_ENABLE) @@ -145,7 +146,31 @@ struct isc_ctrls { u32 hist_minmax[HIST_BAYER][2]; }; -#define ISC_PIPE_LINE_NODE_NUM 11 +#define ISC_PIPE_LINE_NODE_NUM 15 + +/* + * struct isc_reg_offsets - ISC device register offsets + * @csc: Offset for the CSC register + * @cbc: Offset for the CBC register + * @sub422: Offset for the SUB422 register + * @sub420: Offset for the SUB420 register + * @rlp: Offset for the RLP register + * @his: Offset for the HIS related registers + * @dma: Offset for the DMA related registers + * @version: Offset for the version register + * @his_entry: Offset for the HIS entries registers + */ +struct isc_reg_offsets { + u32 csc; + u32 cbc; + u32 sub422; + u32 sub420; + u32 rlp; + u32 his; + u32 dma; + u32 version; + u32 his_entry; +}; /* * struct isc_device - ISC device driver data/config struct @@ -153,6 +178,7 @@ struct isc_ctrls { * @hclock: Hclock clock input (refer datasheet) * @ispck: iscpck clock (refer datasheet) * @isc_clks: ISC clocks + * @dcfg: DMA master configuration, architecture dependent * * @dev: Registered device driver * @v4l2_dev: v4l2 registered device @@ -187,12 +213,46 @@ struct isc_ctrls { * * @current_subdev: current subdevice: the sensor * @subdev_entities: list of subdevice entitites + * + * @gamma_table: pointer to the table with gamma values, has + * gamma_max sets of GAMMA_ENTRIES entries each + * @gamma_max: maximum number of sets of inside the gamma_table + * + * @max_width: maximum frame width, dependent on the internal RAM + * @max_height: maximum frame height, dependent on the internal RAM + * + * @config_dpc: pointer to a function that initializes product + * specific DPC module + * @config_csc: pointer to a function that initializes product + * specific CSC module + * @config_cbc: pointer to a function that initializes product + * specific CBC module + * @config_cc: pointer to a function that initializes product + * specific CC module + * @config_gam: pointer to a function that initializes product + * specific GAMMA module + * @config_rlp: pointer to a function that initializes product + * specific RLP module + * @config_ctrls: pointer to a functoin that initializes product + * specific v4l2 controls. + * + * @adapt_pipeline: pointer to a function that adapts the pipeline bits + * to the product specific pipeline + * + * @offsets: struct holding the product specific register offsets + * @controller_formats: pointer to the array of possible formats that the + * controller can output + * @formats_list: pointer to the array of possible formats that can + * be used as an input to the controller + * @controller_formats_size: size of controller_formats array + * @formats_list_size: size of formats_list array */ struct isc_device { struct regmap *regmap; struct clk *hclock; struct clk *ispck; struct isc_clk isc_clks[2]; + u32 dcfg; struct device *dev; struct v4l2_device v4l2_dev; @@ -245,16 +305,36 @@ struct isc_device { struct v4l2_ctrl *gr_off_ctrl; struct v4l2_ctrl *gb_off_ctrl; }; + +#define GAMMA_ENTRIES 64 + /* pointer to the defined gamma table */ + const u32 (*gamma_table)[GAMMA_ENTRIES]; + u32 gamma_max; + + u32 max_width; + u32 max_height; + + struct { + void (*config_dpc)(struct isc_device *isc); + void (*config_csc)(struct isc_device *isc); + void (*config_cbc)(struct isc_device *isc); + void (*config_cc)(struct isc_device *isc); + void (*config_gam)(struct isc_device *isc); + void (*config_rlp)(struct isc_device *isc); + + void (*config_ctrls)(struct isc_device *isc, + const struct v4l2_ctrl_ops *ops); + + void (*adapt_pipeline)(struct isc_device *isc); + }; + + struct isc_reg_offsets offsets; + const struct isc_format *controller_formats; + struct isc_format *formats_list; + u32 controller_formats_size; + u32 formats_list_size; }; -#define GAMMA_MAX 2 -#define GAMMA_ENTRIES 64 - -#define ATMEL_ISC_NAME "atmel-isc" - -extern struct isc_format formats_list[]; -extern const struct isc_format controller_formats[]; -extern const u32 isc_gamma_table[GAMMA_MAX + 1][GAMMA_ENTRIES]; extern const struct regmap_config isc_regmap_config; extern const struct v4l2_async_notifier_operations isc_async_ops; diff --git a/drivers/media/platform/atmel/atmel-isi.c b/drivers/media/platform/atmel/atmel-isi.c index e392b3efe363..095d80c4f59e 100644 --- a/drivers/media/platform/atmel/atmel-isi.c +++ b/drivers/media/platform/atmel/atmel-isi.c @@ -422,7 +422,9 @@ static int start_streaming(struct vb2_queue *vq, unsigned int count) struct frame_buffer *buf, *node; int ret; - pm_runtime_get_sync(isi->dev); + ret = pm_runtime_resume_and_get(isi->dev); + if (ret < 0) + return ret; /* Enable stream on the sub device */ ret = v4l2_subdev_call(isi->entity.subdev, video, s_stream, 1); @@ -555,7 +557,7 @@ static const struct isi_format *find_format_by_fourcc(struct atmel_isi *isi, } static void isi_try_fse(struct atmel_isi *isi, const struct isi_format *isi_fmt, - struct v4l2_subdev_pad_config *pad_cfg) + struct v4l2_subdev_state *sd_state) { int ret; struct v4l2_subdev_frame_size_enum fse = { @@ -564,17 +566,17 @@ static void isi_try_fse(struct atmel_isi *isi, const struct isi_format *isi_fmt, }; ret = v4l2_subdev_call(isi->entity.subdev, pad, enum_frame_size, - pad_cfg, &fse); + sd_state, &fse); /* * Attempt to obtain format size from subdev. If not available, * just use the maximum ISI can receive. */ if (ret) { - pad_cfg->try_crop.width = MAX_SUPPORT_WIDTH; - pad_cfg->try_crop.height = MAX_SUPPORT_HEIGHT; + sd_state->pads->try_crop.width = MAX_SUPPORT_WIDTH; + sd_state->pads->try_crop.height = MAX_SUPPORT_HEIGHT; } else { - pad_cfg->try_crop.width = fse.max_width; - pad_cfg->try_crop.height = fse.max_height; + sd_state->pads->try_crop.width = fse.max_width; + sd_state->pads->try_crop.height = fse.max_height; } } @@ -584,6 +586,9 @@ static int isi_try_fmt(struct atmel_isi *isi, struct v4l2_format *f, const struct isi_format *isi_fmt; struct v4l2_pix_format *pixfmt = &f->fmt.pix; struct v4l2_subdev_pad_config pad_cfg = {}; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -601,10 +606,10 @@ static int isi_try_fmt(struct atmel_isi *isi, struct v4l2_format *f, v4l2_fill_mbus_format(&format.format, pixfmt, isi_fmt->mbus_code); - isi_try_fse(isi, isi_fmt, &pad_cfg); + isi_try_fse(isi, isi_fmt, &pad_state); ret = v4l2_subdev_call(isi->entity.subdev, pad, set_fmt, - &pad_cfg, &format); + &pad_state, &format); if (ret < 0) return ret; @@ -782,9 +787,10 @@ static int isi_enum_frameintervals(struct file *file, void *fh, return 0; } -static void isi_camera_set_bus_param(struct atmel_isi *isi) +static int isi_camera_set_bus_param(struct atmel_isi *isi) { u32 cfg1 = 0; + int ret; /* set bus param for ISI */ if (isi->pdata.hsync_act_low) @@ -801,12 +807,16 @@ static void isi_camera_set_bus_param(struct atmel_isi *isi) cfg1 |= ISI_CFG1_THMASK_BEATS_16; /* Enable PM and peripheral clock before operate isi registers */ - pm_runtime_get_sync(isi->dev); + ret = pm_runtime_resume_and_get(isi->dev); + if (ret < 0) + return ret; isi_writel(isi, ISI_CTRL, ISI_CTRL_DIS); isi_writel(isi, ISI_CFG1, cfg1); pm_runtime_put(isi->dev); + + return 0; } /* -----------------------------------------------------------------------*/ @@ -1085,7 +1095,11 @@ static int isi_graph_notify_complete(struct v4l2_async_notifier *notifier) dev_err(isi->dev, "No supported mediabus format found\n"); return ret; } - isi_camera_set_bus_param(isi); + ret = isi_camera_set_bus_param(isi); + if (ret) { + dev_err(isi->dev, "Can't wake up device\n"); + return ret; + } ret = isi_set_default_fmt(isi); if (ret) { diff --git a/drivers/media/platform/atmel/atmel-sama5d2-isc.c b/drivers/media/platform/atmel/atmel-sama5d2-isc.c index 61d9885765f4..925aa80a139b 100644 --- a/drivers/media/platform/atmel/atmel-sama5d2-isc.c +++ b/drivers/media/platform/atmel/atmel-sama5d2-isc.c @@ -49,10 +49,262 @@ #include "atmel-isc-regs.h" #include "atmel-isc.h" -#define ISC_MAX_SUPPORT_WIDTH 2592 -#define ISC_MAX_SUPPORT_HEIGHT 1944 +#define ISC_SAMA5D2_MAX_SUPPORT_WIDTH 2592 +#define ISC_SAMA5D2_MAX_SUPPORT_HEIGHT 1944 -#define ISC_CLK_MAX_DIV 255 +#define ISC_SAMA5D2_PIPELINE \ + (WB_ENABLE | CFA_ENABLE | CC_ENABLE | GAM_ENABLES | CSC_ENABLE | \ + CBC_ENABLE | SUB422_ENABLE | SUB420_ENABLE) + +/* This is a list of the formats that the ISC can *output* */ +static const struct isc_format sama5d2_controller_formats[] = { + { + .fourcc = V4L2_PIX_FMT_ARGB444, + }, + { + .fourcc = V4L2_PIX_FMT_ARGB555, + }, + { + .fourcc = V4L2_PIX_FMT_RGB565, + }, + { + .fourcc = V4L2_PIX_FMT_ABGR32, + }, + { + .fourcc = V4L2_PIX_FMT_XBGR32, + }, + { + .fourcc = V4L2_PIX_FMT_YUV420, + }, + { + .fourcc = V4L2_PIX_FMT_YUYV, + }, + { + .fourcc = V4L2_PIX_FMT_YUV422P, + }, + { + .fourcc = V4L2_PIX_FMT_GREY, + }, + { + .fourcc = V4L2_PIX_FMT_Y10, + }, +}; + +/* This is a list of formats that the ISC can receive as *input* */ +static struct isc_format sama5d2_formats_list[] = { + { + .fourcc = V4L2_PIX_FMT_SBGGR8, + .mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_BGBG, + }, + { + .fourcc = V4L2_PIX_FMT_SGBRG8, + .mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_GBGB, + }, + { + .fourcc = V4L2_PIX_FMT_SGRBG8, + .mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_GRGR, + }, + { + .fourcc = V4L2_PIX_FMT_SRGGB8, + .mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_SBGGR10, + .mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_SGBRG10, + .mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_GBGB, + }, + { + .fourcc = V4L2_PIX_FMT_SGRBG10, + .mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_GRGR, + }, + { + .fourcc = V4L2_PIX_FMT_SRGGB10, + .mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_SBGGR12, + .mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_BGBG, + }, + { + .fourcc = V4L2_PIX_FMT_SGBRG12, + .mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_GBGB, + }, + { + .fourcc = V4L2_PIX_FMT_SGRBG12, + .mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_GRGR, + }, + { + .fourcc = V4L2_PIX_FMT_SRGGB12, + .mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_GREY, + .mbus_code = MEDIA_BUS_FMT_Y8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + }, + { + .fourcc = V4L2_PIX_FMT_YUYV, + .mbus_code = MEDIA_BUS_FMT_YUYV8_2X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + }, + { + .fourcc = V4L2_PIX_FMT_RGB565, + .mbus_code = MEDIA_BUS_FMT_RGB565_2X8_LE, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + }, + { + .fourcc = V4L2_PIX_FMT_Y10, + .mbus_code = MEDIA_BUS_FMT_Y10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + }, + +}; + +static void isc_sama5d2_config_csc(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + + /* Convert RGB to YUV */ + regmap_write(regmap, ISC_CSC_YR_YG + isc->offsets.csc, + 0x42 | (0x81 << 16)); + regmap_write(regmap, ISC_CSC_YB_OY + isc->offsets.csc, + 0x19 | (0x10 << 16)); + regmap_write(regmap, ISC_CSC_CBR_CBG + isc->offsets.csc, + 0xFDA | (0xFB6 << 16)); + regmap_write(regmap, ISC_CSC_CBB_OCB + isc->offsets.csc, + 0x70 | (0x80 << 16)); + regmap_write(regmap, ISC_CSC_CRR_CRG + isc->offsets.csc, + 0x70 | (0xFA2 << 16)); + regmap_write(regmap, ISC_CSC_CRB_OCR + isc->offsets.csc, + 0xFEE | (0x80 << 16)); +} + +static void isc_sama5d2_config_cbc(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + + regmap_write(regmap, ISC_CBC_BRIGHT + isc->offsets.cbc, + isc->ctrls.brightness); + regmap_write(regmap, ISC_CBC_CONTRAST + isc->offsets.cbc, + isc->ctrls.contrast); +} + +static void isc_sama5d2_config_cc(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + + /* Configure each register at the neutral fixed point 1.0 or 0.0 */ + regmap_write(regmap, ISC_CC_RR_RG, (1 << 8)); + regmap_write(regmap, ISC_CC_RB_OR, 0); + regmap_write(regmap, ISC_CC_GR_GG, (1 << 8) << 16); + regmap_write(regmap, ISC_CC_GB_OG, 0); + regmap_write(regmap, ISC_CC_BR_BG, 0); + regmap_write(regmap, ISC_CC_BB_OB, (1 << 8)); +} + +static void isc_sama5d2_config_ctrls(struct isc_device *isc, + const struct v4l2_ctrl_ops *ops) +{ + struct isc_ctrls *ctrls = &isc->ctrls; + struct v4l2_ctrl_handler *hdl = &ctrls->handler; + + ctrls->contrast = 256; + + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_CONTRAST, -2048, 2047, 1, 256); +} + +static void isc_sama5d2_config_dpc(struct isc_device *isc) +{ + /* This module is not present on sama5d2 pipeline */ +} + +static void isc_sama5d2_config_gam(struct isc_device *isc) +{ + /* No specific gamma configuration */ +} + +static void isc_sama5d2_config_rlp(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + u32 rlp_mode = isc->config.rlp_cfg_mode; + + regmap_update_bits(regmap, ISC_RLP_CFG + isc->offsets.rlp, + ISC_RLP_CFG_MODE_MASK, rlp_mode); +} + +static void isc_sama5d2_adapt_pipeline(struct isc_device *isc) +{ + isc->try_config.bits_pipeline &= ISC_SAMA5D2_PIPELINE; +} + +/* Gamma table with gamma 1/2.2 */ +static const u32 isc_sama5d2_gamma_table[][GAMMA_ENTRIES] = { + /* 0 --> gamma 1/1.8 */ + { 0x65, 0x66002F, 0x950025, 0xBB0020, 0xDB001D, 0xF8001A, + 0x1130018, 0x12B0017, 0x1420016, 0x1580014, 0x16D0013, 0x1810012, + 0x1940012, 0x1A60012, 0x1B80011, 0x1C90010, 0x1DA0010, 0x1EA000F, + 0x1FA000F, 0x209000F, 0x218000F, 0x227000E, 0x235000E, 0x243000E, + 0x251000E, 0x25F000D, 0x26C000D, 0x279000D, 0x286000D, 0x293000C, + 0x2A0000C, 0x2AC000C, 0x2B8000C, 0x2C4000C, 0x2D0000B, 0x2DC000B, + 0x2E7000B, 0x2F3000B, 0x2FE000B, 0x309000B, 0x314000B, 0x31F000A, + 0x32A000A, 0x334000B, 0x33F000A, 0x349000A, 0x354000A, 0x35E000A, + 0x368000A, 0x372000A, 0x37C000A, 0x386000A, 0x3900009, 0x399000A, + 0x3A30009, 0x3AD0009, 0x3B60009, 0x3BF000A, 0x3C90009, 0x3D20009, + 0x3DB0009, 0x3E40009, 0x3ED0009, 0x3F60009 }, + + /* 1 --> gamma 1/2 */ + { 0x7F, 0x800034, 0xB50028, 0xDE0021, 0x100001E, 0x11E001B, + 0x1390019, 0x1520017, 0x16A0015, 0x1800014, 0x1940014, 0x1A80013, + 0x1BB0012, 0x1CD0011, 0x1DF0010, 0x1EF0010, 0x200000F, 0x20F000F, + 0x21F000E, 0x22D000F, 0x23C000E, 0x24A000E, 0x258000D, 0x265000D, + 0x273000C, 0x27F000D, 0x28C000C, 0x299000C, 0x2A5000C, 0x2B1000B, + 0x2BC000C, 0x2C8000B, 0x2D3000C, 0x2DF000B, 0x2EA000A, 0x2F5000A, + 0x2FF000B, 0x30A000A, 0x314000B, 0x31F000A, 0x329000A, 0x333000A, + 0x33D0009, 0x3470009, 0x350000A, 0x35A0009, 0x363000A, 0x36D0009, + 0x3760009, 0x37F0009, 0x3880009, 0x3910009, 0x39A0009, 0x3A30009, + 0x3AC0008, 0x3B40009, 0x3BD0008, 0x3C60008, 0x3CE0008, 0x3D60009, + 0x3DF0008, 0x3E70008, 0x3EF0008, 0x3F70008 }, + + /* 2 --> gamma 1/2.2 */ + { 0x99, 0x9B0038, 0xD4002A, 0xFF0023, 0x122001F, 0x141001B, + 0x15D0019, 0x1760017, 0x18E0015, 0x1A30015, 0x1B80013, 0x1CC0012, + 0x1DE0011, 0x1F00010, 0x2010010, 0x2110010, 0x221000F, 0x230000F, + 0x23F000E, 0x24D000E, 0x25B000D, 0x269000C, 0x276000C, 0x283000C, + 0x28F000C, 0x29B000C, 0x2A7000C, 0x2B3000B, 0x2BF000B, 0x2CA000B, + 0x2D5000B, 0x2E0000A, 0x2EB000A, 0x2F5000A, 0x2FF000A, 0x30A000A, + 0x3140009, 0x31E0009, 0x327000A, 0x3310009, 0x33A0009, 0x3440009, + 0x34D0009, 0x3560009, 0x35F0009, 0x3680008, 0x3710008, 0x3790009, + 0x3820008, 0x38A0008, 0x3930008, 0x39B0008, 0x3A30008, 0x3AB0008, + 0x3B30008, 0x3BB0008, 0x3C30008, 0x3CB0007, 0x3D20008, 0x3DA0007, + 0x3E20007, 0x3E90007, 0x3F00008, 0x3F80007 }, +}; static int isc_parse_dt(struct device *dev, struct isc_device *isc) { @@ -118,6 +370,7 @@ static int atmel_isc_probe(struct platform_device *pdev) struct isc_subdev_entity *subdev_entity; int irq; int ret; + u32 ver; isc = devm_kzalloc(dev, sizeof(*isc), GFP_KERNEL); if (!isc) @@ -143,13 +396,47 @@ static int atmel_isc_probe(struct platform_device *pdev) return irq; ret = devm_request_irq(dev, irq, isc_interrupt, 0, - ATMEL_ISC_NAME, isc); + "atmel-sama5d2-isc", isc); if (ret < 0) { dev_err(dev, "can't register ISR for IRQ %u (ret=%i)\n", irq, ret); return ret; } + isc->gamma_table = isc_sama5d2_gamma_table; + isc->gamma_max = 2; + + isc->max_width = ISC_SAMA5D2_MAX_SUPPORT_WIDTH; + isc->max_height = ISC_SAMA5D2_MAX_SUPPORT_HEIGHT; + + isc->config_dpc = isc_sama5d2_config_dpc; + isc->config_csc = isc_sama5d2_config_csc; + isc->config_cbc = isc_sama5d2_config_cbc; + isc->config_cc = isc_sama5d2_config_cc; + isc->config_gam = isc_sama5d2_config_gam; + isc->config_rlp = isc_sama5d2_config_rlp; + isc->config_ctrls = isc_sama5d2_config_ctrls; + + isc->adapt_pipeline = isc_sama5d2_adapt_pipeline; + + isc->offsets.csc = ISC_SAMA5D2_CSC_OFFSET; + isc->offsets.cbc = ISC_SAMA5D2_CBC_OFFSET; + isc->offsets.sub422 = ISC_SAMA5D2_SUB422_OFFSET; + isc->offsets.sub420 = ISC_SAMA5D2_SUB420_OFFSET; + isc->offsets.rlp = ISC_SAMA5D2_RLP_OFFSET; + isc->offsets.his = ISC_SAMA5D2_HIS_OFFSET; + isc->offsets.dma = ISC_SAMA5D2_DMA_OFFSET; + isc->offsets.version = ISC_SAMA5D2_VERSION_OFFSET; + isc->offsets.his_entry = ISC_SAMA5D2_HIS_ENTRY_OFFSET; + + isc->controller_formats = sama5d2_controller_formats; + isc->controller_formats_size = ARRAY_SIZE(sama5d2_controller_formats); + isc->formats_list = sama5d2_formats_list; + isc->formats_list_size = ARRAY_SIZE(sama5d2_formats_list); + + /* sama5d2-isc - 8 bits per beat */ + isc->dcfg = ISC_DCFG_YMBSIZE_BEATS8 | ISC_DCFG_CMBSIZE_BEATS8; + ret = isc_pipeline_init(isc); if (ret) return ret; @@ -241,6 +528,9 @@ static int atmel_isc_probe(struct platform_device *pdev) pm_runtime_enable(dev); pm_request_idle(dev); + regmap_read(isc->regmap, ISC_VERSION + isc->offsets.version, &ver); + dev_info(dev, "Microchip ISC version %x\n", ver); + return 0; cleanup_subdev: @@ -319,7 +609,7 @@ static struct platform_driver atmel_isc_driver = { .probe = atmel_isc_probe, .remove = atmel_isc_remove, .driver = { - .name = ATMEL_ISC_NAME, + .name = "atmel-sama5d2-isc", .pm = &atmel_isc_dev_pm_ops, .of_match_table = of_match_ptr(atmel_isc_of_match), }, diff --git a/drivers/media/platform/atmel/atmel-sama7g5-isc.c b/drivers/media/platform/atmel/atmel-sama7g5-isc.c new file mode 100644 index 000000000000..f2785131ff56 --- /dev/null +++ b/drivers/media/platform/atmel/atmel-sama7g5-isc.c @@ -0,0 +1,630 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip eXtended Image Sensor Controller (XISC) driver + * + * Copyright (C) 2019-2021 Microchip Technology, Inc. and its subsidiaries + * + * Author: Eugen Hristev + * + * Sensor-->PFE-->DPC-->WB-->CFA-->CC-->GAM-->VHXS-->CSC-->CBHS-->SUB-->RLP-->DMA-->HIS + * + * ISC video pipeline integrates the following submodules: + * PFE: Parallel Front End to sample the camera sensor input stream + * DPC: Defective Pixel Correction with black offset correction, green disparity + * correction and defective pixel correction (3 modules total) + * WB: Programmable white balance in the Bayer domain + * CFA: Color filter array interpolation module + * CC: Programmable color correction + * GAM: Gamma correction + *VHXS: Vertical and Horizontal Scaler + * CSC: Programmable color space conversion + *CBHS: Contrast Brightness Hue and Saturation control + * SUB: This module performs YCbCr444 to YCbCr420 chrominance subsampling + * RLP: This module performs rounding, range limiting + * and packing of the incoming data + * DMA: This module performs DMA master accesses to write frames to external RAM + * HIS: Histogram module performs statistic counters on the frames + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "atmel-isc-regs.h" +#include "atmel-isc.h" + +#define ISC_SAMA7G5_MAX_SUPPORT_WIDTH 3264 +#define ISC_SAMA7G5_MAX_SUPPORT_HEIGHT 2464 + +#define ISC_SAMA7G5_PIPELINE \ + (WB_ENABLE | CFA_ENABLE | CC_ENABLE | GAM_ENABLES | CSC_ENABLE | \ + CBC_ENABLE | SUB422_ENABLE | SUB420_ENABLE) + +/* This is a list of the formats that the ISC can *output* */ +static const struct isc_format sama7g5_controller_formats[] = { + { + .fourcc = V4L2_PIX_FMT_ARGB444, + }, + { + .fourcc = V4L2_PIX_FMT_ARGB555, + }, + { + .fourcc = V4L2_PIX_FMT_RGB565, + }, + { + .fourcc = V4L2_PIX_FMT_ABGR32, + }, + { + .fourcc = V4L2_PIX_FMT_XBGR32, + }, + { + .fourcc = V4L2_PIX_FMT_YUV420, + }, + { + .fourcc = V4L2_PIX_FMT_UYVY, + }, + { + .fourcc = V4L2_PIX_FMT_VYUY, + }, + { + .fourcc = V4L2_PIX_FMT_YUYV, + }, + { + .fourcc = V4L2_PIX_FMT_YUV422P, + }, + { + .fourcc = V4L2_PIX_FMT_GREY, + }, + { + .fourcc = V4L2_PIX_FMT_Y10, + }, + { + .fourcc = V4L2_PIX_FMT_Y16, + }, +}; + +/* This is a list of formats that the ISC can receive as *input* */ +static struct isc_format sama7g5_formats_list[] = { + { + .fourcc = V4L2_PIX_FMT_SBGGR8, + .mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_BGBG, + }, + { + .fourcc = V4L2_PIX_FMT_SGBRG8, + .mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_GBGB, + }, + { + .fourcc = V4L2_PIX_FMT_SGRBG8, + .mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_GRGR, + }, + { + .fourcc = V4L2_PIX_FMT_SRGGB8, + .mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_SBGGR10, + .mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_SGBRG10, + .mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_GBGB, + }, + { + .fourcc = V4L2_PIX_FMT_SGRBG10, + .mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_GRGR, + }, + { + .fourcc = V4L2_PIX_FMT_SRGGB10, + .mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_SBGGR12, + .mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_BGBG, + }, + { + .fourcc = V4L2_PIX_FMT_SGBRG12, + .mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_GBGB, + }, + { + .fourcc = V4L2_PIX_FMT_SGRBG12, + .mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_GRGR, + }, + { + .fourcc = V4L2_PIX_FMT_SRGGB12, + .mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TWELVE, + .cfa_baycfg = ISC_BAY_CFG_RGRG, + }, + { + .fourcc = V4L2_PIX_FMT_GREY, + .mbus_code = MEDIA_BUS_FMT_Y8_1X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + }, + { + .fourcc = V4L2_PIX_FMT_YUYV, + .mbus_code = MEDIA_BUS_FMT_YUYV8_2X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + }, + { + .fourcc = V4L2_PIX_FMT_UYVY, + .mbus_code = MEDIA_BUS_FMT_YUYV8_2X8, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + }, + { + .fourcc = V4L2_PIX_FMT_RGB565, + .mbus_code = MEDIA_BUS_FMT_RGB565_2X8_LE, + .pfe_cfg0_bps = ISC_PFE_CFG0_BPS_EIGHT, + }, + { + .fourcc = V4L2_PIX_FMT_Y10, + .mbus_code = MEDIA_BUS_FMT_Y10_1X10, + .pfe_cfg0_bps = ISC_PFG_CFG0_BPS_TEN, + }, + +}; + +static void isc_sama7g5_config_csc(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + + /* Convert RGB to YUV */ + regmap_write(regmap, ISC_CSC_YR_YG + isc->offsets.csc, + 0x42 | (0x81 << 16)); + regmap_write(regmap, ISC_CSC_YB_OY + isc->offsets.csc, + 0x19 | (0x10 << 16)); + regmap_write(regmap, ISC_CSC_CBR_CBG + isc->offsets.csc, + 0xFDA | (0xFB6 << 16)); + regmap_write(regmap, ISC_CSC_CBB_OCB + isc->offsets.csc, + 0x70 | (0x80 << 16)); + regmap_write(regmap, ISC_CSC_CRR_CRG + isc->offsets.csc, + 0x70 | (0xFA2 << 16)); + regmap_write(regmap, ISC_CSC_CRB_OCR + isc->offsets.csc, + 0xFEE | (0x80 << 16)); +} + +static void isc_sama7g5_config_cbc(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + + /* Configure what is set via v4l2 ctrls */ + regmap_write(regmap, ISC_CBC_BRIGHT + isc->offsets.cbc, isc->ctrls.brightness); + regmap_write(regmap, ISC_CBC_CONTRAST + isc->offsets.cbc, isc->ctrls.contrast); + /* Configure Hue and Saturation as neutral midpoint */ + regmap_write(regmap, ISC_CBCHS_HUE, 0); + regmap_write(regmap, ISC_CBCHS_SAT, (1 << 4)); +} + +static void isc_sama7g5_config_cc(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + + /* Configure each register at the neutral fixed point 1.0 or 0.0 */ + regmap_write(regmap, ISC_CC_RR_RG, (1 << 8)); + regmap_write(regmap, ISC_CC_RB_OR, 0); + regmap_write(regmap, ISC_CC_GR_GG, (1 << 8) << 16); + regmap_write(regmap, ISC_CC_GB_OG, 0); + regmap_write(regmap, ISC_CC_BR_BG, 0); + regmap_write(regmap, ISC_CC_BB_OB, (1 << 8)); +} + +static void isc_sama7g5_config_ctrls(struct isc_device *isc, + const struct v4l2_ctrl_ops *ops) +{ + struct isc_ctrls *ctrls = &isc->ctrls; + struct v4l2_ctrl_handler *hdl = &ctrls->handler; + + ctrls->contrast = 16; + + v4l2_ctrl_new_std(hdl, ops, V4L2_CID_CONTRAST, -2048, 2047, 1, 16); +} + +static void isc_sama7g5_config_dpc(struct isc_device *isc) +{ + u32 bay_cfg = isc->config.sd_format->cfa_baycfg; + struct regmap *regmap = isc->regmap; + + regmap_update_bits(regmap, ISC_DPC_CFG, ISC_DPC_CFG_BLOFF_MASK, + (64 << ISC_DPC_CFG_BLOFF_SHIFT)); + regmap_update_bits(regmap, ISC_DPC_CFG, ISC_DPC_CFG_BAYCFG_MASK, + (bay_cfg << ISC_DPC_CFG_BAYCFG_SHIFT)); +} + +static void isc_sama7g5_config_gam(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + + regmap_update_bits(regmap, ISC_GAM_CTRL, ISC_GAM_CTRL_BIPART, + ISC_GAM_CTRL_BIPART); +} + +static void isc_sama7g5_config_rlp(struct isc_device *isc) +{ + struct regmap *regmap = isc->regmap; + u32 rlp_mode = isc->config.rlp_cfg_mode; + + regmap_update_bits(regmap, ISC_RLP_CFG + isc->offsets.rlp, + ISC_RLP_CFG_MODE_MASK | ISC_RLP_CFG_LSH | + ISC_RLP_CFG_YMODE_MASK, rlp_mode); +} + +static void isc_sama7g5_adapt_pipeline(struct isc_device *isc) +{ + isc->try_config.bits_pipeline &= ISC_SAMA7G5_PIPELINE; +} + +/* Gamma table with gamma 1/2.2 */ +static const u32 isc_sama7g5_gamma_table[][GAMMA_ENTRIES] = { + /* index 0 --> gamma bipartite */ + { + 0x980, 0x4c0320, 0x650260, 0x7801e0, 0x8701a0, 0x940180, + 0xa00160, 0xab0120, 0xb40120, 0xbd0120, 0xc60100, 0xce0100, + 0xd600e0, 0xdd00e0, 0xe400e0, 0xeb00c0, 0xf100c0, 0xf700c0, + 0xfd00c0, 0x10300a0, 0x10800c0, 0x10e00a0, 0x11300a0, 0x11800a0, + 0x11d00a0, 0x12200a0, 0x12700a0, 0x12c0080, 0x13000a0, 0x1350080, + 0x13900a0, 0x13e0080, 0x1420076, 0x17d0062, 0x1ae0054, 0x1d8004a, + 0x1fd0044, 0x21f003e, 0x23e003a, 0x25b0036, 0x2760032, 0x28f0030, + 0x2a7002e, 0x2be002c, 0x2d4002c, 0x2ea0028, 0x2fe0028, 0x3120026, + 0x3250024, 0x3370024, 0x3490022, 0x35a0022, 0x36b0020, 0x37b0020, + 0x38b0020, 0x39b001e, 0x3aa001e, 0x3b9001c, 0x3c7001c, 0x3d5001c, + 0x3e3001c, 0x3f1001c, 0x3ff001a, 0x40c001a }, +}; + +static int xisc_parse_dt(struct device *dev, struct isc_device *isc) +{ + struct device_node *np = dev->of_node; + struct device_node *epn = NULL; + struct isc_subdev_entity *subdev_entity; + unsigned int flags; + int ret; + bool mipi_mode; + + INIT_LIST_HEAD(&isc->subdev_entities); + + mipi_mode = of_property_read_bool(np, "microchip,mipi-mode"); + + while (1) { + struct v4l2_fwnode_endpoint v4l2_epn = { .bus_type = 0 }; + + epn = of_graph_get_next_endpoint(np, epn); + if (!epn) + return 0; + + ret = v4l2_fwnode_endpoint_parse(of_fwnode_handle(epn), + &v4l2_epn); + if (ret) { + ret = -EINVAL; + dev_err(dev, "Could not parse the endpoint\n"); + break; + } + + subdev_entity = devm_kzalloc(dev, sizeof(*subdev_entity), + GFP_KERNEL); + if (!subdev_entity) { + ret = -ENOMEM; + break; + } + subdev_entity->epn = epn; + + flags = v4l2_epn.bus.parallel.flags; + + if (flags & V4L2_MBUS_HSYNC_ACTIVE_LOW) + subdev_entity->pfe_cfg0 = ISC_PFE_CFG0_HPOL_LOW; + + if (flags & V4L2_MBUS_VSYNC_ACTIVE_LOW) + subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_VPOL_LOW; + + if (flags & V4L2_MBUS_PCLK_SAMPLE_FALLING) + subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_PPOL_LOW; + + if (v4l2_epn.bus_type == V4L2_MBUS_BT656) + subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_CCIR_CRC | + ISC_PFE_CFG0_CCIR656; + + if (mipi_mode) + subdev_entity->pfe_cfg0 |= ISC_PFE_CFG0_MIPI; + + list_add_tail(&subdev_entity->list, &isc->subdev_entities); + } + of_node_put(epn); + + return ret; +} + +static int microchip_xisc_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct isc_device *isc; + struct resource *res; + void __iomem *io_base; + struct isc_subdev_entity *subdev_entity; + int irq; + int ret; + u32 ver; + + isc = devm_kzalloc(dev, sizeof(*isc), GFP_KERNEL); + if (!isc) + return -ENOMEM; + + platform_set_drvdata(pdev, isc); + isc->dev = dev; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + io_base = devm_ioremap_resource(dev, res); + if (IS_ERR(io_base)) + return PTR_ERR(io_base); + + isc->regmap = devm_regmap_init_mmio(dev, io_base, &isc_regmap_config); + if (IS_ERR(isc->regmap)) { + ret = PTR_ERR(isc->regmap); + dev_err(dev, "failed to init register map: %d\n", ret); + return ret; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, isc_interrupt, 0, + "microchip-sama7g5-xisc", isc); + if (ret < 0) { + dev_err(dev, "can't register ISR for IRQ %u (ret=%i)\n", + irq, ret); + return ret; + } + + isc->gamma_table = isc_sama7g5_gamma_table; + isc->gamma_max = 0; + + isc->max_width = ISC_SAMA7G5_MAX_SUPPORT_WIDTH; + isc->max_height = ISC_SAMA7G5_MAX_SUPPORT_HEIGHT; + + isc->config_dpc = isc_sama7g5_config_dpc; + isc->config_csc = isc_sama7g5_config_csc; + isc->config_cbc = isc_sama7g5_config_cbc; + isc->config_cc = isc_sama7g5_config_cc; + isc->config_gam = isc_sama7g5_config_gam; + isc->config_rlp = isc_sama7g5_config_rlp; + isc->config_ctrls = isc_sama7g5_config_ctrls; + + isc->adapt_pipeline = isc_sama7g5_adapt_pipeline; + + isc->offsets.csc = ISC_SAMA7G5_CSC_OFFSET; + isc->offsets.cbc = ISC_SAMA7G5_CBC_OFFSET; + isc->offsets.sub422 = ISC_SAMA7G5_SUB422_OFFSET; + isc->offsets.sub420 = ISC_SAMA7G5_SUB420_OFFSET; + isc->offsets.rlp = ISC_SAMA7G5_RLP_OFFSET; + isc->offsets.his = ISC_SAMA7G5_HIS_OFFSET; + isc->offsets.dma = ISC_SAMA7G5_DMA_OFFSET; + isc->offsets.version = ISC_SAMA7G5_VERSION_OFFSET; + isc->offsets.his_entry = ISC_SAMA7G5_HIS_ENTRY_OFFSET; + + isc->controller_formats = sama7g5_controller_formats; + isc->controller_formats_size = ARRAY_SIZE(sama7g5_controller_formats); + isc->formats_list = sama7g5_formats_list; + isc->formats_list_size = ARRAY_SIZE(sama7g5_formats_list); + + /* sama7g5-isc RAM access port is full AXI4 - 32 bits per beat */ + isc->dcfg = ISC_DCFG_YMBSIZE_BEATS32 | ISC_DCFG_CMBSIZE_BEATS32; + + ret = isc_pipeline_init(isc); + if (ret) + return ret; + + isc->hclock = devm_clk_get(dev, "hclock"); + if (IS_ERR(isc->hclock)) { + ret = PTR_ERR(isc->hclock); + dev_err(dev, "failed to get hclock: %d\n", ret); + return ret; + } + + ret = clk_prepare_enable(isc->hclock); + if (ret) { + dev_err(dev, "failed to enable hclock: %d\n", ret); + return ret; + } + + ret = isc_clk_init(isc); + if (ret) { + dev_err(dev, "failed to init isc clock: %d\n", ret); + goto unprepare_hclk; + } + + isc->ispck = isc->isc_clks[ISC_ISPCK].clk; + + ret = clk_prepare_enable(isc->ispck); + if (ret) { + dev_err(dev, "failed to enable ispck: %d\n", ret); + goto unprepare_hclk; + } + + /* ispck should be greater or equal to hclock */ + ret = clk_set_rate(isc->ispck, clk_get_rate(isc->hclock)); + if (ret) { + dev_err(dev, "failed to set ispck rate: %d\n", ret); + goto unprepare_clk; + } + + ret = v4l2_device_register(dev, &isc->v4l2_dev); + if (ret) { + dev_err(dev, "unable to register v4l2 device.\n"); + goto unprepare_clk; + } + + ret = xisc_parse_dt(dev, isc); + if (ret) { + dev_err(dev, "fail to parse device tree\n"); + goto unregister_v4l2_device; + } + + if (list_empty(&isc->subdev_entities)) { + dev_err(dev, "no subdev found\n"); + ret = -ENODEV; + goto unregister_v4l2_device; + } + + list_for_each_entry(subdev_entity, &isc->subdev_entities, list) { + struct v4l2_async_subdev *asd; + + v4l2_async_notifier_init(&subdev_entity->notifier); + + asd = v4l2_async_notifier_add_fwnode_remote_subdev( + &subdev_entity->notifier, + of_fwnode_handle(subdev_entity->epn), + struct v4l2_async_subdev); + + of_node_put(subdev_entity->epn); + subdev_entity->epn = NULL; + + if (IS_ERR(asd)) { + ret = PTR_ERR(asd); + goto cleanup_subdev; + } + + subdev_entity->notifier.ops = &isc_async_ops; + + ret = v4l2_async_notifier_register(&isc->v4l2_dev, + &subdev_entity->notifier); + if (ret) { + dev_err(dev, "fail to register async notifier\n"); + goto cleanup_subdev; + } + + if (video_is_registered(&isc->video_dev)) + break; + } + + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + pm_request_idle(dev); + + regmap_read(isc->regmap, ISC_VERSION + isc->offsets.version, &ver); + dev_info(dev, "Microchip XISC version %x\n", ver); + + return 0; + +cleanup_subdev: + isc_subdev_cleanup(isc); + +unregister_v4l2_device: + v4l2_device_unregister(&isc->v4l2_dev); + +unprepare_clk: + clk_disable_unprepare(isc->ispck); +unprepare_hclk: + clk_disable_unprepare(isc->hclock); + + isc_clk_cleanup(isc); + + return ret; +} + +static int microchip_xisc_remove(struct platform_device *pdev) +{ + struct isc_device *isc = platform_get_drvdata(pdev); + + pm_runtime_disable(&pdev->dev); + + isc_subdev_cleanup(isc); + + v4l2_device_unregister(&isc->v4l2_dev); + + clk_disable_unprepare(isc->ispck); + clk_disable_unprepare(isc->hclock); + + isc_clk_cleanup(isc); + + return 0; +} + +static int __maybe_unused xisc_runtime_suspend(struct device *dev) +{ + struct isc_device *isc = dev_get_drvdata(dev); + + clk_disable_unprepare(isc->ispck); + clk_disable_unprepare(isc->hclock); + + return 0; +} + +static int __maybe_unused xisc_runtime_resume(struct device *dev) +{ + struct isc_device *isc = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(isc->hclock); + if (ret) + return ret; + + ret = clk_prepare_enable(isc->ispck); + if (ret) + clk_disable_unprepare(isc->hclock); + + return ret; +} + +static const struct dev_pm_ops microchip_xisc_dev_pm_ops = { + SET_RUNTIME_PM_OPS(xisc_runtime_suspend, xisc_runtime_resume, NULL) +}; + +static const struct of_device_id microchip_xisc_of_match[] = { + { .compatible = "microchip,sama7g5-isc" }, + { } +}; +MODULE_DEVICE_TABLE(of, microchip_xisc_of_match); + +static struct platform_driver microchip_xisc_driver = { + .probe = microchip_xisc_probe, + .remove = microchip_xisc_remove, + .driver = { + .name = "microchip-sama7g5-xisc", + .pm = µchip_xisc_dev_pm_ops, + .of_match_table = of_match_ptr(microchip_xisc_of_match), + }, +}; + +module_platform_driver(microchip_xisc_driver); + +MODULE_AUTHOR("Eugen Hristev "); +MODULE_DESCRIPTION("The V4L2 driver for Microchip-XISC"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/media/platform/cadence/cdns-csi2rx.c b/drivers/media/platform/cadence/cdns-csi2rx.c index c68a3eac62cd..f2b4ddd31177 100644 --- a/drivers/media/platform/cadence/cdns-csi2rx.c +++ b/drivers/media/platform/cadence/cdns-csi2rx.c @@ -282,6 +282,7 @@ static int csi2rx_get_resources(struct csi2rx_priv *csi2rx, struct resource *res; unsigned char i; u32 dev_cfg; + int ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); csi2rx->base = devm_ioremap_resource(&pdev->dev, res); @@ -315,7 +316,12 @@ static int csi2rx_get_resources(struct csi2rx_priv *csi2rx, return -EINVAL; } - clk_prepare_enable(csi2rx->p_clk); + ret = clk_prepare_enable(csi2rx->p_clk); + if (ret) { + dev_err(&pdev->dev, "Couldn't prepare and enable P clock\n"); + return ret; + } + dev_cfg = readl(csi2rx->base + CSI2RX_DEVICE_CFG_REG); clk_disable_unprepare(csi2rx->p_clk); diff --git a/drivers/media/platform/cadence/cdns-csi2tx.c b/drivers/media/platform/cadence/cdns-csi2tx.c index e4d08acfbb49..5a67fba73ddd 100644 --- a/drivers/media/platform/cadence/cdns-csi2tx.c +++ b/drivers/media/platform/cadence/cdns-csi2tx.c @@ -156,7 +156,7 @@ static const struct csi2tx_fmt *csi2tx_get_fmt_from_mbus(u32 mbus) } static int csi2tx_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad || code->index >= ARRAY_SIZE(csi2tx_formats)) @@ -169,20 +169,20 @@ static int csi2tx_enum_mbus_code(struct v4l2_subdev *subdev, static struct v4l2_mbus_framefmt * __csi2tx_get_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct csi2tx_priv *csi2tx = v4l2_subdev_to_csi2tx(subdev); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(subdev, cfg, + return v4l2_subdev_get_try_format(subdev, sd_state, fmt->pad); return &csi2tx->pad_fmts[fmt->pad]; } static int csi2tx_get_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { const struct v4l2_mbus_framefmt *format; @@ -191,7 +191,7 @@ static int csi2tx_get_pad_format(struct v4l2_subdev *subdev, if (fmt->pad == CSI2TX_PAD_SOURCE) return -EINVAL; - format = __csi2tx_get_pad_format(subdev, cfg, fmt); + format = __csi2tx_get_pad_format(subdev, sd_state, fmt); if (!format) return -EINVAL; @@ -201,7 +201,7 @@ static int csi2tx_get_pad_format(struct v4l2_subdev *subdev, } static int csi2tx_set_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { const struct v4l2_mbus_framefmt *src_format = &fmt->format; @@ -214,7 +214,7 @@ static int csi2tx_set_pad_format(struct v4l2_subdev *subdev, if (!csi2tx_get_fmt_from_mbus(fmt->format.code)) src_format = &fmt_default; - dst_format = __csi2tx_get_pad_format(subdev, cfg, fmt); + dst_format = __csi2tx_get_pad_format(subdev, sd_state, fmt); if (!dst_format) return -EINVAL; @@ -436,6 +436,7 @@ static int csi2tx_get_resources(struct csi2tx_priv *csi2tx, struct resource *res; unsigned int i; u32 dev_cfg; + int ret; res = platform_get_resource(pdev, IORESOURCE_MEM, 0); csi2tx->base = devm_ioremap_resource(&pdev->dev, res); @@ -454,7 +455,12 @@ static int csi2tx_get_resources(struct csi2tx_priv *csi2tx, return PTR_ERR(csi2tx->esc_clk); } - clk_prepare_enable(csi2tx->p_clk); + ret = clk_prepare_enable(csi2tx->p_clk); + if (ret) { + dev_err(&pdev->dev, "Couldn't prepare and enable p_clk\n"); + return ret; + } + dev_cfg = readl(csi2tx->base + CSI2TX_DEVICE_CONFIG_REG); clk_disable_unprepare(csi2tx->p_clk); diff --git a/drivers/media/platform/coda/coda-common.c b/drivers/media/platform/coda/coda-common.c index bd666c858fa1..0e312b0842d7 100644 --- a/drivers/media/platform/coda/coda-common.c +++ b/drivers/media/platform/coda/coda-common.c @@ -1935,7 +1935,7 @@ int coda_alloc_aux_buf(struct coda_dev *dev, struct coda_aux_buf *buf, if (name && parent) { buf->blob.data = buf->vaddr; buf->blob.size = size; - buf->dentry = debugfs_create_blob(name, 0644, parent, + buf->dentry = debugfs_create_blob(name, 0444, parent, &buf->blob); } @@ -2660,7 +2660,7 @@ static int coda_open(struct file *file) ctx->use_vdoa = false; /* Power up and upload firmware if necessary */ - ret = pm_runtime_get_sync(dev->dev); + ret = pm_runtime_resume_and_get(dev->dev); if (ret < 0) { v4l2_err(&dev->v4l2_dev, "failed to power up: %d\n", ret); goto err_pm_get; @@ -2668,7 +2668,7 @@ static int coda_open(struct file *file) ret = clk_prepare_enable(dev->clk_per); if (ret) - goto err_pm_get; + goto err_clk_enable; ret = clk_prepare_enable(dev->clk_ahb); if (ret) @@ -2707,8 +2707,9 @@ static int coda_open(struct file *file) clk_disable_unprepare(dev->clk_ahb); err_clk_ahb: clk_disable_unprepare(dev->clk_per); -err_pm_get: +err_clk_enable: pm_runtime_put_sync(dev->dev); +err_pm_get: v4l2_fh_del(&ctx->fh); v4l2_fh_exit(&ctx->fh); err_coda_name_init: @@ -3232,7 +3233,7 @@ static int coda_probe(struct platform_device *pdev) memset(dev->iram.vaddr, 0, dev->iram.size); dev->iram.blob.data = dev->iram.vaddr; dev->iram.blob.size = dev->iram.size; - dev->iram.dentry = debugfs_create_blob("iram", 0644, + dev->iram.dentry = debugfs_create_blob("iram", 0444, dev->debugfs_root, &dev->iram.blob); } diff --git a/drivers/media/platform/davinci/vpbe_display.c b/drivers/media/platform/davinci/vpbe_display.c index d19bad997f30..bf3c3e76b921 100644 --- a/drivers/media/platform/davinci/vpbe_display.c +++ b/drivers/media/platform/davinci/vpbe_display.c @@ -47,7 +47,7 @@ static int venc_is_second_field(struct vpbe_display *disp_dev) ret = v4l2_subdev_call(vpbe_dev->venc, core, - ioctl, + command, VENC_GET_FLD, &val); if (ret < 0) { diff --git a/drivers/media/platform/davinci/vpbe_venc.c b/drivers/media/platform/davinci/vpbe_venc.c index 8caa084e5704..bde241c26d79 100644 --- a/drivers/media/platform/davinci/vpbe_venc.c +++ b/drivers/media/platform/davinci/vpbe_venc.c @@ -521,9 +521,7 @@ static int venc_s_routing(struct v4l2_subdev *sd, u32 input, u32 output, return ret; } -static long venc_ioctl(struct v4l2_subdev *sd, - unsigned int cmd, - void *arg) +static long venc_command(struct v4l2_subdev *sd, unsigned int cmd, void *arg) { u32 val; @@ -542,7 +540,7 @@ static long venc_ioctl(struct v4l2_subdev *sd, } static const struct v4l2_subdev_core_ops venc_core_ops = { - .ioctl = venc_ioctl, + .command = venc_command, }; static const struct v4l2_subdev_video_ops venc_video_ops = { diff --git a/drivers/media/platform/davinci/vpif_capture.c b/drivers/media/platform/davinci/vpif_capture.c index 8d2e165bf7de..c034e25dd9aa 100644 --- a/drivers/media/platform/davinci/vpif_capture.c +++ b/drivers/media/platform/davinci/vpif_capture.c @@ -99,7 +99,7 @@ static int vpif_buffer_prepare(struct vb2_buffer *vb) * vpif_buffer_queue_setup : Callback function for buffer setup. * @vq: vb2_queue ptr * @nbuffers: ptr to number of buffers requested by application - * @nplanes:: contains number of distinct video planes needed to hold a frame + * @nplanes: contains number of distinct video planes needed to hold a frame * @sizes: contains the size (in bytes) of each plane. * @alloc_devs: ptr to allocation context * diff --git a/drivers/media/platform/davinci/vpif_display.c b/drivers/media/platform/davinci/vpif_display.c index e5f61d9b221d..59f6b782e104 100644 --- a/drivers/media/platform/davinci/vpif_display.c +++ b/drivers/media/platform/davinci/vpif_display.c @@ -101,7 +101,7 @@ static int vpif_buffer_prepare(struct vb2_buffer *vb) * vpif_buffer_queue_setup : Callback function for buffer setup. * @vq: vb2_queue ptr * @nbuffers: ptr to number of buffers requested by application - * @nplanes:: contains number of distinct video planes needed to hold a frame + * @nplanes: contains number of distinct video planes needed to hold a frame * @sizes: contains the size (in bytes) of each plane. * @alloc_devs: ptr to allocation context * diff --git a/drivers/media/platform/exynos-gsc/gsc-core.c b/drivers/media/platform/exynos-gsc/gsc-core.c index 9f41c2e7097a..f49f3322f835 100644 --- a/drivers/media/platform/exynos-gsc/gsc-core.c +++ b/drivers/media/platform/exynos-gsc/gsc-core.c @@ -1210,18 +1210,19 @@ static int gsc_remove(struct platform_device *pdev) struct gsc_dev *gsc = platform_get_drvdata(pdev); int i; - pm_runtime_get_sync(&pdev->dev); - gsc_unregister_m2m_device(gsc); v4l2_device_unregister(&gsc->v4l2_dev); vb2_dma_contig_clear_max_seg_size(&pdev->dev); - for (i = 0; i < gsc->num_clocks; i++) - clk_disable_unprepare(gsc->clock[i]); - pm_runtime_put_noidle(&pdev->dev); pm_runtime_disable(&pdev->dev); + if (!pm_runtime_status_suspended(&pdev->dev)) + for (i = 0; i < gsc->num_clocks; i++) + clk_disable_unprepare(gsc->clock[i]); + + pm_runtime_set_suspended(&pdev->dev); + dev_dbg(&pdev->dev, "%s driver unloaded\n", pdev->name); return 0; } diff --git a/drivers/media/platform/exynos-gsc/gsc-m2m.c b/drivers/media/platform/exynos-gsc/gsc-m2m.c index 27a3c92c73bc..f1cf847d1cc2 100644 --- a/drivers/media/platform/exynos-gsc/gsc-m2m.c +++ b/drivers/media/platform/exynos-gsc/gsc-m2m.c @@ -56,10 +56,8 @@ static void __gsc_m2m_job_abort(struct gsc_ctx *ctx) static int gsc_m2m_start_streaming(struct vb2_queue *q, unsigned int count) { struct gsc_ctx *ctx = q->drv_priv; - int ret; - ret = pm_runtime_get_sync(&ctx->gsc_dev->pdev->dev); - return ret > 0 ? 0 : ret; + return pm_runtime_resume_and_get(&ctx->gsc_dev->pdev->dev); } static void __gsc_m2m_cleanup_queue(struct gsc_ctx *ctx) diff --git a/drivers/media/platform/exynos4-is/fimc-capture.c b/drivers/media/platform/exynos4-is/fimc-capture.c index 13c838d3f947..7ff4024003f4 100644 --- a/drivers/media/platform/exynos4-is/fimc-capture.c +++ b/drivers/media/platform/exynos4-is/fimc-capture.c @@ -478,11 +478,9 @@ static int fimc_capture_open(struct file *file) goto unlock; set_bit(ST_CAPT_BUSY, &fimc->state); - ret = pm_runtime_get_sync(&fimc->pdev->dev); - if (ret < 0) { - pm_runtime_put_sync(&fimc->pdev->dev); + ret = pm_runtime_resume_and_get(&fimc->pdev->dev); + if (ret < 0) goto unlock; - } ret = v4l2_fh_open(file); if (ret) { @@ -1456,7 +1454,7 @@ void fimc_sensor_notify(struct v4l2_subdev *sd, unsigned int notification, } static int fimc_subdev_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct fimc_fmt *fmt; @@ -1469,7 +1467,7 @@ static int fimc_subdev_enum_mbus_code(struct v4l2_subdev *sd, } static int fimc_subdev_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct fimc_dev *fimc = v4l2_get_subdevdata(sd); @@ -1478,7 +1476,7 @@ static int fimc_subdev_get_fmt(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *mf; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); fmt->format = *mf; return 0; } @@ -1510,7 +1508,7 @@ static int fimc_subdev_get_fmt(struct v4l2_subdev *sd, } static int fimc_subdev_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct fimc_dev *fimc = v4l2_get_subdevdata(sd); @@ -1533,7 +1531,7 @@ static int fimc_subdev_set_fmt(struct v4l2_subdev *sd, mf->colorspace = V4L2_COLORSPACE_JPEG; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *mf = fmt->format; return 0; } @@ -1576,7 +1574,7 @@ static int fimc_subdev_set_fmt(struct v4l2_subdev *sd, } static int fimc_subdev_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct fimc_dev *fimc = v4l2_get_subdevdata(sd); @@ -1603,10 +1601,10 @@ static int fimc_subdev_get_selection(struct v4l2_subdev *sd, return 0; case V4L2_SEL_TGT_CROP: - try_sel = v4l2_subdev_get_try_crop(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_crop(sd, sd_state, sel->pad); break; case V4L2_SEL_TGT_COMPOSE: - try_sel = v4l2_subdev_get_try_compose(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_compose(sd, sd_state, sel->pad); f = &ctx->d_frame; break; default: @@ -1632,7 +1630,7 @@ static int fimc_subdev_get_selection(struct v4l2_subdev *sd, } static int fimc_subdev_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct fimc_dev *fimc = v4l2_get_subdevdata(sd); @@ -1650,10 +1648,10 @@ static int fimc_subdev_set_selection(struct v4l2_subdev *sd, switch (sel->target) { case V4L2_SEL_TGT_CROP: - try_sel = v4l2_subdev_get_try_crop(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_crop(sd, sd_state, sel->pad); break; case V4L2_SEL_TGT_COMPOSE: - try_sel = v4l2_subdev_get_try_compose(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_compose(sd, sd_state, sel->pad); f = &ctx->d_frame; break; default: diff --git a/drivers/media/platform/exynos4-is/fimc-is.c b/drivers/media/platform/exynos4-is/fimc-is.c index 972d9601d236..1b24f5bfc4af 100644 --- a/drivers/media/platform/exynos4-is/fimc-is.c +++ b/drivers/media/platform/exynos4-is/fimc-is.c @@ -828,9 +828,9 @@ static int fimc_is_probe(struct platform_device *pdev) goto err_irq; } - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) - goto err_pm; + goto err_irq; vb2_dma_contig_set_max_seg_size(dev, DMA_BIT_MASK(32)); diff --git a/drivers/media/platform/exynos4-is/fimc-isp-video.c b/drivers/media/platform/exynos4-is/fimc-isp-video.c index 612b9872afc8..83688a7982f7 100644 --- a/drivers/media/platform/exynos4-is/fimc-isp-video.c +++ b/drivers/media/platform/exynos4-is/fimc-isp-video.c @@ -275,7 +275,7 @@ static int isp_video_open(struct file *file) if (ret < 0) goto unlock; - ret = pm_runtime_get_sync(&isp->pdev->dev); + ret = pm_runtime_resume_and_get(&isp->pdev->dev); if (ret < 0) goto rel_fh; @@ -293,7 +293,6 @@ static int isp_video_open(struct file *file) if (!ret) goto unlock; rel_fh: - pm_runtime_put_noidle(&isp->pdev->dev); v4l2_fh_release(file); unlock: mutex_unlock(&isp->video_lock); @@ -306,17 +305,20 @@ static int isp_video_release(struct file *file) struct fimc_is_video *ivc = &isp->video_capture; struct media_entity *entity = &ivc->ve.vdev.entity; struct media_device *mdev = entity->graph_obj.mdev; + bool is_singular_file; mutex_lock(&isp->video_lock); - if (v4l2_fh_is_singular_file(file) && ivc->streaming) { + is_singular_file = v4l2_fh_is_singular_file(file); + + if (is_singular_file && ivc->streaming) { media_pipeline_stop(entity); ivc->streaming = 0; } _vb2_fop_release(file, NULL); - if (v4l2_fh_is_singular_file(file)) { + if (is_singular_file) { fimc_pipeline_call(&ivc->ve, close); mutex_lock(&mdev->graph_mutex); diff --git a/drivers/media/platform/exynos4-is/fimc-isp.c b/drivers/media/platform/exynos4-is/fimc-isp.c index a77c49b18511..855235bea46d 100644 --- a/drivers/media/platform/exynos4-is/fimc-isp.c +++ b/drivers/media/platform/exynos4-is/fimc-isp.c @@ -106,7 +106,7 @@ static const struct media_entity_operations fimc_is_subdev_media_ops = { }; static int fimc_is_subdev_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { const struct fimc_fmt *fmt; @@ -119,14 +119,14 @@ static int fimc_is_subdev_enum_mbus_code(struct v4l2_subdev *sd, } static int fimc_isp_subdev_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct fimc_isp *isp = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *mf = &fmt->format; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - *mf = *v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + *mf = *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); return 0; } @@ -156,7 +156,7 @@ static int fimc_isp_subdev_get_fmt(struct v4l2_subdev *sd, } static void __isp_subdev_try_format(struct fimc_isp *isp, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct v4l2_mbus_framefmt *mf = &fmt->format; @@ -172,8 +172,9 @@ static void __isp_subdev_try_format(struct fimc_isp *isp, mf->code = MEDIA_BUS_FMT_SGRBG10_1X10; } else { if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - format = v4l2_subdev_get_try_format(&isp->subdev, cfg, - FIMC_ISP_SD_PAD_SINK); + format = v4l2_subdev_get_try_format(&isp->subdev, + sd_state, + FIMC_ISP_SD_PAD_SINK); else format = &isp->sink_fmt; @@ -191,7 +192,7 @@ static void __isp_subdev_try_format(struct fimc_isp *isp, } static int fimc_isp_subdev_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct fimc_isp *isp = v4l2_get_subdevdata(sd); @@ -203,10 +204,10 @@ static int fimc_isp_subdev_set_fmt(struct v4l2_subdev *sd, __func__, fmt->pad, mf->code, mf->width, mf->height); mutex_lock(&isp->subdev_lock); - __isp_subdev_try_format(isp, cfg, fmt); + __isp_subdev_try_format(isp, sd_state, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *mf = fmt->format; /* Propagate format to the source pads */ @@ -217,8 +218,10 @@ static int fimc_isp_subdev_set_fmt(struct v4l2_subdev *sd, for (pad = FIMC_ISP_SD_PAD_SRC_FIFO; pad < FIMC_ISP_SD_PADS_NUM; pad++) { format.pad = pad; - __isp_subdev_try_format(isp, cfg, &format); - mf = v4l2_subdev_get_try_format(sd, cfg, pad); + __isp_subdev_try_format(isp, sd_state, + &format); + mf = v4l2_subdev_get_try_format(sd, sd_state, + pad); *mf = format.format; } } @@ -230,7 +233,8 @@ static int fimc_isp_subdev_set_fmt(struct v4l2_subdev *sd, isp->sink_fmt = *mf; format.pad = FIMC_ISP_SD_PAD_SRC_DMA; - __isp_subdev_try_format(isp, cfg, &format); + __isp_subdev_try_format(isp, sd_state, + &format); isp->src_fmt = format.format; __is_set_frame_size(is, &isp->src_fmt); @@ -304,11 +308,10 @@ static int fimc_isp_subdev_s_power(struct v4l2_subdev *sd, int on) pr_debug("on: %d\n", on); if (on) { - ret = pm_runtime_get_sync(&is->pdev->dev); - if (ret < 0) { - pm_runtime_put(&is->pdev->dev); + ret = pm_runtime_resume_and_get(&is->pdev->dev); + if (ret < 0) return ret; - } + set_bit(IS_ST_PWR_ON, &is->state); ret = fimc_is_start_firmware(is); @@ -371,15 +374,18 @@ static int fimc_isp_subdev_open(struct v4l2_subdev *sd, .field = V4L2_FIELD_NONE, }; - format = v4l2_subdev_get_try_format(sd, fh->pad, FIMC_ISP_SD_PAD_SINK); + format = v4l2_subdev_get_try_format(sd, fh->state, + FIMC_ISP_SD_PAD_SINK); *format = fmt; - format = v4l2_subdev_get_try_format(sd, fh->pad, FIMC_ISP_SD_PAD_SRC_FIFO); + format = v4l2_subdev_get_try_format(sd, fh->state, + FIMC_ISP_SD_PAD_SRC_FIFO); fmt.width = DEFAULT_PREVIEW_STILL_WIDTH; fmt.height = DEFAULT_PREVIEW_STILL_HEIGHT; *format = fmt; - format = v4l2_subdev_get_try_format(sd, fh->pad, FIMC_ISP_SD_PAD_SRC_DMA); + format = v4l2_subdev_get_try_format(sd, fh->state, + FIMC_ISP_SD_PAD_SRC_DMA); *format = fmt; return 0; diff --git a/drivers/media/platform/exynos4-is/fimc-lite.c b/drivers/media/platform/exynos4-is/fimc-lite.c index fe20af3a7178..aaa3af0493ce 100644 --- a/drivers/media/platform/exynos4-is/fimc-lite.c +++ b/drivers/media/platform/exynos4-is/fimc-lite.c @@ -469,9 +469,9 @@ static int fimc_lite_open(struct file *file) } set_bit(ST_FLITE_IN_USE, &fimc->state); - ret = pm_runtime_get_sync(&fimc->pdev->dev); + ret = pm_runtime_resume_and_get(&fimc->pdev->dev); if (ret < 0) - goto err_pm; + goto err_in_use; ret = v4l2_fh_open(file); if (ret < 0) @@ -499,6 +499,7 @@ static int fimc_lite_open(struct file *file) v4l2_fh_release(file); err_pm: pm_runtime_put_sync(&fimc->pdev->dev); +err_in_use: clear_bit(ST_FLITE_IN_USE, &fimc->state); unlock: mutex_unlock(&fimc->lock); @@ -549,7 +550,7 @@ static const struct v4l2_file_operations fimc_lite_fops = { */ static const struct fimc_fmt *fimc_lite_subdev_try_fmt(struct fimc_lite *fimc, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct flite_drvdata *dd = fimc->dd; @@ -573,14 +574,16 @@ static const struct fimc_fmt *fimc_lite_subdev_try_fmt(struct fimc_lite *fimc, struct v4l2_rect *rect; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - sink_fmt = v4l2_subdev_get_try_format(&fimc->subdev, cfg, - FLITE_SD_PAD_SINK); + sink_fmt = v4l2_subdev_get_try_format(&fimc->subdev, + sd_state, + FLITE_SD_PAD_SINK); mf->code = sink_fmt->code; mf->colorspace = sink_fmt->colorspace; - rect = v4l2_subdev_get_try_crop(&fimc->subdev, cfg, - FLITE_SD_PAD_SINK); + rect = v4l2_subdev_get_try_crop(&fimc->subdev, + sd_state, + FLITE_SD_PAD_SINK); } else { mf->code = sink->fmt->mbus_code; mf->colorspace = sink->fmt->colorspace; @@ -1001,7 +1004,7 @@ static const struct media_entity_operations fimc_lite_subdev_media_ops = { }; static int fimc_lite_subdev_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { const struct fimc_fmt *fmt; @@ -1015,16 +1018,16 @@ static int fimc_lite_subdev_enum_mbus_code(struct v4l2_subdev *sd, static struct v4l2_mbus_framefmt *__fimc_lite_subdev_get_try_fmt( struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, unsigned int pad) + struct v4l2_subdev_state *sd_state, unsigned int pad) { if (pad != FLITE_SD_PAD_SINK) pad = FLITE_SD_PAD_SOURCE_DMA; - return v4l2_subdev_get_try_format(sd, cfg, pad); + return v4l2_subdev_get_try_format(sd, sd_state, pad); } static int fimc_lite_subdev_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct fimc_lite *fimc = v4l2_get_subdevdata(sd); @@ -1032,7 +1035,7 @@ static int fimc_lite_subdev_get_fmt(struct v4l2_subdev *sd, struct flite_frame *f = &fimc->inp_frame; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = __fimc_lite_subdev_get_try_fmt(sd, cfg, fmt->pad); + mf = __fimc_lite_subdev_get_try_fmt(sd, sd_state, fmt->pad); fmt->format = *mf; return 0; } @@ -1055,7 +1058,7 @@ static int fimc_lite_subdev_get_fmt(struct v4l2_subdev *sd, } static int fimc_lite_subdev_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct fimc_lite *fimc = v4l2_get_subdevdata(sd); @@ -1077,17 +1080,18 @@ static int fimc_lite_subdev_set_fmt(struct v4l2_subdev *sd, return -EBUSY; } - ffmt = fimc_lite_subdev_try_fmt(fimc, cfg, fmt); + ffmt = fimc_lite_subdev_try_fmt(fimc, sd_state, fmt); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { struct v4l2_mbus_framefmt *src_fmt; - mf = __fimc_lite_subdev_get_try_fmt(sd, cfg, fmt->pad); + mf = __fimc_lite_subdev_get_try_fmt(sd, sd_state, fmt->pad); *mf = fmt->format; if (fmt->pad == FLITE_SD_PAD_SINK) { unsigned int pad = FLITE_SD_PAD_SOURCE_DMA; - src_fmt = __fimc_lite_subdev_get_try_fmt(sd, cfg, pad); + src_fmt = __fimc_lite_subdev_get_try_fmt(sd, sd_state, + pad); *src_fmt = *mf; } @@ -1115,7 +1119,7 @@ static int fimc_lite_subdev_set_fmt(struct v4l2_subdev *sd, } static int fimc_lite_subdev_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct fimc_lite *fimc = v4l2_get_subdevdata(sd); @@ -1127,7 +1131,7 @@ static int fimc_lite_subdev_get_selection(struct v4l2_subdev *sd, return -EINVAL; if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { - sel->r = *v4l2_subdev_get_try_crop(sd, cfg, sel->pad); + sel->r = *v4l2_subdev_get_try_crop(sd, sd_state, sel->pad); return 0; } @@ -1150,7 +1154,7 @@ static int fimc_lite_subdev_get_selection(struct v4l2_subdev *sd, } static int fimc_lite_subdev_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct fimc_lite *fimc = v4l2_get_subdevdata(sd); @@ -1164,7 +1168,7 @@ static int fimc_lite_subdev_set_selection(struct v4l2_subdev *sd, fimc_lite_try_crop(fimc, &sel->r); if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_crop(sd, cfg, sel->pad) = sel->r; + *v4l2_subdev_get_try_crop(sd, sd_state, sel->pad) = sel->r; } else { unsigned long flags; spin_lock_irqsave(&fimc->slock, flags); diff --git a/drivers/media/platform/exynos4-is/fimc-m2m.c b/drivers/media/platform/exynos4-is/fimc-m2m.c index c9704a147e5c..df8e2aa454d8 100644 --- a/drivers/media/platform/exynos4-is/fimc-m2m.c +++ b/drivers/media/platform/exynos4-is/fimc-m2m.c @@ -73,17 +73,14 @@ static void fimc_m2m_shutdown(struct fimc_ctx *ctx) static int start_streaming(struct vb2_queue *q, unsigned int count) { struct fimc_ctx *ctx = q->drv_priv; - int ret; - ret = pm_runtime_get_sync(&ctx->fimc_dev->pdev->dev); - return ret > 0 ? 0 : ret; + return pm_runtime_resume_and_get(&ctx->fimc_dev->pdev->dev); } static void stop_streaming(struct vb2_queue *q) { struct fimc_ctx *ctx = q->drv_priv; - fimc_m2m_shutdown(ctx); fimc_m2m_job_finish(ctx, VB2_BUF_STATE_ERROR); pm_runtime_put(&ctx->fimc_dev->pdev->dev); diff --git a/drivers/media/platform/exynos4-is/media-dev.c b/drivers/media/platform/exynos4-is/media-dev.c index 13d192ba4aa6..3b8a24bb724c 100644 --- a/drivers/media/platform/exynos4-is/media-dev.c +++ b/drivers/media/platform/exynos4-is/media-dev.c @@ -512,11 +512,9 @@ static int fimc_md_register_sensor_entities(struct fimc_md *fmd) if (!fmd->pmf) return -ENXIO; - ret = pm_runtime_get_sync(fmd->pmf); - if (ret < 0) { - pm_runtime_put(fmd->pmf); + ret = pm_runtime_resume_and_get(fmd->pmf); + if (ret < 0) return ret; - } fmd->num_sensors = 0; @@ -1286,13 +1284,11 @@ static DEVICE_ATTR(subdev_conf_mode, S_IWUSR | S_IRUGO, static int cam_clk_prepare(struct clk_hw *hw) { struct cam_clk *camclk = to_cam_clk(hw); - int ret; if (camclk->fmd->pmf == NULL) return -ENODEV; - ret = pm_runtime_get_sync(camclk->fmd->pmf); - return ret < 0 ? ret : 0; + return pm_runtime_resume_and_get(camclk->fmd->pmf); } static void cam_clk_unprepare(struct clk_hw *hw) diff --git a/drivers/media/platform/exynos4-is/mipi-csis.c b/drivers/media/platform/exynos4-is/mipi-csis.c index 1aac167abb17..32b23329b033 100644 --- a/drivers/media/platform/exynos4-is/mipi-csis.c +++ b/drivers/media/platform/exynos4-is/mipi-csis.c @@ -494,7 +494,7 @@ static int s5pcsis_s_power(struct v4l2_subdev *sd, int on) struct device *dev = &state->pdev->dev; if (on) - return pm_runtime_get_sync(dev); + return pm_runtime_resume_and_get(dev); return pm_runtime_put_sync(dev); } @@ -509,11 +509,9 @@ static int s5pcsis_s_stream(struct v4l2_subdev *sd, int enable) if (enable) { s5pcsis_clear_counters(state); - ret = pm_runtime_get_sync(&state->pdev->dev); - if (ret && ret != 1) { - pm_runtime_put_noidle(&state->pdev->dev); + ret = pm_runtime_resume_and_get(&state->pdev->dev); + if (ret < 0) return ret; - } } mutex_lock(&state->lock); @@ -535,11 +533,11 @@ static int s5pcsis_s_stream(struct v4l2_subdev *sd, int enable) if (!enable) pm_runtime_put(&state->pdev->dev); - return ret == 1 ? 0 : ret; + return ret; } static int s5pcsis_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(s5pcsis_formats)) @@ -567,23 +565,25 @@ static struct csis_pix_format const *s5pcsis_try_format( } static struct v4l2_mbus_framefmt *__s5pcsis_get_format( - struct csis_state *state, struct v4l2_subdev_pad_config *cfg, + struct csis_state *state, struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return cfg ? v4l2_subdev_get_try_format(&state->sd, cfg, 0) : NULL; + return sd_state ? v4l2_subdev_get_try_format(&state->sd, + sd_state, 0) : NULL; return &state->format; } -static int s5pcsis_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int s5pcsis_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct csis_state *state = sd_to_csis_state(sd); struct csis_pix_format const *csis_fmt; struct v4l2_mbus_framefmt *mf; - mf = __s5pcsis_get_format(state, cfg, fmt->which); + mf = __s5pcsis_get_format(state, sd_state, fmt->which); if (fmt->pad == CSIS_PAD_SOURCE) { if (mf) { @@ -604,13 +604,14 @@ static int s5pcsis_set_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config return 0; } -static int s5pcsis_get_fmt(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int s5pcsis_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct csis_state *state = sd_to_csis_state(sd); struct v4l2_mbus_framefmt *mf; - mf = __s5pcsis_get_format(state, cfg, fmt->which); + mf = __s5pcsis_get_format(state, sd_state, fmt->which); if (!mf) return -EINVAL; diff --git a/drivers/media/platform/imx-jpeg/mxc-jpeg.c b/drivers/media/platform/imx-jpeg/mxc-jpeg.c index 03b9264af068..755138063ee6 100644 --- a/drivers/media/platform/imx-jpeg/mxc-jpeg.c +++ b/drivers/media/platform/imx-jpeg/mxc-jpeg.c @@ -62,7 +62,7 @@ #include "mxc-jpeg-hw.h" #include "mxc-jpeg.h" -static struct mxc_jpeg_fmt mxc_formats[] = { +static const struct mxc_jpeg_fmt mxc_formats[] = { { .name = "JPEG", .fourcc = V4L2_PIX_FMT_JPEG, @@ -341,7 +341,7 @@ static inline struct mxc_jpeg_ctx *mxc_jpeg_fh_to_ctx(struct v4l2_fh *fh) return container_of(fh, struct mxc_jpeg_ctx, fh); } -static int enum_fmt(struct mxc_jpeg_fmt *mxc_formats, int n, +static int enum_fmt(const struct mxc_jpeg_fmt *mxc_formats, int n, struct v4l2_fmtdesc *f, u32 type) { int i, num = 0; @@ -368,13 +368,13 @@ static int enum_fmt(struct mxc_jpeg_fmt *mxc_formats, int n, return 0; } -static struct mxc_jpeg_fmt *mxc_jpeg_find_format(struct mxc_jpeg_ctx *ctx, - u32 pixelformat) +static const struct mxc_jpeg_fmt *mxc_jpeg_find_format(struct mxc_jpeg_ctx *ctx, + u32 pixelformat) { unsigned int k; for (k = 0; k < MXC_JPEG_NUM_FORMATS; k++) { - struct mxc_jpeg_fmt *fmt = &mxc_formats[k]; + const struct mxc_jpeg_fmt *fmt = &mxc_formats[k]; if (fmt->fourcc == pixelformat) return fmt; @@ -1536,7 +1536,7 @@ static int mxc_jpeg_enum_fmt_vid_out(struct file *file, void *priv, MXC_JPEG_FMT_TYPE_RAW); } -static int mxc_jpeg_try_fmt(struct v4l2_format *f, struct mxc_jpeg_fmt *fmt, +static int mxc_jpeg_try_fmt(struct v4l2_format *f, const struct mxc_jpeg_fmt *fmt, struct mxc_jpeg_ctx *ctx, int q_type) { struct v4l2_pix_format_mplane *pix_mp = &f->fmt.pix_mp; @@ -1612,7 +1612,7 @@ static int mxc_jpeg_try_fmt_vid_cap(struct file *file, void *priv, struct mxc_jpeg_ctx *ctx = mxc_jpeg_fh_to_ctx(priv); struct mxc_jpeg_dev *jpeg = ctx->mxc_jpeg; struct device *dev = jpeg->dev; - struct mxc_jpeg_fmt *fmt; + const struct mxc_jpeg_fmt *fmt; u32 fourcc = f->fmt.pix_mp.pixelformat; int q_type = (jpeg->mode == MXC_JPEG_DECODE) ? @@ -1643,7 +1643,7 @@ static int mxc_jpeg_try_fmt_vid_out(struct file *file, void *priv, struct mxc_jpeg_ctx *ctx = mxc_jpeg_fh_to_ctx(priv); struct mxc_jpeg_dev *jpeg = ctx->mxc_jpeg; struct device *dev = jpeg->dev; - struct mxc_jpeg_fmt *fmt; + const struct mxc_jpeg_fmt *fmt; u32 fourcc = f->fmt.pix_mp.pixelformat; int q_type = (jpeg->mode == MXC_JPEG_ENCODE) ? @@ -1890,7 +1890,7 @@ static const struct v4l2_file_operations mxc_jpeg_fops = { .mmap = v4l2_m2m_fop_mmap, }; -static struct v4l2_m2m_ops mxc_jpeg_m2m_ops = { +static const struct v4l2_m2m_ops mxc_jpeg_m2m_ops = { .device_run = mxc_jpeg_device_run, }; diff --git a/drivers/media/platform/imx-jpeg/mxc-jpeg.h b/drivers/media/platform/imx-jpeg/mxc-jpeg.h index 7697de490d2e..4c210852e876 100644 --- a/drivers/media/platform/imx-jpeg/mxc-jpeg.h +++ b/drivers/media/platform/imx-jpeg/mxc-jpeg.h @@ -51,7 +51,7 @@ enum mxc_jpeg_mode { * @flags: flags describing format applicability */ struct mxc_jpeg_fmt { - char *name; + const char *name; u32 fourcc; enum v4l2_jpeg_chroma_subsampling subsampling; int nc; @@ -74,14 +74,14 @@ struct mxc_jpeg_desc { } __packed; struct mxc_jpeg_q_data { - struct mxc_jpeg_fmt *fmt; - u32 sizeimage[MXC_JPEG_MAX_PLANES]; - u32 bytesperline[MXC_JPEG_MAX_PLANES]; - int w; - int w_adjusted; - int h; - int h_adjusted; - unsigned int sequence; + const struct mxc_jpeg_fmt *fmt; + u32 sizeimage[MXC_JPEG_MAX_PLANES]; + u32 bytesperline[MXC_JPEG_MAX_PLANES]; + int w; + int w_adjusted; + int h; + int h_adjusted; + unsigned int sequence; }; struct mxc_jpeg_ctx { diff --git a/drivers/media/platform/marvell-ccic/cafe-driver.c b/drivers/media/platform/marvell-ccic/cafe-driver.c index baac86f3d153..9aa374fa8b36 100644 --- a/drivers/media/platform/marvell-ccic/cafe-driver.c +++ b/drivers/media/platform/marvell-ccic/cafe-driver.c @@ -486,6 +486,7 @@ static int cafe_pci_probe(struct pci_dev *pdev, struct cafe_camera *cam; struct mcam_camera *mcam; struct v4l2_async_subdev *asd; + struct i2c_client *i2c_dev; /* * Start putting together one of our big camera structures. @@ -561,11 +562,16 @@ static int cafe_pci_probe(struct pci_dev *pdev, clkdev_create(mcam->mclk, "xclk", "%d-%04x", i2c_adapter_id(cam->i2c_adapter), ov7670_info.addr); - if (!IS_ERR(i2c_new_client_device(cam->i2c_adapter, &ov7670_info))) { - cam->registered = 1; - return 0; + i2c_dev = i2c_new_client_device(cam->i2c_adapter, &ov7670_info); + if (IS_ERR(i2c_dev)) { + ret = PTR_ERR(i2c_dev); + goto out_mccic_shutdown; } + cam->registered = 1; + return 0; + +out_mccic_shutdown: mccic_shutdown(mcam); out_smbus_shutdown: cafe_smbus_shutdown(cam); diff --git a/drivers/media/platform/marvell-ccic/mcam-core.c b/drivers/media/platform/marvell-ccic/mcam-core.c index 141bf5d97a04..070a0f3fc337 100644 --- a/drivers/media/platform/marvell-ccic/mcam-core.c +++ b/drivers/media/platform/marvell-ccic/mcam-core.c @@ -918,6 +918,7 @@ static int mclk_enable(struct clk_hw *hw) struct mcam_camera *cam = container_of(hw, struct mcam_camera, mclk_hw); int mclk_src; int mclk_div; + int ret; /* * Clock the sensor appropriately. Controller clock should @@ -931,7 +932,9 @@ static int mclk_enable(struct clk_hw *hw) mclk_div = 2; } - pm_runtime_get_sync(cam->dev); + ret = pm_runtime_resume_and_get(cam->dev); + if (ret < 0) + return ret; clk_enable(cam->clk[0]); mcam_reg_write(cam, REG_CLKCTRL, (mclk_src << 29) | mclk_div); mcam_ctlr_power_up(cam); @@ -1347,6 +1350,9 @@ static int mcam_vidioc_try_fmt_vid_cap(struct file *filp, void *priv, struct mcam_format_struct *f; struct v4l2_pix_format *pix = &fmt->fmt.pix; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -1355,7 +1361,7 @@ static int mcam_vidioc_try_fmt_vid_cap(struct file *filp, void *priv, f = mcam_find_format(pix->pixelformat); pix->pixelformat = f->pixelformat; v4l2_fill_mbus_format(&format.format, pix, f->mbus_code); - ret = sensor_call(cam, pad, set_fmt, &pad_cfg, &format); + ret = sensor_call(cam, pad, set_fmt, &pad_state, &format); v4l2_fill_pix_format(pix, &format.format); pix->bytesperline = pix->width * f->bpp; switch (f->pixelformat) { @@ -1611,7 +1617,9 @@ static int mcam_v4l_open(struct file *filp) ret = sensor_call(cam, core, s_power, 1); if (ret) goto out; - pm_runtime_get_sync(cam->dev); + ret = pm_runtime_resume_and_get(cam->dev); + if (ret < 0) + goto out; __mcam_cam_reset(cam); mcam_set_config_needed(cam, 1); } diff --git a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c index 88a23bce569d..a89c7b206eef 100644 --- a/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c +++ b/drivers/media/platform/mtk-jpeg/mtk_jpeg_core.c @@ -920,7 +920,7 @@ static void mtk_jpeg_enc_device_run(void *priv) src_buf = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); dst_buf = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); - ret = pm_runtime_get_sync(jpeg->dev); + ret = pm_runtime_resume_and_get(jpeg->dev); if (ret < 0) goto enc_end; @@ -973,7 +973,7 @@ static void mtk_jpeg_dec_device_run(void *priv) return; } - ret = pm_runtime_get_sync(jpeg->dev); + ret = pm_runtime_resume_and_get(jpeg->dev); if (ret < 0) goto dec_end; diff --git a/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c b/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c index ace4528cdc5e..f14779e7596e 100644 --- a/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c +++ b/drivers/media/platform/mtk-mdp/mtk_mdp_m2m.c @@ -391,12 +391,12 @@ static int mtk_mdp_m2m_start_streaming(struct vb2_queue *q, unsigned int count) struct mtk_mdp_ctx *ctx = q->drv_priv; int ret; - ret = pm_runtime_get_sync(&ctx->mdp_dev->pdev->dev); + ret = pm_runtime_resume_and_get(&ctx->mdp_dev->pdev->dev); if (ret < 0) - mtk_mdp_dbg(1, "[%d] pm_runtime_get_sync failed:%d", + mtk_mdp_dbg(1, "[%d] pm_runtime_resume_and_get failed:%d", ctx->id, ret); - return 0; + return ret; } static void *mtk_mdp_m2m_buf_remove(struct mtk_mdp_ctx *ctx, diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c index 147dfef1638d..f87dc47d9e63 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_drv.c @@ -126,7 +126,9 @@ static int fops_vcodec_open(struct file *file) mtk_vcodec_dec_set_default_params(ctx); if (v4l2_fh_is_singular(&ctx->fh)) { - mtk_vcodec_dec_pw_on(&dev->pm); + ret = mtk_vcodec_dec_pw_on(&dev->pm); + if (ret < 0) + goto err_load_fw; /* * Does nothing if firmware was already loaded. */ diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.c index ddee7046ce42..6038db96f71c 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.c +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.c @@ -88,13 +88,15 @@ void mtk_vcodec_release_dec_pm(struct mtk_vcodec_dev *dev) put_device(dev->pm.larbvdec); } -void mtk_vcodec_dec_pw_on(struct mtk_vcodec_pm *pm) +int mtk_vcodec_dec_pw_on(struct mtk_vcodec_pm *pm) { int ret; - ret = pm_runtime_get_sync(pm->dev); + ret = pm_runtime_resume_and_get(pm->dev); if (ret) - mtk_v4l2_err("pm_runtime_get_sync fail %d", ret); + mtk_v4l2_err("pm_runtime_resume_and_get fail %d", ret); + + return ret; } void mtk_vcodec_dec_pw_off(struct mtk_vcodec_pm *pm) diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.h index 872d8bf8cfaf..280aeaefdb65 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.h +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_dec_pm.h @@ -12,7 +12,7 @@ int mtk_vcodec_init_dec_pm(struct mtk_vcodec_dev *dev); void mtk_vcodec_release_dec_pm(struct mtk_vcodec_dev *dev); -void mtk_vcodec_dec_pw_on(struct mtk_vcodec_pm *pm); +int mtk_vcodec_dec_pw_on(struct mtk_vcodec_pm *pm); void mtk_vcodec_dec_pw_off(struct mtk_vcodec_pm *pm); void mtk_vcodec_dec_clock_on(struct mtk_vcodec_pm *pm); void mtk_vcodec_dec_clock_off(struct mtk_vcodec_pm *pm); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h index d03cca95e99b..c6c7672fecfb 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_drv.h @@ -25,7 +25,7 @@ #define MTK_V4L2_BENCHMARK 0 #define WAIT_INTR_TIMEOUT_MS 1000 -/** +/* * enum mtk_hw_reg_idx - MTK hw register base index */ enum mtk_hw_reg_idx { @@ -49,7 +49,7 @@ enum mtk_hw_reg_idx { NUM_MAX_VCODEC_REG_BASE }; -/** +/* * enum mtk_instance_type - The type of an MTK Vcodec instance. */ enum mtk_instance_type { @@ -74,7 +74,7 @@ enum mtk_instance_state { MTK_STATE_ABORT = 4, }; -/** +/* * enum mtk_encode_param - General encoding parameters type */ enum mtk_encode_param { @@ -92,7 +92,7 @@ enum mtk_fmt_type { MTK_FMT_FRAME = 2, }; -/** +/* * struct mtk_video_fmt - Structure used to store information about pixelformats */ struct mtk_video_fmt { @@ -102,7 +102,7 @@ struct mtk_video_fmt { u32 flags; }; -/** +/* * struct mtk_codec_framesizes - Structure used to store information about * framesizes */ @@ -111,7 +111,7 @@ struct mtk_codec_framesizes { struct v4l2_frmsize_stepwise stepwise; }; -/** +/* * enum mtk_q_type - Type of queue */ enum mtk_q_type { @@ -119,7 +119,7 @@ enum mtk_q_type { MTK_Q_DATA_DST = 1, }; -/** +/* * struct mtk_q_data - Structure used to store information about queue */ struct mtk_q_data { @@ -168,7 +168,7 @@ struct mtk_enc_params { unsigned int force_intra; }; -/** +/* * struct mtk_vcodec_clk_info - Structure used to store clock name */ struct mtk_vcodec_clk_info { @@ -176,7 +176,7 @@ struct mtk_vcodec_clk_info { struct clk *vcodec_clk; }; -/** +/* * struct mtk_vcodec_clk - Structure used to store vcodec clock information */ struct mtk_vcodec_clk { @@ -184,7 +184,7 @@ struct mtk_vcodec_clk { int clk_num; }; -/** +/* * struct mtk_vcodec_pm - Power management data structure */ struct mtk_vcodec_pm { @@ -255,6 +255,7 @@ struct vdec_pic_info { * @ycbcr_enc: enum v4l2_ycbcr_encoding, Y'CbCr encoding * @quantization: enum v4l2_quantization, colorspace quantization * @xfer_func: enum v4l2_xfer_func, colorspace transfer function + * @decoded_frame_cnt: number of decoded frames * @lock: protect variables accessed by V4L2 threads and worker thread such as * mtk_video_dec_buf. */ @@ -302,6 +303,7 @@ struct mtk_vcodec_ctx { enum mtk_chip { MTK_MT8173, MTK_MT8183, + MTK_MT8192, }; /** @@ -310,7 +312,7 @@ enum mtk_chip { * @chip: chip this encoder is compatible with * * @uses_ext: whether the encoder uses the extended firmware messaging format - * @min_birate: minimum supported encoding bitrate + * @min_bitrate: minimum supported encoding bitrate * @max_bitrate: maximum supported encoding bitrate * @capture_formats: array of supported capture formats * @num_capture_formats: number of entries in capture_formats @@ -347,10 +349,12 @@ struct mtk_vcodec_enc_pdata { * @curr_ctx: The context that is waiting for codec hardware * * @reg_base: Mapped address of MTK Vcodec registers. + * @venc_pdata: encoder IC-specific data * * @fw_handler: used to communicate with the firmware. * @id_counter: used to identify current opened instance * + * @decode_workqueue: decode work queue * @encode_workqueue: encode work queue * * @int_cond: used to identify interrupt condition happen diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c index 4831052f475d..416f356af363 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc.c @@ -19,23 +19,30 @@ #define MTK_VENC_MIN_W 160U #define MTK_VENC_MIN_H 128U -#define MTK_VENC_MAX_W 1920U -#define MTK_VENC_MAX_H 1088U +#define MTK_VENC_HD_MAX_W 1920U +#define MTK_VENC_HD_MAX_H 1088U +#define MTK_VENC_4K_MAX_W 3840U +#define MTK_VENC_4K_MAX_H 2176U + #define DFT_CFG_WIDTH MTK_VENC_MIN_W #define DFT_CFG_HEIGHT MTK_VENC_MIN_H #define MTK_MAX_CTRLS_HINT 20 #define MTK_DEFAULT_FRAMERATE_NUM 1001 #define MTK_DEFAULT_FRAMERATE_DENOM 30000 +#define MTK_VENC_4K_CAPABILITY_ENABLE BIT(0) static void mtk_venc_worker(struct work_struct *work); -static const struct v4l2_frmsize_stepwise mtk_venc_framesizes = { - MTK_VENC_MIN_W, MTK_VENC_MAX_W, 16, - MTK_VENC_MIN_H, MTK_VENC_MAX_H, 16, +static const struct v4l2_frmsize_stepwise mtk_venc_hd_framesizes = { + MTK_VENC_MIN_W, MTK_VENC_HD_MAX_W, 16, + MTK_VENC_MIN_H, MTK_VENC_HD_MAX_H, 16, }; -#define NUM_SUPPORTED_FRAMESIZE ARRAY_SIZE(mtk_venc_framesizes) +static const struct v4l2_frmsize_stepwise mtk_venc_4k_framesizes = { + MTK_VENC_MIN_W, MTK_VENC_4K_MAX_W, 16, + MTK_VENC_MIN_H, MTK_VENC_4K_MAX_H, 16, +}; static int vidioc_venc_s_ctrl(struct v4l2_ctrl *ctrl) { @@ -151,17 +158,22 @@ static int vidioc_enum_framesizes(struct file *file, void *fh, struct v4l2_frmsizeenum *fsize) { const struct mtk_video_fmt *fmt; + struct mtk_vcodec_ctx *ctx = fh_to_ctx(fh); if (fsize->index != 0) return -EINVAL; fmt = mtk_venc_find_format(fsize->pixel_format, - fh_to_ctx(fh)->dev->venc_pdata); + ctx->dev->venc_pdata); if (!fmt) return -EINVAL; fsize->type = V4L2_FRMSIZE_TYPE_STEPWISE; - fsize->stepwise = mtk_venc_framesizes; + + if (ctx->dev->enc_capability & MTK_VENC_4K_CAPABILITY_ENABLE) + fsize->stepwise = mtk_venc_4k_framesizes; + else + fsize->stepwise = mtk_venc_hd_framesizes; return 0; } @@ -248,7 +260,7 @@ static struct mtk_q_data *mtk_venc_get_q_data(struct mtk_vcodec_ctx *ctx, /* V4L2 specification suggests the driver corrects the format struct if any of * the dimensions is unsupported */ -static int vidioc_try_fmt(struct v4l2_format *f, +static int vidioc_try_fmt(struct mtk_vcodec_ctx *ctx, struct v4l2_format *f, const struct mtk_video_fmt *fmt) { struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp; @@ -260,13 +272,22 @@ static int vidioc_try_fmt(struct v4l2_format *f, pix_fmt_mp->plane_fmt[0].bytesperline = 0; } else if (f->type == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE) { int tmp_w, tmp_h; + unsigned int max_width, max_height; + + if (ctx->dev->enc_capability & MTK_VENC_4K_CAPABILITY_ENABLE) { + max_width = MTK_VENC_4K_MAX_W; + max_height = MTK_VENC_4K_MAX_H; + } else { + max_width = MTK_VENC_HD_MAX_W; + max_height = MTK_VENC_HD_MAX_H; + } pix_fmt_mp->height = clamp(pix_fmt_mp->height, MTK_VENC_MIN_H, - MTK_VENC_MAX_H); + max_height); pix_fmt_mp->width = clamp(pix_fmt_mp->width, MTK_VENC_MIN_W, - MTK_VENC_MAX_W); + max_width); /* find next closer width align 16, heign align 32, size align * 64 rectangle @@ -275,16 +296,16 @@ static int vidioc_try_fmt(struct v4l2_format *f, tmp_h = pix_fmt_mp->height; v4l_bound_align_image(&pix_fmt_mp->width, MTK_VENC_MIN_W, - MTK_VENC_MAX_W, 4, + max_width, 4, &pix_fmt_mp->height, MTK_VENC_MIN_H, - MTK_VENC_MAX_H, 5, 6); + max_height, 5, 6); if (pix_fmt_mp->width < tmp_w && - (pix_fmt_mp->width + 16) <= MTK_VENC_MAX_W) + (pix_fmt_mp->width + 16) <= max_width) pix_fmt_mp->width += 16; if (pix_fmt_mp->height < tmp_h && - (pix_fmt_mp->height + 32) <= MTK_VENC_MAX_H) + (pix_fmt_mp->height + 32) <= max_height) pix_fmt_mp->height += 32; mtk_v4l2_debug(0, @@ -405,7 +426,7 @@ static int vidioc_venc_s_fmt_cap(struct file *file, void *priv, } q_data->fmt = fmt; - ret = vidioc_try_fmt(f, q_data->fmt); + ret = vidioc_try_fmt(ctx, f, q_data->fmt); if (ret) return ret; @@ -443,7 +464,6 @@ static int vidioc_venc_s_fmt_out(struct file *file, void *priv, struct mtk_q_data *q_data; int ret, i; const struct mtk_video_fmt *fmt; - struct v4l2_pix_format_mplane *pix_fmt_mp = &f->fmt.pix_mp; vq = v4l2_m2m_get_vq(ctx->m2m_ctx, f->type); if (!vq) { @@ -468,20 +488,13 @@ static int vidioc_venc_s_fmt_out(struct file *file, void *priv, f->fmt.pix.pixelformat = fmt->fourcc; } - pix_fmt_mp->height = clamp(pix_fmt_mp->height, - MTK_VENC_MIN_H, - MTK_VENC_MAX_H); - pix_fmt_mp->width = clamp(pix_fmt_mp->width, - MTK_VENC_MIN_W, - MTK_VENC_MAX_W); - - q_data->visible_width = f->fmt.pix_mp.width; - q_data->visible_height = f->fmt.pix_mp.height; - q_data->fmt = fmt; - ret = vidioc_try_fmt(f, q_data->fmt); + ret = vidioc_try_fmt(ctx, f, fmt); if (ret) return ret; + q_data->fmt = fmt; + q_data->visible_width = f->fmt.pix_mp.width; + q_data->visible_height = f->fmt.pix_mp.height; q_data->coded_width = f->fmt.pix_mp.width; q_data->coded_height = f->fmt.pix_mp.height; @@ -553,7 +566,7 @@ static int vidioc_try_fmt_vid_cap_mplane(struct file *file, void *priv, f->fmt.pix_mp.quantization = ctx->quantization; f->fmt.pix_mp.xfer_func = ctx->xfer_func; - return vidioc_try_fmt(f, fmt); + return vidioc_try_fmt(ctx, f, fmt); } static int vidioc_try_fmt_vid_out_mplane(struct file *file, void *priv, @@ -575,7 +588,7 @@ static int vidioc_try_fmt_vid_out_mplane(struct file *file, void *priv, f->fmt.pix_mp.xfer_func = V4L2_XFER_FUNC_DEFAULT; } - return vidioc_try_fmt(f, fmt); + return vidioc_try_fmt(ctx, f, fmt); } static int vidioc_venc_g_selection(struct file *file, void *priv, @@ -1179,16 +1192,16 @@ void mtk_vcodec_enc_set_default_params(struct mtk_vcodec_ctx *ctx) v4l_bound_align_image(&q_data->coded_width, MTK_VENC_MIN_W, - MTK_VENC_MAX_W, 4, + MTK_VENC_HD_MAX_W, 4, &q_data->coded_height, MTK_VENC_MIN_H, - MTK_VENC_MAX_H, 5, 6); + MTK_VENC_HD_MAX_H, 5, 6); if (q_data->coded_width < DFT_CFG_WIDTH && - (q_data->coded_width + 16) <= MTK_VENC_MAX_W) + (q_data->coded_width + 16) <= MTK_VENC_HD_MAX_W) q_data->coded_width += 16; if (q_data->coded_height < DFT_CFG_HEIGHT && - (q_data->coded_height + 32) <= MTK_VENC_MAX_H) + (q_data->coded_height + 32) <= MTK_VENC_HD_MAX_H) q_data->coded_height += 32; q_data->sizeimage[0] = @@ -1218,6 +1231,12 @@ int mtk_vcodec_enc_ctrls_setup(struct mtk_vcodec_ctx *ctx) { const struct v4l2_ctrl_ops *ops = &mtk_vcodec_enc_ctrl_ops; struct v4l2_ctrl_handler *handler = &ctx->ctrl_hdl; + u8 h264_max_level; + + if (ctx->dev->enc_capability & MTK_VENC_4K_CAPABILITY_ENABLE) + h264_max_level = V4L2_MPEG_VIDEO_H264_LEVEL_5_1; + else + h264_max_level = V4L2_MPEG_VIDEO_H264_LEVEL_4_2; v4l2_ctrl_handler_init(handler, MTK_MAX_CTRLS_HINT); @@ -1248,8 +1267,9 @@ int mtk_vcodec_enc_ctrls_setup(struct mtk_vcodec_ctx *ctx) V4L2_MPEG_VIDEO_H264_PROFILE_HIGH, 0, V4L2_MPEG_VIDEO_H264_PROFILE_HIGH); v4l2_ctrl_new_std_menu(handler, ops, V4L2_CID_MPEG_VIDEO_H264_LEVEL, - V4L2_MPEG_VIDEO_H264_LEVEL_4_2, - 0, V4L2_MPEG_VIDEO_H264_LEVEL_4_0); + h264_max_level, + 0, V4L2_MPEG_VIDEO_H264_LEVEL_4_0); + if (handler->error) { mtk_v4l2_err("Init control handler fail %d", handler->error); diff --git a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c index 7d7b8cfc2cc5..45d1870c83dd 100644 --- a/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c +++ b/drivers/media/platform/mtk-vcodec/mtk_vcodec_enc_drv.c @@ -361,6 +361,9 @@ static int mtk_vcodec_probe(struct platform_device *pdev) goto err_event_workq; } + if (of_get_property(pdev->dev.of_node, "dma-ranges", NULL)) + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(34)); + ret = video_register_device(vfd_enc, VFL_TYPE_VIDEO, 1); if (ret) { mtk_v4l2_err("Failed to register video device"); @@ -422,12 +425,26 @@ static const struct mtk_vcodec_enc_pdata mt8183_pdata = { .core_id = VENC_SYS, }; +static const struct mtk_vcodec_enc_pdata mt8192_pdata = { + .chip = MTK_MT8192, + .uses_ext = true, + /* MT8192 supports the same capture formats as MT8183 */ + .capture_formats = mtk_video_formats_capture_mt8183, + .num_capture_formats = ARRAY_SIZE(mtk_video_formats_capture_mt8183), + /* MT8192 supports the same output formats as MT8173 */ + .output_formats = mtk_video_formats_output_mt8173, + .num_output_formats = ARRAY_SIZE(mtk_video_formats_output_mt8173), + .min_bitrate = 64, + .max_bitrate = 100000000, + .core_id = VENC_SYS, +}; static const struct of_device_id mtk_vcodec_enc_match[] = { {.compatible = "mediatek,mt8173-vcodec-enc", .data = &mt8173_avc_pdata}, {.compatible = "mediatek,mt8173-vcodec-enc-vp8", .data = &mt8173_vp8_pdata}, {.compatible = "mediatek,mt8183-vcodec-enc", .data = &mt8183_pdata}, + {.compatible = "mediatek,mt8192-vcodec-enc", .data = &mt8192_pdata}, {}, }; MODULE_DEVICE_TABLE(of, mtk_vcodec_enc_match); diff --git a/drivers/media/platform/mtk-vcodec/vdec_ipi_msg.h b/drivers/media/platform/mtk-vcodec/vdec_ipi_msg.h index 47a1c1c0fd04..68e8d5cb16d7 100644 --- a/drivers/media/platform/mtk-vcodec/vdec_ipi_msg.h +++ b/drivers/media/platform/mtk-vcodec/vdec_ipi_msg.h @@ -7,7 +7,7 @@ #ifndef _VDEC_IPI_MSG_H_ #define _VDEC_IPI_MSG_H_ -/** +/* * enum vdec_ipi_msgid - message id between AP and VPU * @AP_IPIMSG_XXX : AP to VPU cmd message id * @VPU_IPIMSG_XXX_ACK : VPU ack AP cmd message id diff --git a/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c b/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c index d0123dfc5f93..b6a4f2074fa5 100644 --- a/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c +++ b/drivers/media/platform/mtk-vcodec/venc/venc_h264_if.c @@ -215,6 +215,10 @@ static unsigned int h264_get_level(struct venc_h264_inst *inst, return 41; case V4L2_MPEG_VIDEO_H264_LEVEL_4_2: return 42; + case V4L2_MPEG_VIDEO_H264_LEVEL_5_0: + return 50; + case V4L2_MPEG_VIDEO_H264_LEVEL_5_1: + return 51; default: mtk_vcodec_debug(inst, "unsupported level %d", level); return 31; diff --git a/drivers/media/platform/mtk-vcodec/venc_ipi_msg.h b/drivers/media/platform/mtk-vcodec/venc_ipi_msg.h index 5f53d4255c36..587a2cf15b76 100644 --- a/drivers/media/platform/mtk-vcodec/venc_ipi_msg.h +++ b/drivers/media/platform/mtk-vcodec/venc_ipi_msg.h @@ -12,7 +12,7 @@ #define AP_IPIMSG_VENC_BASE 0xC000 #define VPU_IPIMSG_VENC_BASE 0xD000 -/** +/* * enum venc_ipi_msg_id - message id between AP and VPU * (ipi stands for inter-processor interrupt) * @AP_IPIMSG_ENC_XXX: AP to VPU cmd message id @@ -111,7 +111,7 @@ struct venc_ap_ipi_msg_deinit { uint32_t vpu_inst_addr; }; -/** +/* * enum venc_ipi_msg_status - VPU ack AP cmd status */ enum venc_ipi_msg_status { diff --git a/drivers/media/platform/mtk-vpu/mtk_vpu.c b/drivers/media/platform/mtk-vpu/mtk_vpu.c index c8a56271b259..ec290dde59cf 100644 --- a/drivers/media/platform/mtk-vpu/mtk_vpu.c +++ b/drivers/media/platform/mtk-vpu/mtk_vpu.c @@ -821,13 +821,11 @@ static int mtk_vpu_probe(struct platform_device *pdev) return -ENOMEM; vpu->dev = &pdev->dev; - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "tcm"); - vpu->reg.tcm = devm_ioremap_resource(dev, res); + vpu->reg.tcm = devm_platform_ioremap_resource_byname(pdev, "tcm"); if (IS_ERR((__force void *)vpu->reg.tcm)) return PTR_ERR((__force void *)vpu->reg.tcm); - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cfg_reg"); - vpu->reg.cfg = devm_ioremap_resource(dev, res); + vpu->reg.cfg = devm_platform_ioremap_resource_byname(pdev, "cfg_reg"); if (IS_ERR((__force void *)vpu->reg.cfg)) return PTR_ERR((__force void *)vpu->reg.cfg); @@ -987,6 +985,12 @@ static int mtk_vpu_suspend(struct device *dev) return ret; } + if (!vpu_running(vpu)) { + vpu_clock_disable(vpu); + clk_unprepare(vpu->clk); + return 0; + } + mutex_lock(&vpu->vpu_mutex); /* disable vpu timer interrupt */ vpu_cfg_writel(vpu, vpu_cfg_readl(vpu, VPU_INT_STATUS) | VPU_IDLE_STATE, diff --git a/drivers/media/platform/omap3isp/ispccdc.c b/drivers/media/platform/omap3isp/ispccdc.c index 4e8905ef362f..108b5e9f82cb 100644 --- a/drivers/media/platform/omap3isp/ispccdc.c +++ b/drivers/media/platform/omap3isp/ispccdc.c @@ -29,7 +29,8 @@ #define CCDC_MIN_HEIGHT 32 static struct v4l2_mbus_framefmt * -__ccdc_get_format(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg, +__ccdc_get_format(struct isp_ccdc_device *ccdc, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which); static const unsigned int ccdc_fmts[] = { @@ -1936,21 +1937,25 @@ static int ccdc_set_stream(struct v4l2_subdev *sd, int enable) } static struct v4l2_mbus_framefmt * -__ccdc_get_format(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg, +__ccdc_get_format(struct isp_ccdc_device *ccdc, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&ccdc->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&ccdc->subdev, sd_state, + pad); else return &ccdc->formats[pad]; } static struct v4l2_rect * -__ccdc_get_crop(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg, +__ccdc_get_crop(struct isp_ccdc_device *ccdc, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_crop(&ccdc->subdev, cfg, CCDC_PAD_SOURCE_OF); + return v4l2_subdev_get_try_crop(&ccdc->subdev, sd_state, + CCDC_PAD_SOURCE_OF); else return &ccdc->crop; } @@ -1963,7 +1968,8 @@ __ccdc_get_crop(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg * @fmt: Format */ static void -ccdc_try_format(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg, +ccdc_try_format(struct isp_ccdc_device *ccdc, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) { @@ -1999,7 +2005,8 @@ ccdc_try_format(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg case CCDC_PAD_SOURCE_OF: pixelcode = fmt->code; field = fmt->field; - *fmt = *__ccdc_get_format(ccdc, cfg, CCDC_PAD_SINK, which); + *fmt = *__ccdc_get_format(ccdc, sd_state, CCDC_PAD_SINK, + which); /* In SYNC mode the bridge converts YUV formats from 2X8 to * 1X16. In BT.656 no such conversion occurs. As we don't know @@ -2024,7 +2031,7 @@ ccdc_try_format(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg } /* Hardcode the output size to the crop rectangle size. */ - crop = __ccdc_get_crop(ccdc, cfg, which); + crop = __ccdc_get_crop(ccdc, sd_state, which); fmt->width = crop->width; fmt->height = crop->height; @@ -2041,7 +2048,8 @@ ccdc_try_format(struct isp_ccdc_device *ccdc, struct v4l2_subdev_pad_config *cfg break; case CCDC_PAD_SOURCE_VP: - *fmt = *__ccdc_get_format(ccdc, cfg, CCDC_PAD_SINK, which); + *fmt = *__ccdc_get_format(ccdc, sd_state, CCDC_PAD_SINK, + which); /* The video port interface truncates the data to 10 bits. */ info = omap3isp_video_format_info(fmt->code); @@ -2118,7 +2126,7 @@ static void ccdc_try_crop(struct isp_ccdc_device *ccdc, * return -EINVAL or zero on success */ static int ccdc_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct isp_ccdc_device *ccdc = v4l2_get_subdevdata(sd); @@ -2133,7 +2141,7 @@ static int ccdc_enum_mbus_code(struct v4l2_subdev *sd, break; case CCDC_PAD_SOURCE_OF: - format = __ccdc_get_format(ccdc, cfg, code->pad, + format = __ccdc_get_format(ccdc, sd_state, code->pad, code->which); if (format->code == MEDIA_BUS_FMT_YUYV8_2X8 || @@ -2164,7 +2172,7 @@ static int ccdc_enum_mbus_code(struct v4l2_subdev *sd, if (code->index != 0) return -EINVAL; - format = __ccdc_get_format(ccdc, cfg, code->pad, + format = __ccdc_get_format(ccdc, sd_state, code->pad, code->which); /* A pixel code equal to 0 means that the video port doesn't @@ -2184,7 +2192,7 @@ static int ccdc_enum_mbus_code(struct v4l2_subdev *sd, } static int ccdc_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct isp_ccdc_device *ccdc = v4l2_get_subdevdata(sd); @@ -2196,7 +2204,7 @@ static int ccdc_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - ccdc_try_format(ccdc, cfg, fse->pad, &format, fse->which); + ccdc_try_format(ccdc, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -2206,7 +2214,7 @@ static int ccdc_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - ccdc_try_format(ccdc, cfg, fse->pad, &format, fse->which); + ccdc_try_format(ccdc, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -2224,7 +2232,8 @@ static int ccdc_enum_frame_size(struct v4l2_subdev *sd, * * Return 0 on success or a negative error code otherwise. */ -static int ccdc_get_selection(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int ccdc_get_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct isp_ccdc_device *ccdc = v4l2_get_subdevdata(sd); @@ -2240,12 +2249,13 @@ static int ccdc_get_selection(struct v4l2_subdev *sd, struct v4l2_subdev_pad_con sel->r.width = INT_MAX; sel->r.height = INT_MAX; - format = __ccdc_get_format(ccdc, cfg, CCDC_PAD_SINK, sel->which); + format = __ccdc_get_format(ccdc, sd_state, CCDC_PAD_SINK, + sel->which); ccdc_try_crop(ccdc, format, &sel->r); break; case V4L2_SEL_TGT_CROP: - sel->r = *__ccdc_get_crop(ccdc, cfg, sel->which); + sel->r = *__ccdc_get_crop(ccdc, sd_state, sel->which); break; default: @@ -2266,7 +2276,8 @@ static int ccdc_get_selection(struct v4l2_subdev *sd, struct v4l2_subdev_pad_con * * Return 0 on success or a negative error code otherwise. */ -static int ccdc_set_selection(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int ccdc_set_selection(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct isp_ccdc_device *ccdc = v4l2_get_subdevdata(sd); @@ -2285,17 +2296,19 @@ static int ccdc_set_selection(struct v4l2_subdev *sd, struct v4l2_subdev_pad_con * rectangle. */ if (sel->flags & V4L2_SEL_FLAG_KEEP_CONFIG) { - sel->r = *__ccdc_get_crop(ccdc, cfg, sel->which); + sel->r = *__ccdc_get_crop(ccdc, sd_state, sel->which); return 0; } - format = __ccdc_get_format(ccdc, cfg, CCDC_PAD_SINK, sel->which); + format = __ccdc_get_format(ccdc, sd_state, CCDC_PAD_SINK, sel->which); ccdc_try_crop(ccdc, format, &sel->r); - *__ccdc_get_crop(ccdc, cfg, sel->which) = sel->r; + *__ccdc_get_crop(ccdc, sd_state, sel->which) = sel->r; /* Update the source format. */ - format = __ccdc_get_format(ccdc, cfg, CCDC_PAD_SOURCE_OF, sel->which); - ccdc_try_format(ccdc, cfg, CCDC_PAD_SOURCE_OF, format, sel->which); + format = __ccdc_get_format(ccdc, sd_state, CCDC_PAD_SOURCE_OF, + sel->which); + ccdc_try_format(ccdc, sd_state, CCDC_PAD_SOURCE_OF, format, + sel->which); return 0; } @@ -2309,13 +2322,14 @@ static int ccdc_set_selection(struct v4l2_subdev *sd, struct v4l2_subdev_pad_con * Return 0 on success or -EINVAL if the pad is invalid or doesn't correspond * to the format type. */ -static int ccdc_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int ccdc_get_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_ccdc_device *ccdc = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ccdc_get_format(ccdc, cfg, fmt->pad, fmt->which); + format = __ccdc_get_format(ccdc, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -2332,24 +2346,25 @@ static int ccdc_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config * Return 0 on success or -EINVAL if the pad is invalid or doesn't correspond * to the format type. */ -static int ccdc_set_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int ccdc_set_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_ccdc_device *ccdc = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; - format = __ccdc_get_format(ccdc, cfg, fmt->pad, fmt->which); + format = __ccdc_get_format(ccdc, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - ccdc_try_format(ccdc, cfg, fmt->pad, &fmt->format, fmt->which); + ccdc_try_format(ccdc, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == CCDC_PAD_SINK) { /* Reset the crop rectangle. */ - crop = __ccdc_get_crop(ccdc, cfg, fmt->which); + crop = __ccdc_get_crop(ccdc, sd_state, fmt->which); crop->left = 0; crop->top = 0; crop->width = fmt->format.width; @@ -2358,16 +2373,16 @@ static int ccdc_set_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config ccdc_try_crop(ccdc, &fmt->format, crop); /* Update the source formats. */ - format = __ccdc_get_format(ccdc, cfg, CCDC_PAD_SOURCE_OF, + format = __ccdc_get_format(ccdc, sd_state, CCDC_PAD_SOURCE_OF, fmt->which); *format = fmt->format; - ccdc_try_format(ccdc, cfg, CCDC_PAD_SOURCE_OF, format, + ccdc_try_format(ccdc, sd_state, CCDC_PAD_SOURCE_OF, format, fmt->which); - format = __ccdc_get_format(ccdc, cfg, CCDC_PAD_SOURCE_VP, + format = __ccdc_get_format(ccdc, sd_state, CCDC_PAD_SOURCE_VP, fmt->which); *format = fmt->format; - ccdc_try_format(ccdc, cfg, CCDC_PAD_SOURCE_VP, format, + ccdc_try_format(ccdc, sd_state, CCDC_PAD_SOURCE_VP, format, fmt->which); } @@ -2454,7 +2469,7 @@ static int ccdc_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10; format.format.width = 4096; format.format.height = 4096; - ccdc_set_format(sd, fh ? fh->pad : NULL, &format); + ccdc_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/media/platform/omap3isp/ispccp2.c b/drivers/media/platform/omap3isp/ispccp2.c index d0a49cdfd22d..acb58b6ddba1 100644 --- a/drivers/media/platform/omap3isp/ispccp2.c +++ b/drivers/media/platform/omap3isp/ispccp2.c @@ -618,11 +618,13 @@ static const unsigned int ccp2_fmts[] = { * return format structure or NULL on error */ static struct v4l2_mbus_framefmt * -__ccp2_get_format(struct isp_ccp2_device *ccp2, struct v4l2_subdev_pad_config *cfg, - unsigned int pad, enum v4l2_subdev_format_whence which) +__ccp2_get_format(struct isp_ccp2_device *ccp2, + struct v4l2_subdev_state *sd_state, + unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&ccp2->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&ccp2->subdev, sd_state, + pad); else return &ccp2->formats[pad]; } @@ -636,7 +638,8 @@ __ccp2_get_format(struct isp_ccp2_device *ccp2, struct v4l2_subdev_pad_config *c * @which : wanted subdev format */ static void ccp2_try_format(struct isp_ccp2_device *ccp2, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, + unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) { @@ -670,7 +673,8 @@ static void ccp2_try_format(struct isp_ccp2_device *ccp2, * When CCP2 write to memory feature will be added this * should be changed properly. */ - format = __ccp2_get_format(ccp2, cfg, CCP2_PAD_SINK, which); + format = __ccp2_get_format(ccp2, sd_state, CCP2_PAD_SINK, + which); memcpy(fmt, format, sizeof(*fmt)); fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10; break; @@ -688,7 +692,7 @@ static void ccp2_try_format(struct isp_ccp2_device *ccp2, * return -EINVAL or zero on success */ static int ccp2_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct isp_ccp2_device *ccp2 = v4l2_get_subdevdata(sd); @@ -703,8 +707,8 @@ static int ccp2_enum_mbus_code(struct v4l2_subdev *sd, if (code->index != 0) return -EINVAL; - format = __ccp2_get_format(ccp2, cfg, CCP2_PAD_SINK, - code->which); + format = __ccp2_get_format(ccp2, sd_state, CCP2_PAD_SINK, + code->which); code->code = format->code; } @@ -712,7 +716,7 @@ static int ccp2_enum_mbus_code(struct v4l2_subdev *sd, } static int ccp2_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct isp_ccp2_device *ccp2 = v4l2_get_subdevdata(sd); @@ -724,7 +728,7 @@ static int ccp2_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - ccp2_try_format(ccp2, cfg, fse->pad, &format, fse->which); + ccp2_try_format(ccp2, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -734,7 +738,7 @@ static int ccp2_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - ccp2_try_format(ccp2, cfg, fse->pad, &format, fse->which); + ccp2_try_format(ccp2, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -748,13 +752,14 @@ static int ccp2_enum_frame_size(struct v4l2_subdev *sd, * @fmt : pointer to v4l2 subdev format structure * return -EINVAL or zero on success */ -static int ccp2_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *fmt) +static int ccp2_get_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) { struct isp_ccp2_device *ccp2 = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ccp2_get_format(ccp2, cfg, fmt->pad, fmt->which); + format = __ccp2_get_format(ccp2, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -769,25 +774,27 @@ static int ccp2_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config * @fmt : pointer to v4l2 subdev format structure * returns zero */ -static int ccp2_set_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *fmt) +static int ccp2_set_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *fmt) { struct isp_ccp2_device *ccp2 = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ccp2_get_format(ccp2, cfg, fmt->pad, fmt->which); + format = __ccp2_get_format(ccp2, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - ccp2_try_format(ccp2, cfg, fmt->pad, &fmt->format, fmt->which); + ccp2_try_format(ccp2, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == CCP2_PAD_SINK) { - format = __ccp2_get_format(ccp2, cfg, CCP2_PAD_SOURCE, + format = __ccp2_get_format(ccp2, sd_state, CCP2_PAD_SOURCE, fmt->which); *format = fmt->format; - ccp2_try_format(ccp2, cfg, CCP2_PAD_SOURCE, format, fmt->which); + ccp2_try_format(ccp2, sd_state, CCP2_PAD_SOURCE, format, + fmt->which); } return 0; @@ -812,7 +819,7 @@ static int ccp2_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10; format.format.width = 4096; format.format.height = 4096; - ccp2_set_format(sd, fh ? fh->pad : NULL, &format); + ccp2_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/media/platform/omap3isp/ispcsi2.c b/drivers/media/platform/omap3isp/ispcsi2.c index fd493c5e4e24..6302e0c94034 100644 --- a/drivers/media/platform/omap3isp/ispcsi2.c +++ b/drivers/media/platform/omap3isp/ispcsi2.c @@ -827,17 +827,20 @@ static const struct isp_video_operations csi2_ispvideo_ops = { */ static struct v4l2_mbus_framefmt * -__csi2_get_format(struct isp_csi2_device *csi2, struct v4l2_subdev_pad_config *cfg, +__csi2_get_format(struct isp_csi2_device *csi2, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&csi2->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&csi2->subdev, sd_state, + pad); else return &csi2->formats[pad]; } static void -csi2_try_format(struct isp_csi2_device *csi2, struct v4l2_subdev_pad_config *cfg, +csi2_try_format(struct isp_csi2_device *csi2, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) { @@ -867,7 +870,8 @@ csi2_try_format(struct isp_csi2_device *csi2, struct v4l2_subdev_pad_config *cfg * compression. */ pixelcode = fmt->code; - format = __csi2_get_format(csi2, cfg, CSI2_PAD_SINK, which); + format = __csi2_get_format(csi2, sd_state, CSI2_PAD_SINK, + which); memcpy(fmt, format, sizeof(*fmt)); /* @@ -893,7 +897,7 @@ csi2_try_format(struct isp_csi2_device *csi2, struct v4l2_subdev_pad_config *cfg * return -EINVAL or zero on success */ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct isp_csi2_device *csi2 = v4l2_get_subdevdata(sd); @@ -906,7 +910,7 @@ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, code->code = csi2_input_fmts[code->index]; } else { - format = __csi2_get_format(csi2, cfg, CSI2_PAD_SINK, + format = __csi2_get_format(csi2, sd_state, CSI2_PAD_SINK, code->which); switch (code->index) { case 0: @@ -930,7 +934,7 @@ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, } static int csi2_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct isp_csi2_device *csi2 = v4l2_get_subdevdata(sd); @@ -942,7 +946,7 @@ static int csi2_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - csi2_try_format(csi2, cfg, fse->pad, &format, fse->which); + csi2_try_format(csi2, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -952,7 +956,7 @@ static int csi2_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - csi2_try_format(csi2, cfg, fse->pad, &format, fse->which); + csi2_try_format(csi2, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -966,13 +970,14 @@ static int csi2_enum_frame_size(struct v4l2_subdev *sd, * @fmt: pointer to v4l2 subdev format structure * return -EINVAL or zero on success */ -static int csi2_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int csi2_get_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_csi2_device *csi2 = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csi2_get_format(csi2, cfg, fmt->pad, fmt->which); + format = __csi2_get_format(csi2, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -987,25 +992,27 @@ static int csi2_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config * @fmt: pointer to v4l2 subdev format structure * return -EINVAL or zero on success */ -static int csi2_set_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int csi2_set_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_csi2_device *csi2 = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csi2_get_format(csi2, cfg, fmt->pad, fmt->which); + format = __csi2_get_format(csi2, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - csi2_try_format(csi2, cfg, fmt->pad, &fmt->format, fmt->which); + csi2_try_format(csi2, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == CSI2_PAD_SINK) { - format = __csi2_get_format(csi2, cfg, CSI2_PAD_SOURCE, + format = __csi2_get_format(csi2, sd_state, CSI2_PAD_SOURCE, fmt->which); *format = fmt->format; - csi2_try_format(csi2, cfg, CSI2_PAD_SOURCE, format, fmt->which); + csi2_try_format(csi2, sd_state, CSI2_PAD_SOURCE, format, + fmt->which); } return 0; @@ -1030,7 +1037,7 @@ static int csi2_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10; format.format.width = 4096; format.format.height = 4096; - csi2_set_format(sd, fh ? fh->pad : NULL, &format); + csi2_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/media/platform/omap3isp/isppreview.c b/drivers/media/platform/omap3isp/isppreview.c index 607b7685c982..53aedec7990d 100644 --- a/drivers/media/platform/omap3isp/isppreview.c +++ b/drivers/media/platform/omap3isp/isppreview.c @@ -1679,21 +1679,25 @@ static int preview_set_stream(struct v4l2_subdev *sd, int enable) } static struct v4l2_mbus_framefmt * -__preview_get_format(struct isp_prev_device *prev, struct v4l2_subdev_pad_config *cfg, +__preview_get_format(struct isp_prev_device *prev, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&prev->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&prev->subdev, sd_state, + pad); else return &prev->formats[pad]; } static struct v4l2_rect * -__preview_get_crop(struct isp_prev_device *prev, struct v4l2_subdev_pad_config *cfg, +__preview_get_crop(struct isp_prev_device *prev, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_crop(&prev->subdev, cfg, PREV_PAD_SINK); + return v4l2_subdev_get_try_crop(&prev->subdev, sd_state, + PREV_PAD_SINK); else return &prev->crop; } @@ -1729,7 +1733,8 @@ static const unsigned int preview_output_fmts[] = { * engine limits and the format and crop rectangles on other pads. */ static void preview_try_format(struct isp_prev_device *prev, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, + unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) { @@ -1770,7 +1775,8 @@ static void preview_try_format(struct isp_prev_device *prev, case PREV_PAD_SOURCE: pixelcode = fmt->code; - *fmt = *__preview_get_format(prev, cfg, PREV_PAD_SINK, which); + *fmt = *__preview_get_format(prev, sd_state, PREV_PAD_SINK, + which); switch (pixelcode) { case MEDIA_BUS_FMT_YUYV8_1X16: @@ -1788,7 +1794,7 @@ static void preview_try_format(struct isp_prev_device *prev, * is not supported yet, hardcode the output size to the crop * rectangle size. */ - crop = __preview_get_crop(prev, cfg, which); + crop = __preview_get_crop(prev, sd_state, which); fmt->width = crop->width; fmt->height = crop->height; @@ -1862,7 +1868,7 @@ static void preview_try_crop(struct isp_prev_device *prev, * return -EINVAL or zero on success */ static int preview_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { switch (code->pad) { @@ -1886,7 +1892,7 @@ static int preview_enum_mbus_code(struct v4l2_subdev *sd, } static int preview_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct isp_prev_device *prev = v4l2_get_subdevdata(sd); @@ -1898,7 +1904,7 @@ static int preview_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - preview_try_format(prev, cfg, fse->pad, &format, fse->which); + preview_try_format(prev, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -1908,7 +1914,7 @@ static int preview_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - preview_try_format(prev, cfg, fse->pad, &format, fse->which); + preview_try_format(prev, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -1926,7 +1932,7 @@ static int preview_enum_frame_size(struct v4l2_subdev *sd, * Return 0 on success or a negative error code otherwise. */ static int preview_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct isp_prev_device *prev = v4l2_get_subdevdata(sd); @@ -1942,13 +1948,13 @@ static int preview_get_selection(struct v4l2_subdev *sd, sel->r.width = INT_MAX; sel->r.height = INT_MAX; - format = __preview_get_format(prev, cfg, PREV_PAD_SINK, + format = __preview_get_format(prev, sd_state, PREV_PAD_SINK, sel->which); preview_try_crop(prev, format, &sel->r); break; case V4L2_SEL_TGT_CROP: - sel->r = *__preview_get_crop(prev, cfg, sel->which); + sel->r = *__preview_get_crop(prev, sd_state, sel->which); break; default: @@ -1969,7 +1975,7 @@ static int preview_get_selection(struct v4l2_subdev *sd, * Return 0 on success or a negative error code otherwise. */ static int preview_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct isp_prev_device *prev = v4l2_get_subdevdata(sd); @@ -1988,17 +1994,20 @@ static int preview_set_selection(struct v4l2_subdev *sd, * rectangle. */ if (sel->flags & V4L2_SEL_FLAG_KEEP_CONFIG) { - sel->r = *__preview_get_crop(prev, cfg, sel->which); + sel->r = *__preview_get_crop(prev, sd_state, sel->which); return 0; } - format = __preview_get_format(prev, cfg, PREV_PAD_SINK, sel->which); + format = __preview_get_format(prev, sd_state, PREV_PAD_SINK, + sel->which); preview_try_crop(prev, format, &sel->r); - *__preview_get_crop(prev, cfg, sel->which) = sel->r; + *__preview_get_crop(prev, sd_state, sel->which) = sel->r; /* Update the source format. */ - format = __preview_get_format(prev, cfg, PREV_PAD_SOURCE, sel->which); - preview_try_format(prev, cfg, PREV_PAD_SOURCE, format, sel->which); + format = __preview_get_format(prev, sd_state, PREV_PAD_SOURCE, + sel->which); + preview_try_format(prev, sd_state, PREV_PAD_SOURCE, format, + sel->which); return 0; } @@ -2010,13 +2019,14 @@ static int preview_set_selection(struct v4l2_subdev *sd, * @fmt: pointer to v4l2 subdev format structure * return -EINVAL or zero on success */ -static int preview_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int preview_get_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_prev_device *prev = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __preview_get_format(prev, cfg, fmt->pad, fmt->which); + format = __preview_get_format(prev, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -2031,24 +2041,25 @@ static int preview_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_con * @fmt: pointer to v4l2 subdev format structure * return -EINVAL or zero on success */ -static int preview_set_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int preview_set_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_prev_device *prev = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; - format = __preview_get_format(prev, cfg, fmt->pad, fmt->which); + format = __preview_get_format(prev, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - preview_try_format(prev, cfg, fmt->pad, &fmt->format, fmt->which); + preview_try_format(prev, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == PREV_PAD_SINK) { /* Reset the crop rectangle. */ - crop = __preview_get_crop(prev, cfg, fmt->which); + crop = __preview_get_crop(prev, sd_state, fmt->which); crop->left = 0; crop->top = 0; crop->width = fmt->format.width; @@ -2057,9 +2068,9 @@ static int preview_set_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_con preview_try_crop(prev, &fmt->format, crop); /* Update the source format. */ - format = __preview_get_format(prev, cfg, PREV_PAD_SOURCE, + format = __preview_get_format(prev, sd_state, PREV_PAD_SOURCE, fmt->which); - preview_try_format(prev, cfg, PREV_PAD_SOURCE, format, + preview_try_format(prev, sd_state, PREV_PAD_SOURCE, format, fmt->which); } @@ -2086,7 +2097,7 @@ static int preview_init_formats(struct v4l2_subdev *sd, format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10; format.format.width = 4096; format.format.height = 4096; - preview_set_format(sd, fh ? fh->pad : NULL, &format); + preview_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/media/platform/omap3isp/ispresizer.c b/drivers/media/platform/omap3isp/ispresizer.c index 78d9dd7ea2da..ed2fb0c7a57e 100644 --- a/drivers/media/platform/omap3isp/ispresizer.c +++ b/drivers/media/platform/omap3isp/ispresizer.c @@ -114,11 +114,12 @@ static const struct isprsz_coef filter_coefs = { * return zero */ static struct v4l2_mbus_framefmt * -__resizer_get_format(struct isp_res_device *res, struct v4l2_subdev_pad_config *cfg, +__resizer_get_format(struct isp_res_device *res, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&res->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&res->subdev, sd_state, pad); else return &res->formats[pad]; } @@ -130,11 +131,13 @@ __resizer_get_format(struct isp_res_device *res, struct v4l2_subdev_pad_config * * @which : wanted subdev crop rectangle */ static struct v4l2_rect * -__resizer_get_crop(struct isp_res_device *res, struct v4l2_subdev_pad_config *cfg, +__resizer_get_crop(struct isp_res_device *res, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_crop(&res->subdev, cfg, RESZ_PAD_SINK); + return v4l2_subdev_get_try_crop(&res->subdev, sd_state, + RESZ_PAD_SINK); else return &res->crop.request; } @@ -1220,7 +1223,7 @@ static void resizer_try_crop(const struct v4l2_mbus_framefmt *sink, * Return 0 on success or a negative error code otherwise. */ static int resizer_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct isp_res_device *res = v4l2_get_subdevdata(sd); @@ -1231,9 +1234,9 @@ static int resizer_get_selection(struct v4l2_subdev *sd, if (sel->pad != RESZ_PAD_SINK) return -EINVAL; - format_sink = __resizer_get_format(res, cfg, RESZ_PAD_SINK, + format_sink = __resizer_get_format(res, sd_state, RESZ_PAD_SINK, sel->which); - format_source = __resizer_get_format(res, cfg, RESZ_PAD_SOURCE, + format_source = __resizer_get_format(res, sd_state, RESZ_PAD_SOURCE, sel->which); switch (sel->target) { @@ -1248,7 +1251,7 @@ static int resizer_get_selection(struct v4l2_subdev *sd, break; case V4L2_SEL_TGT_CROP: - sel->r = *__resizer_get_crop(res, cfg, sel->which); + sel->r = *__resizer_get_crop(res, sd_state, sel->which); resizer_calc_ratios(res, &sel->r, format_source, &ratio); break; @@ -1273,7 +1276,7 @@ static int resizer_get_selection(struct v4l2_subdev *sd, * Return 0 on success or a negative error code otherwise. */ static int resizer_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct isp_res_device *res = v4l2_get_subdevdata(sd); @@ -1287,9 +1290,9 @@ static int resizer_set_selection(struct v4l2_subdev *sd, sel->pad != RESZ_PAD_SINK) return -EINVAL; - format_sink = __resizer_get_format(res, cfg, RESZ_PAD_SINK, + format_sink = __resizer_get_format(res, sd_state, RESZ_PAD_SINK, sel->which); - format_source = *__resizer_get_format(res, cfg, RESZ_PAD_SOURCE, + format_source = *__resizer_get_format(res, sd_state, RESZ_PAD_SOURCE, sel->which); dev_dbg(isp->dev, "%s(%s): req %ux%u -> (%d,%d)/%ux%u -> %ux%u\n", @@ -1307,7 +1310,7 @@ static int resizer_set_selection(struct v4l2_subdev *sd, * stored the mangled rectangle. */ resizer_try_crop(format_sink, &format_source, &sel->r); - *__resizer_get_crop(res, cfg, sel->which) = sel->r; + *__resizer_get_crop(res, sd_state, sel->which) = sel->r; resizer_calc_ratios(res, &sel->r, &format_source, &ratio); dev_dbg(isp->dev, "%s(%s): got %ux%u -> (%d,%d)/%ux%u -> %ux%u\n", @@ -1317,7 +1320,8 @@ static int resizer_set_selection(struct v4l2_subdev *sd, format_source.width, format_source.height); if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { - *__resizer_get_format(res, cfg, RESZ_PAD_SOURCE, sel->which) = + *__resizer_get_format(res, sd_state, RESZ_PAD_SOURCE, + sel->which) = format_source; return 0; } @@ -1328,7 +1332,7 @@ static int resizer_set_selection(struct v4l2_subdev *sd, */ spin_lock_irqsave(&res->lock, flags); - *__resizer_get_format(res, cfg, RESZ_PAD_SOURCE, sel->which) = + *__resizer_get_format(res, sd_state, RESZ_PAD_SOURCE, sel->which) = format_source; res->ratio = ratio; @@ -1371,7 +1375,8 @@ static unsigned int resizer_max_in_width(struct isp_res_device *res) * @which : wanted subdev format */ static void resizer_try_format(struct isp_res_device *res, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, + unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) { @@ -1392,10 +1397,11 @@ static void resizer_try_format(struct isp_res_device *res, break; case RESZ_PAD_SOURCE: - format = __resizer_get_format(res, cfg, RESZ_PAD_SINK, which); + format = __resizer_get_format(res, sd_state, RESZ_PAD_SINK, + which); fmt->code = format->code; - crop = *__resizer_get_crop(res, cfg, which); + crop = *__resizer_get_crop(res, sd_state, which); resizer_calc_ratios(res, &crop, fmt, &ratio); break; } @@ -1412,7 +1418,7 @@ static void resizer_try_format(struct isp_res_device *res, * return -EINVAL or zero on success */ static int resizer_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct isp_res_device *res = v4l2_get_subdevdata(sd); @@ -1427,7 +1433,7 @@ static int resizer_enum_mbus_code(struct v4l2_subdev *sd, if (code->index != 0) return -EINVAL; - format = __resizer_get_format(res, cfg, RESZ_PAD_SINK, + format = __resizer_get_format(res, sd_state, RESZ_PAD_SINK, code->which); code->code = format->code; } @@ -1436,7 +1442,7 @@ static int resizer_enum_mbus_code(struct v4l2_subdev *sd, } static int resizer_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct isp_res_device *res = v4l2_get_subdevdata(sd); @@ -1448,7 +1454,7 @@ static int resizer_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - resizer_try_format(res, cfg, fse->pad, &format, fse->which); + resizer_try_format(res, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -1458,7 +1464,7 @@ static int resizer_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - resizer_try_format(res, cfg, fse->pad, &format, fse->which); + resizer_try_format(res, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -1472,13 +1478,14 @@ static int resizer_enum_frame_size(struct v4l2_subdev *sd, * @fmt : pointer to v4l2 subdev format structure * return -EINVAL or zero on success */ -static int resizer_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int resizer_get_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_res_device *res = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __resizer_get_format(res, cfg, fmt->pad, fmt->which); + format = __resizer_get_format(res, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -1493,33 +1500,34 @@ static int resizer_get_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_con * @fmt : pointer to v4l2 subdev format structure * return -EINVAL or zero on success */ -static int resizer_set_format(struct v4l2_subdev *sd, struct v4l2_subdev_pad_config *cfg, +static int resizer_set_format(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct isp_res_device *res = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; - format = __resizer_get_format(res, cfg, fmt->pad, fmt->which); + format = __resizer_get_format(res, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - resizer_try_format(res, cfg, fmt->pad, &fmt->format, fmt->which); + resizer_try_format(res, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; if (fmt->pad == RESZ_PAD_SINK) { /* reset crop rectangle */ - crop = __resizer_get_crop(res, cfg, fmt->which); + crop = __resizer_get_crop(res, sd_state, fmt->which); crop->left = 0; crop->top = 0; crop->width = fmt->format.width; crop->height = fmt->format.height; /* Propagate the format from sink to source */ - format = __resizer_get_format(res, cfg, RESZ_PAD_SOURCE, + format = __resizer_get_format(res, sd_state, RESZ_PAD_SOURCE, fmt->which); *format = fmt->format; - resizer_try_format(res, cfg, RESZ_PAD_SOURCE, format, + resizer_try_format(res, sd_state, RESZ_PAD_SOURCE, format, fmt->which); } @@ -1570,7 +1578,7 @@ static int resizer_init_formats(struct v4l2_subdev *sd, format.format.code = MEDIA_BUS_FMT_YUYV8_1X16; format.format.width = 4096; format.format.height = 4096; - resizer_set_format(sd, fh ? fh->pad : NULL, &format); + resizer_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/media/platform/pxa_camera.c b/drivers/media/platform/pxa_camera.c index dd510ee9b58a..ec4c010644ca 100644 --- a/drivers/media/platform/pxa_camera.c +++ b/drivers/media/platform/pxa_camera.c @@ -1792,6 +1792,9 @@ static int pxac_vidioc_try_fmt_vid_cap(struct file *filp, void *priv, const struct pxa_camera_format_xlate *xlate; struct v4l2_pix_format *pix = &f->fmt.pix; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -1816,7 +1819,7 @@ static int pxac_vidioc_try_fmt_vid_cap(struct file *filp, void *priv, pixfmt == V4L2_PIX_FMT_YUV422P ? 4 : 0); v4l2_fill_mbus_format(mf, pix, xlate->code); - ret = sensor_call(pcdev, pad, set_fmt, &pad_cfg, &format); + ret = sensor_call(pcdev, pad, set_fmt, &pad_state, &format); if (ret < 0) return ret; diff --git a/drivers/media/platform/qcom/camss/camss-csid.c b/drivers/media/platform/qcom/camss/camss-csid.c index cc11fbfdae13..a1637b78568b 100644 --- a/drivers/media/platform/qcom/camss/camss-csid.c +++ b/drivers/media/platform/qcom/camss/camss-csid.c @@ -156,11 +156,9 @@ static int csid_set_power(struct v4l2_subdev *sd, int on) int ret; if (on) { - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put_sync(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) return ret; - } ret = regulator_enable(csid->vdda); if (ret < 0) { @@ -247,12 +245,13 @@ static int csid_set_stream(struct v4l2_subdev *sd, int enable) */ static struct v4l2_mbus_framefmt * __csid_get_format(struct csid_device *csid, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&csid->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&csid->subdev, sd_state, + pad); return &csid->fmt[pad]; } @@ -266,7 +265,7 @@ __csid_get_format(struct csid_device *csid, * @which: wanted subdev format */ static void csid_try_format(struct csid_device *csid, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) @@ -299,7 +298,7 @@ static void csid_try_format(struct csid_device *csid, /* keep pad formats in sync */ u32 code = fmt->code; - *fmt = *__csid_get_format(csid, cfg, + *fmt = *__csid_get_format(csid, sd_state, MSM_CSID_PAD_SINK, which); fmt->code = csid->ops->src_pad_code(csid, fmt->code, 0, code); } else { @@ -333,7 +332,7 @@ static void csid_try_format(struct csid_device *csid, * return -EINVAL or zero on success */ static int csid_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct csid_device *csid = v4l2_get_subdevdata(sd); @@ -347,7 +346,7 @@ static int csid_enum_mbus_code(struct v4l2_subdev *sd, if (csid->testgen_mode->cur.val == 0) { struct v4l2_mbus_framefmt *sink_fmt; - sink_fmt = __csid_get_format(csid, cfg, + sink_fmt = __csid_get_format(csid, sd_state, MSM_CSID_PAD_SINK, code->which); @@ -374,7 +373,7 @@ static int csid_enum_mbus_code(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int csid_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct csid_device *csid = v4l2_get_subdevdata(sd); @@ -386,7 +385,7 @@ static int csid_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - csid_try_format(csid, cfg, fse->pad, &format, fse->which); + csid_try_format(csid, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -396,7 +395,7 @@ static int csid_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - csid_try_format(csid, cfg, fse->pad, &format, fse->which); + csid_try_format(csid, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -412,13 +411,13 @@ static int csid_enum_frame_size(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int csid_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct csid_device *csid = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csid_get_format(csid, cfg, fmt->pad, fmt->which); + format = __csid_get_format(csid, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -436,26 +435,26 @@ static int csid_get_format(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int csid_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct csid_device *csid = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csid_get_format(csid, cfg, fmt->pad, fmt->which); + format = __csid_get_format(csid, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - csid_try_format(csid, cfg, fmt->pad, &fmt->format, fmt->which); + csid_try_format(csid, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == MSM_CSID_PAD_SINK) { - format = __csid_get_format(csid, cfg, MSM_CSID_PAD_SRC, + format = __csid_get_format(csid, sd_state, MSM_CSID_PAD_SRC, fmt->which); *format = fmt->format; - csid_try_format(csid, cfg, MSM_CSID_PAD_SRC, format, + csid_try_format(csid, sd_state, MSM_CSID_PAD_SRC, format, fmt->which); } @@ -484,7 +483,7 @@ static int csid_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) } }; - return csid_set_format(sd, fh ? fh->pad : NULL, &format); + return csid_set_format(sd, fh ? fh->state : NULL, &format); } /* @@ -566,8 +565,7 @@ int msm_csid_subdev_init(struct camss *camss, struct csid_device *csid, /* Memory */ - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]); - csid->base = devm_ioremap_resource(dev, r); + csid->base = devm_platform_ioremap_resource_byname(pdev, res->reg[0]); if (IS_ERR(csid->base)) return PTR_ERR(csid->base); @@ -584,14 +582,13 @@ int msm_csid_subdev_init(struct camss *camss, struct csid_device *csid, snprintf(csid->irq_name, sizeof(csid->irq_name), "%s_%s%d", dev_name(dev), MSM_CSID_NAME, csid->id); ret = devm_request_irq(dev, csid->irq, csid->ops->isr, - IRQF_TRIGGER_RISING, csid->irq_name, csid); + IRQF_TRIGGER_RISING | IRQF_NO_AUTOEN, + csid->irq_name, csid); if (ret < 0) { dev_err(dev, "request_irq failed: %d\n", ret); return ret; } - disable_irq(csid->irq); - /* Clocks */ csid->nclocks = 0; diff --git a/drivers/media/platform/qcom/camss/camss-csiphy.c b/drivers/media/platform/qcom/camss/camss-csiphy.c index b3c3bf19e522..24eec16197e7 100644 --- a/drivers/media/platform/qcom/camss/camss-csiphy.c +++ b/drivers/media/platform/qcom/camss/camss-csiphy.c @@ -197,11 +197,9 @@ static int csiphy_set_power(struct v4l2_subdev *sd, int on) if (on) { int ret; - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put_sync(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) return ret; - } ret = csiphy_set_clock_rates(csiphy); if (ret < 0) { @@ -340,12 +338,13 @@ static int csiphy_set_stream(struct v4l2_subdev *sd, int enable) */ static struct v4l2_mbus_framefmt * __csiphy_get_format(struct csiphy_device *csiphy, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&csiphy->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&csiphy->subdev, sd_state, + pad); return &csiphy->fmt[pad]; } @@ -359,7 +358,7 @@ __csiphy_get_format(struct csiphy_device *csiphy, * @which: wanted subdev format */ static void csiphy_try_format(struct csiphy_device *csiphy, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) @@ -389,7 +388,8 @@ static void csiphy_try_format(struct csiphy_device *csiphy, case MSM_CSIPHY_PAD_SRC: /* Set and return a format same as sink pad */ - *fmt = *__csiphy_get_format(csiphy, cfg, MSM_CSID_PAD_SINK, + *fmt = *__csiphy_get_format(csiphy, sd_state, + MSM_CSID_PAD_SINK, which); break; @@ -404,7 +404,7 @@ static void csiphy_try_format(struct csiphy_device *csiphy, * return -EINVAL or zero on success */ static int csiphy_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct csiphy_device *csiphy = v4l2_get_subdevdata(sd); @@ -419,7 +419,8 @@ static int csiphy_enum_mbus_code(struct v4l2_subdev *sd, if (code->index > 0) return -EINVAL; - format = __csiphy_get_format(csiphy, cfg, MSM_CSIPHY_PAD_SINK, + format = __csiphy_get_format(csiphy, sd_state, + MSM_CSIPHY_PAD_SINK, code->which); code->code = format->code; @@ -436,7 +437,7 @@ static int csiphy_enum_mbus_code(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int csiphy_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct csiphy_device *csiphy = v4l2_get_subdevdata(sd); @@ -448,7 +449,7 @@ static int csiphy_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - csiphy_try_format(csiphy, cfg, fse->pad, &format, fse->which); + csiphy_try_format(csiphy, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -458,7 +459,7 @@ static int csiphy_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - csiphy_try_format(csiphy, cfg, fse->pad, &format, fse->which); + csiphy_try_format(csiphy, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -474,13 +475,13 @@ static int csiphy_enum_frame_size(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int csiphy_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct csiphy_device *csiphy = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csiphy_get_format(csiphy, cfg, fmt->pad, fmt->which); + format = __csiphy_get_format(csiphy, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -498,26 +499,29 @@ static int csiphy_get_format(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int csiphy_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct csiphy_device *csiphy = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csiphy_get_format(csiphy, cfg, fmt->pad, fmt->which); + format = __csiphy_get_format(csiphy, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - csiphy_try_format(csiphy, cfg, fmt->pad, &fmt->format, fmt->which); + csiphy_try_format(csiphy, sd_state, fmt->pad, &fmt->format, + fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == MSM_CSIPHY_PAD_SINK) { - format = __csiphy_get_format(csiphy, cfg, MSM_CSIPHY_PAD_SRC, + format = __csiphy_get_format(csiphy, sd_state, + MSM_CSIPHY_PAD_SRC, fmt->which); *format = fmt->format; - csiphy_try_format(csiphy, cfg, MSM_CSIPHY_PAD_SRC, format, + csiphy_try_format(csiphy, sd_state, MSM_CSIPHY_PAD_SRC, + format, fmt->which); } @@ -547,7 +551,7 @@ static int csiphy_init_formats(struct v4l2_subdev *sd, } }; - return csiphy_set_format(sd, fh ? fh->pad : NULL, &format); + return csiphy_set_format(sd, fh ? fh->state : NULL, &format); } /* @@ -591,16 +595,14 @@ int msm_csiphy_subdev_init(struct camss *camss, /* Memory */ - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]); - csiphy->base = devm_ioremap_resource(dev, r); + csiphy->base = devm_platform_ioremap_resource_byname(pdev, res->reg[0]); if (IS_ERR(csiphy->base)) return PTR_ERR(csiphy->base); if (camss->version == CAMSS_8x16 || camss->version == CAMSS_8x96) { - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, - res->reg[1]); - csiphy->base_clk_mux = devm_ioremap_resource(dev, r); + csiphy->base_clk_mux = + devm_platform_ioremap_resource_byname(pdev, res->reg[1]); if (IS_ERR(csiphy->base_clk_mux)) return PTR_ERR(csiphy->base_clk_mux); } else { @@ -621,14 +623,13 @@ int msm_csiphy_subdev_init(struct camss *camss, dev_name(dev), MSM_CSIPHY_NAME, csiphy->id); ret = devm_request_irq(dev, csiphy->irq, csiphy->ops->isr, - IRQF_TRIGGER_RISING, csiphy->irq_name, csiphy); + IRQF_TRIGGER_RISING | IRQF_NO_AUTOEN, + csiphy->irq_name, csiphy); if (ret < 0) { dev_err(dev, "request_irq failed: %d\n", ret); return ret; } - disable_irq(csiphy->irq); - /* Clocks */ csiphy->nclocks = 0; diff --git a/drivers/media/platform/qcom/camss/camss-ispif.c b/drivers/media/platform/qcom/camss/camss-ispif.c index 37611c8861da..ba5d65f6ef34 100644 --- a/drivers/media/platform/qcom/camss/camss-ispif.c +++ b/drivers/media/platform/qcom/camss/camss-ispif.c @@ -372,11 +372,9 @@ static int ispif_set_power(struct v4l2_subdev *sd, int on) goto exit; } - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put_sync(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) goto exit; - } ret = camss_enable_clocks(ispif->nclocks, ispif->clock, dev); if (ret < 0) { @@ -876,12 +874,13 @@ static int ispif_set_stream(struct v4l2_subdev *sd, int enable) */ static struct v4l2_mbus_framefmt * __ispif_get_format(struct ispif_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&line->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&line->subdev, sd_state, + pad); return &line->fmt[pad]; } @@ -895,7 +894,7 @@ __ispif_get_format(struct ispif_line *line, * @which: wanted subdev format */ static void ispif_try_format(struct ispif_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) @@ -925,7 +924,7 @@ static void ispif_try_format(struct ispif_line *line, case MSM_ISPIF_PAD_SRC: /* Set and return a format same as sink pad */ - *fmt = *__ispif_get_format(line, cfg, MSM_ISPIF_PAD_SINK, + *fmt = *__ispif_get_format(line, sd_state, MSM_ISPIF_PAD_SINK, which); break; @@ -942,7 +941,7 @@ static void ispif_try_format(struct ispif_line *line, * return -EINVAL or zero on success */ static int ispif_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct ispif_line *line = v4l2_get_subdevdata(sd); @@ -957,7 +956,8 @@ static int ispif_enum_mbus_code(struct v4l2_subdev *sd, if (code->index > 0) return -EINVAL; - format = __ispif_get_format(line, cfg, MSM_ISPIF_PAD_SINK, + format = __ispif_get_format(line, sd_state, + MSM_ISPIF_PAD_SINK, code->which); code->code = format->code; @@ -974,7 +974,7 @@ static int ispif_enum_mbus_code(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int ispif_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct ispif_line *line = v4l2_get_subdevdata(sd); @@ -986,7 +986,7 @@ static int ispif_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - ispif_try_format(line, cfg, fse->pad, &format, fse->which); + ispif_try_format(line, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -996,7 +996,7 @@ static int ispif_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - ispif_try_format(line, cfg, fse->pad, &format, fse->which); + ispif_try_format(line, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -1012,13 +1012,13 @@ static int ispif_enum_frame_size(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int ispif_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ispif_line *line = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ispif_get_format(line, cfg, fmt->pad, fmt->which); + format = __ispif_get_format(line, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -1036,26 +1036,26 @@ static int ispif_get_format(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int ispif_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct ispif_line *line = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ispif_get_format(line, cfg, fmt->pad, fmt->which); + format = __ispif_get_format(line, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - ispif_try_format(line, cfg, fmt->pad, &fmt->format, fmt->which); + ispif_try_format(line, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == MSM_ISPIF_PAD_SINK) { - format = __ispif_get_format(line, cfg, MSM_ISPIF_PAD_SRC, + format = __ispif_get_format(line, sd_state, MSM_ISPIF_PAD_SRC, fmt->which); *format = fmt->format; - ispif_try_format(line, cfg, MSM_ISPIF_PAD_SRC, format, + ispif_try_format(line, sd_state, MSM_ISPIF_PAD_SRC, format, fmt->which); } @@ -1084,7 +1084,7 @@ static int ispif_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) } }; - return ispif_set_format(sd, fh ? fh->pad : NULL, &format); + return ispif_set_format(sd, fh ? fh->state : NULL, &format); } /* @@ -1143,13 +1143,11 @@ int msm_ispif_subdev_init(struct camss *camss, /* Memory */ - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]); - ispif->base = devm_ioremap_resource(dev, r); + ispif->base = devm_platform_ioremap_resource_byname(pdev, res->reg[0]); if (IS_ERR(ispif->base)) return PTR_ERR(ispif->base); - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[1]); - ispif->base_clk_mux = devm_ioremap_resource(dev, r); + ispif->base_clk_mux = devm_platform_ioremap_resource_byname(pdev, res->reg[1]); if (IS_ERR(ispif->base_clk_mux)) return PTR_ERR(ispif->base_clk_mux); diff --git a/drivers/media/platform/qcom/camss/camss-vfe.c b/drivers/media/platform/qcom/camss/camss-vfe.c index 15695fd466c4..e0f3a36f3f3f 100644 --- a/drivers/media/platform/qcom/camss/camss-vfe.c +++ b/drivers/media/platform/qcom/camss/camss-vfe.c @@ -584,9 +584,9 @@ static int vfe_get(struct vfe_device *vfe) if (ret < 0) goto error_pm_domain; - ret = pm_runtime_get_sync(vfe->camss->dev); + ret = pm_runtime_resume_and_get(vfe->camss->dev); if (ret < 0) - goto error_pm_runtime_get; + goto error_domain_off; ret = vfe_set_clock_rates(vfe); if (ret < 0) @@ -620,6 +620,7 @@ static int vfe_get(struct vfe_device *vfe) error_pm_runtime_get: pm_runtime_put_sync(vfe->camss->dev); +error_domain_off: vfe->ops->pm_domain_off(vfe); error_pm_domain: @@ -762,12 +763,13 @@ static int vfe_set_stream(struct v4l2_subdev *sd, int enable) */ static struct v4l2_mbus_framefmt * __vfe_get_format(struct vfe_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&line->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&line->subdev, sd_state, + pad); return &line->fmt[pad]; } @@ -782,11 +784,11 @@ __vfe_get_format(struct vfe_line *line, */ static struct v4l2_rect * __vfe_get_compose(struct vfe_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_compose(&line->subdev, cfg, + return v4l2_subdev_get_try_compose(&line->subdev, sd_state, MSM_VFE_PAD_SINK); return &line->compose; @@ -802,11 +804,11 @@ __vfe_get_compose(struct vfe_line *line, */ static struct v4l2_rect * __vfe_get_crop(struct vfe_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_crop(&line->subdev, cfg, + return v4l2_subdev_get_try_crop(&line->subdev, sd_state, MSM_VFE_PAD_SRC); return &line->crop; @@ -821,7 +823,7 @@ __vfe_get_crop(struct vfe_line *line, * @which: wanted subdev format */ static void vfe_try_format(struct vfe_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) @@ -853,14 +855,15 @@ static void vfe_try_format(struct vfe_line *line, /* Set and return a format same as sink pad */ code = fmt->code; - *fmt = *__vfe_get_format(line, cfg, MSM_VFE_PAD_SINK, which); + *fmt = *__vfe_get_format(line, sd_state, MSM_VFE_PAD_SINK, + which); fmt->code = vfe_src_pad_code(line, fmt->code, 0, code); if (line->id == VFE_LINE_PIX) { struct v4l2_rect *rect; - rect = __vfe_get_crop(line, cfg, which); + rect = __vfe_get_crop(line, sd_state, which); fmt->width = rect->width; fmt->height = rect->height; @@ -880,13 +883,13 @@ static void vfe_try_format(struct vfe_line *line, * @which: wanted subdev format */ static void vfe_try_compose(struct vfe_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_rect *rect, enum v4l2_subdev_format_whence which) { struct v4l2_mbus_framefmt *fmt; - fmt = __vfe_get_format(line, cfg, MSM_VFE_PAD_SINK, which); + fmt = __vfe_get_format(line, sd_state, MSM_VFE_PAD_SINK, which); if (rect->width > fmt->width) rect->width = fmt->width; @@ -919,13 +922,13 @@ static void vfe_try_compose(struct vfe_line *line, * @which: wanted subdev format */ static void vfe_try_crop(struct vfe_line *line, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_rect *rect, enum v4l2_subdev_format_whence which) { struct v4l2_rect *compose; - compose = __vfe_get_compose(line, cfg, which); + compose = __vfe_get_compose(line, sd_state, which); if (rect->width > compose->width) rect->width = compose->width; @@ -963,7 +966,7 @@ static void vfe_try_crop(struct vfe_line *line, * return -EINVAL or zero on success */ static int vfe_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct vfe_line *line = v4l2_get_subdevdata(sd); @@ -976,7 +979,7 @@ static int vfe_enum_mbus_code(struct v4l2_subdev *sd, } else { struct v4l2_mbus_framefmt *sink_fmt; - sink_fmt = __vfe_get_format(line, cfg, MSM_VFE_PAD_SINK, + sink_fmt = __vfe_get_format(line, sd_state, MSM_VFE_PAD_SINK, code->which); code->code = vfe_src_pad_code(line, sink_fmt->code, @@ -997,7 +1000,7 @@ static int vfe_enum_mbus_code(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int vfe_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct vfe_line *line = v4l2_get_subdevdata(sd); @@ -1009,7 +1012,7 @@ static int vfe_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - vfe_try_format(line, cfg, fse->pad, &format, fse->which); + vfe_try_format(line, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -1019,7 +1022,7 @@ static int vfe_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - vfe_try_format(line, cfg, fse->pad, &format, fse->which); + vfe_try_format(line, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -1035,13 +1038,13 @@ static int vfe_enum_frame_size(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int vfe_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vfe_line *line = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __vfe_get_format(line, cfg, fmt->pad, fmt->which); + format = __vfe_get_format(line, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; @@ -1051,7 +1054,7 @@ static int vfe_get_format(struct v4l2_subdev *sd, } static int vfe_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel); /* @@ -1063,17 +1066,17 @@ static int vfe_set_selection(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int vfe_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vfe_line *line = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __vfe_get_format(line, cfg, fmt->pad, fmt->which); + format = __vfe_get_format(line, sd_state, fmt->pad, fmt->which); if (format == NULL) return -EINVAL; - vfe_try_format(line, cfg, fmt->pad, &fmt->format, fmt->which); + vfe_try_format(line, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; if (fmt->pad == MSM_VFE_PAD_SINK) { @@ -1081,11 +1084,11 @@ static int vfe_set_format(struct v4l2_subdev *sd, int ret; /* Propagate the format from sink to source */ - format = __vfe_get_format(line, cfg, MSM_VFE_PAD_SRC, + format = __vfe_get_format(line, sd_state, MSM_VFE_PAD_SRC, fmt->which); *format = fmt->format; - vfe_try_format(line, cfg, MSM_VFE_PAD_SRC, format, + vfe_try_format(line, sd_state, MSM_VFE_PAD_SRC, format, fmt->which); if (line->id != VFE_LINE_PIX) @@ -1097,7 +1100,7 @@ static int vfe_set_format(struct v4l2_subdev *sd, sel.target = V4L2_SEL_TGT_COMPOSE; sel.r.width = fmt->format.width; sel.r.height = fmt->format.height; - ret = vfe_set_selection(sd, cfg, &sel); + ret = vfe_set_selection(sd, sd_state, &sel); if (ret < 0) return ret; } @@ -1114,7 +1117,7 @@ static int vfe_set_format(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int vfe_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vfe_line *line = v4l2_get_subdevdata(sd); @@ -1130,7 +1133,7 @@ static int vfe_get_selection(struct v4l2_subdev *sd, case V4L2_SEL_TGT_COMPOSE_BOUNDS: fmt.pad = sel->pad; fmt.which = sel->which; - ret = vfe_get_format(sd, cfg, &fmt); + ret = vfe_get_format(sd, sd_state, &fmt); if (ret < 0) return ret; @@ -1140,7 +1143,7 @@ static int vfe_get_selection(struct v4l2_subdev *sd, sel->r.height = fmt.format.height; break; case V4L2_SEL_TGT_COMPOSE: - rect = __vfe_get_compose(line, cfg, sel->which); + rect = __vfe_get_compose(line, sd_state, sel->which); if (rect == NULL) return -EINVAL; @@ -1152,7 +1155,7 @@ static int vfe_get_selection(struct v4l2_subdev *sd, else if (sel->pad == MSM_VFE_PAD_SRC) switch (sel->target) { case V4L2_SEL_TGT_CROP_BOUNDS: - rect = __vfe_get_compose(line, cfg, sel->which); + rect = __vfe_get_compose(line, sd_state, sel->which); if (rect == NULL) return -EINVAL; @@ -1162,7 +1165,7 @@ static int vfe_get_selection(struct v4l2_subdev *sd, sel->r.height = rect->height; break; case V4L2_SEL_TGT_CROP: - rect = __vfe_get_crop(line, cfg, sel->which); + rect = __vfe_get_crop(line, sd_state, sel->which); if (rect == NULL) return -EINVAL; @@ -1184,7 +1187,7 @@ static int vfe_get_selection(struct v4l2_subdev *sd, * Return -EINVAL or zero on success */ static int vfe_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vfe_line *line = v4l2_get_subdevdata(sd); @@ -1198,11 +1201,11 @@ static int vfe_set_selection(struct v4l2_subdev *sd, sel->pad == MSM_VFE_PAD_SINK) { struct v4l2_subdev_selection crop = { 0 }; - rect = __vfe_get_compose(line, cfg, sel->which); + rect = __vfe_get_compose(line, sd_state, sel->which); if (rect == NULL) return -EINVAL; - vfe_try_compose(line, cfg, &sel->r, sel->which); + vfe_try_compose(line, sd_state, &sel->r, sel->which); *rect = sel->r; /* Reset source crop selection */ @@ -1210,28 +1213,28 @@ static int vfe_set_selection(struct v4l2_subdev *sd, crop.pad = MSM_VFE_PAD_SRC; crop.target = V4L2_SEL_TGT_CROP; crop.r = *rect; - ret = vfe_set_selection(sd, cfg, &crop); + ret = vfe_set_selection(sd, sd_state, &crop); } else if (sel->target == V4L2_SEL_TGT_CROP && sel->pad == MSM_VFE_PAD_SRC) { struct v4l2_subdev_format fmt = { 0 }; - rect = __vfe_get_crop(line, cfg, sel->which); + rect = __vfe_get_crop(line, sd_state, sel->which); if (rect == NULL) return -EINVAL; - vfe_try_crop(line, cfg, &sel->r, sel->which); + vfe_try_crop(line, sd_state, &sel->r, sel->which); *rect = sel->r; /* Reset source pad format width and height */ fmt.which = sel->which; fmt.pad = MSM_VFE_PAD_SRC; - ret = vfe_get_format(sd, cfg, &fmt); + ret = vfe_get_format(sd, sd_state, &fmt); if (ret < 0) return ret; fmt.format.width = rect->width; fmt.format.height = rect->height; - ret = vfe_set_format(sd, cfg, &fmt); + ret = vfe_set_format(sd, sd_state, &fmt); } else { ret = -EINVAL; } @@ -1261,7 +1264,7 @@ static int vfe_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) } }; - return vfe_set_format(sd, fh ? fh->pad : NULL, &format); + return vfe_set_format(sd, fh ? fh->state : NULL, &format); } /* @@ -1301,8 +1304,7 @@ int msm_vfe_subdev_init(struct camss *camss, struct vfe_device *vfe, /* Memory */ - r = platform_get_resource_byname(pdev, IORESOURCE_MEM, res->reg[0]); - vfe->base = devm_ioremap_resource(dev, r); + vfe->base = devm_platform_ioremap_resource_byname(pdev, res->reg[0]); if (IS_ERR(vfe->base)) { dev_err(dev, "could not map memory\n"); return PTR_ERR(vfe->base); diff --git a/drivers/media/platform/qcom/venus/core.c b/drivers/media/platform/qcom/venus/core.c index 54bac7ec14c5..91b15842c555 100644 --- a/drivers/media/platform/qcom/venus/core.c +++ b/drivers/media/platform/qcom/venus/core.c @@ -78,22 +78,32 @@ static const struct hfi_core_ops venus_core_ops = { .event_notify = venus_event_notify, }; +#define RPM_WAIT_FOR_IDLE_MAX_ATTEMPTS 10 + static void venus_sys_error_handler(struct work_struct *work) { struct venus_core *core = container_of(work, struct venus_core, work.work); - int ret = 0; + int ret, i, max_attempts = RPM_WAIT_FOR_IDLE_MAX_ATTEMPTS; + const char *err_msg = ""; + bool failed = false; - pm_runtime_get_sync(core->dev); + ret = pm_runtime_get_sync(core->dev); + if (ret < 0) { + err_msg = "resume runtime PM"; + max_attempts = 0; + failed = true; + } hfi_core_deinit(core, true); - dev_warn(core->dev, "system error has occurred, starting recovery!\n"); - mutex_lock(&core->lock); - while (pm_runtime_active(core->dev_dec) || pm_runtime_active(core->dev_enc)) + for (i = 0; i < max_attempts; i++) { + if (!pm_runtime_active(core->dev_dec) && !pm_runtime_active(core->dev_enc)) + break; msleep(10); + } venus_shutdown(core); @@ -101,31 +111,55 @@ static void venus_sys_error_handler(struct work_struct *work) pm_runtime_put_sync(core->dev); - while (core->pmdomains[0] && pm_runtime_active(core->pmdomains[0])) + for (i = 0; i < max_attempts; i++) { + if (!core->pmdomains[0] || !pm_runtime_active(core->pmdomains[0])) + break; usleep_range(1000, 1500); + } hfi_reinit(core); - pm_runtime_get_sync(core->dev); + ret = pm_runtime_get_sync(core->dev); + if (ret < 0) { + err_msg = "resume runtime PM"; + failed = true; + } - ret |= venus_boot(core); - ret |= hfi_core_resume(core, true); + ret = venus_boot(core); + if (ret && !failed) { + err_msg = "boot Venus"; + failed = true; + } + + ret = hfi_core_resume(core, true); + if (ret && !failed) { + err_msg = "resume HFI"; + failed = true; + } enable_irq(core->irq); mutex_unlock(&core->lock); - ret |= hfi_core_init(core); + ret = hfi_core_init(core); + if (ret && !failed) { + err_msg = "init HFI"; + failed = true; + } pm_runtime_put_sync(core->dev); - if (ret) { + if (failed) { disable_irq_nosync(core->irq); - dev_warn(core->dev, "recovery failed (%d)\n", ret); + dev_warn_ratelimited(core->dev, + "System error has occurred, recovery failed to %s\n", + err_msg); schedule_delayed_work(&core->work, msecs_to_jiffies(10)); return; } + dev_warn(core->dev, "system error has occurred (recovered)\n"); + mutex_lock(&core->lock); core->sys_error = false; mutex_unlock(&core->lock); diff --git a/drivers/media/platform/qcom/venus/core.h b/drivers/media/platform/qcom/venus/core.h index 745f226a523f..8df2d497d706 100644 --- a/drivers/media/platform/qcom/venus/core.h +++ b/drivers/media/platform/qcom/venus/core.h @@ -155,7 +155,6 @@ struct venus_core { struct clk *vcodec1_clks[VIDC_VCODEC_CLKS_NUM_MAX]; struct icc_path *video_path; struct icc_path *cpucfg_path; - struct opp_table *opp_table; bool has_opp_table; struct device *pmdomains[VIDC_PMDOMAINS_NUM_MAX]; struct device_link *opp_dl_venus; @@ -293,6 +292,7 @@ struct clock_data { unsigned long freq; unsigned long vpp_freq; unsigned long vsp_freq; + unsigned long low_power_freq; }; #define to_venus_buffer(ptr) container_of(ptr, struct venus_buffer, vb) @@ -316,6 +316,10 @@ struct venus_ts_metadata { struct v4l2_timecode tc; }; +enum venus_inst_modes { + VENUS_LOW_POWER = BIT(0), +}; + /** * struct venus_inst - holds per instance parameters * @@ -445,6 +449,7 @@ struct venus_inst { unsigned int pic_struct; bool next_buf_last; bool drain_active; + enum venus_inst_modes flags; }; #define IS_V1(core) ((core)->res->hfi_version == HFI_VERSION_1XX) diff --git a/drivers/media/platform/qcom/venus/helpers.c b/drivers/media/platform/qcom/venus/helpers.c index b813d6dba481..1fe6d463dc99 100644 --- a/drivers/media/platform/qcom/venus/helpers.c +++ b/drivers/media/platform/qcom/venus/helpers.c @@ -595,8 +595,7 @@ static int platform_get_bufreq(struct venus_inst *inst, u32 buftype, params.dec.is_secondary_output = inst->opb_buftype == HFI_BUFFER_OUTPUT2; params.dec.is_interlaced = - inst->pic_struct != HFI_INTERLACE_FRAME_PROGRESSIVE ? - true : false; + inst->pic_struct != HFI_INTERLACE_FRAME_PROGRESSIVE; } else { params.width = inst->out_width; params.height = inst->out_height; @@ -1627,6 +1626,8 @@ int venus_helper_session_init(struct venus_inst *inst) session_type); inst->clk_data.vsp_freq = hfi_platform_get_codec_vsp_freq(version, codec, session_type); + inst->clk_data.low_power_freq = hfi_platform_get_codec_lp_freq(version, codec, + session_type); return 0; } diff --git a/drivers/media/platform/qcom/venus/hfi_cmds.c b/drivers/media/platform/qcom/venus/hfi_cmds.c index 11a8347e5f5c..f51024786991 100644 --- a/drivers/media/platform/qcom/venus/hfi_cmds.c +++ b/drivers/media/platform/qcom/venus/hfi_cmds.c @@ -3,6 +3,7 @@ * Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. * Copyright (C) 2017 Linaro Ltd. */ +#include #include #include @@ -27,7 +28,7 @@ void pkt_sys_idle_indicator(struct hfi_sys_set_property_pkt *pkt, u32 enable) { struct hfi_enable *hfi = (struct hfi_enable *)&pkt->data[1]; - pkt->hdr.size = sizeof(*pkt) + sizeof(*hfi) + sizeof(u32); + pkt->hdr.size = struct_size(pkt, data, 1) + sizeof(*hfi); pkt->hdr.pkt_type = HFI_CMD_SYS_SET_PROPERTY; pkt->num_properties = 1; pkt->data[0] = HFI_PROPERTY_SYS_IDLE_INDICATOR; @@ -39,7 +40,7 @@ void pkt_sys_debug_config(struct hfi_sys_set_property_pkt *pkt, u32 mode, { struct hfi_debug_config *hfi; - pkt->hdr.size = sizeof(*pkt) + sizeof(*hfi) + sizeof(u32); + pkt->hdr.size = struct_size(pkt, data, 1) + sizeof(*hfi); pkt->hdr.pkt_type = HFI_CMD_SYS_SET_PROPERTY; pkt->num_properties = 1; pkt->data[0] = HFI_PROPERTY_SYS_DEBUG_CONFIG; @@ -50,7 +51,7 @@ void pkt_sys_debug_config(struct hfi_sys_set_property_pkt *pkt, u32 mode, void pkt_sys_coverage_config(struct hfi_sys_set_property_pkt *pkt, u32 mode) { - pkt->hdr.size = sizeof(*pkt) + sizeof(u32); + pkt->hdr.size = struct_size(pkt, data, 2); pkt->hdr.pkt_type = HFI_CMD_SYS_SET_PROPERTY; pkt->num_properties = 1; pkt->data[0] = HFI_PROPERTY_SYS_CONFIG_COVERAGE; @@ -116,7 +117,7 @@ void pkt_sys_power_control(struct hfi_sys_set_property_pkt *pkt, u32 enable) { struct hfi_enable *hfi = (struct hfi_enable *)&pkt->data[1]; - pkt->hdr.size = sizeof(*pkt) + sizeof(*hfi) + sizeof(u32); + pkt->hdr.size = struct_size(pkt, data, 1) + sizeof(*hfi); pkt->hdr.pkt_type = HFI_CMD_SYS_SET_PROPERTY; pkt->num_properties = 1; pkt->data[0] = HFI_PROPERTY_SYS_CODEC_POWER_PLANE_CTRL; @@ -1226,6 +1227,17 @@ pkt_session_set_property_4xx(struct hfi_session_set_property_pkt *pkt, pkt->shdr.hdr.size += sizeof(u32) + sizeof(*hdr10); break; } + case HFI_PROPERTY_PARAM_VDEC_CONCEAL_COLOR: { + struct hfi_conceal_color_v4 *color = prop_data; + u32 *in = pdata; + + color->conceal_color_8bit = *in & 0xff; + color->conceal_color_8bit |= ((*in >> 10) & 0xff) << 8; + color->conceal_color_8bit |= ((*in >> 20) & 0xff) << 16; + color->conceal_color_10bit = *in; + pkt->shdr.hdr.size += sizeof(u32) + sizeof(*color); + break; + } case HFI_PROPERTY_CONFIG_VENC_MAX_BITRATE: case HFI_PROPERTY_CONFIG_VDEC_POST_LOOP_DEBLOCKER: @@ -1279,17 +1291,6 @@ pkt_session_set_property_6xx(struct hfi_session_set_property_pkt *pkt, pkt->shdr.hdr.size += sizeof(u32) + sizeof(*cq); break; } - case HFI_PROPERTY_PARAM_VDEC_CONCEAL_COLOR: { - struct hfi_conceal_color_v4 *color = prop_data; - u32 *in = pdata; - - color->conceal_color_8bit = *in & 0xff; - color->conceal_color_8bit |= ((*in >> 10) & 0xff) << 8; - color->conceal_color_8bit |= ((*in >> 20) & 0xff) << 16; - color->conceal_color_10bit = *in; - pkt->shdr.hdr.size += sizeof(u32) + sizeof(*color); - break; - } default: return pkt_session_set_property_4xx(pkt, cookie, ptype, pdata); } diff --git a/drivers/media/platform/qcom/venus/hfi_cmds.h b/drivers/media/platform/qcom/venus/hfi_cmds.h index 83705e237f1c..327ed90a2788 100644 --- a/drivers/media/platform/qcom/venus/hfi_cmds.h +++ b/drivers/media/platform/qcom/venus/hfi_cmds.h @@ -68,7 +68,7 @@ struct hfi_sys_release_resource_pkt { struct hfi_sys_set_property_pkt { struct hfi_pkt_hdr hdr; u32 num_properties; - u32 data[1]; + u32 data[]; }; struct hfi_sys_get_property_pkt { diff --git a/drivers/media/platform/qcom/venus/hfi_helper.h b/drivers/media/platform/qcom/venus/hfi_helper.h index 63cd347a62da..b0a9beb4163c 100644 --- a/drivers/media/platform/qcom/venus/hfi_helper.h +++ b/drivers/media/platform/qcom/venus/hfi_helper.h @@ -415,9 +415,6 @@ #define HFI_BUFFER_MODE_RING 0x1000002 #define HFI_BUFFER_MODE_DYNAMIC 0x1000003 -#define HFI_VENC_PERFMODE_MAX_QUALITY 0x1 -#define HFI_VENC_PERFMODE_POWER_SAVE 0x2 - /* * HFI_PROPERTY_SYS_COMMON_START * HFI_DOMAIN_BASE_COMMON + HFI_ARCH_COMMON_OFFSET + 0x0000 @@ -848,6 +845,13 @@ struct hfi_framesize { u32 height; }; +#define HFI_VENC_PERFMODE_MAX_QUALITY 0x1 +#define HFI_VENC_PERFMODE_POWER_SAVE 0x2 + +struct hfi_perf_mode { + u32 video_perf_mode; +}; + #define VIDC_CORE_ID_DEFAULT 0 #define VIDC_CORE_ID_1 1 #define VIDC_CORE_ID_2 2 diff --git a/drivers/media/platform/qcom/venus/hfi_msgs.c b/drivers/media/platform/qcom/venus/hfi_msgs.c index a2d436d407b2..d9fde66f6fa8 100644 --- a/drivers/media/platform/qcom/venus/hfi_msgs.c +++ b/drivers/media/platform/qcom/venus/hfi_msgs.c @@ -251,11 +251,11 @@ sys_get_prop_image_version(struct device *dev, req_bytes = pkt->hdr.size - sizeof(*pkt); - if (req_bytes < VER_STR_SZ || !pkt->data[1] || pkt->num_properties > 1) + if (req_bytes < VER_STR_SZ || !pkt->data[0] || pkt->num_properties > 1) /* bad packet */ return; - img_ver = (u8 *)&pkt->data[1]; + img_ver = pkt->data; dev_dbg(dev, VDBGL "F/W version: %s\n", img_ver); @@ -277,7 +277,7 @@ static void hfi_sys_property_info(struct venus_core *core, return; } - switch (pkt->data[0]) { + switch (pkt->property) { case HFI_PROPERTY_SYS_IMAGE_VERSION: sys_get_prop_image_version(dev, pkt); break; @@ -338,7 +338,7 @@ session_get_prop_profile_level(struct hfi_msg_session_property_info_pkt *pkt, /* bad packet */ return HFI_ERR_SESSION_INVALID_PARAMETER; - hfi = (struct hfi_profile_level *)&pkt->data[1]; + hfi = (struct hfi_profile_level *)&pkt->data[0]; profile_level->profile = hfi->profile; profile_level->level = hfi->level; @@ -355,11 +355,11 @@ session_get_prop_buf_req(struct hfi_msg_session_property_info_pkt *pkt, req_bytes = pkt->shdr.hdr.size - sizeof(*pkt); - if (!req_bytes || req_bytes % sizeof(*buf_req) || !pkt->data[1]) + if (!req_bytes || req_bytes % sizeof(*buf_req) || !pkt->data[0]) /* bad packet */ return HFI_ERR_SESSION_INVALID_PARAMETER; - buf_req = (struct hfi_buffer_requirements *)&pkt->data[1]; + buf_req = (struct hfi_buffer_requirements *)&pkt->data[0]; if (!buf_req) return HFI_ERR_SESSION_INVALID_PARAMETER; @@ -391,7 +391,7 @@ static void hfi_session_prop_info(struct venus_core *core, goto done; } - switch (pkt->data[0]) { + switch (pkt->property) { case HFI_PROPERTY_CONFIG_BUFFER_REQUIREMENTS: memset(hprop->bufreq, 0, sizeof(hprop->bufreq)); error = session_get_prop_buf_req(pkt, hprop->bufreq); @@ -404,7 +404,7 @@ static void hfi_session_prop_info(struct venus_core *core, case HFI_PROPERTY_CONFIG_VDEC_ENTROPY: break; default: - dev_dbg(dev, VDBGM "unknown property id:%x\n", pkt->data[0]); + dev_dbg(dev, VDBGM "unknown property id:%x\n", pkt->property); return; } diff --git a/drivers/media/platform/qcom/venus/hfi_msgs.h b/drivers/media/platform/qcom/venus/hfi_msgs.h index 526d9f5b487b..510513697335 100644 --- a/drivers/media/platform/qcom/venus/hfi_msgs.h +++ b/drivers/media/platform/qcom/venus/hfi_msgs.h @@ -113,7 +113,8 @@ struct hfi_msg_sys_ping_ack_pkt { struct hfi_msg_sys_property_info_pkt { struct hfi_pkt_hdr hdr; u32 num_properties; - u32 data[1]; + u32 property; + u8 data[]; }; struct hfi_msg_session_load_resources_done_pkt { @@ -233,7 +234,8 @@ struct hfi_msg_session_parse_sequence_header_done_pkt { struct hfi_msg_session_property_info_pkt { struct hfi_session_hdr_pkt shdr; u32 num_properties; - u32 data[1]; + u32 property; + u8 data[]; }; struct hfi_msg_session_release_resources_done_pkt { diff --git a/drivers/media/platform/qcom/venus/hfi_platform.c b/drivers/media/platform/qcom/venus/hfi_platform.c index 8f47804e973f..f5b4e1f4764f 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform.c +++ b/drivers/media/platform/qcom/venus/hfi_platform.c @@ -50,6 +50,22 @@ hfi_platform_get_codec_vsp_freq(enum hfi_version version, u32 codec, u32 session return freq; } +unsigned long +hfi_platform_get_codec_lp_freq(enum hfi_version version, u32 codec, u32 session_type) +{ + const struct hfi_platform *plat; + unsigned long freq = 0; + + plat = hfi_platform_get(version); + if (!plat) + return 0; + + if (plat->codec_lp_freq) + freq = plat->codec_lp_freq(session_type, codec); + + return freq; +} + u8 hfi_platform_num_vpp_pipes(enum hfi_version version) { const struct hfi_platform *plat; diff --git a/drivers/media/platform/qcom/venus/hfi_platform.h b/drivers/media/platform/qcom/venus/hfi_platform.h index 3819bb2b36bd..2dbe608c53af 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform.h +++ b/drivers/media/platform/qcom/venus/hfi_platform.h @@ -43,11 +43,13 @@ struct hfi_platform_codec_freq_data { u32 session_type; unsigned long vpp_freq; unsigned long vsp_freq; + unsigned long low_power_freq; }; struct hfi_platform { unsigned long (*codec_vpp_freq)(u32 session_type, u32 codec); unsigned long (*codec_vsp_freq)(u32 session_type, u32 codec); + unsigned long (*codec_lp_freq)(u32 session_type, u32 codec); void (*codecs)(u32 *enc_codecs, u32 *dec_codecs, u32 *count); const struct hfi_plat_caps *(*capabilities)(unsigned int *entries); u8 (*num_vpp_pipes)(void); @@ -63,5 +65,7 @@ unsigned long hfi_platform_get_codec_vpp_freq(enum hfi_version version, u32 code u32 session_type); unsigned long hfi_platform_get_codec_vsp_freq(enum hfi_version version, u32 codec, u32 session_type); +unsigned long hfi_platform_get_codec_lp_freq(enum hfi_version version, u32 codec, + u32 session_type); u8 hfi_platform_num_vpp_pipes(enum hfi_version version); #endif diff --git a/drivers/media/platform/qcom/venus/hfi_platform_v4.c b/drivers/media/platform/qcom/venus/hfi_platform_v4.c index 3848bb6d7408..3f7f5277a50e 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform_v4.c +++ b/drivers/media/platform/qcom/venus/hfi_platform_v4.c @@ -262,14 +262,14 @@ static void get_codecs(u32 *enc_codecs, u32 *dec_codecs, u32 *count) } static const struct hfi_platform_codec_freq_data codec_freq_data[] = { - { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_ENC, 675, 10 }, - { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_ENC, 675, 10 }, - { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_ENC, 675, 10 }, - { V4L2_PIX_FMT_MPEG2, VIDC_SESSION_TYPE_DEC, 200, 10 }, - { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_DEC, 200, 10 }, - { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_DEC, 200, 10 }, - { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_DEC, 200, 10 }, - { V4L2_PIX_FMT_VP9, VIDC_SESSION_TYPE_DEC, 200, 10 }, + { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_ENC, 675, 10, 320 }, + { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_ENC, 675, 10, 320 }, + { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_ENC, 675, 10, 320 }, + { V4L2_PIX_FMT_MPEG2, VIDC_SESSION_TYPE_DEC, 200, 10, 200 }, + { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_DEC, 200, 10, 200 }, + { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_DEC, 200, 10, 200 }, + { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_DEC, 200, 10, 200 }, + { V4L2_PIX_FMT_VP9, VIDC_SESSION_TYPE_DEC, 200, 10, 200 }, }; static const struct hfi_platform_codec_freq_data * @@ -311,9 +311,21 @@ static unsigned long codec_vsp_freq(u32 session_type, u32 codec) return 0; } +static unsigned long codec_lp_freq(u32 session_type, u32 codec) +{ + const struct hfi_platform_codec_freq_data *data; + + data = get_codec_freq_data(session_type, codec); + if (data) + return data->low_power_freq; + + return 0; +} + const struct hfi_platform hfi_plat_v4 = { .codec_vpp_freq = codec_vpp_freq, .codec_vsp_freq = codec_vsp_freq, + .codec_lp_freq = codec_lp_freq, .codecs = get_codecs, .capabilities = get_capabilities, }; diff --git a/drivers/media/platform/qcom/venus/hfi_platform_v6.c b/drivers/media/platform/qcom/venus/hfi_platform_v6.c index dd1a03911b6c..d8243b22568a 100644 --- a/drivers/media/platform/qcom/venus/hfi_platform_v6.c +++ b/drivers/media/platform/qcom/venus/hfi_platform_v6.c @@ -262,14 +262,14 @@ static void get_codecs(u32 *enc_codecs, u32 *dec_codecs, u32 *count) } static const struct hfi_platform_codec_freq_data codec_freq_data[] = { - { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_ENC, 675, 25 }, - { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_ENC, 675, 25 }, - { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_ENC, 675, 60 }, - { V4L2_PIX_FMT_MPEG2, VIDC_SESSION_TYPE_DEC, 200, 25 }, - { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_DEC, 200, 25 }, - { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_DEC, 200, 25 }, - { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_DEC, 200, 60 }, - { V4L2_PIX_FMT_VP9, VIDC_SESSION_TYPE_DEC, 200, 60 }, + { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_ENC, 675, 25, 320 }, + { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_ENC, 675, 25, 320 }, + { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_ENC, 675, 60, 320 }, + { V4L2_PIX_FMT_MPEG2, VIDC_SESSION_TYPE_DEC, 200, 25, 200 }, + { V4L2_PIX_FMT_H264, VIDC_SESSION_TYPE_DEC, 200, 25, 200 }, + { V4L2_PIX_FMT_HEVC, VIDC_SESSION_TYPE_DEC, 200, 25, 200 }, + { V4L2_PIX_FMT_VP8, VIDC_SESSION_TYPE_DEC, 200, 60, 200 }, + { V4L2_PIX_FMT_VP9, VIDC_SESSION_TYPE_DEC, 200, 60, 200 }, }; static const struct hfi_platform_codec_freq_data * @@ -311,6 +311,17 @@ static unsigned long codec_vsp_freq(u32 session_type, u32 codec) return 0; } +static unsigned long codec_lp_freq(u32 session_type, u32 codec) +{ + const struct hfi_platform_codec_freq_data *data; + + data = get_codec_freq_data(session_type, codec); + if (data) + return data->low_power_freq; + + return 0; +} + static u8 num_vpp_pipes(void) { return 4; @@ -319,6 +330,7 @@ static u8 num_vpp_pipes(void) const struct hfi_platform hfi_plat_v6 = { .codec_vpp_freq = codec_vpp_freq, .codec_vsp_freq = codec_vsp_freq, + .codec_lp_freq = codec_lp_freq, .codecs = get_codecs, .capabilities = get_capabilities, .num_vpp_pipes = num_vpp_pipes, diff --git a/drivers/media/platform/qcom/venus/pm_helpers.c b/drivers/media/platform/qcom/venus/pm_helpers.c index c7e1ebec47ee..3e2345eb47f7 100644 --- a/drivers/media/platform/qcom/venus/pm_helpers.c +++ b/drivers/media/platform/qcom/venus/pm_helpers.c @@ -300,16 +300,15 @@ static int core_get_v1(struct venus_core *core) if (ret) return ret; - core->opp_table = dev_pm_opp_set_clkname(core->dev, "core"); - if (IS_ERR(core->opp_table)) - return PTR_ERR(core->opp_table); + ret = devm_pm_opp_set_clkname(core->dev, "core"); + if (ret) + return ret; return 0; } static void core_put_v1(struct venus_core *core) { - dev_pm_opp_put_clkname(core->opp_table); } static int core_power_v1(struct venus_core *core, int on) @@ -524,8 +523,50 @@ static int poweron_coreid(struct venus_core *core, unsigned int coreid_mask) return 0; } +static inline int power_save_mode_enable(struct venus_inst *inst, + bool enable) +{ + struct venc_controls *enc_ctr = &inst->controls.enc; + const u32 ptype = HFI_PROPERTY_CONFIG_VENC_PERF_MODE; + u32 venc_mode; + int ret = 0; + + if (inst->session_type != VIDC_SESSION_TYPE_ENC) + return 0; + + if (enc_ctr->bitrate_mode == V4L2_MPEG_VIDEO_BITRATE_MODE_CQ) + enable = false; + + venc_mode = enable ? HFI_VENC_PERFMODE_POWER_SAVE : + HFI_VENC_PERFMODE_MAX_QUALITY; + + ret = hfi_session_set_property(inst, ptype, &venc_mode); + if (ret) + return ret; + + inst->flags = enable ? inst->flags | VENUS_LOW_POWER : + inst->flags & ~VENUS_LOW_POWER; + + return ret; +} + +static int move_core_to_power_save_mode(struct venus_core *core, + u32 core_id) +{ + struct venus_inst *inst = NULL; + + mutex_lock(&core->lock); + list_for_each_entry(inst, &core->instances, list) { + if (inst->clk_data.core_id == core_id && + inst->session_type == VIDC_SESSION_TYPE_ENC) + power_save_mode_enable(inst, true); + } + mutex_unlock(&core->lock); + return 0; +} + static void -min_loaded_core(struct venus_inst *inst, u32 *min_coreid, u32 *min_load) +min_loaded_core(struct venus_inst *inst, u32 *min_coreid, u32 *min_load, bool low_power) { u32 mbs_per_sec, load, core1_load = 0, core2_load = 0; u32 cores_max = core_num_max(inst); @@ -543,7 +584,14 @@ min_loaded_core(struct venus_inst *inst, u32 *min_coreid, u32 *min_load) if (inst_pos->state != INST_START) continue; - vpp_freq = inst_pos->clk_data.vpp_freq; + if (inst->session_type == VIDC_SESSION_TYPE_DEC) + vpp_freq = inst_pos->clk_data.vpp_freq; + else if (inst->session_type == VIDC_SESSION_TYPE_ENC) + vpp_freq = low_power ? inst_pos->clk_data.vpp_freq : + inst_pos->clk_data.low_power_freq; + else + continue; + coreid = inst_pos->clk_data.core_id; mbs_per_sec = load_per_instance(inst_pos); @@ -575,9 +623,11 @@ static int decide_core(struct venus_inst *inst) { const u32 ptype = HFI_PROPERTY_CONFIG_VIDEOCORES_USAGE; struct venus_core *core = inst->core; - u32 min_coreid, min_load, inst_load; + u32 min_coreid, min_load, cur_inst_load; + u32 min_lp_coreid, min_lp_load, cur_inst_lp_load; struct hfi_videocores_usage_type cu; unsigned long max_freq; + int ret = 0; if (legacy_binding) { if (inst->session_type == VIDC_SESSION_TYPE_DEC) @@ -591,23 +641,43 @@ static int decide_core(struct venus_inst *inst) if (inst->clk_data.core_id != VIDC_CORE_ID_DEFAULT) return 0; - inst_load = load_per_instance(inst); - inst_load *= inst->clk_data.vpp_freq; + cur_inst_load = load_per_instance(inst); + cur_inst_load *= inst->clk_data.vpp_freq; + /*TODO : divide this inst->load by work_route */ + + cur_inst_lp_load = load_per_instance(inst); + cur_inst_lp_load *= inst->clk_data.low_power_freq; + /*TODO : divide this inst->load by work_route */ + max_freq = core->res->freq_tbl[0].freq; - min_loaded_core(inst, &min_coreid, &min_load); + min_loaded_core(inst, &min_coreid, &min_load, false); + min_loaded_core(inst, &min_lp_coreid, &min_lp_load, true); - if ((inst_load + min_load) > max_freq) { - dev_warn(core->dev, "HW is overloaded, needed: %u max: %lu\n", - inst_load, max_freq); + if (cur_inst_load + min_load <= max_freq) { + inst->clk_data.core_id = min_coreid; + cu.video_core_enable_mask = min_coreid; + } else if (cur_inst_lp_load + min_load <= max_freq) { + /* Move current instance to LP and return */ + inst->clk_data.core_id = min_coreid; + cu.video_core_enable_mask = min_coreid; + power_save_mode_enable(inst, true); + } else if (cur_inst_lp_load + min_lp_load <= max_freq) { + /* Move all instances to LP mode and return */ + inst->clk_data.core_id = min_lp_coreid; + cu.video_core_enable_mask = min_lp_coreid; + move_core_to_power_save_mode(core, min_lp_coreid); + } else { + dev_warn(core->dev, "HW can't support this load"); return -EINVAL; } - inst->clk_data.core_id = min_coreid; - cu.video_core_enable_mask = min_coreid; - done: - return hfi_session_set_property(inst, ptype, &cu); + ret = hfi_session_set_property(inst, ptype, &cu); + if (ret) + return ret; + + return ret; } static int acquire_core(struct venus_inst *inst) @@ -788,7 +858,6 @@ static int venc_power_v4(struct device *dev, int on) static int vcodec_domains_get(struct venus_core *core) { int ret; - struct opp_table *opp_table; struct device **opp_virt_dev; struct device *dev = core->dev; const struct venus_resources *res = core->res; @@ -811,11 +880,9 @@ static int vcodec_domains_get(struct venus_core *core) return 0; /* Attach the power domain for setting performance state */ - opp_table = dev_pm_opp_attach_genpd(dev, res->opp_pmdomain, &opp_virt_dev); - if (IS_ERR(opp_table)) { - ret = PTR_ERR(opp_table); + ret = devm_pm_opp_attach_genpd(dev, res->opp_pmdomain, &opp_virt_dev); + if (ret) goto opp_attach_err; - } core->opp_pmdomain = *opp_virt_dev; core->opp_dl_venus = device_link_add(dev, core->opp_pmdomain, @@ -824,13 +891,11 @@ static int vcodec_domains_get(struct venus_core *core) DL_FLAG_STATELESS); if (!core->opp_dl_venus) { ret = -ENODEV; - goto opp_dl_add_err; + goto opp_attach_err; } return 0; -opp_dl_add_err: - dev_pm_opp_detach_genpd(core->opp_table); opp_attach_err: for (i = 0; i < res->vcodec_pmdomains_num; i++) { if (IS_ERR_OR_NULL(core->pmdomains[i])) @@ -861,8 +926,6 @@ static void vcodec_domains_put(struct venus_core *core) if (core->opp_dl_venus) device_link_del(core->opp_dl_venus); - - dev_pm_opp_detach_genpd(core->opp_table); } static int core_resets_reset(struct venus_core *core) @@ -941,45 +1004,33 @@ static int core_get_v4(struct venus_core *core) if (legacy_binding) return 0; - core->opp_table = dev_pm_opp_set_clkname(dev, "core"); - if (IS_ERR(core->opp_table)) - return PTR_ERR(core->opp_table); + ret = devm_pm_opp_set_clkname(dev, "core"); + if (ret) + return ret; if (core->res->opp_pmdomain) { - ret = dev_pm_opp_of_add_table(dev); + ret = devm_pm_opp_of_add_table(dev); if (!ret) { core->has_opp_table = true; } else if (ret != -ENODEV) { dev_err(dev, "invalid OPP table in device tree\n"); - dev_pm_opp_put_clkname(core->opp_table); return ret; } } ret = vcodec_domains_get(core); - if (ret) { - if (core->has_opp_table) - dev_pm_opp_of_remove_table(dev); - dev_pm_opp_put_clkname(core->opp_table); + if (ret) return ret; - } return 0; } static void core_put_v4(struct venus_core *core) { - struct device *dev = core->dev; - if (legacy_binding) return; vcodec_domains_put(core); - - if (core->has_opp_table) - dev_pm_opp_of_remove_table(dev); - dev_pm_opp_put_clkname(core->opp_table); - } static int core_power_v4(struct venus_core *core, int on) @@ -990,9 +1041,8 @@ static int core_power_v4(struct venus_core *core, int on) if (on == POWER_ON) { if (pmctrl) { - ret = pm_runtime_get_sync(pmctrl); + ret = pm_runtime_resume_and_get(pmctrl); if (ret < 0) { - pm_runtime_put_noidle(pmctrl); return ret; } } @@ -1026,7 +1076,7 @@ static int core_power_v4(struct venus_core *core, int on) static unsigned long calculate_inst_freq(struct venus_inst *inst, unsigned long filled_len) { - unsigned long vpp_freq = 0, vsp_freq = 0; + unsigned long vpp_freq_per_mb = 0, vpp_freq = 0, vsp_freq = 0; u32 fps = (u32)inst->fps; u32 mbs_per_sec; @@ -1035,7 +1085,12 @@ static unsigned long calculate_inst_freq(struct venus_inst *inst, if (inst->state != INST_START) return 0; - vpp_freq = mbs_per_sec * inst->clk_data.vpp_freq; + if (inst->session_type == VIDC_SESSION_TYPE_ENC) + vpp_freq_per_mb = inst->flags & VENUS_LOW_POWER ? + inst->clk_data.low_power_freq : + inst->clk_data.vpp_freq; + + vpp_freq = mbs_per_sec * vpp_freq_per_mb; /* 21 / 20 is overhead factor */ vpp_freq += vpp_freq / 20; vsp_freq = mbs_per_sec * inst->clk_data.vsp_freq; diff --git a/drivers/media/platform/qcom/venus/vdec.c b/drivers/media/platform/qcom/venus/vdec.c index ddb7cd39424e..198e47eb63f4 100644 --- a/drivers/media/platform/qcom/venus/vdec.c +++ b/drivers/media/platform/qcom/venus/vdec.c @@ -568,10 +568,10 @@ static int vdec_pm_get(struct venus_inst *inst) int ret; mutex_lock(&core->pm_lock); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); mutex_unlock(&core->pm_lock); - return ret < 0 ? ret : 0; + return ret; } static int vdec_pm_put(struct venus_inst *inst, bool autosuspend) @@ -601,7 +601,7 @@ static int vdec_pm_get_put(struct venus_inst *inst) mutex_lock(&core->pm_lock); if (pm_runtime_suspended(dev)) { - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) goto error; diff --git a/drivers/media/platform/qcom/venus/venc.c b/drivers/media/platform/qcom/venus/venc.c index 4a7291f934b6..8dd49d4f124c 100644 --- a/drivers/media/platform/qcom/venus/venc.c +++ b/drivers/media/platform/qcom/venus/venc.c @@ -1205,9 +1205,9 @@ static int venc_open(struct file *file) venus_helper_init_instance(inst); - ret = pm_runtime_get_sync(core->dev_enc); + ret = pm_runtime_resume_and_get(core->dev_enc); if (ret < 0) - goto err_put_sync; + goto err_free; ret = venc_ctrl_init(inst); if (ret) @@ -1252,6 +1252,7 @@ static int venc_open(struct file *file) venc_ctrl_deinit(inst); err_put_sync: pm_runtime_put_sync(core->dev_enc); +err_free: kfree(inst); return ret; } diff --git a/drivers/media/platform/rcar-fcp.c b/drivers/media/platform/rcar-fcp.c index 5c03318ae07b..eb59a3ba6d0f 100644 --- a/drivers/media/platform/rcar-fcp.c +++ b/drivers/media/platform/rcar-fcp.c @@ -96,18 +96,10 @@ EXPORT_SYMBOL_GPL(rcar_fcp_get_device); */ int rcar_fcp_enable(struct rcar_fcp_device *fcp) { - int ret; - if (!fcp) return 0; - ret = pm_runtime_get_sync(fcp->dev); - if (ret < 0) { - pm_runtime_put_noidle(fcp->dev); - return ret; - } - - return 0; + return pm_runtime_resume_and_get(fcp->dev); } EXPORT_SYMBOL_GPL(rcar_fcp_enable); diff --git a/drivers/media/platform/rcar-vin/rcar-core.c b/drivers/media/platform/rcar-vin/rcar-core.c index cb3025992817..33957cc9118c 100644 --- a/drivers/media/platform/rcar-vin/rcar-core.c +++ b/drivers/media/platform/rcar-vin/rcar-core.c @@ -1362,6 +1362,10 @@ static const struct of_device_id rvin_of_id_table[] = { .compatible = "renesas,vin-r8a7796", .data = &rcar_info_r8a7796, }, + { + .compatible = "renesas,vin-r8a77961", + .data = &rcar_info_r8a7796, + }, { .compatible = "renesas,vin-r8a77965", .data = &rcar_info_r8a77965, diff --git a/drivers/media/platform/rcar-vin/rcar-csi2.c b/drivers/media/platform/rcar-vin/rcar-csi2.c index e06cd512aba2..e28eff039688 100644 --- a/drivers/media/platform/rcar-vin/rcar-csi2.c +++ b/drivers/media/platform/rcar-vin/rcar-csi2.c @@ -320,10 +320,12 @@ static const struct rcar_csi2_format rcar_csi2_formats[] = { { .code = MEDIA_BUS_FMT_YUYV8_1X16, .datatype = 0x1e, .bpp = 16 }, { .code = MEDIA_BUS_FMT_UYVY8_2X8, .datatype = 0x1e, .bpp = 16 }, { .code = MEDIA_BUS_FMT_YUYV10_2X10, .datatype = 0x1e, .bpp = 20 }, + { .code = MEDIA_BUS_FMT_Y10_1X10, .datatype = 0x2b, .bpp = 10 }, { .code = MEDIA_BUS_FMT_SBGGR8_1X8, .datatype = 0x2a, .bpp = 8 }, { .code = MEDIA_BUS_FMT_SGBRG8_1X8, .datatype = 0x2a, .bpp = 8 }, { .code = MEDIA_BUS_FMT_SGRBG8_1X8, .datatype = 0x2a, .bpp = 8 }, { .code = MEDIA_BUS_FMT_SRGGB8_1X8, .datatype = 0x2a, .bpp = 8 }, + { .code = MEDIA_BUS_FMT_Y8_1X8, .datatype = 0x2a, .bpp = 8 }, }; static const struct rcar_csi2_format *rcsi2_code_to_fmt(unsigned int code) @@ -406,10 +408,17 @@ static void rcsi2_enter_standby(struct rcar_csi2 *priv) pm_runtime_put(priv->dev); } -static void rcsi2_exit_standby(struct rcar_csi2 *priv) +static int rcsi2_exit_standby(struct rcar_csi2 *priv) { - pm_runtime_get_sync(priv->dev); + int ret; + + ret = pm_runtime_resume_and_get(priv->dev); + if (ret < 0) + return ret; + reset_control_deassert(priv->rstc); + + return 0; } static int rcsi2_wait_phy_start(struct rcar_csi2 *priv, @@ -657,7 +666,9 @@ static int rcsi2_start(struct rcar_csi2 *priv) { int ret; - rcsi2_exit_standby(priv); + ret = rcsi2_exit_standby(priv); + if (ret < 0) + return ret; ret = rcsi2_start_receiver(priv); if (ret) { @@ -708,7 +719,7 @@ static int rcsi2_s_stream(struct v4l2_subdev *sd, int enable) } static int rcsi2_set_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct rcar_csi2 *priv = sd_to_csi2(sd); @@ -720,7 +731,7 @@ static int rcsi2_set_pad_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) { priv->mf = format->format; } else { - framefmt = v4l2_subdev_get_try_format(sd, cfg, 0); + framefmt = v4l2_subdev_get_try_format(sd, sd_state, 0); *framefmt = format->format; } @@ -728,7 +739,7 @@ static int rcsi2_set_pad_format(struct v4l2_subdev *sd, } static int rcsi2_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct rcar_csi2 *priv = sd_to_csi2(sd); @@ -736,7 +747,7 @@ static int rcsi2_get_pad_format(struct v4l2_subdev *sd, if (format->which == V4L2_SUBDEV_FORMAT_ACTIVE) format->format = priv->mf; else - format->format = *v4l2_subdev_get_try_format(sd, cfg, 0); + format->format = *v4l2_subdev_get_try_format(sd, sd_state, 0); return 0; } @@ -1112,6 +1123,11 @@ static const struct rcar_csi2_info rcar_csi2_info_r8a7796 = { .num_channels = 4, }; +static const struct rcar_csi2_info rcar_csi2_info_r8a77961 = { + .hsfreqrange = hsfreqrange_m3w_h3es1, + .num_channels = 4, +}; + static const struct rcar_csi2_info rcar_csi2_info_r8a77965 = { .init_phtw = rcsi2_init_phtw_h3_v3h_m3n, .hsfreqrange = hsfreqrange_h3_v3h_m3n, @@ -1164,6 +1180,10 @@ static const struct of_device_id rcar_csi2_of_table[] = { .compatible = "renesas,r8a7796-csi2", .data = &rcar_csi2_info_r8a7796, }, + { + .compatible = "renesas,r8a77961-csi2", + .data = &rcar_csi2_info_r8a77961, + }, { .compatible = "renesas,r8a77965-csi2", .data = &rcar_csi2_info_r8a77965, diff --git a/drivers/media/platform/rcar-vin/rcar-dma.c b/drivers/media/platform/rcar-vin/rcar-dma.c index f30dafbdf61c..f5f722ab1d4e 100644 --- a/drivers/media/platform/rcar-vin/rcar-dma.c +++ b/drivers/media/platform/rcar-vin/rcar-dma.c @@ -1458,11 +1458,9 @@ int rvin_set_channel_routing(struct rvin_dev *vin, u8 chsel) u32 vnmc; int ret; - ret = pm_runtime_get_sync(vin->dev); - if (ret < 0) { - pm_runtime_put_noidle(vin->dev); + ret = pm_runtime_resume_and_get(vin->dev); + if (ret < 0) return ret; - } /* Make register writes take effect immediately. */ vnmc = rvin_read(vin, VNMC_REG); diff --git a/drivers/media/platform/rcar-vin/rcar-v4l2.c b/drivers/media/platform/rcar-vin/rcar-v4l2.c index 457a65bf6b66..cca15a10c0b3 100644 --- a/drivers/media/platform/rcar-vin/rcar-v4l2.c +++ b/drivers/media/platform/rcar-vin/rcar-v4l2.c @@ -243,7 +243,7 @@ static int rvin_try_format(struct rvin_dev *vin, u32 which, struct v4l2_rect *src_rect) { struct v4l2_subdev *sd = vin_to_source(vin); - struct v4l2_subdev_pad_config *pad_cfg; + struct v4l2_subdev_state *sd_state; struct v4l2_subdev_format format = { .which = which, .pad = vin->parallel.source_pad, @@ -252,8 +252,8 @@ static int rvin_try_format(struct rvin_dev *vin, u32 which, u32 width, height; int ret; - pad_cfg = v4l2_subdev_alloc_pad_config(sd); - if (pad_cfg == NULL) + sd_state = v4l2_subdev_alloc_state(sd); + if (sd_state == NULL) return -ENOMEM; if (!rvin_format_from_pixel(vin, pix->pixelformat)) @@ -266,7 +266,7 @@ static int rvin_try_format(struct rvin_dev *vin, u32 which, width = pix->width; height = pix->height; - ret = v4l2_subdev_call(sd, pad, set_fmt, pad_cfg, &format); + ret = v4l2_subdev_call(sd, pad, set_fmt, sd_state, &format); if (ret < 0 && ret != -ENOIOCTLCMD) goto done; ret = 0; @@ -288,7 +288,7 @@ static int rvin_try_format(struct rvin_dev *vin, u32 which, rvin_format_align(vin, pix); done: - v4l2_subdev_free_pad_config(pad_cfg); + v4l2_subdev_free_state(sd_state); return ret; } @@ -870,11 +870,9 @@ static int rvin_open(struct file *file) struct rvin_dev *vin = video_drvdata(file); int ret; - ret = pm_runtime_get_sync(vin->dev); - if (ret < 0) { - pm_runtime_put_noidle(vin->dev); + ret = pm_runtime_resume_and_get(vin->dev); + if (ret < 0) return ret; - } ret = mutex_lock_interruptible(&vin->lock); if (ret) diff --git a/drivers/media/platform/rcar_fdp1.c b/drivers/media/platform/rcar_fdp1.c index 01c1fbb97bf6..89aac60066d9 100644 --- a/drivers/media/platform/rcar_fdp1.c +++ b/drivers/media/platform/rcar_fdp1.c @@ -2117,9 +2117,7 @@ static int fdp1_open(struct file *file) if (ctx->hdl.error) { ret = ctx->hdl.error; - v4l2_ctrl_handler_free(&ctx->hdl); - kfree(ctx); - goto done; + goto error_ctx; } ctx->fh.ctrl_handler = &ctx->hdl; @@ -2133,20 +2131,27 @@ static int fdp1_open(struct file *file) if (IS_ERR(ctx->fh.m2m_ctx)) { ret = PTR_ERR(ctx->fh.m2m_ctx); - - v4l2_ctrl_handler_free(&ctx->hdl); - kfree(ctx); - goto done; + goto error_ctx; } /* Perform any power management required */ - pm_runtime_get_sync(fdp1->dev); + ret = pm_runtime_resume_and_get(fdp1->dev); + if (ret < 0) + goto error_pm; v4l2_fh_add(&ctx->fh); dprintk(fdp1, "Created instance: %p, m2m_ctx: %p\n", ctx, ctx->fh.m2m_ctx); + mutex_unlock(&fdp1->dev_mutex); + return 0; + +error_pm: + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); +error_ctx: + v4l2_ctrl_handler_free(&ctx->hdl); + kfree(ctx); done: mutex_unlock(&fdp1->dev_mutex); return ret; @@ -2351,7 +2356,9 @@ static int fdp1_probe(struct platform_device *pdev) /* Power up the cells to read HW */ pm_runtime_enable(&pdev->dev); - pm_runtime_get_sync(fdp1->dev); + ret = pm_runtime_resume_and_get(fdp1->dev); + if (ret < 0) + goto disable_pm; hw_version = fdp1_read(fdp1, FD1_IP_INTDATA); switch (hw_version) { @@ -2380,6 +2387,9 @@ static int fdp1_probe(struct platform_device *pdev) return 0; +disable_pm: + pm_runtime_disable(fdp1->dev); + release_m2m: v4l2_m2m_release(fdp1->m2m_dev); diff --git a/drivers/media/platform/rcar_jpu.c b/drivers/media/platform/rcar_jpu.c index a7c198c17deb..f57158bf2b11 100644 --- a/drivers/media/platform/rcar_jpu.c +++ b/drivers/media/platform/rcar_jpu.c @@ -42,7 +42,7 @@ /* * Align JPEG header end to cache line to make sure we will not have any issues - * with cache; additionally to requerment (33.3.27 R01UH0501EJ0100 Rev.1.00) + * with cache; additionally to requirement (33.3.27 R01UH0501EJ0100 Rev.1.00) */ #define JPU_JPEG_HDR_SIZE (ALIGN(0x258, L1_CACHE_BYTES)) #define JPU_JPEG_MAX_BYTES_PER_PIXEL 2 /* 16 bit precision format */ @@ -121,7 +121,7 @@ #define JCCMD_JEND (1 << 2) #define JCCMD_JSRT (1 << 0) -/* JPEG code quantanization table number register */ +/* JPEG code quantization table number register */ #define JCQTN 0x0c #define JCQTN_SHIFT(t) (((t) - 1) << 1) @@ -1644,7 +1644,7 @@ static int jpu_probe(struct platform_device *pdev) goto device_register_rollback; } - /* fill in qantization and Huffman tables for encoder */ + /* fill in quantization and Huffman tables for encoder */ for (i = 0; i < JPU_MAX_QUALITY; i++) jpu_generate_hdr(i, (unsigned char *)jpeg_hdrs[i]); diff --git a/drivers/media/platform/renesas-ceu.c b/drivers/media/platform/renesas-ceu.c index cd137101d41e..f432032c7084 100644 --- a/drivers/media/platform/renesas-ceu.c +++ b/drivers/media/platform/renesas-ceu.c @@ -794,6 +794,9 @@ static int __ceu_try_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt struct v4l2_pix_format_mplane *pix = &v4l2_fmt->fmt.pix_mp; struct v4l2_subdev *v4l2_sd = ceu_sd->v4l2_sd; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; const struct ceu_fmt *ceu_fmt; u32 mbus_code_old; u32 mbus_code; @@ -850,13 +853,13 @@ static int __ceu_try_fmt(struct ceu_device *ceudev, struct v4l2_format *v4l2_fmt * time. */ sd_format.format.code = mbus_code; - ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, &pad_cfg, &sd_format); + ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, &pad_state, &sd_format); if (ret) { if (ret == -EINVAL) { /* fallback */ sd_format.format.code = mbus_code_old; ret = v4l2_subdev_call(v4l2_sd, pad, set_fmt, - &pad_cfg, &sd_format); + &pad_state, &sd_format); } if (ret) @@ -1099,10 +1102,10 @@ static int ceu_open(struct file *file) mutex_lock(&ceudev->mlock); /* Causes soft-reset and sensor power on on first open */ - pm_runtime_get_sync(ceudev->dev); + ret = pm_runtime_resume_and_get(ceudev->dev); mutex_unlock(&ceudev->mlock); - return 0; + return ret; } static int ceu_release(struct file *file) diff --git a/drivers/media/platform/rockchip/rga/rga-buf.c b/drivers/media/platform/rockchip/rga/rga-buf.c index bf9a75b75083..81508ed5abf3 100644 --- a/drivers/media/platform/rockchip/rga/rga-buf.c +++ b/drivers/media/platform/rockchip/rga/rga-buf.c @@ -79,9 +79,8 @@ static int rga_buf_start_streaming(struct vb2_queue *q, unsigned int count) struct rockchip_rga *rga = ctx->rga; int ret; - ret = pm_runtime_get_sync(rga->dev); + ret = pm_runtime_resume_and_get(rga->dev); if (ret < 0) { - pm_runtime_put_noidle(rga->dev); rga_buf_return_buffers(q, VB2_BUF_STATE_QUEUED); return ret; } diff --git a/drivers/media/platform/rockchip/rga/rga.c b/drivers/media/platform/rockchip/rga/rga.c index 9d122429706e..bf3fd71ec3af 100644 --- a/drivers/media/platform/rockchip/rga/rga.c +++ b/drivers/media/platform/rockchip/rga/rga.c @@ -866,7 +866,9 @@ static int rga_probe(struct platform_device *pdev) goto unreg_video_dev; } - pm_runtime_get_sync(rga->dev); + ret = pm_runtime_resume_and_get(rga->dev); + if (ret < 0) + goto unreg_video_dev; rga->version.major = (rga_read(rga, RGA_VERSION_INFO) >> 24) & 0xFF; rga->version.minor = (rga_read(rga, RGA_VERSION_INFO) >> 20) & 0x0F; diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c index 5f6c9d1623e4..60cd2200e7ae 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-capture.c @@ -830,8 +830,8 @@ static void rkisp1_return_all_buffers(struct rkisp1_capture *cap, } /* - * Most of registers inside rockchip ISP1 have shadow register since - * they must be not be changed during processing a frame. + * Most registers inside the rockchip ISP1 have shadow register since + * they must not be changed while processing a frame. * Usually, each sub-module updates its shadow register after * processing the last pixel of a frame. */ @@ -847,14 +847,14 @@ static void rkisp1_cap_stream_enable(struct rkisp1_capture *cap) spin_lock_irq(&cap->buf.lock); rkisp1_set_next_buf(cap); cap->ops->enable(cap); - /* It's safe to config ACTIVE and SHADOW regs for the + /* It's safe to configure ACTIVE and SHADOW registers for the * first stream. While when the second is starting, do NOT - * force update because it also update the first one. + * force update because it also updates the first one. * - * The latter case would drop one more buf(that is 2) since - * there's not buf in shadow when the second FE received. This's - * also required because the second FE maybe corrupt especially - * when run at 120fps. + * The latter case would drop one more buffer(that is 2) since + * there's no buffer in a shadow register when the second FE received. + * This's also required because the second FE maybe corrupt + * especially when run at 120fps. */ if (!other->is_streaming) { /* force cfg update */ @@ -1003,9 +1003,8 @@ rkisp1_vb2_start_streaming(struct vb2_queue *queue, unsigned int count) if (ret) goto err_pipeline_stop; - ret = pm_runtime_get_sync(cap->rkisp1->dev); + ret = pm_runtime_resume_and_get(cap->rkisp1->dev); if (ret < 0) { - pm_runtime_put_noidle(cap->rkisp1->dev); dev_err(cap->rkisp1->dev, "power up failed %d\n", ret); goto err_destroy_dummy; } diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c index 2e5b57e3aedc..d596bc040005 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-isp.c @@ -208,24 +208,30 @@ static struct v4l2_subdev *rkisp1_get_remote_sensor(struct v4l2_subdev *sd) static struct v4l2_mbus_framefmt * rkisp1_isp_get_pad_fmt(struct rkisp1_isp *isp, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { + struct v4l2_subdev_state state = { + .pads = isp->pad_cfg + }; if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&isp->sd, cfg, pad); + return v4l2_subdev_get_try_format(&isp->sd, sd_state, pad); else - return v4l2_subdev_get_try_format(&isp->sd, isp->pad_cfg, pad); + return v4l2_subdev_get_try_format(&isp->sd, &state, pad); } static struct v4l2_rect * rkisp1_isp_get_pad_crop(struct rkisp1_isp *isp, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { + struct v4l2_subdev_state state = { + .pads = isp->pad_cfg + }; if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_crop(&isp->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&isp->sd, sd_state, pad); else - return v4l2_subdev_get_try_crop(&isp->sd, isp->pad_cfg, pad); + return v4l2_subdev_get_try_crop(&isp->sd, &state, pad); } /* ---------------------------------------------------------------------------- @@ -561,7 +567,7 @@ static void rkisp1_isp_start(struct rkisp1_device *rkisp1) */ static int rkisp1_isp_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { unsigned int i, dir; @@ -601,7 +607,7 @@ static int rkisp1_isp_enum_mbus_code(struct v4l2_subdev *sd, } static int rkisp1_isp_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { const struct rkisp1_isp_mbus_info *mbus_info; @@ -634,37 +640,37 @@ static int rkisp1_isp_enum_frame_size(struct v4l2_subdev *sd, } static int rkisp1_isp_init_config(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_mbus_framefmt *sink_fmt, *src_fmt; struct v4l2_rect *sink_crop, *src_crop; - sink_fmt = v4l2_subdev_get_try_format(sd, cfg, + sink_fmt = v4l2_subdev_get_try_format(sd, sd_state, RKISP1_ISP_PAD_SINK_VIDEO); sink_fmt->width = RKISP1_DEFAULT_WIDTH; sink_fmt->height = RKISP1_DEFAULT_HEIGHT; sink_fmt->field = V4L2_FIELD_NONE; sink_fmt->code = RKISP1_DEF_SINK_PAD_FMT; - sink_crop = v4l2_subdev_get_try_crop(sd, cfg, + sink_crop = v4l2_subdev_get_try_crop(sd, sd_state, RKISP1_ISP_PAD_SINK_VIDEO); sink_crop->width = RKISP1_DEFAULT_WIDTH; sink_crop->height = RKISP1_DEFAULT_HEIGHT; sink_crop->left = 0; sink_crop->top = 0; - src_fmt = v4l2_subdev_get_try_format(sd, cfg, + src_fmt = v4l2_subdev_get_try_format(sd, sd_state, RKISP1_ISP_PAD_SOURCE_VIDEO); *src_fmt = *sink_fmt; src_fmt->code = RKISP1_DEF_SRC_PAD_FMT; - src_crop = v4l2_subdev_get_try_crop(sd, cfg, + src_crop = v4l2_subdev_get_try_crop(sd, sd_state, RKISP1_ISP_PAD_SOURCE_VIDEO); *src_crop = *sink_crop; - sink_fmt = v4l2_subdev_get_try_format(sd, cfg, + sink_fmt = v4l2_subdev_get_try_format(sd, sd_state, RKISP1_ISP_PAD_SINK_PARAMS); - src_fmt = v4l2_subdev_get_try_format(sd, cfg, + src_fmt = v4l2_subdev_get_try_format(sd, sd_state, RKISP1_ISP_PAD_SOURCE_STATS); sink_fmt->width = 0; sink_fmt->height = 0; @@ -676,7 +682,7 @@ static int rkisp1_isp_init_config(struct v4l2_subdev *sd, } static void rkisp1_isp_set_src_fmt(struct rkisp1_isp *isp, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_mbus_framefmt *format, unsigned int which) { @@ -684,9 +690,9 @@ static void rkisp1_isp_set_src_fmt(struct rkisp1_isp *isp, struct v4l2_mbus_framefmt *src_fmt; const struct v4l2_rect *src_crop; - src_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, + src_fmt = rkisp1_isp_get_pad_fmt(isp, sd_state, RKISP1_ISP_PAD_SOURCE_VIDEO, which); - src_crop = rkisp1_isp_get_pad_crop(isp, cfg, + src_crop = rkisp1_isp_get_pad_crop(isp, sd_state, RKISP1_ISP_PAD_SOURCE_VIDEO, which); src_fmt->code = format->code; @@ -717,17 +723,17 @@ static void rkisp1_isp_set_src_fmt(struct rkisp1_isp *isp, } static void rkisp1_isp_set_src_crop(struct rkisp1_isp *isp, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_rect *r, unsigned int which) { struct v4l2_mbus_framefmt *src_fmt; const struct v4l2_rect *sink_crop; struct v4l2_rect *src_crop; - src_crop = rkisp1_isp_get_pad_crop(isp, cfg, + src_crop = rkisp1_isp_get_pad_crop(isp, sd_state, RKISP1_ISP_PAD_SOURCE_VIDEO, which); - sink_crop = rkisp1_isp_get_pad_crop(isp, cfg, + sink_crop = rkisp1_isp_get_pad_crop(isp, sd_state, RKISP1_ISP_PAD_SINK_VIDEO, which); @@ -740,21 +746,23 @@ static void rkisp1_isp_set_src_crop(struct rkisp1_isp *isp, *r = *src_crop; /* Propagate to out format */ - src_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, + src_fmt = rkisp1_isp_get_pad_fmt(isp, sd_state, RKISP1_ISP_PAD_SOURCE_VIDEO, which); - rkisp1_isp_set_src_fmt(isp, cfg, src_fmt, which); + rkisp1_isp_set_src_fmt(isp, sd_state, src_fmt, which); } static void rkisp1_isp_set_sink_crop(struct rkisp1_isp *isp, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_rect *r, unsigned int which) { struct v4l2_rect *sink_crop, *src_crop; struct v4l2_mbus_framefmt *sink_fmt; - sink_crop = rkisp1_isp_get_pad_crop(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + sink_crop = rkisp1_isp_get_pad_crop(isp, sd_state, + RKISP1_ISP_PAD_SINK_VIDEO, which); - sink_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + sink_fmt = rkisp1_isp_get_pad_fmt(isp, sd_state, + RKISP1_ISP_PAD_SINK_VIDEO, which); sink_crop->left = ALIGN(r->left, 2); @@ -766,13 +774,13 @@ static void rkisp1_isp_set_sink_crop(struct rkisp1_isp *isp, *r = *sink_crop; /* Propagate to out crop */ - src_crop = rkisp1_isp_get_pad_crop(isp, cfg, + src_crop = rkisp1_isp_get_pad_crop(isp, sd_state, RKISP1_ISP_PAD_SOURCE_VIDEO, which); - rkisp1_isp_set_src_crop(isp, cfg, src_crop, which); + rkisp1_isp_set_src_crop(isp, sd_state, src_crop, which); } static void rkisp1_isp_set_sink_fmt(struct rkisp1_isp *isp, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_mbus_framefmt *format, unsigned int which) { @@ -780,7 +788,8 @@ static void rkisp1_isp_set_sink_fmt(struct rkisp1_isp *isp, struct v4l2_mbus_framefmt *sink_fmt; struct v4l2_rect *sink_crop; - sink_fmt = rkisp1_isp_get_pad_fmt(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + sink_fmt = rkisp1_isp_get_pad_fmt(isp, sd_state, + RKISP1_ISP_PAD_SINK_VIDEO, which); sink_fmt->code = format->code; mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); @@ -801,36 +810,40 @@ static void rkisp1_isp_set_sink_fmt(struct rkisp1_isp *isp, *format = *sink_fmt; /* Propagate to in crop */ - sink_crop = rkisp1_isp_get_pad_crop(isp, cfg, RKISP1_ISP_PAD_SINK_VIDEO, + sink_crop = rkisp1_isp_get_pad_crop(isp, sd_state, + RKISP1_ISP_PAD_SINK_VIDEO, which); - rkisp1_isp_set_sink_crop(isp, cfg, sink_crop, which); + rkisp1_isp_set_sink_crop(isp, sd_state, sink_crop, which); } static int rkisp1_isp_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct rkisp1_isp *isp = container_of(sd, struct rkisp1_isp, sd); mutex_lock(&isp->ops_lock); - fmt->format = *rkisp1_isp_get_pad_fmt(isp, cfg, fmt->pad, fmt->which); + fmt->format = *rkisp1_isp_get_pad_fmt(isp, sd_state, fmt->pad, + fmt->which); mutex_unlock(&isp->ops_lock); return 0; } static int rkisp1_isp_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct rkisp1_isp *isp = container_of(sd, struct rkisp1_isp, sd); mutex_lock(&isp->ops_lock); if (fmt->pad == RKISP1_ISP_PAD_SINK_VIDEO) - rkisp1_isp_set_sink_fmt(isp, cfg, &fmt->format, fmt->which); + rkisp1_isp_set_sink_fmt(isp, sd_state, &fmt->format, + fmt->which); else if (fmt->pad == RKISP1_ISP_PAD_SOURCE_VIDEO) - rkisp1_isp_set_src_fmt(isp, cfg, &fmt->format, fmt->which); + rkisp1_isp_set_src_fmt(isp, sd_state, &fmt->format, + fmt->which); else - fmt->format = *rkisp1_isp_get_pad_fmt(isp, cfg, fmt->pad, + fmt->format = *rkisp1_isp_get_pad_fmt(isp, sd_state, fmt->pad, fmt->which); mutex_unlock(&isp->ops_lock); @@ -838,7 +851,7 @@ static int rkisp1_isp_set_fmt(struct v4l2_subdev *sd, } static int rkisp1_isp_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct rkisp1_isp *isp = container_of(sd, struct rkisp1_isp, sd); @@ -854,20 +867,20 @@ static int rkisp1_isp_get_selection(struct v4l2_subdev *sd, if (sel->pad == RKISP1_ISP_PAD_SINK_VIDEO) { struct v4l2_mbus_framefmt *fmt; - fmt = rkisp1_isp_get_pad_fmt(isp, cfg, sel->pad, + fmt = rkisp1_isp_get_pad_fmt(isp, sd_state, sel->pad, sel->which); sel->r.height = fmt->height; sel->r.width = fmt->width; sel->r.left = 0; sel->r.top = 0; } else { - sel->r = *rkisp1_isp_get_pad_crop(isp, cfg, - RKISP1_ISP_PAD_SINK_VIDEO, - sel->which); + sel->r = *rkisp1_isp_get_pad_crop(isp, sd_state, + RKISP1_ISP_PAD_SINK_VIDEO, + sel->which); } break; case V4L2_SEL_TGT_CROP: - sel->r = *rkisp1_isp_get_pad_crop(isp, cfg, sel->pad, + sel->r = *rkisp1_isp_get_pad_crop(isp, sd_state, sel->pad, sel->which); break; default: @@ -878,7 +891,7 @@ static int rkisp1_isp_get_selection(struct v4l2_subdev *sd, } static int rkisp1_isp_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct rkisp1_device *rkisp1 = @@ -893,9 +906,9 @@ static int rkisp1_isp_set_selection(struct v4l2_subdev *sd, sel->pad, sel->r.left, sel->r.top, sel->r.width, sel->r.height); mutex_lock(&isp->ops_lock); if (sel->pad == RKISP1_ISP_PAD_SINK_VIDEO) - rkisp1_isp_set_sink_crop(isp, cfg, &sel->r, sel->which); + rkisp1_isp_set_sink_crop(isp, sd_state, &sel->r, sel->which); else if (sel->pad == RKISP1_ISP_PAD_SOURCE_VIDEO) - rkisp1_isp_set_src_crop(isp, cfg, &sel->r, sel->which); + rkisp1_isp_set_src_crop(isp, sd_state, &sel->r, sel->which); else ret = -EINVAL; @@ -1037,6 +1050,9 @@ static const struct v4l2_subdev_ops rkisp1_isp_ops = { int rkisp1_isp_register(struct rkisp1_device *rkisp1) { + struct v4l2_subdev_state state = { + .pads = rkisp1->isp.pad_cfg + }; struct rkisp1_isp *isp = &rkisp1->isp; struct media_pad *pads = isp->pads; struct v4l2_subdev *sd = &isp->sd; @@ -1069,7 +1085,7 @@ int rkisp1_isp_register(struct rkisp1_device *rkisp1) goto err_cleanup_media_entity; } - rkisp1_isp_init_config(sd, rkisp1->isp.pad_cfg); + rkisp1_isp_init_config(sd, &state); return 0; err_cleanup_media_entity: diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c index b6beddd988d0..529c6e21815f 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-params.c @@ -1258,7 +1258,10 @@ void rkisp1_params_configure(struct rkisp1_params *params, rkisp1_params_config_parameter(params); } -/* Not called when the camera active, thus not isr protection. */ +/* + * Not called when the camera is active, therefore there is no need to acquire + * a lock. + */ void rkisp1_params_disable(struct rkisp1_params *params) { rkisp1_param_clear_bits(params, RKISP1_CIF_ISP_DPCC_MODE, diff --git a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c index 79deed8adcea..2070f4b06705 100644 --- a/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c +++ b/drivers/media/platform/rockchip/rkisp1/rkisp1-resizer.c @@ -180,24 +180,30 @@ static const struct rkisp1_rsz_config rkisp1_rsz_config_sp = { static struct v4l2_mbus_framefmt * rkisp1_rsz_get_pad_fmt(struct rkisp1_resizer *rsz, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { + struct v4l2_subdev_state state = { + .pads = rsz->pad_cfg + }; if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&rsz->sd, cfg, pad); + return v4l2_subdev_get_try_format(&rsz->sd, sd_state, pad); else - return v4l2_subdev_get_try_format(&rsz->sd, rsz->pad_cfg, pad); + return v4l2_subdev_get_try_format(&rsz->sd, &state, pad); } static struct v4l2_rect * rkisp1_rsz_get_pad_crop(struct rkisp1_resizer *rsz, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { + struct v4l2_subdev_state state = { + .pads = rsz->pad_cfg + }; if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_crop(&rsz->sd, cfg, pad); + return v4l2_subdev_get_try_crop(&rsz->sd, sd_state, pad); else - return v4l2_subdev_get_try_crop(&rsz->sd, rsz->pad_cfg, pad); + return v4l2_subdev_get_try_crop(&rsz->sd, &state, pad); } /* ---------------------------------------------------------------------------- @@ -451,12 +457,15 @@ static void rkisp1_rsz_config(struct rkisp1_resizer *rsz, */ static int rkisp1_rsz_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct rkisp1_resizer *rsz = container_of(sd, struct rkisp1_resizer, sd); struct v4l2_subdev_pad_config dummy_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &dummy_cfg + }; u32 pad = code->pad; int ret; @@ -481,7 +490,7 @@ static int rkisp1_rsz_enum_mbus_code(struct v4l2_subdev *sd, /* supported mbus codes on the sink pad are the same as isp src pad */ code->pad = RKISP1_ISP_PAD_SOURCE_VIDEO; ret = v4l2_subdev_call(&rsz->rkisp1->isp.sd, pad, enum_mbus_code, - &dummy_cfg, code); + &pad_state, code); /* restore pad */ code->pad = pad; @@ -490,24 +499,27 @@ static int rkisp1_rsz_enum_mbus_code(struct v4l2_subdev *sd, } static int rkisp1_rsz_init_config(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_mbus_framefmt *sink_fmt, *src_fmt; struct v4l2_rect *sink_crop; - sink_fmt = v4l2_subdev_get_try_format(sd, cfg, RKISP1_RSZ_PAD_SRC); + sink_fmt = v4l2_subdev_get_try_format(sd, sd_state, + RKISP1_RSZ_PAD_SRC); sink_fmt->width = RKISP1_DEFAULT_WIDTH; sink_fmt->height = RKISP1_DEFAULT_HEIGHT; sink_fmt->field = V4L2_FIELD_NONE; sink_fmt->code = RKISP1_DEF_FMT; - sink_crop = v4l2_subdev_get_try_crop(sd, cfg, RKISP1_RSZ_PAD_SINK); + sink_crop = v4l2_subdev_get_try_crop(sd, sd_state, + RKISP1_RSZ_PAD_SINK); sink_crop->width = RKISP1_DEFAULT_WIDTH; sink_crop->height = RKISP1_DEFAULT_HEIGHT; sink_crop->left = 0; sink_crop->top = 0; - src_fmt = v4l2_subdev_get_try_format(sd, cfg, RKISP1_RSZ_PAD_SINK); + src_fmt = v4l2_subdev_get_try_format(sd, sd_state, + RKISP1_RSZ_PAD_SINK); *src_fmt = *sink_fmt; /* NOTE: there is no crop in the source pad, only in the sink */ @@ -516,15 +528,17 @@ static int rkisp1_rsz_init_config(struct v4l2_subdev *sd, } static void rkisp1_rsz_set_src_fmt(struct rkisp1_resizer *rsz, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_mbus_framefmt *format, unsigned int which) { const struct rkisp1_isp_mbus_info *sink_mbus_info; struct v4l2_mbus_framefmt *src_fmt, *sink_fmt; - sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, which); - src_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SRC, which); + sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, sd_state, RKISP1_RSZ_PAD_SINK, + which); + src_fmt = rkisp1_rsz_get_pad_fmt(rsz, sd_state, RKISP1_RSZ_PAD_SRC, + which); sink_mbus_info = rkisp1_isp_mbus_info_get(sink_fmt->code); /* for YUV formats, userspace can change the mbus code on the src pad if it is supported */ @@ -543,7 +557,7 @@ static void rkisp1_rsz_set_src_fmt(struct rkisp1_resizer *rsz, } static void rkisp1_rsz_set_sink_crop(struct rkisp1_resizer *rsz, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_rect *r, unsigned int which) { @@ -551,8 +565,10 @@ static void rkisp1_rsz_set_sink_crop(struct rkisp1_resizer *rsz, struct v4l2_mbus_framefmt *sink_fmt; struct v4l2_rect *sink_crop; - sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, which); - sink_crop = rkisp1_rsz_get_pad_crop(rsz, cfg, RKISP1_RSZ_PAD_SINK, + sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, sd_state, RKISP1_RSZ_PAD_SINK, + which); + sink_crop = rkisp1_rsz_get_pad_crop(rsz, sd_state, + RKISP1_RSZ_PAD_SINK, which); /* Not crop for MP bayer raw data */ @@ -579,7 +595,7 @@ static void rkisp1_rsz_set_sink_crop(struct rkisp1_resizer *rsz, } static void rkisp1_rsz_set_sink_fmt(struct rkisp1_resizer *rsz, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_mbus_framefmt *format, unsigned int which) { @@ -587,9 +603,12 @@ static void rkisp1_rsz_set_sink_fmt(struct rkisp1_resizer *rsz, struct v4l2_mbus_framefmt *sink_fmt, *src_fmt; struct v4l2_rect *sink_crop; - sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, which); - src_fmt = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SRC, which); - sink_crop = rkisp1_rsz_get_pad_crop(rsz, cfg, RKISP1_RSZ_PAD_SINK, + sink_fmt = rkisp1_rsz_get_pad_fmt(rsz, sd_state, RKISP1_RSZ_PAD_SINK, + which); + src_fmt = rkisp1_rsz_get_pad_fmt(rsz, sd_state, RKISP1_RSZ_PAD_SRC, + which); + sink_crop = rkisp1_rsz_get_pad_crop(rsz, sd_state, + RKISP1_RSZ_PAD_SINK, which); if (rsz->id == RKISP1_SELFPATH) sink_fmt->code = MEDIA_BUS_FMT_YUYV8_2X8; @@ -617,24 +636,25 @@ static void rkisp1_rsz_set_sink_fmt(struct rkisp1_resizer *rsz, *format = *sink_fmt; /* Update sink crop */ - rkisp1_rsz_set_sink_crop(rsz, cfg, sink_crop, which); + rkisp1_rsz_set_sink_crop(rsz, sd_state, sink_crop, which); } static int rkisp1_rsz_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct rkisp1_resizer *rsz = container_of(sd, struct rkisp1_resizer, sd); mutex_lock(&rsz->ops_lock); - fmt->format = *rkisp1_rsz_get_pad_fmt(rsz, cfg, fmt->pad, fmt->which); + fmt->format = *rkisp1_rsz_get_pad_fmt(rsz, sd_state, fmt->pad, + fmt->which); mutex_unlock(&rsz->ops_lock); return 0; } static int rkisp1_rsz_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct rkisp1_resizer *rsz = @@ -642,16 +662,18 @@ static int rkisp1_rsz_set_fmt(struct v4l2_subdev *sd, mutex_lock(&rsz->ops_lock); if (fmt->pad == RKISP1_RSZ_PAD_SINK) - rkisp1_rsz_set_sink_fmt(rsz, cfg, &fmt->format, fmt->which); + rkisp1_rsz_set_sink_fmt(rsz, sd_state, &fmt->format, + fmt->which); else - rkisp1_rsz_set_src_fmt(rsz, cfg, &fmt->format, fmt->which); + rkisp1_rsz_set_src_fmt(rsz, sd_state, &fmt->format, + fmt->which); mutex_unlock(&rsz->ops_lock); return 0; } static int rkisp1_rsz_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct rkisp1_resizer *rsz = @@ -665,7 +687,8 @@ static int rkisp1_rsz_get_selection(struct v4l2_subdev *sd, mutex_lock(&rsz->ops_lock); switch (sel->target) { case V4L2_SEL_TGT_CROP_BOUNDS: - mf_sink = rkisp1_rsz_get_pad_fmt(rsz, cfg, RKISP1_RSZ_PAD_SINK, + mf_sink = rkisp1_rsz_get_pad_fmt(rsz, sd_state, + RKISP1_RSZ_PAD_SINK, sel->which); sel->r.height = mf_sink->height; sel->r.width = mf_sink->width; @@ -673,7 +696,8 @@ static int rkisp1_rsz_get_selection(struct v4l2_subdev *sd, sel->r.top = 0; break; case V4L2_SEL_TGT_CROP: - sel->r = *rkisp1_rsz_get_pad_crop(rsz, cfg, RKISP1_RSZ_PAD_SINK, + sel->r = *rkisp1_rsz_get_pad_crop(rsz, sd_state, + RKISP1_RSZ_PAD_SINK, sel->which); break; default: @@ -685,7 +709,7 @@ static int rkisp1_rsz_get_selection(struct v4l2_subdev *sd, } static int rkisp1_rsz_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct rkisp1_resizer *rsz = @@ -698,7 +722,7 @@ static int rkisp1_rsz_set_selection(struct v4l2_subdev *sd, sel->pad, sel->r.left, sel->r.top, sel->r.width, sel->r.height); mutex_lock(&rsz->ops_lock); - rkisp1_rsz_set_sink_crop(rsz, cfg, &sel->r, sel->which); + rkisp1_rsz_set_sink_crop(rsz, sd_state, &sel->r, sel->which); mutex_unlock(&rsz->ops_lock); return 0; @@ -764,6 +788,9 @@ static void rkisp1_rsz_unregister(struct rkisp1_resizer *rsz) static int rkisp1_rsz_register(struct rkisp1_resizer *rsz) { + struct v4l2_subdev_state state = { + .pads = rsz->pad_cfg + }; static const char * const dev_names[] = { RKISP1_RSZ_MP_DEV_NAME, RKISP1_RSZ_SP_DEV_NAME @@ -802,7 +829,7 @@ static int rkisp1_rsz_register(struct rkisp1_resizer *rsz) goto err_cleanup_media_entity; } - rkisp1_rsz_init_config(sd, rsz->pad_cfg); + rkisp1_rsz_init_config(sd, &state); return 0; err_cleanup_media_entity: diff --git a/drivers/media/platform/s3c-camif/camif-capture.c b/drivers/media/platform/s3c-camif/camif-capture.c index 9ca49af29542..140854ab4dd8 100644 --- a/drivers/media/platform/s3c-camif/camif-capture.c +++ b/drivers/media/platform/s3c-camif/camif-capture.c @@ -547,7 +547,7 @@ static int s3c_camif_open(struct file *file) if (ret < 0) goto unlock; - ret = pm_runtime_get_sync(camif->dev); + ret = pm_runtime_resume_and_get(camif->dev); if (ret < 0) goto err_pm; @@ -1199,7 +1199,7 @@ static const u32 camif_mbus_formats[] = { */ static int s3c_camif_subdev_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(camif_mbus_formats)) @@ -1210,14 +1210,14 @@ static int s3c_camif_subdev_enum_mbus_code(struct v4l2_subdev *sd, } static int s3c_camif_subdev_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct camif_dev *camif = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *mf = &fmt->format; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); fmt->format = *mf; return 0; } @@ -1278,7 +1278,7 @@ static void __camif_subdev_try_format(struct camif_dev *camif, } static int s3c_camif_subdev_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct camif_dev *camif = v4l2_get_subdevdata(sd); @@ -1306,7 +1306,7 @@ static int s3c_camif_subdev_set_fmt(struct v4l2_subdev *sd, __camif_subdev_try_format(camif, mf, fmt->pad); if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); *mf = fmt->format; mutex_unlock(&camif->lock); return 0; @@ -1345,7 +1345,7 @@ static int s3c_camif_subdev_set_fmt(struct v4l2_subdev *sd, } static int s3c_camif_subdev_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct camif_dev *camif = v4l2_get_subdevdata(sd); @@ -1358,7 +1358,7 @@ static int s3c_camif_subdev_get_selection(struct v4l2_subdev *sd, return -EINVAL; if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { - sel->r = *v4l2_subdev_get_try_crop(sd, cfg, sel->pad); + sel->r = *v4l2_subdev_get_try_crop(sd, sd_state, sel->pad); return 0; } @@ -1432,7 +1432,7 @@ static void __camif_try_crop(struct camif_dev *camif, struct v4l2_rect *r) } static int s3c_camif_subdev_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct camif_dev *camif = v4l2_get_subdevdata(sd); @@ -1446,7 +1446,7 @@ static int s3c_camif_subdev_set_selection(struct v4l2_subdev *sd, __camif_try_crop(camif, &sel->r); if (sel->which == V4L2_SUBDEV_FORMAT_TRY) { - *v4l2_subdev_get_try_crop(sd, cfg, sel->pad) = sel->r; + *v4l2_subdev_get_try_crop(sd, sd_state, sel->pad) = sel->r; } else { unsigned long flags; unsigned int i; diff --git a/drivers/media/platform/s3c-camif/camif-core.c b/drivers/media/platform/s3c-camif/camif-core.c index 4c3c00d59c92..e1d51fd3e700 100644 --- a/drivers/media/platform/s3c-camif/camif-core.c +++ b/drivers/media/platform/s3c-camif/camif-core.c @@ -460,9 +460,9 @@ static int s3c_camif_probe(struct platform_device *pdev) pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) - goto err_pm; + goto err_disable; ret = camif_media_dev_init(camif); if (ret < 0) @@ -502,6 +502,7 @@ static int s3c_camif_probe(struct platform_device *pdev) camif_unregister_media_entities(camif); err_pm: pm_runtime_put(dev); +err_disable: pm_runtime_disable(dev); camif_clk_put(camif); err_clk: diff --git a/drivers/media/platform/s5p-g2d/g2d.c b/drivers/media/platform/s5p-g2d/g2d.c index 15bcb7f6e113..1cb5eaabf340 100644 --- a/drivers/media/platform/s5p-g2d/g2d.c +++ b/drivers/media/platform/s5p-g2d/g2d.c @@ -276,6 +276,9 @@ static int g2d_release(struct file *file) struct g2d_dev *dev = video_drvdata(file); struct g2d_ctx *ctx = fh2ctx(file->private_data); + mutex_lock(&dev->mutex); + v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); + mutex_unlock(&dev->mutex); v4l2_ctrl_handler_free(&ctx->ctrl_handler); v4l2_fh_del(&ctx->fh); v4l2_fh_exit(&ctx->fh); diff --git a/drivers/media/platform/s5p-jpeg/jpeg-core.c b/drivers/media/platform/s5p-jpeg/jpeg-core.c index 026111505f5a..d402e456f27d 100644 --- a/drivers/media/platform/s5p-jpeg/jpeg-core.c +++ b/drivers/media/platform/s5p-jpeg/jpeg-core.c @@ -2566,11 +2566,8 @@ static void s5p_jpeg_buf_queue(struct vb2_buffer *vb) static int s5p_jpeg_start_streaming(struct vb2_queue *q, unsigned int count) { struct s5p_jpeg_ctx *ctx = vb2_get_drv_priv(q); - int ret; - ret = pm_runtime_get_sync(ctx->jpeg->dev); - - return ret > 0 ? 0 : ret; + return pm_runtime_resume_and_get(ctx->jpeg->dev); } static void s5p_jpeg_stop_streaming(struct vb2_queue *q) diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c b/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c index a92a9ca6e87e..c1d3bda8385b 100644 --- a/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c +++ b/drivers/media/platform/s5p-mfc/s5p_mfc_dec.c @@ -172,6 +172,7 @@ static struct mfc_control controls[] = { .type = V4L2_CTRL_TYPE_INTEGER, .minimum = 0, .maximum = 16383, + .step = 1, .default_value = 0, }, { diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc_pm.c b/drivers/media/platform/s5p-mfc/s5p_mfc_pm.c index 62d2320a7218..88b7d33c9197 100644 --- a/drivers/media/platform/s5p-mfc/s5p_mfc_pm.c +++ b/drivers/media/platform/s5p-mfc/s5p_mfc_pm.c @@ -78,11 +78,9 @@ int s5p_mfc_power_on(void) { int i, ret = 0; - ret = pm_runtime_get_sync(pm->device); - if (ret < 0) { - pm_runtime_put_noidle(pm->device); + ret = pm_runtime_resume_and_get(pm->device); + if (ret < 0) return ret; - } /* clock control */ for (i = 0; i < pm->num_clocks; i++) { diff --git a/drivers/media/platform/sh_vou.c b/drivers/media/platform/sh_vou.c index 4ac48441f22c..ca4310e26c49 100644 --- a/drivers/media/platform/sh_vou.c +++ b/drivers/media/platform/sh_vou.c @@ -1133,7 +1133,11 @@ static int sh_vou_open(struct file *file) if (v4l2_fh_is_singular_file(file) && vou_dev->status == SH_VOU_INITIALISING) { /* First open */ - pm_runtime_get_sync(vou_dev->v4l2_dev.dev); + err = pm_runtime_resume_and_get(vou_dev->v4l2_dev.dev); + if (err < 0) { + v4l2_fh_release(file); + goto done_open; + } err = sh_vou_hw_init(vou_dev); if (err < 0) { pm_runtime_put(vou_dev->v4l2_dev.dev); diff --git a/drivers/media/platform/sti/bdisp/Makefile b/drivers/media/platform/sti/bdisp/Makefile index caf7ccd193ea..39ade0a34723 100644 --- a/drivers/media/platform/sti/bdisp/Makefile +++ b/drivers/media/platform/sti/bdisp/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_VIDEO_STI_BDISP) := bdisp.o +obj-$(CONFIG_VIDEO_STI_BDISP) += bdisp.o bdisp-objs := bdisp-v4l2.o bdisp-hw.o bdisp-debug.o diff --git a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c index 060ca85f64d5..6413cd279125 100644 --- a/drivers/media/platform/sti/bdisp/bdisp-v4l2.c +++ b/drivers/media/platform/sti/bdisp/bdisp-v4l2.c @@ -499,7 +499,7 @@ static int bdisp_start_streaming(struct vb2_queue *q, unsigned int count) { struct bdisp_ctx *ctx = q->drv_priv; struct vb2_v4l2_buffer *buf; - int ret = pm_runtime_get_sync(ctx->bdisp_dev->dev); + int ret = pm_runtime_resume_and_get(ctx->bdisp_dev->dev); if (ret < 0) { dev_err(ctx->bdisp_dev->dev, "failed to set runtime PM\n"); @@ -1318,7 +1318,6 @@ static int bdisp_probe(struct platform_device *pdev) res = platform_get_resource(pdev, IORESOURCE_MEM, 0); bdisp->regs = devm_ioremap_resource(dev, res); if (IS_ERR(bdisp->regs)) { - dev_err(dev, "failed to get regs\n"); ret = PTR_ERR(bdisp->regs); goto err_wq; } @@ -1364,10 +1363,10 @@ static int bdisp_probe(struct platform_device *pdev) /* Power management */ pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "failed to set PM\n"); - goto err_pm; + goto err_remove; } /* Filters */ @@ -1395,6 +1394,7 @@ static int bdisp_probe(struct platform_device *pdev) bdisp_hw_free_filters(bdisp->dev); err_pm: pm_runtime_put(dev); +err_remove: bdisp_debugfs_remove(bdisp); v4l2_device_unregister(&bdisp->v4l2_dev); err_clk: diff --git a/drivers/media/platform/sti/delta/Makefile b/drivers/media/platform/sti/delta/Makefile index 92b37e216f00..32412fa4c632 100644 --- a/drivers/media/platform/sti/delta/Makefile +++ b/drivers/media/platform/sti/delta/Makefile @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_VIDEO_STI_DELTA_DRIVER) := st-delta.o +obj-$(CONFIG_VIDEO_STI_DELTA_DRIVER) += st-delta.o st-delta-y := delta-v4l2.o delta-mem.o delta-ipc.o delta-debug.o # MJPEG support diff --git a/drivers/media/platform/sti/delta/delta-v4l2.c b/drivers/media/platform/sti/delta/delta-v4l2.c index c691b3d81549..c887a31ebb54 100644 --- a/drivers/media/platform/sti/delta/delta-v4l2.c +++ b/drivers/media/platform/sti/delta/delta-v4l2.c @@ -954,10 +954,8 @@ static void delta_run_work(struct work_struct *work) /* enable the hardware */ if (!dec->pm) { ret = delta_get_sync(ctx); - if (ret) { - delta_put_autosuspend(ctx); + if (ret) goto err; - } } /* decode this access unit */ @@ -1009,7 +1007,6 @@ static void delta_run_work(struct work_struct *work) dev_err(delta->dev, "%s NULL decoded frame\n", ctx->name); - ret = -EIO; goto out; } @@ -1277,9 +1274,9 @@ int delta_get_sync(struct delta_ctx *ctx) int ret = 0; /* enable the hardware */ - ret = pm_runtime_get_sync(delta->dev); + ret = pm_runtime_resume_and_get(delta->dev); if (ret < 0) { - dev_err(delta->dev, "%s pm_runtime_get_sync failed (%d)\n", + dev_err(delta->dev, "%s pm_runtime_resume_and_get failed (%d)\n", __func__, ret); return ret; } diff --git a/drivers/media/platform/sti/hva/Makefile b/drivers/media/platform/sti/hva/Makefile index 74b41ec52f97..b5a5478bdd01 100644 --- a/drivers/media/platform/sti/hva/Makefile +++ b/drivers/media/platform/sti/hva/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_VIDEO_STI_HVA) := st-hva.o +obj-$(CONFIG_VIDEO_STI_HVA) += st-hva.o st-hva-y := hva-v4l2.o hva-hw.o hva-mem.o hva-h264.o st-hva-$(CONFIG_VIDEO_STI_HVA_DEBUGFS) += hva-debugfs.o diff --git a/drivers/media/platform/sti/hva/hva-hw.c b/drivers/media/platform/sti/hva/hva-hw.c index f59811e27f51..30fb1aa4a351 100644 --- a/drivers/media/platform/sti/hva/hva-hw.c +++ b/drivers/media/platform/sti/hva/hva-hw.c @@ -130,8 +130,7 @@ static irqreturn_t hva_hw_its_irq_thread(int irq, void *arg) ctx_id = (hva->sts_reg & 0xFF00) >> 8; if (ctx_id >= HVA_MAX_INSTANCES) { dev_err(dev, "%s %s: bad context identifier: %d\n", - ctx->name, __func__, ctx_id); - ctx->hw_err = true; + HVA_PREFIX, __func__, ctx_id); goto out; } @@ -270,9 +269,8 @@ static unsigned long int hva_hw_get_ip_version(struct hva_dev *hva) struct device *dev = hva_to_dev(hva); unsigned long int version; - if (pm_runtime_get_sync(dev) < 0) { + if (pm_runtime_resume_and_get(dev) < 0) { dev_err(dev, "%s failed to get pm_runtime\n", HVA_PREFIX); - pm_runtime_put_noidle(dev); mutex_unlock(&hva->protect_mutex); return -EFAULT; } @@ -386,10 +384,10 @@ int hva_hw_probe(struct platform_device *pdev, struct hva_dev *hva) pm_runtime_set_suspended(dev); pm_runtime_enable(dev); - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "%s failed to set PM\n", HVA_PREFIX); - goto err_pm; + goto err_clk; } /* check IP hardware version */ @@ -462,6 +460,7 @@ int hva_hw_execute_task(struct hva_ctx *ctx, enum hva_hw_cmd_type cmd, u8 client_id = ctx->id; int ret; u32 reg = 0; + bool got_pm = false; mutex_lock(&hva->protect_mutex); @@ -469,12 +468,13 @@ int hva_hw_execute_task(struct hva_ctx *ctx, enum hva_hw_cmd_type cmd, enable_irq(hva->irq_its); enable_irq(hva->irq_err); - if (pm_runtime_get_sync(dev) < 0) { + if (pm_runtime_resume_and_get(dev) < 0) { dev_err(dev, "%s failed to get pm_runtime\n", ctx->name); ctx->sys_errors++; ret = -EFAULT; goto out; } + got_pm = true; reg = readl_relaxed(hva->regs + HVA_HIF_REG_CLK_GATING); switch (cmd) { @@ -537,7 +537,8 @@ int hva_hw_execute_task(struct hva_ctx *ctx, enum hva_hw_cmd_type cmd, dev_dbg(dev, "%s unknown command 0x%x\n", ctx->name, cmd); } - pm_runtime_put_autosuspend(dev); + if (got_pm) + pm_runtime_put_autosuspend(dev); mutex_unlock(&hva->protect_mutex); return ret; @@ -553,9 +554,8 @@ void hva_hw_dump_regs(struct hva_dev *hva, struct seq_file *s) mutex_lock(&hva->protect_mutex); - if (pm_runtime_get_sync(dev) < 0) { + if (pm_runtime_resume_and_get(dev) < 0) { seq_puts(s, "Cannot wake up IP\n"); - pm_runtime_put_noidle(dev); mutex_unlock(&hva->protect_mutex); return; } diff --git a/drivers/media/platform/stm32/stm32-dcmi.c b/drivers/media/platform/stm32/stm32-dcmi.c index d9b4ad0abf0c..d914ccef9831 100644 --- a/drivers/media/platform/stm32/stm32-dcmi.c +++ b/drivers/media/platform/stm32/stm32-dcmi.c @@ -600,7 +600,7 @@ static struct media_entity *dcmi_find_source(struct stm32_dcmi *dcmi) } static int dcmi_pipeline_s_fmt(struct stm32_dcmi *dcmi, - struct v4l2_subdev_pad_config *pad_cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct media_entity *entity = &dcmi->source->entity; @@ -642,7 +642,7 @@ static int dcmi_pipeline_s_fmt(struct stm32_dcmi *dcmi, format->format.width, format->format.height); fmt.pad = pad->index; - ret = v4l2_subdev_call(subdev, pad, set_fmt, pad_cfg, &fmt); + ret = v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &fmt); if (ret < 0) { dev_err(dcmi->dev, "%s: Failed to set format 0x%x %ux%u on \"%s\":%d pad (%d)\n", __func__, format->format.code, @@ -723,11 +723,11 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count) u32 val = 0; int ret; - ret = pm_runtime_get_sync(dcmi->dev); + ret = pm_runtime_resume_and_get(dcmi->dev); if (ret < 0) { dev_err(dcmi->dev, "%s: Failed to start streaming, cannot get sync (%d)\n", __func__, ret); - goto err_pm_put; + goto err_unlocked; } ret = media_pipeline_start(&dcmi->vdev->entity, &dcmi->pipeline); @@ -848,6 +848,7 @@ static int dcmi_start_streaming(struct vb2_queue *vq, unsigned int count) err_pm_put: pm_runtime_put(dcmi->dev); +err_unlocked: spin_lock_irq(&dcmi->irqlock); /* * Return all buffers to vb2 in QUEUED state. @@ -977,6 +978,9 @@ static int dcmi_try_fmt(struct stm32_dcmi *dcmi, struct v4l2_format *f, struct dcmi_framesize sd_fsize; struct v4l2_pix_format *pix = &f->fmt.pix; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -1012,7 +1016,7 @@ static int dcmi_try_fmt(struct stm32_dcmi *dcmi, struct v4l2_format *f, v4l2_fill_mbus_format(&format.format, pix, sd_fmt->mbus_code); ret = v4l2_subdev_call(dcmi->source, pad, set_fmt, - &pad_cfg, &format); + &pad_state, &format); if (ret < 0) return ret; @@ -1162,6 +1166,9 @@ static int dcmi_set_sensor_format(struct stm32_dcmi *dcmi, .which = V4L2_SUBDEV_FORMAT_TRY, }; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; int ret; sd_fmt = find_format_by_fourcc(dcmi, pix->pixelformat); @@ -1175,7 +1182,7 @@ static int dcmi_set_sensor_format(struct stm32_dcmi *dcmi, v4l2_fill_mbus_format(&format.format, pix, sd_fmt->mbus_code); ret = v4l2_subdev_call(dcmi->source, pad, set_fmt, - &pad_cfg, &format); + &pad_state, &format); if (ret < 0) return ret; diff --git a/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c b/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c index 4785faddf630..3872027ed2fa 100644 --- a/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c +++ b/drivers/media/platform/sunxi/sun4i-csi/sun4i_v4l2.c @@ -206,9 +206,9 @@ static int sun4i_csi_open(struct file *file) if (ret) return ret; - ret = pm_runtime_get_sync(csi->dev); + ret = pm_runtime_resume_and_get(csi->dev); if (ret < 0) - goto err_pm_put; + goto err_unlock; ret = v4l2_pipeline_pm_get(&csi->vdev.entity); if (ret) @@ -227,6 +227,8 @@ static int sun4i_csi_open(struct file *file) err_pm_put: pm_runtime_put(csi->dev); + +err_unlock: mutex_unlock(&csi->lock); return ret; @@ -269,25 +271,26 @@ static const struct v4l2_mbus_framefmt sun4i_csi_pad_fmt_default = { }; static int sun4i_csi_subdev_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_mbus_framefmt *fmt; - fmt = v4l2_subdev_get_try_format(subdev, cfg, CSI_SUBDEV_SINK); + fmt = v4l2_subdev_get_try_format(subdev, sd_state, CSI_SUBDEV_SINK); *fmt = sun4i_csi_pad_fmt_default; return 0; } static int sun4i_csi_subdev_get_fmt(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct sun4i_csi *csi = container_of(subdev, struct sun4i_csi, subdev); struct v4l2_mbus_framefmt *subdev_fmt; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - subdev_fmt = v4l2_subdev_get_try_format(subdev, cfg, fmt->pad); + subdev_fmt = v4l2_subdev_get_try_format(subdev, sd_state, + fmt->pad); else subdev_fmt = &csi->subdev_fmt; @@ -297,14 +300,15 @@ static int sun4i_csi_subdev_get_fmt(struct v4l2_subdev *subdev, } static int sun4i_csi_subdev_set_fmt(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct sun4i_csi *csi = container_of(subdev, struct sun4i_csi, subdev); struct v4l2_mbus_framefmt *subdev_fmt; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - subdev_fmt = v4l2_subdev_get_try_format(subdev, cfg, fmt->pad); + subdev_fmt = v4l2_subdev_get_try_format(subdev, sd_state, + fmt->pad); else subdev_fmt = &csi->subdev_fmt; @@ -323,7 +327,7 @@ static int sun4i_csi_subdev_set_fmt(struct v4l2_subdev *subdev, static int sun4i_csi_subdev_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *mbus) { if (mbus->index >= ARRAY_SIZE(sun4i_csi_formats)) diff --git a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c index 3181d0781b61..07b2161392d2 100644 --- a/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c +++ b/drivers/media/platform/sunxi/sun6i-csi/sun6i_video.c @@ -481,8 +481,10 @@ static int sun6i_video_open(struct file *file) goto fh_release; /* check if already powered */ - if (!v4l2_fh_is_singular_file(file)) + if (!v4l2_fh_is_singular_file(file)) { + ret = -EBUSY; goto unlock; + } ret = sun6i_csi_set_power(video->csi, true); if (ret < 0) diff --git a/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c b/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c index 3f81dd17755c..fbcca59a0517 100644 --- a/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c +++ b/drivers/media/platform/sunxi/sun8i-rotate/sun8i_rotate.c @@ -494,7 +494,7 @@ static int rotate_start_streaming(struct vb2_queue *vq, unsigned int count) struct device *dev = ctx->dev->dev; int ret; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err(dev, "Failed to enable module\n"); diff --git a/drivers/media/platform/ti-vpe/cal-camerarx.c b/drivers/media/platform/ti-vpe/cal-camerarx.c index cbe6114908de..124a4e2bdefe 100644 --- a/drivers/media/platform/ti-vpe/cal-camerarx.c +++ b/drivers/media/platform/ti-vpe/cal-camerarx.c @@ -586,12 +586,12 @@ static inline struct cal_camerarx *to_cal_camerarx(struct v4l2_subdev *sd) static struct v4l2_mbus_framefmt * cal_camerarx_get_pad_format(struct cal_camerarx *phy, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&phy->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&phy->subdev, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &phy->formats[pad]; default: @@ -611,7 +611,7 @@ static int cal_camerarx_sd_s_stream(struct v4l2_subdev *sd, int enable) } static int cal_camerarx_sd_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct cal_camerarx *phy = to_cal_camerarx(sd); @@ -623,7 +623,7 @@ static int cal_camerarx_sd_enum_mbus_code(struct v4l2_subdev *sd, if (code->index > 0) return -EINVAL; - fmt = cal_camerarx_get_pad_format(phy, cfg, + fmt = cal_camerarx_get_pad_format(phy, sd_state, CAL_CAMERARX_PAD_SINK, code->which); code->code = fmt->code; @@ -639,7 +639,7 @@ static int cal_camerarx_sd_enum_mbus_code(struct v4l2_subdev *sd, } static int cal_camerarx_sd_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct cal_camerarx *phy = to_cal_camerarx(sd); @@ -652,7 +652,7 @@ static int cal_camerarx_sd_enum_frame_size(struct v4l2_subdev *sd, if (fse->pad == CAL_CAMERARX_PAD_SOURCE) { struct v4l2_mbus_framefmt *fmt; - fmt = cal_camerarx_get_pad_format(phy, cfg, + fmt = cal_camerarx_get_pad_format(phy, sd_state, CAL_CAMERARX_PAD_SINK, fse->which); if (fse->code != fmt->code) @@ -679,20 +679,21 @@ static int cal_camerarx_sd_enum_frame_size(struct v4l2_subdev *sd, } static int cal_camerarx_sd_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct cal_camerarx *phy = to_cal_camerarx(sd); struct v4l2_mbus_framefmt *fmt; - fmt = cal_camerarx_get_pad_format(phy, cfg, format->pad, format->which); + fmt = cal_camerarx_get_pad_format(phy, sd_state, format->pad, + format->which); format->format = *fmt; return 0; } static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct cal_camerarx *phy = to_cal_camerarx(sd); @@ -702,7 +703,7 @@ static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, /* No transcoding, source and sink formats must match. */ if (format->pad == CAL_CAMERARX_PAD_SOURCE) - return cal_camerarx_sd_get_fmt(sd, cfg, format); + return cal_camerarx_sd_get_fmt(sd, sd_state, format); /* * Default to the first format is the requested media bus code isn't @@ -727,11 +728,13 @@ static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, format->format.code = fmtinfo->code; /* Store the format and propagate it to the source pad. */ - fmt = cal_camerarx_get_pad_format(phy, cfg, CAL_CAMERARX_PAD_SINK, + fmt = cal_camerarx_get_pad_format(phy, sd_state, + CAL_CAMERARX_PAD_SINK, format->which); *fmt = format->format; - fmt = cal_camerarx_get_pad_format(phy, cfg, CAL_CAMERARX_PAD_SOURCE, + fmt = cal_camerarx_get_pad_format(phy, sd_state, + CAL_CAMERARX_PAD_SOURCE, format->which); *fmt = format->format; @@ -742,11 +745,11 @@ static int cal_camerarx_sd_set_fmt(struct v4l2_subdev *sd, } static int cal_camerarx_sd_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format format = { - .which = cfg ? V4L2_SUBDEV_FORMAT_TRY - : V4L2_SUBDEV_FORMAT_ACTIVE, + .which = sd_state ? V4L2_SUBDEV_FORMAT_TRY + : V4L2_SUBDEV_FORMAT_ACTIVE, .pad = CAL_CAMERARX_PAD_SINK, .format = { .width = 640, @@ -760,7 +763,7 @@ static int cal_camerarx_sd_init_cfg(struct v4l2_subdev *sd, }, }; - return cal_camerarx_sd_set_fmt(sd, cfg, &format); + return cal_camerarx_sd_set_fmt(sd, sd_state, &format); } static const struct v4l2_subdev_video_ops cal_camerarx_video_ops = { diff --git a/drivers/media/platform/ti-vpe/cal-video.c b/drivers/media/platform/ti-vpe/cal-video.c index 7b7436a355ee..15fb5360cf13 100644 --- a/drivers/media/platform/ti-vpe/cal-video.c +++ b/drivers/media/platform/ti-vpe/cal-video.c @@ -700,7 +700,9 @@ static int cal_start_streaming(struct vb2_queue *vq, unsigned int count) addr = vb2_dma_contig_plane_dma_addr(&buf->vb.vb2_buf, 0); - pm_runtime_get_sync(ctx->cal->dev); + ret = pm_runtime_resume_and_get(ctx->cal->dev); + if (ret < 0) + goto error_pipeline; cal_ctx_set_dma_addr(ctx, addr); cal_ctx_start(ctx); diff --git a/drivers/media/platform/ti-vpe/cal.c b/drivers/media/platform/ti-vpe/cal.c index 2e2bef91b2b0..76fe7a8b33f6 100644 --- a/drivers/media/platform/ti-vpe/cal.c +++ b/drivers/media/platform/ti-vpe/cal.c @@ -1024,7 +1024,7 @@ static int cal_probe(struct platform_device *pdev) /* Read the revision and hardware info to verify hardware access. */ pm_runtime_enable(&pdev->dev); - ret = pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); if (ret) goto error_pm_runtime; @@ -1098,10 +1098,11 @@ static int cal_remove(struct platform_device *pdev) { struct cal_dev *cal = platform_get_drvdata(pdev); unsigned int i; + int ret; cal_dbg(1, cal, "Removing %s\n", CAL_MODULE_NAME); - pm_runtime_get_sync(&pdev->dev); + ret = pm_runtime_resume_and_get(&pdev->dev); cal_media_unregister(cal); @@ -1115,7 +1116,8 @@ static int cal_remove(struct platform_device *pdev) for (i = 0; i < cal->data->num_csi2_phy; i++) cal_camerarx_destroy(cal->phy[i]); - pm_runtime_put_sync(&pdev->dev); + if (ret >= 0) + pm_runtime_put_sync(&pdev->dev); pm_runtime_disable(&pdev->dev); return 0; diff --git a/drivers/media/platform/ti-vpe/vpe.c b/drivers/media/platform/ti-vpe/vpe.c index 10251b787674..5b1c5d96a407 100644 --- a/drivers/media/platform/ti-vpe/vpe.c +++ b/drivers/media/platform/ti-vpe/vpe.c @@ -2471,11 +2471,9 @@ static int vpe_runtime_get(struct platform_device *pdev) dev_dbg(&pdev->dev, "vpe_runtime_get\n"); - r = pm_runtime_get_sync(&pdev->dev); + r = pm_runtime_resume_and_get(&pdev->dev); WARN_ON(r < 0); - if (r) - pm_runtime_put_noidle(&pdev->dev); - return r < 0 ? r : 0; + return r; } static void vpe_runtime_put(struct platform_device *pdev) @@ -2580,7 +2578,7 @@ static int vpe_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); ret = vpe_runtime_get(pdev); - if (ret) + if (ret < 0) goto rel_m2m; /* Perform clk enable followed by reset */ diff --git a/drivers/media/platform/via-camera.c b/drivers/media/platform/via-camera.c index ed0ad68c5c48..3655573e8581 100644 --- a/drivers/media/platform/via-camera.c +++ b/drivers/media/platform/via-camera.c @@ -844,6 +844,9 @@ static int viacam_do_try_fmt(struct via_camera *cam, { int ret; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -852,7 +855,7 @@ static int viacam_do_try_fmt(struct via_camera *cam, upix->pixelformat = f->pixelformat; viacam_fmt_pre(upix, spix); v4l2_fill_mbus_format(&format.format, spix, f->mbus_code); - ret = sensor_call(cam, pad, set_fmt, &pad_cfg, &format); + ret = sensor_call(cam, pad, set_fmt, &pad_state, &format); v4l2_fill_pix_format(spix, &format.format); viacam_fmt_post(upix, spix); return ret; diff --git a/drivers/media/platform/video-mux.c b/drivers/media/platform/video-mux.c index 133122e38515..905005e271ca 100644 --- a/drivers/media/platform/video-mux.c +++ b/drivers/media/platform/video-mux.c @@ -140,14 +140,14 @@ static const struct v4l2_subdev_video_ops video_mux_subdev_video_ops = { static struct v4l2_mbus_framefmt * __video_mux_get_pad_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { struct video_mux *vmux = v4l2_subdev_to_video_mux(sd); switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(sd, cfg, pad); + return v4l2_subdev_get_try_format(sd, sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &vmux->format_mbus[pad]; default: @@ -156,14 +156,15 @@ __video_mux_get_pad_format(struct v4l2_subdev *sd, } static int video_mux_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct video_mux *vmux = v4l2_subdev_to_video_mux(sd); mutex_lock(&vmux->lock); - sdformat->format = *__video_mux_get_pad_format(sd, cfg, sdformat->pad, + sdformat->format = *__video_mux_get_pad_format(sd, sd_state, + sdformat->pad, sdformat->which); mutex_unlock(&vmux->lock); @@ -172,7 +173,7 @@ static int video_mux_get_format(struct v4l2_subdev *sd, } static int video_mux_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct video_mux *vmux = v4l2_subdev_to_video_mux(sd); @@ -180,12 +181,13 @@ static int video_mux_set_format(struct v4l2_subdev *sd, struct media_pad *pad = &vmux->pads[sdformat->pad]; u16 source_pad = sd->entity.num_pads - 1; - mbusformat = __video_mux_get_pad_format(sd, cfg, sdformat->pad, - sdformat->which); + mbusformat = __video_mux_get_pad_format(sd, sd_state, sdformat->pad, + sdformat->which); if (!mbusformat) return -EINVAL; - source_mbusformat = __video_mux_get_pad_format(sd, cfg, source_pad, + source_mbusformat = __video_mux_get_pad_format(sd, sd_state, + source_pad, sdformat->which); if (!source_mbusformat) return -EINVAL; @@ -310,7 +312,7 @@ static int video_mux_set_format(struct v4l2_subdev *sd, } static int video_mux_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct video_mux *vmux = v4l2_subdev_to_video_mux(sd); struct v4l2_mbus_framefmt *mbusformat; @@ -319,7 +321,7 @@ static int video_mux_init_cfg(struct v4l2_subdev *sd, mutex_lock(&vmux->lock); for (i = 0; i < sd->entity.num_pads; i++) { - mbusformat = v4l2_subdev_get_try_format(sd, cfg, i); + mbusformat = v4l2_subdev_get_try_format(sd, sd_state, i); *mbusformat = video_mux_format_mbus_default; } @@ -362,7 +364,7 @@ static int video_mux_async_register(struct video_mux *vmux, for (i = 0; i < num_input_pads; i++) { struct v4l2_async_subdev *asd; - struct fwnode_handle *ep; + struct fwnode_handle *ep, *remote_ep; ep = fwnode_graph_get_endpoint_by_id( dev_fwnode(vmux->subdev.dev), i, 0, @@ -370,6 +372,14 @@ static int video_mux_async_register(struct video_mux *vmux, if (!ep) continue; + /* Skip dangling endpoints for backwards compatibility */ + remote_ep = fwnode_graph_get_remote_endpoint(ep); + if (!remote_ep) { + fwnode_handle_put(ep); + continue; + } + fwnode_handle_put(remote_ep); + asd = v4l2_async_notifier_add_fwnode_remote_subdev( &vmux->notifier, ep, struct v4l2_async_subdev); diff --git a/drivers/media/platform/vsp1/vsp1_brx.c b/drivers/media/platform/vsp1/vsp1_brx.c index 2d86c718a5cf..89385b4cabe5 100644 --- a/drivers/media/platform/vsp1/vsp1_brx.c +++ b/drivers/media/platform/vsp1/vsp1_brx.c @@ -65,7 +65,7 @@ static const struct v4l2_ctrl_ops brx_ctrl_ops = { */ static int brx_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { static const unsigned int codes[] = { @@ -73,12 +73,12 @@ static int brx_enum_mbus_code(struct v4l2_subdev *subdev, MEDIA_BUS_FMT_AYUV8_1X32, }; - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, codes, ARRAY_SIZE(codes)); } static int brx_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index) @@ -97,14 +97,14 @@ static int brx_enum_frame_size(struct v4l2_subdev *subdev, } static struct v4l2_rect *brx_get_compose(struct vsp1_brx *brx, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad) { - return v4l2_subdev_get_try_compose(&brx->entity.subdev, cfg, pad); + return v4l2_subdev_get_try_compose(&brx->entity.subdev, sd_state, pad); } static void brx_try_format(struct vsp1_brx *brx, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt) { struct v4l2_mbus_framefmt *format; @@ -119,7 +119,7 @@ static void brx_try_format(struct vsp1_brx *brx, default: /* The BRx can't perform format conversion. */ - format = vsp1_entity_get_pad_format(&brx->entity, config, + format = vsp1_entity_get_pad_format(&brx->entity, sd_state, BRX_PAD_SINK(0)); fmt->code = format->code; break; @@ -132,17 +132,18 @@ static void brx_try_format(struct vsp1_brx *brx, } static int brx_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vsp1_brx *brx = to_brx(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; mutex_lock(&brx->entity.lock); - config = vsp1_entity_get_pad_config(&brx->entity, cfg, fmt->which); + config = vsp1_entity_get_pad_config(&brx->entity, sd_state, + fmt->which); if (!config) { ret = -EINVAL; goto done; @@ -181,11 +182,11 @@ static int brx_set_format(struct v4l2_subdev *subdev, } static int brx_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_brx *brx = to_brx(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; if (sel->pad == brx->entity.source_pad) return -EINVAL; @@ -199,7 +200,7 @@ static int brx_get_selection(struct v4l2_subdev *subdev, return 0; case V4L2_SEL_TGT_COMPOSE: - config = vsp1_entity_get_pad_config(&brx->entity, cfg, + config = vsp1_entity_get_pad_config(&brx->entity, sd_state, sel->which); if (!config) return -EINVAL; @@ -215,11 +216,11 @@ static int brx_get_selection(struct v4l2_subdev *subdev, } static int brx_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_brx *brx = to_brx(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; struct v4l2_rect *compose; int ret = 0; @@ -232,7 +233,8 @@ static int brx_set_selection(struct v4l2_subdev *subdev, mutex_lock(&brx->entity.lock); - config = vsp1_entity_get_pad_config(&brx->entity, cfg, sel->which); + config = vsp1_entity_get_pad_config(&brx->entity, sd_state, + sel->which); if (!config) { ret = -EINVAL; goto done; diff --git a/drivers/media/platform/vsp1/vsp1_clu.c b/drivers/media/platform/vsp1/vsp1_clu.c index a47b23bf5abf..c5217fee24f1 100644 --- a/drivers/media/platform/vsp1/vsp1_clu.c +++ b/drivers/media/platform/vsp1/vsp1_clu.c @@ -123,27 +123,28 @@ static const unsigned int clu_codes[] = { }; static int clu_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, clu_codes, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, clu_codes, ARRAY_SIZE(clu_codes)); } static int clu_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { - return vsp1_subdev_enum_frame_size(subdev, cfg, fse, CLU_MIN_SIZE, + return vsp1_subdev_enum_frame_size(subdev, sd_state, fse, + CLU_MIN_SIZE, CLU_MIN_SIZE, CLU_MAX_SIZE, CLU_MAX_SIZE); } static int clu_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - return vsp1_subdev_set_pad_format(subdev, cfg, fmt, clu_codes, + return vsp1_subdev_set_pad_format(subdev, sd_state, fmt, clu_codes, ARRAY_SIZE(clu_codes), CLU_MIN_SIZE, CLU_MIN_SIZE, CLU_MAX_SIZE, CLU_MAX_SIZE); diff --git a/drivers/media/platform/vsp1/vsp1_drv.c b/drivers/media/platform/vsp1/vsp1_drv.c index aa66e4f5f3f3..de442d6c9926 100644 --- a/drivers/media/platform/vsp1/vsp1_drv.c +++ b/drivers/media/platform/vsp1/vsp1_drv.c @@ -559,15 +559,7 @@ static int vsp1_device_init(struct vsp1_device *vsp1) */ int vsp1_device_get(struct vsp1_device *vsp1) { - int ret; - - ret = pm_runtime_get_sync(vsp1->dev); - if (ret < 0) { - pm_runtime_put_noidle(vsp1->dev); - return ret; - } - - return 0; + return pm_runtime_resume_and_get(vsp1->dev); } /* diff --git a/drivers/media/platform/vsp1/vsp1_entity.c b/drivers/media/platform/vsp1/vsp1_entity.c index aa9d2286056e..6f51e5c75543 100644 --- a/drivers/media/platform/vsp1/vsp1_entity.c +++ b/drivers/media/platform/vsp1/vsp1_entity.c @@ -103,7 +103,7 @@ void vsp1_entity_configure_partition(struct vsp1_entity *entity, /** * vsp1_entity_get_pad_config - Get the pad configuration for an entity * @entity: the entity - * @cfg: the TRY pad configuration + * @sd_state: the TRY state * @which: configuration selector (ACTIVE or TRY) * * When called with which set to V4L2_SUBDEV_FORMAT_ACTIVE the caller must hold @@ -114,9 +114,9 @@ void vsp1_entity_configure_partition(struct vsp1_entity *entity, * and simply returned when requested. The ACTIVE configuration comes from the * entity structure. */ -struct v4l2_subdev_pad_config * +struct v4l2_subdev_state * vsp1_entity_get_pad_config(struct vsp1_entity *entity, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { switch (which) { @@ -124,14 +124,14 @@ vsp1_entity_get_pad_config(struct vsp1_entity *entity, return entity->config; case V4L2_SUBDEV_FORMAT_TRY: default: - return cfg; + return sd_state; } } /** * vsp1_entity_get_pad_format - Get a pad format from storage for an entity * @entity: the entity - * @cfg: the configuration storage + * @sd_state: the state storage * @pad: the pad number * * Return the format stored in the given configuration for an entity's pad. The @@ -139,16 +139,16 @@ vsp1_entity_get_pad_config(struct vsp1_entity *entity, */ struct v4l2_mbus_framefmt * vsp1_entity_get_pad_format(struct vsp1_entity *entity, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad) { - return v4l2_subdev_get_try_format(&entity->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&entity->subdev, sd_state, pad); } /** * vsp1_entity_get_pad_selection - Get a pad selection from storage for entity * @entity: the entity - * @cfg: the configuration storage + * @sd_state: the state storage * @pad: the pad number * @target: the selection target * @@ -158,14 +158,16 @@ vsp1_entity_get_pad_format(struct vsp1_entity *entity, */ struct v4l2_rect * vsp1_entity_get_pad_selection(struct vsp1_entity *entity, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, unsigned int target) { switch (target) { case V4L2_SEL_TGT_COMPOSE: - return v4l2_subdev_get_try_compose(&entity->subdev, cfg, pad); + return v4l2_subdev_get_try_compose(&entity->subdev, sd_state, + pad); case V4L2_SEL_TGT_CROP: - return v4l2_subdev_get_try_crop(&entity->subdev, cfg, pad); + return v4l2_subdev_get_try_crop(&entity->subdev, sd_state, + pad); default: return NULL; } @@ -180,7 +182,7 @@ vsp1_entity_get_pad_selection(struct vsp1_entity *entity, * function can be used as a handler for the subdev pad::init_cfg operation. */ int vsp1_entity_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_subdev_format format; unsigned int pad; @@ -189,10 +191,10 @@ int vsp1_entity_init_cfg(struct v4l2_subdev *subdev, memset(&format, 0, sizeof(format)); format.pad = pad; - format.which = cfg ? V4L2_SUBDEV_FORMAT_TRY + format.which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; - v4l2_subdev_call(subdev, pad, set_fmt, cfg, &format); + v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &format); } return 0; @@ -208,13 +210,13 @@ int vsp1_entity_init_cfg(struct v4l2_subdev *subdev, * a direct drop-in for the operation handler. */ int vsp1_subdev_get_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vsp1_entity *entity = to_vsp1_entity(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; - config = vsp1_entity_get_pad_config(entity, cfg, fmt->which); + config = vsp1_entity_get_pad_config(entity, sd_state, fmt->which); if (!config) return -EINVAL; @@ -239,7 +241,7 @@ int vsp1_subdev_get_pad_format(struct v4l2_subdev *subdev, * the sink pad. */ int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code, const unsigned int *codes, unsigned int ncodes) { @@ -251,7 +253,7 @@ int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev, code->code = codes[code->index]; } else { - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; /* @@ -261,7 +263,8 @@ int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev, if (code->index) return -EINVAL; - config = vsp1_entity_get_pad_config(entity, cfg, code->which); + config = vsp1_entity_get_pad_config(entity, sd_state, + code->which); if (!config) return -EINVAL; @@ -290,17 +293,17 @@ int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev, * source pad size identical to the sink pad. */ int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse, unsigned int min_width, unsigned int min_height, unsigned int max_width, unsigned int max_height) { struct vsp1_entity *entity = to_vsp1_entity(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; - config = vsp1_entity_get_pad_config(entity, cfg, fse->which); + config = vsp1_entity_get_pad_config(entity, sd_state, fse->which); if (!config) return -EINVAL; @@ -353,14 +356,14 @@ int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev, * source pad. */ int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt, const unsigned int *codes, unsigned int ncodes, unsigned int min_width, unsigned int min_height, unsigned int max_width, unsigned int max_height) { struct vsp1_entity *entity = to_vsp1_entity(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; struct v4l2_rect *selection; unsigned int i; @@ -368,7 +371,7 @@ int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev, mutex_lock(&entity->lock); - config = vsp1_entity_get_pad_config(entity, cfg, fmt->which); + config = vsp1_entity_get_pad_config(entity, sd_state, fmt->which); if (!config) { ret = -EINVAL; goto done; @@ -672,7 +675,7 @@ int vsp1_entity_init(struct vsp1_device *vsp1, struct vsp1_entity *entity, * Allocate the pad configuration to store formats and selection * rectangles. */ - entity->config = v4l2_subdev_alloc_pad_config(&entity->subdev); + entity->config = v4l2_subdev_alloc_state(&entity->subdev); if (entity->config == NULL) { media_entity_cleanup(&entity->subdev.entity); return -ENOMEM; @@ -687,6 +690,6 @@ void vsp1_entity_destroy(struct vsp1_entity *entity) entity->ops->destroy(entity); if (entity->subdev.ctrl_handler) v4l2_ctrl_handler_free(entity->subdev.ctrl_handler); - v4l2_subdev_free_pad_config(entity->config); + v4l2_subdev_free_state(entity->config); media_entity_cleanup(&entity->subdev.entity); } diff --git a/drivers/media/platform/vsp1/vsp1_entity.h b/drivers/media/platform/vsp1/vsp1_entity.h index a1ceb37bb837..f22724439cdc 100644 --- a/drivers/media/platform/vsp1/vsp1_entity.h +++ b/drivers/media/platform/vsp1/vsp1_entity.h @@ -115,7 +115,7 @@ struct vsp1_entity { unsigned int sink_pad; struct v4l2_subdev subdev; - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct mutex lock; /* Protects the pad config */ }; @@ -136,20 +136,20 @@ int vsp1_entity_link_setup(struct media_entity *entity, const struct media_pad *local, const struct media_pad *remote, u32 flags); -struct v4l2_subdev_pad_config * +struct v4l2_subdev_state * vsp1_entity_get_pad_config(struct vsp1_entity *entity, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which); struct v4l2_mbus_framefmt * vsp1_entity_get_pad_format(struct vsp1_entity *entity, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad); struct v4l2_rect * vsp1_entity_get_pad_selection(struct vsp1_entity *entity, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, unsigned int target); int vsp1_entity_init_cfg(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg); + struct v4l2_subdev_state *sd_state); void vsp1_entity_route_setup(struct vsp1_entity *entity, struct vsp1_pipeline *pipe, @@ -173,20 +173,20 @@ void vsp1_entity_configure_partition(struct vsp1_entity *entity, struct media_pad *vsp1_entity_remote_pad(struct media_pad *pad); int vsp1_subdev_get_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt); int vsp1_subdev_set_pad_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt, const unsigned int *codes, unsigned int ncodes, unsigned int min_width, unsigned int min_height, unsigned int max_width, unsigned int max_height); int vsp1_subdev_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code, const unsigned int *codes, unsigned int ncodes); int vsp1_subdev_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse, unsigned int min_w, unsigned int min_h, unsigned int max_w, unsigned int max_h); diff --git a/drivers/media/platform/vsp1/vsp1_histo.c b/drivers/media/platform/vsp1/vsp1_histo.c index a91e142bcb94..5e5013d2cd2a 100644 --- a/drivers/media/platform/vsp1/vsp1_histo.c +++ b/drivers/media/platform/vsp1/vsp1_histo.c @@ -170,7 +170,7 @@ static const struct vb2_ops histo_video_queue_qops = { */ static int histo_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct vsp1_histogram *histo = subdev_to_histo(subdev); @@ -180,28 +180,30 @@ static int histo_enum_mbus_code(struct v4l2_subdev *subdev, return 0; } - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, histo->formats, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, + histo->formats, histo->num_formats); } static int histo_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->pad != HISTO_PAD_SINK) return -EINVAL; - return vsp1_subdev_enum_frame_size(subdev, cfg, fse, HISTO_MIN_SIZE, + return vsp1_subdev_enum_frame_size(subdev, sd_state, fse, + HISTO_MIN_SIZE, HISTO_MIN_SIZE, HISTO_MAX_SIZE, HISTO_MAX_SIZE); } static int histo_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_histogram *histo = subdev_to_histo(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; int ret = 0; @@ -211,7 +213,8 @@ static int histo_get_selection(struct v4l2_subdev *subdev, mutex_lock(&histo->entity.lock); - config = vsp1_entity_get_pad_config(&histo->entity, cfg, sel->which); + config = vsp1_entity_get_pad_config(&histo->entity, sd_state, + sel->which); if (!config) { ret = -EINVAL; goto done; @@ -256,15 +259,15 @@ static int histo_get_selection(struct v4l2_subdev *subdev, } static int histo_set_crop(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, - struct v4l2_subdev_selection *sel) + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_selection *sel) { struct vsp1_histogram *histo = subdev_to_histo(subdev); struct v4l2_mbus_framefmt *format; struct v4l2_rect *selection; /* The crop rectangle must be inside the input frame. */ - format = vsp1_entity_get_pad_format(&histo->entity, config, + format = vsp1_entity_get_pad_format(&histo->entity, sd_state, HISTO_PAD_SINK); sel->r.left = clamp_t(unsigned int, sel->r.left, 0, format->width - 1); sel->r.top = clamp_t(unsigned int, sel->r.top, 0, format->height - 1); @@ -274,11 +277,11 @@ static int histo_set_crop(struct v4l2_subdev *subdev, format->height - sel->r.top); /* Set the crop rectangle and reset the compose rectangle. */ - selection = vsp1_entity_get_pad_selection(&histo->entity, config, + selection = vsp1_entity_get_pad_selection(&histo->entity, sd_state, sel->pad, V4L2_SEL_TGT_CROP); *selection = sel->r; - selection = vsp1_entity_get_pad_selection(&histo->entity, config, + selection = vsp1_entity_get_pad_selection(&histo->entity, sd_state, sel->pad, V4L2_SEL_TGT_COMPOSE); *selection = sel->r; @@ -287,7 +290,7 @@ static int histo_set_crop(struct v4l2_subdev *subdev, } static int histo_set_compose(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_histogram *histo = subdev_to_histo(subdev); @@ -303,7 +306,8 @@ static int histo_set_compose(struct v4l2_subdev *subdev, sel->r.left = 0; sel->r.top = 0; - crop = vsp1_entity_get_pad_selection(&histo->entity, config, sel->pad, + crop = vsp1_entity_get_pad_selection(&histo->entity, sd_state, + sel->pad, V4L2_SEL_TGT_CROP); /* @@ -329,7 +333,7 @@ static int histo_set_compose(struct v4l2_subdev *subdev, ratio = 1 << (crop->height * 2 / sel->r.height / 3); sel->r.height = crop->height / ratio; - compose = vsp1_entity_get_pad_selection(&histo->entity, config, + compose = vsp1_entity_get_pad_selection(&histo->entity, sd_state, sel->pad, V4L2_SEL_TGT_COMPOSE); *compose = sel->r; @@ -338,11 +342,11 @@ static int histo_set_compose(struct v4l2_subdev *subdev, } static int histo_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_histogram *histo = subdev_to_histo(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; int ret; if (sel->pad != HISTO_PAD_SINK) @@ -350,7 +354,8 @@ static int histo_set_selection(struct v4l2_subdev *subdev, mutex_lock(&histo->entity.lock); - config = vsp1_entity_get_pad_config(&histo->entity, cfg, sel->which); + config = vsp1_entity_get_pad_config(&histo->entity, sd_state, + sel->which); if (!config) { ret = -EINVAL; goto done; @@ -369,7 +374,7 @@ static int histo_set_selection(struct v4l2_subdev *subdev, } static int histo_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { if (fmt->pad == HISTO_PAD_SOURCE) { @@ -381,19 +386,19 @@ static int histo_get_format(struct v4l2_subdev *subdev, return 0; } - return vsp1_subdev_get_pad_format(subdev, cfg, fmt); + return vsp1_subdev_get_pad_format(subdev, sd_state, fmt); } static int histo_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vsp1_histogram *histo = subdev_to_histo(subdev); if (fmt->pad != HISTO_PAD_SINK) - return histo_get_format(subdev, cfg, fmt); + return histo_get_format(subdev, sd_state, fmt); - return vsp1_subdev_set_pad_format(subdev, cfg, fmt, + return vsp1_subdev_set_pad_format(subdev, sd_state, fmt, histo->formats, histo->num_formats, HISTO_MIN_SIZE, HISTO_MIN_SIZE, HISTO_MAX_SIZE, HISTO_MAX_SIZE); diff --git a/drivers/media/platform/vsp1/vsp1_hsit.c b/drivers/media/platform/vsp1/vsp1_hsit.c index d5ebd9d08c8a..361a870380c2 100644 --- a/drivers/media/platform/vsp1/vsp1_hsit.c +++ b/drivers/media/platform/vsp1/vsp1_hsit.c @@ -34,7 +34,7 @@ static inline void vsp1_hsit_write(struct vsp1_hsit *hsit, */ static int hsit_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct vsp1_hsit *hsit = to_hsit(subdev); @@ -52,26 +52,28 @@ static int hsit_enum_mbus_code(struct v4l2_subdev *subdev, } static int hsit_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { - return vsp1_subdev_enum_frame_size(subdev, cfg, fse, HSIT_MIN_SIZE, + return vsp1_subdev_enum_frame_size(subdev, sd_state, fse, + HSIT_MIN_SIZE, HSIT_MIN_SIZE, HSIT_MAX_SIZE, HSIT_MAX_SIZE); } static int hsit_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vsp1_hsit *hsit = to_hsit(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; mutex_lock(&hsit->entity.lock); - config = vsp1_entity_get_pad_config(&hsit->entity, cfg, fmt->which); + config = vsp1_entity_get_pad_config(&hsit->entity, sd_state, + fmt->which); if (!config) { ret = -EINVAL; goto done; diff --git a/drivers/media/platform/vsp1/vsp1_lif.c b/drivers/media/platform/vsp1/vsp1_lif.c index 14ed5d7bd061..6a6857ac9327 100644 --- a/drivers/media/platform/vsp1/vsp1_lif.c +++ b/drivers/media/platform/vsp1/vsp1_lif.c @@ -40,27 +40,28 @@ static const unsigned int lif_codes[] = { }; static int lif_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, lif_codes, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, lif_codes, ARRAY_SIZE(lif_codes)); } static int lif_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { - return vsp1_subdev_enum_frame_size(subdev, cfg, fse, LIF_MIN_SIZE, + return vsp1_subdev_enum_frame_size(subdev, sd_state, fse, + LIF_MIN_SIZE, LIF_MIN_SIZE, LIF_MAX_SIZE, LIF_MAX_SIZE); } static int lif_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - return vsp1_subdev_set_pad_format(subdev, cfg, fmt, lif_codes, + return vsp1_subdev_set_pad_format(subdev, sd_state, fmt, lif_codes, ARRAY_SIZE(lif_codes), LIF_MIN_SIZE, LIF_MIN_SIZE, LIF_MAX_SIZE, LIF_MAX_SIZE); diff --git a/drivers/media/platform/vsp1/vsp1_lut.c b/drivers/media/platform/vsp1/vsp1_lut.c index 9f88842d7048..ac6802a325f5 100644 --- a/drivers/media/platform/vsp1/vsp1_lut.c +++ b/drivers/media/platform/vsp1/vsp1_lut.c @@ -99,27 +99,28 @@ static const unsigned int lut_codes[] = { }; static int lut_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, lut_codes, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, lut_codes, ARRAY_SIZE(lut_codes)); } static int lut_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { - return vsp1_subdev_enum_frame_size(subdev, cfg, fse, LUT_MIN_SIZE, + return vsp1_subdev_enum_frame_size(subdev, sd_state, fse, + LUT_MIN_SIZE, LUT_MIN_SIZE, LUT_MAX_SIZE, LUT_MAX_SIZE); } static int lut_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - return vsp1_subdev_set_pad_format(subdev, cfg, fmt, lut_codes, + return vsp1_subdev_set_pad_format(subdev, sd_state, fmt, lut_codes, ARRAY_SIZE(lut_codes), LUT_MIN_SIZE, LUT_MIN_SIZE, LUT_MAX_SIZE, LUT_MAX_SIZE); diff --git a/drivers/media/platform/vsp1/vsp1_rwpf.c b/drivers/media/platform/vsp1/vsp1_rwpf.c index 049bdd958e56..22a82d218152 100644 --- a/drivers/media/platform/vsp1/vsp1_rwpf.c +++ b/drivers/media/platform/vsp1/vsp1_rwpf.c @@ -17,9 +17,9 @@ #define RWPF_MIN_HEIGHT 1 struct v4l2_rect *vsp1_rwpf_get_crop(struct vsp1_rwpf *rwpf, - struct v4l2_subdev_pad_config *config) + struct v4l2_subdev_state *sd_state) { - return v4l2_subdev_get_try_crop(&rwpf->entity.subdev, config, + return v4l2_subdev_get_try_crop(&rwpf->entity.subdev, sd_state, RWPF_PAD_SINK); } @@ -28,7 +28,7 @@ struct v4l2_rect *vsp1_rwpf_get_crop(struct vsp1_rwpf *rwpf, */ static int vsp1_rwpf_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { static const unsigned int codes[] = { @@ -46,28 +46,30 @@ static int vsp1_rwpf_enum_mbus_code(struct v4l2_subdev *subdev, } static int vsp1_rwpf_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct vsp1_rwpf *rwpf = to_rwpf(subdev); - return vsp1_subdev_enum_frame_size(subdev, cfg, fse, RWPF_MIN_WIDTH, + return vsp1_subdev_enum_frame_size(subdev, sd_state, fse, + RWPF_MIN_WIDTH, RWPF_MIN_HEIGHT, rwpf->max_width, rwpf->max_height); } static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vsp1_rwpf *rwpf = to_rwpf(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; mutex_lock(&rwpf->entity.lock); - config = vsp1_entity_get_pad_config(&rwpf->entity, cfg, fmt->which); + config = vsp1_entity_get_pad_config(&rwpf->entity, sd_state, + fmt->which); if (!config) { ret = -EINVAL; goto done; @@ -128,11 +130,11 @@ static int vsp1_rwpf_set_format(struct v4l2_subdev *subdev, } static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_rwpf *rwpf = to_rwpf(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; @@ -145,7 +147,8 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev, mutex_lock(&rwpf->entity.lock); - config = vsp1_entity_get_pad_config(&rwpf->entity, cfg, sel->which); + config = vsp1_entity_get_pad_config(&rwpf->entity, sd_state, + sel->which); if (!config) { ret = -EINVAL; goto done; @@ -176,11 +179,11 @@ static int vsp1_rwpf_get_selection(struct v4l2_subdev *subdev, } static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_rwpf *rwpf = to_rwpf(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; struct v4l2_rect *crop; int ret = 0; @@ -197,7 +200,8 @@ static int vsp1_rwpf_set_selection(struct v4l2_subdev *subdev, mutex_lock(&rwpf->entity.lock); - config = vsp1_entity_get_pad_config(&rwpf->entity, cfg, sel->which); + config = vsp1_entity_get_pad_config(&rwpf->entity, sd_state, + sel->which); if (!config) { ret = -EINVAL; goto done; diff --git a/drivers/media/platform/vsp1/vsp1_rwpf.h b/drivers/media/platform/vsp1/vsp1_rwpf.h index 2f3582590618..eac5c04c2239 100644 --- a/drivers/media/platform/vsp1/vsp1_rwpf.h +++ b/drivers/media/platform/vsp1/vsp1_rwpf.h @@ -84,6 +84,6 @@ int vsp1_rwpf_init_ctrls(struct vsp1_rwpf *rwpf, unsigned int ncontrols); extern const struct v4l2_subdev_pad_ops vsp1_rwpf_pad_ops; struct v4l2_rect *vsp1_rwpf_get_crop(struct vsp1_rwpf *rwpf, - struct v4l2_subdev_pad_config *config); + struct v4l2_subdev_state *sd_state); #endif /* __VSP1_RWPF_H__ */ diff --git a/drivers/media/platform/vsp1/vsp1_sru.c b/drivers/media/platform/vsp1/vsp1_sru.c index 2b65457ee12f..b614a2aea461 100644 --- a/drivers/media/platform/vsp1/vsp1_sru.c +++ b/drivers/media/platform/vsp1/vsp1_sru.c @@ -106,7 +106,7 @@ static const struct v4l2_ctrl_config sru_intensity_control = { */ static int sru_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { static const unsigned int codes[] = { @@ -114,20 +114,21 @@ static int sru_enum_mbus_code(struct v4l2_subdev *subdev, MEDIA_BUS_FMT_AYUV8_1X32, }; - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, codes, ARRAY_SIZE(codes)); } static int sru_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct vsp1_sru *sru = to_sru(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; - config = vsp1_entity_get_pad_config(&sru->entity, cfg, fse->which); + config = vsp1_entity_get_pad_config(&sru->entity, sd_state, + fse->which); if (!config) return -EINVAL; @@ -164,7 +165,7 @@ static int sru_enum_frame_size(struct v4l2_subdev *subdev, } static void sru_try_format(struct vsp1_sru *sru, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt) { struct v4l2_mbus_framefmt *format; @@ -184,7 +185,7 @@ static void sru_try_format(struct vsp1_sru *sru, case SRU_PAD_SOURCE: /* The SRU can't perform format conversion. */ - format = vsp1_entity_get_pad_format(&sru->entity, config, + format = vsp1_entity_get_pad_format(&sru->entity, sd_state, SRU_PAD_SINK); fmt->code = format->code; @@ -216,17 +217,18 @@ static void sru_try_format(struct vsp1_sru *sru, } static int sru_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vsp1_sru *sru = to_sru(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; mutex_lock(&sru->entity.lock); - config = vsp1_entity_get_pad_config(&sru->entity, cfg, fmt->which); + config = vsp1_entity_get_pad_config(&sru->entity, sd_state, + fmt->which); if (!config) { ret = -EINVAL; goto done; diff --git a/drivers/media/platform/vsp1/vsp1_uds.c b/drivers/media/platform/vsp1/vsp1_uds.c index 5fc04c082d1a..1c290cda005a 100644 --- a/drivers/media/platform/vsp1/vsp1_uds.c +++ b/drivers/media/platform/vsp1/vsp1_uds.c @@ -111,7 +111,7 @@ static unsigned int uds_compute_ratio(unsigned int input, unsigned int output) */ static int uds_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { static const unsigned int codes[] = { @@ -119,20 +119,21 @@ static int uds_enum_mbus_code(struct v4l2_subdev *subdev, MEDIA_BUS_FMT_AYUV8_1X32, }; - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, codes, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, codes, ARRAY_SIZE(codes)); } static int uds_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct vsp1_uds *uds = to_uds(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; - config = vsp1_entity_get_pad_config(&uds->entity, cfg, fse->which); + config = vsp1_entity_get_pad_config(&uds->entity, sd_state, + fse->which); if (!config) return -EINVAL; @@ -164,7 +165,7 @@ static int uds_enum_frame_size(struct v4l2_subdev *subdev, } static void uds_try_format(struct vsp1_uds *uds, - struct v4l2_subdev_pad_config *config, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt) { struct v4l2_mbus_framefmt *format; @@ -184,7 +185,7 @@ static void uds_try_format(struct vsp1_uds *uds, case UDS_PAD_SOURCE: /* The UDS scales but can't perform format conversion. */ - format = vsp1_entity_get_pad_format(&uds->entity, config, + format = vsp1_entity_get_pad_format(&uds->entity, sd_state, UDS_PAD_SINK); fmt->code = format->code; @@ -200,17 +201,18 @@ static void uds_try_format(struct vsp1_uds *uds, } static int uds_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vsp1_uds *uds = to_uds(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; mutex_lock(&uds->entity.lock); - config = vsp1_entity_get_pad_config(&uds->entity, cfg, fmt->which); + config = vsp1_entity_get_pad_config(&uds->entity, sd_state, + fmt->which); if (!config) { ret = -EINVAL; goto done; diff --git a/drivers/media/platform/vsp1/vsp1_uif.c b/drivers/media/platform/vsp1/vsp1_uif.c index 467d1072577b..83d7f17df80e 100644 --- a/drivers/media/platform/vsp1/vsp1_uif.c +++ b/drivers/media/platform/vsp1/vsp1_uif.c @@ -54,38 +54,39 @@ static const unsigned int uif_codes[] = { }; static int uif_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { - return vsp1_subdev_enum_mbus_code(subdev, cfg, code, uif_codes, + return vsp1_subdev_enum_mbus_code(subdev, sd_state, code, uif_codes, ARRAY_SIZE(uif_codes)); } static int uif_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { - return vsp1_subdev_enum_frame_size(subdev, cfg, fse, UIF_MIN_SIZE, + return vsp1_subdev_enum_frame_size(subdev, sd_state, fse, + UIF_MIN_SIZE, UIF_MIN_SIZE, UIF_MAX_SIZE, UIF_MAX_SIZE); } static int uif_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - return vsp1_subdev_set_pad_format(subdev, cfg, fmt, uif_codes, + return vsp1_subdev_set_pad_format(subdev, sd_state, fmt, uif_codes, ARRAY_SIZE(uif_codes), UIF_MIN_SIZE, UIF_MIN_SIZE, UIF_MAX_SIZE, UIF_MAX_SIZE); } static int uif_get_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_uif *uif = to_uif(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; int ret = 0; @@ -94,7 +95,8 @@ static int uif_get_selection(struct v4l2_subdev *subdev, mutex_lock(&uif->entity.lock); - config = vsp1_entity_get_pad_config(&uif->entity, cfg, sel->which); + config = vsp1_entity_get_pad_config(&uif->entity, sd_state, + sel->which); if (!config) { ret = -EINVAL; goto done; @@ -127,11 +129,11 @@ static int uif_get_selection(struct v4l2_subdev *subdev, } static int uif_set_selection(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vsp1_uif *uif = to_uif(subdev); - struct v4l2_subdev_pad_config *config; + struct v4l2_subdev_state *config; struct v4l2_mbus_framefmt *format; struct v4l2_rect *selection; int ret = 0; @@ -142,7 +144,8 @@ static int uif_set_selection(struct v4l2_subdev *subdev, mutex_lock(&uif->entity.lock); - config = vsp1_entity_get_pad_config(&uif->entity, cfg, sel->which); + config = vsp1_entity_get_pad_config(&uif->entity, sd_state, + sel->which); if (!config) { ret = -EINVAL; goto done; diff --git a/drivers/media/platform/xilinx/xilinx-csi2rxss.c b/drivers/media/platform/xilinx/xilinx-csi2rxss.c index fff7ddec6745..b1baf9d7b6ec 100644 --- a/drivers/media/platform/xilinx/xilinx-csi2rxss.c +++ b/drivers/media/platform/xilinx/xilinx-csi2rxss.c @@ -681,12 +681,13 @@ static int xcsi2rxss_s_stream(struct v4l2_subdev *sd, int enable) static struct v4l2_mbus_framefmt * __xcsi2rxss_get_pad_format(struct xcsi2rxss_state *xcsi2rxss, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&xcsi2rxss->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&xcsi2rxss->subdev, + sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &xcsi2rxss->format; default: @@ -697,7 +698,7 @@ __xcsi2rxss_get_pad_format(struct xcsi2rxss_state *xcsi2rxss, /** * xcsi2rxss_init_cfg - Initialise the pad format config to default * @sd: Pointer to V4L2 Sub device structure - * @cfg: Pointer to sub device pad information structure + * @sd_state: Pointer to sub device state structure * * This function is used to initialize the pad format with the default * values. @@ -705,7 +706,7 @@ __xcsi2rxss_get_pad_format(struct xcsi2rxss_state *xcsi2rxss, * Return: 0 on success */ static int xcsi2rxss_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct xcsi2rxss_state *xcsi2rxss = to_xcsi2rxssstate(sd); struct v4l2_mbus_framefmt *format; @@ -713,7 +714,7 @@ static int xcsi2rxss_init_cfg(struct v4l2_subdev *sd, mutex_lock(&xcsi2rxss->lock); for (i = 0; i < XCSI_MEDIA_PADS; i++) { - format = v4l2_subdev_get_try_format(sd, cfg, i); + format = v4l2_subdev_get_try_format(sd, sd_state, i); *format = xcsi2rxss->default_format; } mutex_unlock(&xcsi2rxss->lock); @@ -724,7 +725,7 @@ static int xcsi2rxss_init_cfg(struct v4l2_subdev *sd, /** * xcsi2rxss_get_format - Get the pad format * @sd: Pointer to V4L2 Sub device structure - * @cfg: Pointer to sub device pad information structure + * @sd_state: Pointer to sub device state structure * @fmt: Pointer to pad level media bus format * * This function is used to get the pad format information. @@ -732,13 +733,14 @@ static int xcsi2rxss_init_cfg(struct v4l2_subdev *sd, * Return: 0 on success */ static int xcsi2rxss_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct xcsi2rxss_state *xcsi2rxss = to_xcsi2rxssstate(sd); mutex_lock(&xcsi2rxss->lock); - fmt->format = *__xcsi2rxss_get_pad_format(xcsi2rxss, cfg, fmt->pad, + fmt->format = *__xcsi2rxss_get_pad_format(xcsi2rxss, sd_state, + fmt->pad, fmt->which); mutex_unlock(&xcsi2rxss->lock); @@ -748,7 +750,7 @@ static int xcsi2rxss_get_format(struct v4l2_subdev *sd, /** * xcsi2rxss_set_format - This is used to set the pad format * @sd: Pointer to V4L2 Sub device structure - * @cfg: Pointer to sub device pad information structure + * @sd_state: Pointer to sub device state structure * @fmt: Pointer to pad level media bus format * * This function is used to set the pad format. Since the pad format is fixed @@ -759,7 +761,7 @@ static int xcsi2rxss_get_format(struct v4l2_subdev *sd, * Return: 0 on success */ static int xcsi2rxss_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct xcsi2rxss_state *xcsi2rxss = to_xcsi2rxssstate(sd); @@ -773,7 +775,7 @@ static int xcsi2rxss_set_format(struct v4l2_subdev *sd, * CSI format cannot be changed at runtime. * Ensure that format to set is copied to over to CSI pad format */ - __format = __xcsi2rxss_get_pad_format(xcsi2rxss, cfg, + __format = __xcsi2rxss_get_pad_format(xcsi2rxss, sd_state, fmt->pad, fmt->which); /* only sink pad format can be updated */ @@ -811,7 +813,7 @@ static int xcsi2rxss_set_format(struct v4l2_subdev *sd, * Return: -EINVAL or zero on success */ static int xcsi2rxss_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct xcsi2rxss_state *state = to_xcsi2rxssstate(sd); diff --git a/drivers/media/platform/xilinx/xilinx-dma.c b/drivers/media/platform/xilinx/xilinx-dma.c index 2a56201cb853..338c3661d809 100644 --- a/drivers/media/platform/xilinx/xilinx-dma.c +++ b/drivers/media/platform/xilinx/xilinx-dma.c @@ -26,7 +26,6 @@ #include "xilinx-vip.h" #include "xilinx-vipp.h" -#define XVIP_DMA_DEF_FORMAT V4L2_PIX_FMT_YUYV #define XVIP_DMA_DEF_WIDTH 1920 #define XVIP_DMA_DEF_HEIGHT 1080 @@ -549,8 +548,6 @@ __xvip_dma_try_format(struct xvip_dma *dma, struct v4l2_pix_format *pix, * requested format isn't supported. */ info = xvip_get_format_by_fourcc(pix->pixelformat); - if (IS_ERR(info)) - info = xvip_get_format_by_fourcc(XVIP_DMA_DEF_FORMAT); pix->pixelformat = info->fourcc; pix->field = V4L2_FIELD_NONE; @@ -660,7 +657,7 @@ int xvip_dma_init(struct xvip_composite_device *xdev, struct xvip_dma *dma, INIT_LIST_HEAD(&dma->queued_bufs); spin_lock_init(&dma->queued_lock); - dma->fmtinfo = xvip_get_format_by_fourcc(XVIP_DMA_DEF_FORMAT); + dma->fmtinfo = xvip_get_format_by_fourcc(V4L2_PIX_FMT_YUYV); dma->format.pixelformat = dma->fmtinfo->fourcc; dma->format.colorspace = V4L2_COLORSPACE_SRGB; dma->format.field = V4L2_FIELD_NONE; diff --git a/drivers/media/platform/xilinx/xilinx-tpg.c b/drivers/media/platform/xilinx/xilinx-tpg.c index ed01bedb5db6..0f2d5a0edf0c 100644 --- a/drivers/media/platform/xilinx/xilinx-tpg.c +++ b/drivers/media/platform/xilinx/xilinx-tpg.c @@ -251,12 +251,13 @@ static int xtpg_s_stream(struct v4l2_subdev *subdev, int enable) static struct v4l2_mbus_framefmt * __xtpg_get_pad_format(struct xtpg_device *xtpg, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, u32 which) { switch (which) { case V4L2_SUBDEV_FORMAT_TRY: - return v4l2_subdev_get_try_format(&xtpg->xvip.subdev, cfg, pad); + return v4l2_subdev_get_try_format(&xtpg->xvip.subdev, + sd_state, pad); case V4L2_SUBDEV_FORMAT_ACTIVE: return &xtpg->formats[pad]; default: @@ -265,25 +266,26 @@ __xtpg_get_pad_format(struct xtpg_device *xtpg, } static int xtpg_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct xtpg_device *xtpg = to_tpg(subdev); - fmt->format = *__xtpg_get_pad_format(xtpg, cfg, fmt->pad, fmt->which); + fmt->format = *__xtpg_get_pad_format(xtpg, sd_state, fmt->pad, + fmt->which); return 0; } static int xtpg_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct xtpg_device *xtpg = to_tpg(subdev); struct v4l2_mbus_framefmt *__format; u32 bayer_phase; - __format = __xtpg_get_pad_format(xtpg, cfg, fmt->pad, fmt->which); + __format = __xtpg_get_pad_format(xtpg, sd_state, fmt->pad, fmt->which); /* In two pads mode the source pad format is always identical to the * sink pad format. @@ -306,7 +308,8 @@ static int xtpg_set_format(struct v4l2_subdev *subdev, /* Propagate the format to the source pad. */ if (xtpg->npads == 2) { - __format = __xtpg_get_pad_format(xtpg, cfg, 1, fmt->which); + __format = __xtpg_get_pad_format(xtpg, sd_state, 1, + fmt->which); *__format = fmt->format; } @@ -318,12 +321,12 @@ static int xtpg_set_format(struct v4l2_subdev *subdev, */ static int xtpg_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct v4l2_mbus_framefmt *format; - format = v4l2_subdev_get_try_format(subdev, cfg, fse->pad); + format = v4l2_subdev_get_try_format(subdev, sd_state, fse->pad); if (fse->index || fse->code != format->code) return -EINVAL; @@ -351,11 +354,11 @@ static int xtpg_open(struct v4l2_subdev *subdev, struct v4l2_subdev_fh *fh) struct xtpg_device *xtpg = to_tpg(subdev); struct v4l2_mbus_framefmt *format; - format = v4l2_subdev_get_try_format(subdev, fh->pad, 0); + format = v4l2_subdev_get_try_format(subdev, fh->state, 0); *format = xtpg->default_format; if (xtpg->npads == 2) { - format = v4l2_subdev_get_try_format(subdev, fh->pad, 1); + format = v4l2_subdev_get_try_format(subdev, fh->state, 1); *format = xtpg->default_format; } diff --git a/drivers/media/platform/xilinx/xilinx-vip.c b/drivers/media/platform/xilinx/xilinx-vip.c index 6ad61b08a31a..425a32dd5d19 100644 --- a/drivers/media/platform/xilinx/xilinx-vip.c +++ b/drivers/media/platform/xilinx/xilinx-vip.c @@ -70,8 +70,8 @@ EXPORT_SYMBOL_GPL(xvip_get_format_by_code); * @fourcc: the format 4CC * * Return: a pointer to the format information structure corresponding to the - * given V4L2 format @fourcc, or ERR_PTR if no corresponding format can be - * found. + * given V4L2 format @fourcc. If not found, return a pointer to the first + * available format (V4L2_PIX_FMT_YUYV). */ const struct xvip_video_format *xvip_get_format_by_fourcc(u32 fourcc) { @@ -84,7 +84,7 @@ const struct xvip_video_format *xvip_get_format_by_fourcc(u32 fourcc) return format; } - return ERR_PTR(-EINVAL); + return &xvip_video_formats[0]; } EXPORT_SYMBOL_GPL(xvip_get_format_by_fourcc); @@ -234,7 +234,7 @@ EXPORT_SYMBOL_GPL(xvip_cleanup_resources); /** * xvip_enum_mbus_code - Enumerate the media format code * @subdev: V4L2 subdevice - * @cfg: V4L2 subdev pad configuration + * @sd_state: V4L2 subdev state * @code: returning media bus code * * Enumerate the media bus code of the subdevice. Return the corresponding @@ -246,7 +246,7 @@ EXPORT_SYMBOL_GPL(xvip_cleanup_resources); * is not valid. */ int xvip_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct v4l2_mbus_framefmt *format; @@ -260,7 +260,7 @@ int xvip_enum_mbus_code(struct v4l2_subdev *subdev, if (code->index) return -EINVAL; - format = v4l2_subdev_get_try_format(subdev, cfg, code->pad); + format = v4l2_subdev_get_try_format(subdev, sd_state, code->pad); code->code = format->code; @@ -271,7 +271,7 @@ EXPORT_SYMBOL_GPL(xvip_enum_mbus_code); /** * xvip_enum_frame_size - Enumerate the media bus frame size * @subdev: V4L2 subdevice - * @cfg: V4L2 subdev pad configuration + * @sd_state: V4L2 subdev state * @fse: returning media bus frame size * * This function is a drop-in implementation of the subdev enum_frame_size pad @@ -284,7 +284,7 @@ EXPORT_SYMBOL_GPL(xvip_enum_mbus_code); * if the index or the code is not valid. */ int xvip_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct v4l2_mbus_framefmt *format; @@ -295,7 +295,7 @@ int xvip_enum_frame_size(struct v4l2_subdev *subdev, if (fse->which == V4L2_SUBDEV_FORMAT_ACTIVE) return -EINVAL; - format = v4l2_subdev_get_try_format(subdev, cfg, fse->pad); + format = v4l2_subdev_get_try_format(subdev, sd_state, fse->pad); if (fse->index || fse->code != format->code) return -EINVAL; diff --git a/drivers/media/platform/xilinx/xilinx-vip.h b/drivers/media/platform/xilinx/xilinx-vip.h index a528a32ea1dc..d0b0e0600952 100644 --- a/drivers/media/platform/xilinx/xilinx-vip.h +++ b/drivers/media/platform/xilinx/xilinx-vip.h @@ -125,10 +125,10 @@ const struct xvip_video_format *xvip_of_get_format(struct device_node *node); void xvip_set_format_size(struct v4l2_mbus_framefmt *format, const struct v4l2_subdev_format *fmt); int xvip_enum_mbus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code); int xvip_enum_frame_size(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse); static inline u32 xvip_read(struct xvip_device *xvip, u32 addr) diff --git a/drivers/media/radio/si4713/radio-platform-si4713.c b/drivers/media/radio/si4713/radio-platform-si4713.c index a7dfe5f55c18..433f9642786d 100644 --- a/drivers/media/radio/si4713/radio-platform-si4713.c +++ b/drivers/media/radio/si4713/radio-platform-si4713.c @@ -110,7 +110,7 @@ static long radio_si4713_default(struct file *file, void *p, ioctl, cmd, arg); } -static struct v4l2_ioctl_ops radio_si4713_ioctl_ops = { +static const struct v4l2_ioctl_ops radio_si4713_ioctl_ops = { .vidioc_querycap = radio_si4713_querycap, .vidioc_g_modulator = radio_si4713_g_modulator, .vidioc_s_modulator = radio_si4713_s_modulator, diff --git a/drivers/media/rc/Kconfig b/drivers/media/rc/Kconfig index f016b35c2b17..d0a8326b75c2 100644 --- a/drivers/media/rc/Kconfig +++ b/drivers/media/rc/Kconfig @@ -19,7 +19,6 @@ source "drivers/media/rc/keymaps/Kconfig" config LIRC bool "LIRC user interface" - depends on RC_CORE help Enable this option to enable the Linux Infrared Remote Control user interface (e.g. /dev/lirc*). This interface @@ -41,12 +40,10 @@ config BPF_LIRC_MODE2 menuconfig RC_DECODERS bool "Remote controller decoders" - depends on RC_CORE if RC_DECODERS config IR_NEC_DECODER tristate "Enable IR raw decoder for the NEC protocol" - depends on RC_CORE select BITREVERSE help @@ -55,7 +52,6 @@ config IR_NEC_DECODER config IR_RC5_DECODER tristate "Enable IR raw decoder for the RC-5 protocol" - depends on RC_CORE select BITREVERSE help @@ -64,7 +60,6 @@ config IR_RC5_DECODER config IR_RC6_DECODER tristate "Enable IR raw decoder for the RC6 protocol" - depends on RC_CORE select BITREVERSE help @@ -73,7 +68,6 @@ config IR_RC6_DECODER config IR_JVC_DECODER tristate "Enable IR raw decoder for the JVC protocol" - depends on RC_CORE select BITREVERSE help @@ -82,7 +76,6 @@ config IR_JVC_DECODER config IR_SONY_DECODER tristate "Enable IR raw decoder for the Sony protocol" - depends on RC_CORE select BITREVERSE help @@ -91,7 +84,6 @@ config IR_SONY_DECODER config IR_SANYO_DECODER tristate "Enable IR raw decoder for the Sanyo protocol" - depends on RC_CORE select BITREVERSE help @@ -101,7 +93,6 @@ config IR_SANYO_DECODER config IR_SHARP_DECODER tristate "Enable IR raw decoder for the Sharp protocol" - depends on RC_CORE select BITREVERSE help @@ -111,7 +102,6 @@ config IR_SHARP_DECODER config IR_MCE_KBD_DECODER tristate "Enable IR raw decoder for the MCE keyboard/mouse protocol" - depends on RC_CORE select BITREVERSE help @@ -121,7 +111,6 @@ config IR_MCE_KBD_DECODER config IR_XMP_DECODER tristate "Enable IR raw decoder for the XMP protocol" - depends on RC_CORE select BITREVERSE help @@ -130,7 +119,6 @@ config IR_XMP_DECODER config IR_IMON_DECODER tristate "Enable IR raw decoder for the iMON protocol" - depends on RC_CORE help Enable this option if you have iMON PAD or Antec Veris infrared remote control and you would like to use it with a raw IR @@ -138,7 +126,6 @@ config IR_IMON_DECODER config IR_RCMM_DECODER tristate "Enable IR raw decoder for the RC-MM protocol" - depends on RC_CORE help Enable this option when you have IR with RC-MM protocol, and you need the software decoder. The driver supports 12, @@ -153,15 +140,12 @@ endif #RC_DECODERS menuconfig RC_DEVICES bool "Remote Controller devices" - depends on RC_CORE if RC_DEVICES config RC_ATI_REMOTE tristate "ATI / X10 based USB RF remote controls" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB help Say Y here if you want to use an X10 based USB remote control. These are RF remotes with USB receivers. @@ -179,7 +163,6 @@ config RC_ATI_REMOTE config IR_ENE tristate "ENE eHome Receiver/Transceiver (pnp id: ENE0100/ENE02xxx)" depends on PNP || COMPILE_TEST - depends on RC_CORE help Say Y here to enable support for integrated infrared receiver /transceiver made by ENE. @@ -192,7 +175,6 @@ config IR_ENE config IR_HIX5HD2 tristate "Hisilicon hix5hd2 IR remote control" - depends on RC_CORE depends on OF || COMPILE_TEST help Say Y here if you want to use hisilicon hix5hd2 remote control. @@ -203,9 +185,7 @@ config IR_HIX5HD2 config IR_IMON tristate "SoundGraph iMON Receiver and Display" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB help Say Y here if you want to use a SoundGraph iMON (aka Antec Veris) IR Receiver and/or LCD/VFD/VGA display. @@ -215,9 +195,7 @@ config IR_IMON config IR_IMON_RAW tristate "SoundGraph iMON Receiver (early raw IR models)" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB help Say Y here if you want to use a SoundGraph iMON IR Receiver, early raw models. @@ -227,9 +205,7 @@ config IR_IMON_RAW config IR_MCEUSB tristate "Windows Media Center Ed. eHome Infrared Transceiver" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB help Say Y here if you want to use a Windows Media Center Edition eHome Infrared Transceiver. @@ -240,7 +216,6 @@ config IR_MCEUSB config IR_ITE_CIR tristate "ITE Tech Inc. IT8712/IT8512 Consumer Infrared Transceiver" depends on PNP || COMPILE_TEST - depends on RC_CORE help Say Y here to enable support for integrated infrared receivers /transceivers made by ITE Tech Inc. These are found in @@ -253,7 +228,6 @@ config IR_ITE_CIR config IR_FINTEK tristate "Fintek Consumer Infrared Transceiver" depends on PNP || COMPILE_TEST - depends on RC_CORE help Say Y here to enable support for integrated infrared receiver /transceiver made by Fintek. This chip is found on assorted @@ -264,7 +238,6 @@ config IR_FINTEK config IR_MESON tristate "Amlogic Meson IR remote receiver" - depends on RC_CORE depends on ARCH_MESON || COMPILE_TEST help Say Y if you want to use the IR remote receiver available @@ -275,7 +248,6 @@ config IR_MESON config IR_MTK tristate "Mediatek IR remote receiver" - depends on RC_CORE depends on ARCH_MEDIATEK || COMPILE_TEST help Say Y if you want to use the IR remote receiver available @@ -287,7 +259,6 @@ config IR_MTK config IR_NUVOTON tristate "Nuvoton w836x7hg Consumer Infrared Transceiver" depends on PNP || COMPILE_TEST - depends on RC_CORE help Say Y here to enable support for integrated infrared receiver /transceiver made by Nuvoton (formerly Winbond). This chip is @@ -299,11 +270,9 @@ config IR_NUVOTON config IR_REDRAT3 tristate "RedRat3 IR Transceiver" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE + depends on USB select NEW_LEDS select LEDS_CLASS - select USB help Say Y here if you want to use a RedRat3 Infrared Transceiver. @@ -322,9 +291,7 @@ config IR_SPI config IR_STREAMZAP tristate "Streamzap PC Remote IR Receiver" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB help Say Y here if you want to use a Streamzap PC Remote Infrared Receiver. @@ -335,7 +302,6 @@ config IR_STREAMZAP config IR_WINBOND_CIR tristate "Winbond IR remote control" depends on (X86 && PNP) || COMPILE_TEST - depends on RC_CORE select NEW_LEDS select LEDS_CLASS select BITREVERSE @@ -350,9 +316,7 @@ config IR_WINBOND_CIR config IR_IGORPLUGUSB tristate "IgorPlug-USB IR Receiver" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB help Say Y here if you want to use the IgorPlug-USB IR Receiver by Igor Cesko. This device is included on the Fit-PC2. @@ -365,9 +329,7 @@ config IR_IGORPLUGUSB config IR_IGUANA tristate "IguanaWorks USB IR Transceiver" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB help Say Y here if you want to use the IguanaWorks USB IR Transceiver. Both infrared receive and send are supported. If you want to @@ -381,9 +343,7 @@ config IR_IGUANA config IR_TTUSBIR tristate "TechnoTrend USB IR Receiver" - depends on USB_ARCH_HAS_HCD - depends on RC_CORE - select USB + depends on USB select NEW_LEDS select LEDS_CLASS help @@ -407,7 +367,6 @@ source "drivers/media/rc/img-ir/Kconfig" config RC_LOOPBACK tristate "Remote Control Loopback Driver" - depends on RC_CORE help Say Y here if you want support for the remote control loopback driver which allows TX data to be sent back as RX data. @@ -420,7 +379,6 @@ config RC_LOOPBACK config IR_GPIO_CIR tristate "GPIO IR remote control" - depends on RC_CORE depends on (OF && GPIOLIB) || COMPILE_TEST help Say Y if you want to use GPIO based IR Receiver. @@ -430,7 +388,6 @@ config IR_GPIO_CIR config IR_GPIO_TX tristate "GPIO IR Bit Banging Transmitter" - depends on RC_CORE depends on LIRC depends on (OF && GPIOLIB) || COMPILE_TEST help @@ -442,7 +399,6 @@ config IR_GPIO_TX config IR_PWM_TX tristate "PWM IR transmitter" - depends on RC_CORE depends on LIRC depends on PWM depends on OF || COMPILE_TEST @@ -455,7 +411,6 @@ config IR_PWM_TX config RC_ST tristate "ST remote control receiver" - depends on RC_CORE depends on ARCH_STI || COMPILE_TEST help Say Y here if you want support for ST remote control driver @@ -466,7 +421,6 @@ config RC_ST config IR_SUNXI tristate "SUNXI IR remote control" - depends on RC_CORE depends on ARCH_SUNXI || COMPILE_TEST help Say Y if you want to use sunXi internal IR Controller @@ -476,7 +430,6 @@ config IR_SUNXI config IR_SERIAL tristate "Homebrew Serial Port Receiver" - depends on RC_CORE help Say Y if you want to use Homebrew Serial Port Receivers and Transceivers. @@ -492,28 +445,15 @@ config IR_SERIAL_TRANSMITTER config IR_SIR tristate "Built-in SIR IrDA port" - depends on RC_CORE help Say Y if you want to use a IrDA SIR port Transceivers. To compile this driver as a module, choose M here: the module will be called sir-ir. -config IR_TANGO - tristate "Sigma Designs SMP86xx IR decoder" - depends on RC_CORE - depends on ARCH_TANGO || COMPILE_TEST - help - Adds support for the HW IR decoder embedded on Sigma Designs - Tango-based systems (SMP86xx, SMP87xx). - The HW decoder supports NEC, RC-5, RC-6 IR protocols. - When compiled as a module, look for tango-ir. - config RC_XBOX_DVD tristate "Xbox DVD Movie Playback Kit" - depends on RC_CORE - depends on USB_ARCH_HAS_HCD - select USB + depends on USB help Say Y here if you want to use the Xbox DVD Movie Playback Kit. These are IR remotes with USB receivers for the Original Xbox (2001). @@ -523,8 +463,7 @@ config RC_XBOX_DVD config IR_TOY tristate "Infrared Toy and IR Droid" - depends on RC_CORE - depends on USB_ARCH_HAS_HCD + depends on USB help Say Y here if you want to use the Infrared Toy or IR Droid, USB versions. diff --git a/drivers/media/rc/Makefile b/drivers/media/rc/Makefile index f31002288f7c..692e9b6b203f 100644 --- a/drivers/media/rc/Makefile +++ b/drivers/media/rc/Makefile @@ -48,6 +48,5 @@ obj-$(CONFIG_IR_IMG) += img-ir/ obj-$(CONFIG_IR_SERIAL) += serial_ir.o obj-$(CONFIG_IR_SIR) += sir_ir.o obj-$(CONFIG_IR_MTK) += mtk-cir.o -obj-$(CONFIG_IR_TANGO) += tango-ir.o obj-$(CONFIG_RC_XBOX_DVD) += xbox_remote.o obj-$(CONFIG_IR_TOY) += ir_toy.o diff --git a/drivers/media/rc/imon.c b/drivers/media/rc/imon.c index a7962ca2ac8e..2ca4e86c7b9f 100644 --- a/drivers/media/rc/imon.c +++ b/drivers/media/rc/imon.c @@ -780,7 +780,7 @@ static int send_set_imon_clock(struct imon_context *ictx, /* * These are the sysfs functions to handle the association on the iMON 2.4G LT. */ -static ssize_t show_associate_remote(struct device *d, +static ssize_t associate_remote_show(struct device *d, struct device_attribute *attr, char *buf) { @@ -800,7 +800,7 @@ static ssize_t show_associate_remote(struct device *d, return strlen(buf); } -static ssize_t store_associate_remote(struct device *d, +static ssize_t associate_remote_store(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { @@ -822,7 +822,7 @@ static ssize_t store_associate_remote(struct device *d, /* * sysfs functions to control internal imon clock */ -static ssize_t show_imon_clock(struct device *d, +static ssize_t imon_clock_show(struct device *d, struct device_attribute *attr, char *buf) { struct imon_context *ictx = dev_get_drvdata(d); @@ -848,7 +848,7 @@ static ssize_t show_imon_clock(struct device *d, return len; } -static ssize_t store_imon_clock(struct device *d, +static ssize_t imon_clock_store(struct device *d, struct device_attribute *attr, const char *buf, size_t count) { @@ -895,11 +895,8 @@ static ssize_t store_imon_clock(struct device *d, } -static DEVICE_ATTR(imon_clock, S_IWUSR | S_IRUGO, show_imon_clock, - store_imon_clock); - -static DEVICE_ATTR(associate_remote, S_IWUSR | S_IRUGO, show_associate_remote, - store_associate_remote); +static DEVICE_ATTR_RW(imon_clock); +static DEVICE_ATTR_RW(associate_remote); static struct attribute *imon_display_sysfs_entries[] = { &dev_attr_imon_clock.attr, diff --git a/drivers/media/rc/ite-cir.h b/drivers/media/rc/ite-cir.h index ce7a40b10828..4b4294d77555 100644 --- a/drivers/media/rc/ite-cir.h +++ b/drivers/media/rc/ite-cir.h @@ -167,7 +167,7 @@ struct ite_dev { * hardware data obtained from: * * IT8712F - * Environment Control – Low Pin Count Input / Output + * Environment Control - Low Pin Count Input / Output * (EC - LPC I/O) * Preliminary Specification V0. 81 */ diff --git a/drivers/media/rc/keymaps/Makefile b/drivers/media/rc/keymaps/Makefile index 50b2833dbe4f..5fe5c9e1a46d 100644 --- a/drivers/media/rc/keymaps/Makefile +++ b/drivers/media/rc/keymaps/Makefile @@ -23,6 +23,7 @@ obj-$(CONFIG_RC_MAP) += rc-adstech-dvb-t-pci.o \ rc-budget-ci-old.o \ rc-cinergy-1400.o \ rc-cinergy.o \ + rc-ct-90405.o \ rc-d680-dmb.o \ rc-delock-61959.o \ rc-dib0700-nec.o \ @@ -100,7 +101,6 @@ obj-$(CONFIG_RC_MAP) += rc-adstech-dvb-t-pci.o \ rc-reddo.o \ rc-snapstream-firefly.o \ rc-streamzap.o \ - rc-tango.o \ rc-tanix-tx3mini.o \ rc-tanix-tx5max.o \ rc-tbs-nec.o \ diff --git a/drivers/media/rc/keymaps/rc-ct-90405.c b/drivers/media/rc/keymaps/rc-ct-90405.c new file mode 100644 index 000000000000..8914c83c9d9f --- /dev/null +++ b/drivers/media/rc/keymaps/rc-ct-90405.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Toshiba CT-90405 remote controller keytable + * + * Copyright (C) 2021 Alexander Voronov + */ + +#include +#include + +static struct rc_map_table ct_90405[] = { + { 0x4014, KEY_SWITCHVIDEOMODE }, + { 0x4012, KEY_POWER }, + { 0x4044, KEY_TV }, + { 0x40be43, KEY_3D_MODE }, + { 0x400c, KEY_SUBTITLE }, + { 0x4001, KEY_NUMERIC_1 }, + { 0x4002, KEY_NUMERIC_2 }, + { 0x4003, KEY_NUMERIC_3 }, + { 0x4004, KEY_NUMERIC_4 }, + { 0x4005, KEY_NUMERIC_5 }, + { 0x4006, KEY_NUMERIC_6 }, + { 0x4007, KEY_NUMERIC_7 }, + { 0x4008, KEY_NUMERIC_8 }, + { 0x4009, KEY_NUMERIC_9 }, + { 0x4062, KEY_AUDIO_DESC }, + { 0x4000, KEY_NUMERIC_0 }, + { 0x401a, KEY_VOLUMEUP }, + { 0x401e, KEY_VOLUMEDOWN }, + { 0x4016, KEY_INFO }, + { 0x4010, KEY_MUTE }, + { 0x401b, KEY_CHANNELUP }, + { 0x401f, KEY_CHANNELDOWN }, + { 0x40da, KEY_VENDOR }, + { 0x4066, KEY_PLAYER }, + { 0x4017, KEY_TEXT }, + { 0x4047, KEY_LIST }, + { 0x4073, KEY_PAGEUP }, + { 0x4045, KEY_PROGRAM }, + { 0x4043, KEY_EXIT }, + { 0x4074, KEY_PAGEDOWN }, + { 0x4064, KEY_BACK }, + { 0x405b, KEY_MENU }, + { 0x4019, KEY_UP }, + { 0x4040, KEY_RIGHT }, + { 0x401d, KEY_DOWN }, + { 0x4042, KEY_LEFT }, + { 0x4021, KEY_OK }, + { 0x4053, KEY_REWIND }, + { 0x4067, KEY_PLAY }, + { 0x400d, KEY_FASTFORWARD }, + { 0x4054, KEY_PREVIOUS }, + { 0x4068, KEY_STOP }, + { 0x406a, KEY_PAUSE }, + { 0x4015, KEY_NEXT }, + { 0x4048, KEY_RED }, + { 0x4049, KEY_GREEN }, + { 0x404a, KEY_YELLOW }, + { 0x404b, KEY_BLUE }, + { 0x406f, KEY_RECORD } +}; + +static struct rc_map_list ct_90405_map = { + .map = { + .scan = ct_90405, + .size = ARRAY_SIZE(ct_90405), + .rc_proto = RC_PROTO_NEC, + .name = RC_MAP_CT_90405, + } +}; + +static int __init init_rc_map_ct_90405(void) +{ + return rc_map_register(&ct_90405_map); +} + +static void __exit exit_rc_map_ct_90405(void) +{ + rc_map_unregister(&ct_90405_map); +} + +module_init(init_rc_map_ct_90405) +module_exit(exit_rc_map_ct_90405) + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Alexander Voronov "); diff --git a/drivers/media/rc/keymaps/rc-tango.c b/drivers/media/rc/keymaps/rc-tango.c deleted file mode 100644 index 2b9cef6ef5b5..000000000000 --- a/drivers/media/rc/keymaps/rc-tango.c +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2017 Sigma Designs - */ - -#include -#include - -static struct rc_map_table tango_table[] = { - { 0x4cb4a, KEY_POWER }, - { 0x4cb48, KEY_FILE }, - { 0x4cb0f, KEY_SETUP }, - { 0x4cb4d, KEY_SUSPEND }, - { 0x4cb4e, KEY_VOLUMEUP }, - { 0x4cb44, KEY_EJECTCD }, - { 0x4cb13, KEY_TV }, - { 0x4cb51, KEY_MUTE }, - { 0x4cb52, KEY_VOLUMEDOWN }, - - { 0x4cb41, KEY_NUMERIC_1 }, - { 0x4cb03, KEY_NUMERIC_2 }, - { 0x4cb42, KEY_NUMERIC_3 }, - { 0x4cb45, KEY_NUMERIC_4 }, - { 0x4cb07, KEY_NUMERIC_5 }, - { 0x4cb46, KEY_NUMERIC_6 }, - { 0x4cb55, KEY_NUMERIC_7 }, - { 0x4cb17, KEY_NUMERIC_8 }, - { 0x4cb56, KEY_NUMERIC_9 }, - { 0x4cb1b, KEY_NUMERIC_0 }, - { 0x4cb59, KEY_DELETE }, - { 0x4cb5a, KEY_CAPSLOCK }, - - { 0x4cb47, KEY_BACK }, - { 0x4cb05, KEY_SWITCHVIDEOMODE }, - { 0x4cb06, KEY_UP }, - { 0x4cb43, KEY_LEFT }, - { 0x4cb01, KEY_RIGHT }, - { 0x4cb0a, KEY_DOWN }, - { 0x4cb02, KEY_ENTER }, - { 0x4cb4b, KEY_INFO }, - { 0x4cb09, KEY_HOME }, - - { 0x4cb53, KEY_MENU }, - { 0x4cb12, KEY_PREVIOUS }, - { 0x4cb50, KEY_PLAY }, - { 0x4cb11, KEY_NEXT }, - { 0x4cb4f, KEY_TITLE }, - { 0x4cb0e, KEY_REWIND }, - { 0x4cb4c, KEY_STOP }, - { 0x4cb0d, KEY_FORWARD }, - { 0x4cb57, KEY_MEDIA_REPEAT }, - { 0x4cb16, KEY_ANGLE }, - { 0x4cb54, KEY_PAUSE }, - { 0x4cb15, KEY_SLOW }, - { 0x4cb5b, KEY_TIME }, - { 0x4cb1a, KEY_AUDIO }, - { 0x4cb58, KEY_SUBTITLE }, - { 0x4cb19, KEY_ZOOM }, - - { 0x4cb5f, KEY_RED }, - { 0x4cb1e, KEY_GREEN }, - { 0x4cb5c, KEY_YELLOW }, - { 0x4cb1d, KEY_BLUE }, -}; - -static struct rc_map_list tango_map = { - .map = { - .scan = tango_table, - .size = ARRAY_SIZE(tango_table), - .rc_proto = RC_PROTO_NECX, - .name = RC_MAP_TANGO, - } -}; - -static int __init init_rc_map_tango(void) -{ - return rc_map_register(&tango_map); -} - -static void __exit exit_rc_map_tango(void) -{ - rc_map_unregister(&tango_map); -} - -module_init(init_rc_map_tango) -module_exit(exit_rc_map_tango) - -MODULE_AUTHOR("Sigma Designs"); -MODULE_LICENSE("GPL"); diff --git a/drivers/media/rc/st_rc.c b/drivers/media/rc/st_rc.c index 3237fef5d502..d79d1e3996b2 100644 --- a/drivers/media/rc/st_rc.c +++ b/drivers/media/rc/st_rc.c @@ -157,8 +157,9 @@ static irqreturn_t st_rc_rx_interrupt(int irq, void *data) return IRQ_HANDLED; } -static void st_rc_hardware_init(struct st_rc_device *dev) +static int st_rc_hardware_init(struct st_rc_device *dev) { + int ret; int baseclock, freqdiff; unsigned int rx_max_symbol_per = MAX_SYMB_TIME; unsigned int rx_sampling_freq_div; @@ -166,7 +167,12 @@ static void st_rc_hardware_init(struct st_rc_device *dev) /* Enable the IP */ reset_control_deassert(dev->rstc); - clk_prepare_enable(dev->sys_clock); + ret = clk_prepare_enable(dev->sys_clock); + if (ret) { + dev_err(dev->dev, "Failed to prepare/enable system clock\n"); + return ret; + } + baseclock = clk_get_rate(dev->sys_clock); /* IRB input pins are inverted internally from high to low. */ @@ -184,6 +190,8 @@ static void st_rc_hardware_init(struct st_rc_device *dev) } writel(rx_max_symbol_per, dev->rx_base + IRB_MAX_SYM_PERIOD); + + return 0; } static int st_rc_remove(struct platform_device *pdev) @@ -287,7 +295,9 @@ static int st_rc_probe(struct platform_device *pdev) rc_dev->dev = dev; platform_set_drvdata(pdev, rc_dev); - st_rc_hardware_init(rc_dev); + ret = st_rc_hardware_init(rc_dev); + if (ret) + goto err; rdev->allowed_protocols = RC_PROTO_BIT_ALL_IR_DECODER; /* rx sampling rate is 10Mhz */ @@ -359,6 +369,7 @@ static int st_rc_suspend(struct device *dev) static int st_rc_resume(struct device *dev) { + int ret; struct st_rc_device *rc_dev = dev_get_drvdata(dev); struct rc_dev *rdev = rc_dev->rdev; @@ -367,7 +378,10 @@ static int st_rc_resume(struct device *dev) rc_dev->irq_wake = 0; } else { pinctrl_pm_select_default_state(dev); - st_rc_hardware_init(rc_dev); + ret = st_rc_hardware_init(rc_dev); + if (ret) + return ret; + if (rdev->users) { writel(IRB_RX_INTS, rc_dev->rx_base + IRB_RX_INT_EN); writel(0x01, rc_dev->rx_base + IRB_RX_EN); diff --git a/drivers/media/rc/tango-ir.c b/drivers/media/rc/tango-ir.c deleted file mode 100644 index b8eb5bc4d9be..000000000000 --- a/drivers/media/rc/tango-ir.c +++ /dev/null @@ -1,267 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Copyright (C) 2015 Mans Rullgard - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#define DRIVER_NAME "tango-ir" - -#define IR_NEC_CTRL 0x00 -#define IR_NEC_DATA 0x04 -#define IR_CTRL 0x08 -#define IR_RC5_CLK_DIV 0x0c -#define IR_RC5_DATA 0x10 -#define IR_INT 0x14 - -#define NEC_TIME_BASE 560 -#define RC5_TIME_BASE 1778 - -#define RC6_CTRL 0x00 -#define RC6_CLKDIV 0x04 -#define RC6_DATA0 0x08 -#define RC6_DATA1 0x0c -#define RC6_DATA2 0x10 -#define RC6_DATA3 0x14 -#define RC6_DATA4 0x18 - -#define RC6_CARRIER 36000 -#define RC6_TIME_BASE 16 - -#define NEC_CAP(n) ((n) << 24) -#define GPIO_SEL(n) ((n) << 16) -#define DISABLE_NEC (BIT(4) | BIT(8)) -#define ENABLE_RC5 (BIT(0) | BIT(9)) -#define ENABLE_RC6 (BIT(0) | BIT(7)) -#define ACK_IR_INT (BIT(0) | BIT(1)) -#define ACK_RC6_INT (BIT(31)) - -#define NEC_ANY (RC_PROTO_BIT_NEC | RC_PROTO_BIT_NECX | RC_PROTO_BIT_NEC32) - -struct tango_ir { - void __iomem *rc5_base; - void __iomem *rc6_base; - struct rc_dev *rc; - struct clk *clk; -}; - -static void tango_ir_handle_nec(struct tango_ir *ir) -{ - u32 v, code; - enum rc_proto proto; - - v = readl_relaxed(ir->rc5_base + IR_NEC_DATA); - if (!v) { - rc_repeat(ir->rc); - return; - } - - code = ir_nec_bytes_to_scancode(v, v >> 8, v >> 16, v >> 24, &proto); - rc_keydown(ir->rc, proto, code, 0); -} - -static void tango_ir_handle_rc5(struct tango_ir *ir) -{ - u32 data, field, toggle, addr, cmd, code; - - data = readl_relaxed(ir->rc5_base + IR_RC5_DATA); - if (data & BIT(31)) - return; - - field = data >> 12 & 1; - toggle = data >> 11 & 1; - addr = data >> 6 & 0x1f; - cmd = (data & 0x3f) | (field ^ 1) << 6; - - code = RC_SCANCODE_RC5(addr, cmd); - rc_keydown(ir->rc, RC_PROTO_RC5, code, toggle); -} - -static void tango_ir_handle_rc6(struct tango_ir *ir) -{ - u32 data0, data1, toggle, mode, addr, cmd, code; - - data0 = readl_relaxed(ir->rc6_base + RC6_DATA0); - data1 = readl_relaxed(ir->rc6_base + RC6_DATA1); - - mode = data0 >> 1 & 7; - if (mode != 0) - return; - - toggle = data0 & 1; - addr = data0 >> 16; - cmd = data1; - - code = RC_SCANCODE_RC6_0(addr, cmd); - rc_keydown(ir->rc, RC_PROTO_RC6_0, code, toggle); -} - -static irqreturn_t tango_ir_irq(int irq, void *dev_id) -{ - struct tango_ir *ir = dev_id; - unsigned int rc5_stat; - unsigned int rc6_stat; - - rc5_stat = readl_relaxed(ir->rc5_base + IR_INT); - writel_relaxed(rc5_stat, ir->rc5_base + IR_INT); - - rc6_stat = readl_relaxed(ir->rc6_base + RC6_CTRL); - writel_relaxed(rc6_stat, ir->rc6_base + RC6_CTRL); - - if (!(rc5_stat & 3) && !(rc6_stat & BIT(31))) - return IRQ_NONE; - - if (rc5_stat & BIT(0)) - tango_ir_handle_rc5(ir); - - if (rc5_stat & BIT(1)) - tango_ir_handle_nec(ir); - - if (rc6_stat & BIT(31)) - tango_ir_handle_rc6(ir); - - return IRQ_HANDLED; -} - -static int tango_change_protocol(struct rc_dev *dev, u64 *rc_type) -{ - struct tango_ir *ir = dev->priv; - u32 rc5_ctrl = DISABLE_NEC; - u32 rc6_ctrl = 0; - - if (*rc_type & NEC_ANY) - rc5_ctrl = 0; - - if (*rc_type & RC_PROTO_BIT_RC5) - rc5_ctrl |= ENABLE_RC5; - - if (*rc_type & RC_PROTO_BIT_RC6_0) - rc6_ctrl = ENABLE_RC6; - - writel_relaxed(rc5_ctrl, ir->rc5_base + IR_CTRL); - writel_relaxed(rc6_ctrl, ir->rc6_base + RC6_CTRL); - - return 0; -} - -static int tango_ir_probe(struct platform_device *pdev) -{ - const char *map_name = RC_MAP_TANGO; - struct device *dev = &pdev->dev; - struct rc_dev *rc; - struct tango_ir *ir; - u64 clkrate, clkdiv; - int irq, err; - u32 val; - - irq = platform_get_irq(pdev, 0); - if (irq <= 0) - return -EINVAL; - - ir = devm_kzalloc(dev, sizeof(*ir), GFP_KERNEL); - if (!ir) - return -ENOMEM; - - ir->rc5_base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(ir->rc5_base)) - return PTR_ERR(ir->rc5_base); - - ir->rc6_base = devm_platform_ioremap_resource(pdev, 1); - if (IS_ERR(ir->rc6_base)) - return PTR_ERR(ir->rc6_base); - - ir->clk = devm_clk_get(dev, NULL); - if (IS_ERR(ir->clk)) - return PTR_ERR(ir->clk); - - rc = devm_rc_allocate_device(dev, RC_DRIVER_SCANCODE); - if (!rc) - return -ENOMEM; - - of_property_read_string(dev->of_node, "linux,rc-map-name", &map_name); - - rc->device_name = DRIVER_NAME; - rc->driver_name = DRIVER_NAME; - rc->input_phys = DRIVER_NAME "/input0"; - rc->map_name = map_name; - rc->allowed_protocols = NEC_ANY | RC_PROTO_BIT_RC5 | RC_PROTO_BIT_RC6_0; - rc->change_protocol = tango_change_protocol; - rc->priv = ir; - ir->rc = rc; - - err = clk_prepare_enable(ir->clk); - if (err) - return err; - - clkrate = clk_get_rate(ir->clk); - - clkdiv = clkrate * NEC_TIME_BASE; - do_div(clkdiv, 1000000); - - val = NEC_CAP(31) | GPIO_SEL(12) | clkdiv; - writel_relaxed(val, ir->rc5_base + IR_NEC_CTRL); - - clkdiv = clkrate * RC5_TIME_BASE; - do_div(clkdiv, 1000000); - - writel_relaxed(DISABLE_NEC, ir->rc5_base + IR_CTRL); - writel_relaxed(clkdiv, ir->rc5_base + IR_RC5_CLK_DIV); - writel_relaxed(ACK_IR_INT, ir->rc5_base + IR_INT); - - clkdiv = clkrate * RC6_TIME_BASE; - do_div(clkdiv, RC6_CARRIER); - - writel_relaxed(ACK_RC6_INT, ir->rc6_base + RC6_CTRL); - writel_relaxed((clkdiv >> 2) << 18 | clkdiv, ir->rc6_base + RC6_CLKDIV); - - err = devm_request_irq(dev, irq, tango_ir_irq, IRQF_SHARED, - dev_name(dev), ir); - if (err) - goto err_clk; - - err = devm_rc_register_device(dev, rc); - if (err) - goto err_clk; - - platform_set_drvdata(pdev, ir); - return 0; - -err_clk: - clk_disable_unprepare(ir->clk); - return err; -} - -static int tango_ir_remove(struct platform_device *pdev) -{ - struct tango_ir *ir = platform_get_drvdata(pdev); - - clk_disable_unprepare(ir->clk); - return 0; -} - -static const struct of_device_id tango_ir_dt_ids[] = { - { .compatible = "sigma,smp8642-ir" }, - { } -}; -MODULE_DEVICE_TABLE(of, tango_ir_dt_ids); - -static struct platform_driver tango_ir_driver = { - .probe = tango_ir_probe, - .remove = tango_ir_remove, - .driver = { - .name = DRIVER_NAME, - .of_match_table = tango_ir_dt_ids, - }, -}; -module_platform_driver(tango_ir_driver); - -MODULE_DESCRIPTION("SMP86xx IR decoder driver"); -MODULE_AUTHOR("Mans Rullgard "); -MODULE_LICENSE("GPL"); diff --git a/drivers/media/spi/cxd2880-spi.c b/drivers/media/spi/cxd2880-spi.c index 931ec0727cd3..e5094fff04c5 100644 --- a/drivers/media/spi/cxd2880-spi.c +++ b/drivers/media/spi/cxd2880-spi.c @@ -147,7 +147,7 @@ static int cxd2880_spi_read_ts(struct spi_device *spi, ret = spi_sync(spi, &message); if (ret) - pr_err("spi_write_then_read failed\n"); + pr_err("spi_sync failed\n"); return ret; } @@ -401,7 +401,7 @@ static int cxd2880_start_feed(struct dvb_demux_feed *feed) dvb_spi, "cxd2880_ts_read"); if (IS_ERR(dvb_spi->cxd2880_ts_read_thread)) { - pr_err("kthread_run failed/\n"); + pr_err("kthread_run failed\n"); kfree(dvb_spi->ts_buf); dvb_spi->ts_buf = NULL; memset(&dvb_spi->filter_config, 0, @@ -448,7 +448,7 @@ static int cxd2880_stop_feed(struct dvb_demux_feed *feed) * in dvb_spi->all_pid_feed_count. */ if (dvb_spi->all_pid_feed_count <= 0) { - pr_err("PID %d not found.\n", feed->pid); + pr_err("PID %d not found\n", feed->pid); return -EINVAL; } dvb_spi->all_pid_feed_count--; @@ -485,7 +485,7 @@ static int cxd2880_stop_feed(struct dvb_demux_feed *feed) ret_stop = kthread_stop(dvb_spi->cxd2880_ts_read_thread); if (ret_stop) { - pr_err("'kthread_stop failed. (%d)\n", ret_stop); + pr_err("kthread_stop failed. (%d)\n", ret_stop); ret = ret_stop; } kfree(dvb_spi->ts_buf); @@ -512,7 +512,7 @@ cxd2880_spi_probe(struct spi_device *spi) struct cxd2880_config config; if (!spi) { - pr_err("invalid arg.\n"); + pr_err("invalid arg\n"); return -EINVAL; } @@ -596,7 +596,7 @@ cxd2880_spi_probe(struct spi_device *spi) ret = dvb_spi->demux.dmx.connect_frontend(&dvb_spi->demux.dmx, &dvb_spi->dmx_fe); if (ret < 0) { - pr_err("dvb_register_frontend() failed\n"); + pr_err("connect_frontend() failed\n"); goto fail_fe_conn; } diff --git a/drivers/media/test-drivers/vim2m.c b/drivers/media/test-drivers/vim2m.c index a24624353f9e..d714fe50afe5 100644 --- a/drivers/media/test-drivers/vim2m.c +++ b/drivers/media/test-drivers/vim2m.c @@ -624,11 +624,6 @@ static void device_work(struct work_struct *w) curr_ctx = container_of(w, struct vim2m_ctx, work_run.work); - if (!curr_ctx) { - pr_err("Instance released before the end of transaction\n"); - return; - } - vim2m_dev = curr_ctx->dev; src_vb = v4l2_m2m_src_buf_remove(curr_ctx->fh.m2m_ctx); diff --git a/drivers/media/test-drivers/vimc/vimc-debayer.c b/drivers/media/test-drivers/vimc/vimc-debayer.c index c3f6fef34f68..2d06cdbacc76 100644 --- a/drivers/media/test-drivers/vimc/vimc-debayer.c +++ b/drivers/media/test-drivers/vimc/vimc-debayer.c @@ -150,17 +150,17 @@ static bool vimc_deb_src_code_is_valid(u32 code) } static int vimc_deb_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *mf; unsigned int i; - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); *mf = sink_fmt_default; for (i = 1; i < sd->entity.num_pads; i++) { - mf = v4l2_subdev_get_try_format(sd, cfg, i); + mf = v4l2_subdev_get_try_format(sd, sd_state, i); *mf = sink_fmt_default; mf->code = vdeb->src_code; } @@ -169,7 +169,7 @@ static int vimc_deb_init_cfg(struct v4l2_subdev *sd, } static int vimc_deb_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (VIMC_IS_SRC(code->pad)) { @@ -188,7 +188,7 @@ static int vimc_deb_enum_mbus_code(struct v4l2_subdev *sd, } static int vimc_deb_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { if (fse->index) @@ -213,14 +213,14 @@ static int vimc_deb_enum_frame_size(struct v4l2_subdev *sd, } static int vimc_deb_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd); /* Get the current sink format */ fmt->format = fmt->which == V4L2_SUBDEV_FORMAT_TRY ? - *v4l2_subdev_get_try_format(sd, cfg, 0) : + *v4l2_subdev_get_try_format(sd, sd_state, 0) : vdeb->sink_fmt; /* Set the right code for the source pad */ @@ -251,7 +251,7 @@ static void vimc_deb_adjust_sink_fmt(struct v4l2_mbus_framefmt *fmt) } static int vimc_deb_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vimc_deb_device *vdeb = v4l2_get_subdevdata(sd); @@ -266,8 +266,8 @@ static int vimc_deb_set_fmt(struct v4l2_subdev *sd, sink_fmt = &vdeb->sink_fmt; src_code = &vdeb->src_code; } else { - sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); - src_code = &v4l2_subdev_get_try_format(sd, cfg, 1)->code; + sink_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); + src_code = &v4l2_subdev_get_try_format(sd, sd_state, 1)->code; } /* diff --git a/drivers/media/test-drivers/vimc/vimc-scaler.c b/drivers/media/test-drivers/vimc/vimc-scaler.c index 121fa7d62a2e..06880dd0b6ac 100644 --- a/drivers/media/test-drivers/vimc/vimc-scaler.c +++ b/drivers/media/test-drivers/vimc/vimc-scaler.c @@ -84,20 +84,20 @@ static void vimc_sca_adjust_sink_crop(struct v4l2_rect *r, } static int vimc_sca_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_mbus_framefmt *mf; struct v4l2_rect *r; unsigned int i; - mf = v4l2_subdev_get_try_format(sd, cfg, 0); + mf = v4l2_subdev_get_try_format(sd, sd_state, 0); *mf = sink_fmt_default; - r = v4l2_subdev_get_try_crop(sd, cfg, 0); + r = v4l2_subdev_get_try_crop(sd, sd_state, 0); *r = crop_rect_default; for (i = 1; i < sd->entity.num_pads; i++) { - mf = v4l2_subdev_get_try_format(sd, cfg, i); + mf = v4l2_subdev_get_try_format(sd, sd_state, i); *mf = sink_fmt_default; mf->width = mf->width * sca_mult; mf->height = mf->height * sca_mult; @@ -107,7 +107,7 @@ static int vimc_sca_init_cfg(struct v4l2_subdev *sd, } static int vimc_sca_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { u32 mbus_code = vimc_mbus_code_by_index(code->index); @@ -128,7 +128,7 @@ static int vimc_sca_enum_mbus_code(struct v4l2_subdev *sd, } static int vimc_sca_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { const struct vimc_pix_map *vpix; @@ -156,7 +156,7 @@ static int vimc_sca_enum_frame_size(struct v4l2_subdev *sd, } static int vimc_sca_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); @@ -164,8 +164,8 @@ static int vimc_sca_get_fmt(struct v4l2_subdev *sd, /* Get the current sink format */ if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - format->format = *v4l2_subdev_get_try_format(sd, cfg, 0); - crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); + format->format = *v4l2_subdev_get_try_format(sd, sd_state, 0); + crop_rect = v4l2_subdev_get_try_crop(sd, sd_state, 0); } else { format->format = vsca->sink_fmt; crop_rect = &vsca->crop_rect; @@ -201,7 +201,7 @@ static void vimc_sca_adjust_sink_fmt(struct v4l2_mbus_framefmt *fmt) } static int vimc_sca_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); @@ -216,8 +216,8 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd, sink_fmt = &vsca->sink_fmt; crop_rect = &vsca->crop_rect; } else { - sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); - crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); + sink_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); + crop_rect = v4l2_subdev_get_try_crop(sd, sd_state, 0); } /* @@ -254,7 +254,7 @@ static int vimc_sca_set_fmt(struct v4l2_subdev *sd, } static int vimc_sca_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); @@ -268,8 +268,8 @@ static int vimc_sca_get_selection(struct v4l2_subdev *sd, sink_fmt = &vsca->sink_fmt; crop_rect = &vsca->crop_rect; } else { - sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); - crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); + sink_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); + crop_rect = v4l2_subdev_get_try_crop(sd, sd_state, 0); } switch (sel->target) { @@ -287,7 +287,7 @@ static int vimc_sca_get_selection(struct v4l2_subdev *sd, } static int vimc_sca_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct vimc_sca_device *vsca = v4l2_get_subdevdata(sd); @@ -305,8 +305,8 @@ static int vimc_sca_set_selection(struct v4l2_subdev *sd, crop_rect = &vsca->crop_rect; sink_fmt = &vsca->sink_fmt; } else { - crop_rect = v4l2_subdev_get_try_crop(sd, cfg, 0); - sink_fmt = v4l2_subdev_get_try_format(sd, cfg, 0); + crop_rect = v4l2_subdev_get_try_crop(sd, sd_state, 0); + sink_fmt = v4l2_subdev_get_try_format(sd, sd_state, 0); } switch (sel->target) { diff --git a/drivers/media/test-drivers/vimc/vimc-sensor.c b/drivers/media/test-drivers/vimc/vimc-sensor.c index ba5db5a150b4..74ab79cadb5d 100644 --- a/drivers/media/test-drivers/vimc/vimc-sensor.c +++ b/drivers/media/test-drivers/vimc/vimc-sensor.c @@ -42,14 +42,14 @@ static const struct v4l2_mbus_framefmt fmt_default = { }; static int vimc_sen_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { unsigned int i; for (i = 0; i < sd->entity.num_pads; i++) { struct v4l2_mbus_framefmt *mf; - mf = v4l2_subdev_get_try_format(sd, cfg, i); + mf = v4l2_subdev_get_try_format(sd, sd_state, i); *mf = fmt_default; } @@ -57,7 +57,7 @@ static int vimc_sen_init_cfg(struct v4l2_subdev *sd, } static int vimc_sen_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { u32 mbus_code = vimc_mbus_code_by_index(code->index); @@ -71,7 +71,7 @@ static int vimc_sen_enum_mbus_code(struct v4l2_subdev *sd, } static int vimc_sen_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { const struct vimc_pix_map *vpix; @@ -93,14 +93,14 @@ static int vimc_sen_enum_frame_size(struct v4l2_subdev *sd, } static int vimc_sen_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vimc_sen_device *vsen = container_of(sd, struct vimc_sen_device, sd); fmt->format = fmt->which == V4L2_SUBDEV_FORMAT_TRY ? - *v4l2_subdev_get_try_format(sd, cfg, fmt->pad) : + *v4l2_subdev_get_try_format(sd, sd_state, fmt->pad) : vsen->mbus_format; return 0; @@ -146,7 +146,7 @@ static void vimc_sen_adjust_fmt(struct v4l2_mbus_framefmt *fmt) } static int vimc_sen_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct vimc_sen_device *vsen = v4l2_get_subdevdata(sd); @@ -159,7 +159,7 @@ static int vimc_sen_set_fmt(struct v4l2_subdev *sd, mf = &vsen->mbus_format; } else { - mf = v4l2_subdev_get_try_format(sd, cfg, fmt->pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, fmt->pad); } /* Set the new format */ diff --git a/drivers/media/test-drivers/vivid/vivid-core.c b/drivers/media/test-drivers/vivid/vivid-core.c index ca0ebf6ad9cc..d2bd2653cf54 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.c +++ b/drivers/media/test-drivers/vivid/vivid-core.c @@ -656,6 +656,46 @@ static const struct v4l2_file_operations vivid_radio_fops = { .unlocked_ioctl = video_ioctl2, }; +static int vidioc_reqbufs(struct file *file, void *priv, + struct v4l2_requestbuffers *p) +{ + struct video_device *vdev = video_devdata(file); + int r; + + /* + * Sliced and raw VBI capture share the same queue so we must + * change the type. + */ + if (p->type == V4L2_BUF_TYPE_SLICED_VBI_CAPTURE || + p->type == V4L2_BUF_TYPE_VBI_CAPTURE) { + r = vb2_queue_change_type(vdev->queue, p->type); + if (r) + return r; + } + + return vb2_ioctl_reqbufs(file, priv, p); +} + +static int vidioc_create_bufs(struct file *file, void *priv, + struct v4l2_create_buffers *p) +{ + struct video_device *vdev = video_devdata(file); + int r; + + /* + * Sliced and raw VBI capture share the same queue so we must + * change the type. + */ + if (p->format.type == V4L2_BUF_TYPE_SLICED_VBI_CAPTURE || + p->format.type == V4L2_BUF_TYPE_VBI_CAPTURE) { + r = vb2_queue_change_type(vdev->queue, p->format.type); + if (r) + return r; + } + + return vb2_ioctl_create_bufs(file, priv, p); +} + static const struct v4l2_ioctl_ops vivid_ioctl_ops = { .vidioc_querycap = vidioc_querycap, @@ -717,8 +757,8 @@ static const struct v4l2_ioctl_ops vivid_ioctl_ops = { .vidioc_g_fbuf = vidioc_g_fbuf, .vidioc_s_fbuf = vidioc_s_fbuf, - .vidioc_reqbufs = vb2_ioctl_reqbufs, - .vidioc_create_bufs = vb2_ioctl_create_bufs, + .vidioc_reqbufs = vidioc_reqbufs, + .vidioc_create_bufs = vidioc_create_bufs, .vidioc_prepare_buf = vb2_ioctl_prepare_buf, .vidioc_querybuf = vb2_ioctl_querybuf, .vidioc_qbuf = vb2_ioctl_qbuf, diff --git a/drivers/media/test-drivers/vivid/vivid-core.h b/drivers/media/test-drivers/vivid/vivid-core.h index cdff6cd264d0..1e3c4f5a9413 100644 --- a/drivers/media/test-drivers/vivid/vivid-core.h +++ b/drivers/media/test-drivers/vivid/vivid-core.h @@ -429,7 +429,6 @@ struct vivid_dev { u32 vbi_cap_seq_start; u32 vbi_cap_seq_count; bool vbi_cap_streaming; - bool stream_sliced_vbi_cap; u32 meta_cap_seq_start; u32 meta_cap_seq_count; bool meta_cap_streaming; diff --git a/drivers/media/test-drivers/vivid/vivid-kthread-cap.c b/drivers/media/test-drivers/vivid/vivid-kthread-cap.c index c0dc609c1358..9da730ccfa94 100644 --- a/drivers/media/test-drivers/vivid/vivid-kthread-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-kthread-cap.c @@ -752,7 +752,7 @@ static noinline_for_stack void vivid_thread_vid_cap_tick(struct vivid_dev *dev, v4l2_ctrl_request_setup(vbi_cap_buf->vb.vb2_buf.req_obj.req, &dev->ctrl_hdl_vbi_cap); - if (dev->stream_sliced_vbi_cap) + if (vbi_cap_buf->vb.vb2_buf.type == V4L2_BUF_TYPE_SLICED_VBI_CAPTURE) vivid_sliced_vbi_cap_process(dev, vbi_cap_buf); else vivid_raw_vbi_cap_process(dev, vbi_cap_buf); diff --git a/drivers/media/test-drivers/vivid/vivid-sdr-cap.c b/drivers/media/test-drivers/vivid/vivid-sdr-cap.c index a1e52708b7ca..265db2114671 100644 --- a/drivers/media/test-drivers/vivid/vivid-sdr-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-sdr-cap.c @@ -455,7 +455,6 @@ int vidioc_g_fmt_sdr_cap(struct file *file, void *fh, struct v4l2_format *f) f->fmt.sdr.pixelformat = dev->sdr_pixelformat; f->fmt.sdr.buffersize = dev->sdr_buffersize; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); return 0; } @@ -468,7 +467,6 @@ int vidioc_s_fmt_sdr_cap(struct file *file, void *fh, struct v4l2_format *f) if (vb2_is_busy(q)) return -EBUSY; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < ARRAY_SIZE(formats); i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { dev->sdr_pixelformat = formats[i].pixelformat; @@ -488,7 +486,6 @@ int vidioc_try_fmt_sdr_cap(struct file *file, void *fh, struct v4l2_format *f) { int i; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < ARRAY_SIZE(formats); i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { f->fmt.sdr.buffersize = formats[i].buffersize; diff --git a/drivers/media/test-drivers/vivid/vivid-vbi-cap.c b/drivers/media/test-drivers/vivid/vivid-vbi-cap.c index 1a9348eea781..b65b02eeeb97 100644 --- a/drivers/media/test-drivers/vivid/vivid-vbi-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vbi-cap.c @@ -255,10 +255,8 @@ int vidioc_s_fmt_vbi_cap(struct file *file, void *priv, if (ret) return ret; - if (dev->stream_sliced_vbi_cap && vb2_is_busy(&dev->vb_vbi_cap_q)) + if (f->type != V4L2_BUF_TYPE_VBI_CAPTURE && vb2_is_busy(&dev->vb_vbi_cap_q)) return -EBUSY; - dev->stream_sliced_vbi_cap = false; - dev->vbi_cap_dev.queue->type = V4L2_BUF_TYPE_VBI_CAPTURE; return 0; } @@ -322,11 +320,9 @@ int vidioc_s_fmt_sliced_vbi_cap(struct file *file, void *fh, struct v4l2_format if (ret) return ret; - if (!dev->stream_sliced_vbi_cap && vb2_is_busy(&dev->vb_vbi_cap_q)) + if (fmt->type != V4L2_BUF_TYPE_SLICED_VBI_CAPTURE && vb2_is_busy(&dev->vb_vbi_cap_q)) return -EBUSY; dev->service_set_cap = vbi->service_set; - dev->stream_sliced_vbi_cap = true; - dev->vbi_cap_dev.queue->type = V4L2_BUF_TYPE_SLICED_VBI_CAPTURE; return 0; } diff --git a/drivers/media/usb/Kconfig b/drivers/media/usb/Kconfig index 00feadb217d8..f97153df3c84 100644 --- a/drivers/media/usb/Kconfig +++ b/drivers/media/usb/Kconfig @@ -1,10 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only -# This Kconfig option is also used by the legacy av7110 driver -config TTPCI_EEPROM - tristate - depends on I2C - if USB && MEDIA_SUPPORT menuconfig MEDIA_USB_SUPPORT diff --git a/drivers/media/usb/airspy/airspy.c b/drivers/media/usb/airspy/airspy.c index 751703db06f5..7a81be7970b2 100644 --- a/drivers/media/usb/airspy/airspy.c +++ b/drivers/media/usb/airspy/airspy.c @@ -632,7 +632,6 @@ static int airspy_g_fmt_sdr_cap(struct file *file, void *priv, f->fmt.sdr.pixelformat = s->pixelformat; f->fmt.sdr.buffersize = s->buffersize; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); return 0; } @@ -647,7 +646,6 @@ static int airspy_s_fmt_sdr_cap(struct file *file, void *priv, if (vb2_is_busy(q)) return -EBUSY; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < NUM_FORMATS; i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { s->pixelformat = formats[i].pixelformat; @@ -670,7 +668,6 @@ static int airspy_try_fmt_sdr_cap(struct file *file, void *priv, { int i; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < NUM_FORMATS; i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { f->fmt.sdr.buffersize = formats[i].buffersize; diff --git a/drivers/media/usb/au0828/au0828-core.c b/drivers/media/usb/au0828/au0828-core.c index a8a72d5fbd12..caefac07af92 100644 --- a/drivers/media/usb/au0828/au0828-core.c +++ b/drivers/media/usb/au0828/au0828-core.c @@ -199,8 +199,8 @@ static int au0828_media_device_init(struct au0828_dev *dev, struct media_device *mdev; mdev = media_device_usb_allocate(udev, KBUILD_MODNAME, THIS_MODULE); - if (!mdev) - return -ENOMEM; + if (IS_ERR(mdev)) + return PTR_ERR(mdev); dev->media_dev = mdev; #endif diff --git a/drivers/media/usb/cpia2/cpia2.h b/drivers/media/usb/cpia2/cpia2.h index 50835f5f7512..57b7f1ea68da 100644 --- a/drivers/media/usb/cpia2/cpia2.h +++ b/drivers/media/usb/cpia2/cpia2.h @@ -429,6 +429,7 @@ int cpia2_send_command(struct camera_data *cam, struct cpia2_command *cmd); int cpia2_do_command(struct camera_data *cam, unsigned int command, unsigned char direction, unsigned char param); +void cpia2_deinit_camera_struct(struct camera_data *cam, struct usb_interface *intf); struct camera_data *cpia2_init_camera_struct(struct usb_interface *intf); int cpia2_init_camera(struct camera_data *cam); int cpia2_allocate_buffers(struct camera_data *cam); diff --git a/drivers/media/usb/cpia2/cpia2_core.c b/drivers/media/usb/cpia2/cpia2_core.c index e747548ab286..b5a2d06fb356 100644 --- a/drivers/media/usb/cpia2/cpia2_core.c +++ b/drivers/media/usb/cpia2/cpia2_core.c @@ -2163,6 +2163,18 @@ static void reset_camera_struct(struct camera_data *cam) cam->height = cam->params.roi.height; } +/****************************************************************************** + * + * cpia2_init_camera_struct + * + * Deinitialize camera struct + *****************************************************************************/ +void cpia2_deinit_camera_struct(struct camera_data *cam, struct usb_interface *intf) +{ + v4l2_device_unregister(&cam->v4l2_dev); + kfree(cam); +} + /****************************************************************************** * * cpia2_init_camera_struct diff --git a/drivers/media/usb/cpia2/cpia2_usb.c b/drivers/media/usb/cpia2/cpia2_usb.c index 3ab80a7b4498..76aac06f9fb8 100644 --- a/drivers/media/usb/cpia2/cpia2_usb.c +++ b/drivers/media/usb/cpia2/cpia2_usb.c @@ -844,15 +844,13 @@ static int cpia2_usb_probe(struct usb_interface *intf, ret = set_alternate(cam, USBIF_CMDONLY); if (ret < 0) { ERR("%s: usb_set_interface error (ret = %d)\n", __func__, ret); - kfree(cam); - return ret; + goto alt_err; } if((ret = cpia2_init_camera(cam)) < 0) { ERR("%s: failed to initialize cpia2 camera (ret = %d)\n", __func__, ret); - kfree(cam); - return ret; + goto alt_err; } LOG(" CPiA Version: %d.%02d (%d.%d)\n", cam->params.version.firmware_revision_hi, @@ -872,11 +870,14 @@ static int cpia2_usb_probe(struct usb_interface *intf, ret = cpia2_register_camera(cam); if (ret < 0) { ERR("%s: Failed to register cpia2 camera (ret = %d)\n", __func__, ret); - kfree(cam); - return ret; + goto alt_err; } return 0; + +alt_err: + cpia2_deinit_camera_struct(cam, intf); + return ret; } /****************************************************************************** diff --git a/drivers/media/usb/cpia2/cpia2_v4l.c b/drivers/media/usb/cpia2/cpia2_v4l.c index 69d5c628a797..926ecfc9b64a 100644 --- a/drivers/media/usb/cpia2/cpia2_v4l.c +++ b/drivers/media/usb/cpia2/cpia2_v4l.c @@ -140,10 +140,10 @@ static ssize_t cpia2_v4l_read(struct file *file, char __user *buf, size_t count, loff_t *off) { struct camera_data *cam = video_drvdata(file); - int noblock = file->f_flags&O_NONBLOCK; + int noblock = file->f_flags & O_NONBLOCK; ssize_t ret; - if(!cam) + if (!cam) return -EINVAL; if (mutex_lock_interruptible(&cam->v4l2_lock)) @@ -153,7 +153,6 @@ static ssize_t cpia2_v4l_read(struct file *file, char __user *buf, size_t count, return ret; } - /****************************************************************************** * * cpia2_v4l_poll @@ -170,7 +169,6 @@ static __poll_t cpia2_v4l_poll(struct file *filp, struct poll_table_struct *wait return res; } - static int sync(struct camera_data *cam, int frame_nr) { struct framebuf *frame = &cam->buffers[frame_nr]; @@ -247,8 +245,8 @@ static int cpia2_querycap(struct file *file, void *fh, struct v4l2_capability *v break; } - if (usb_make_path(cam->dev, vc->bus_info, sizeof(vc->bus_info)) <0) - memset(vc->bus_info,0, sizeof(vc->bus_info)); + if (usb_make_path(cam->dev, vc->bus_info, sizeof(vc->bus_info)) < 0) + memset(vc->bus_info, 0, sizeof(vc->bus_info)); return 0; } @@ -289,7 +287,7 @@ static int cpia2_s_input(struct file *file, void *fh, unsigned int i) *****************************************************************************/ static int cpia2_enum_fmt_vid_cap(struct file *file, void *fh, - struct v4l2_fmtdesc *f) + struct v4l2_fmtdesc *f) { if (f->index > 1) return -EINVAL; @@ -310,13 +308,13 @@ static int cpia2_enum_fmt_vid_cap(struct file *file, void *fh, *****************************************************************************/ static int cpia2_try_fmt_vid_cap(struct file *file, void *fh, - struct v4l2_format *f) + struct v4l2_format *f) { struct camera_data *cam = video_drvdata(file); if (f->fmt.pix.pixelformat != V4L2_PIX_FMT_MJPEG && f->fmt.pix.pixelformat != V4L2_PIX_FMT_JPEG) - return -EINVAL; + return -EINVAL; f->fmt.pix.field = V4L2_FIELD_NONE; f->fmt.pix.bytesperline = 0; @@ -371,19 +369,20 @@ static int cpia2_try_fmt_vid_cap(struct file *file, void *fh, *****************************************************************************/ static int cpia2_s_fmt_vid_cap(struct file *file, void *_fh, - struct v4l2_format *f) + struct v4l2_format *f) { struct camera_data *cam = video_drvdata(file); int err, frame; err = cpia2_try_fmt_vid_cap(file, _fh, f); - if(err != 0) + if (err != 0) return err; cam->pixelformat = f->fmt.pix.pixelformat; /* NOTE: This should be set to 1 for MJPEG, but some apps don't handle - * the missing Huffman table properly. */ + * the missing Huffman table properly. + */ cam->params.compression.inhibit_htables = 0; /*f->fmt.pix.pixelformat == V4L2_PIX_FMT_MJPEG;*/ @@ -421,7 +420,7 @@ static int cpia2_s_fmt_vid_cap(struct file *file, void *_fh, *****************************************************************************/ static int cpia2_g_fmt_vid_cap(struct file *file, void *fh, - struct v4l2_format *f) + struct v4l2_format *f) { struct camera_data *cam = video_drvdata(file); @@ -547,9 +546,8 @@ static const struct { }; static int cpia2_enum_framesizes(struct file *file, void *fh, - struct v4l2_frmsizeenum *fsize) + struct v4l2_frmsizeenum *fsize) { - if (fsize->pixel_format != V4L2_PIX_FMT_MJPEG && fsize->pixel_format != V4L2_PIX_FMT_JPEG) return -EINVAL; @@ -563,7 +561,7 @@ static int cpia2_enum_framesizes(struct file *file, void *fh, } static int cpia2_enum_frameintervals(struct file *file, void *fh, - struct v4l2_frmivalenum *fival) + struct v4l2_frmivalenum *fival) { struct camera_data *cam = video_drvdata(file); int max = ARRAY_SIZE(framerate_controls) - 1; @@ -665,19 +663,18 @@ static int cpia2_g_jpegcomp(struct file *file, void *fh, struct v4l2_jpegcompres parms->quality = 80; // TODO: Can this be made meaningful? parms->jpeg_markers = V4L2_JPEG_MARKER_DQT | V4L2_JPEG_MARKER_DRI; - if(!cam->params.compression.inhibit_htables) { + if (!cam->params.compression.inhibit_htables) parms->jpeg_markers |= V4L2_JPEG_MARKER_DHT; - } parms->APPn = cam->APPn; parms->APP_len = cam->APP_len; - if(cam->APP_len > 0) { + if (cam->APP_len > 0) { memcpy(parms->APP_data, cam->APP_data, cam->APP_len); parms->jpeg_markers |= V4L2_JPEG_MARKER_APP; } parms->COM_len = cam->COM_len; - if(cam->COM_len > 0) { + if (cam->COM_len > 0) { memcpy(parms->COM_data, cam->COM_data, cam->COM_len); parms->jpeg_markers |= JPEG_MARKER_COM; } @@ -698,7 +695,7 @@ static int cpia2_g_jpegcomp(struct file *file, void *fh, struct v4l2_jpegcompres *****************************************************************************/ static int cpia2_s_jpegcomp(struct file *file, void *fh, - const struct v4l2_jpegcompression *parms) + const struct v4l2_jpegcompression *parms) { struct camera_data *cam = video_drvdata(file); @@ -708,9 +705,9 @@ static int cpia2_s_jpegcomp(struct file *file, void *fh, cam->params.compression.inhibit_htables = !(parms->jpeg_markers & V4L2_JPEG_MARKER_DHT); - if(parms->APP_len != 0) { - if(parms->APP_len > 0 && - parms->APP_len <= sizeof(cam->APP_data) && + if (parms->APP_len != 0) { + if (parms->APP_len > 0 && + parms->APP_len <= sizeof(cam->APP_data) && parms->APPn >= 0 && parms->APPn <= 15) { cam->APPn = parms->APPn; cam->APP_len = parms->APP_len; @@ -724,9 +721,9 @@ static int cpia2_s_jpegcomp(struct file *file, void *fh, cam->APP_len = 0; } - if(parms->COM_len != 0) { - if(parms->COM_len > 0 && - parms->COM_len <= sizeof(cam->COM_data)) { + if (parms->COM_len != 0) { + if (parms->COM_len > 0 && + parms->COM_len <= sizeof(cam->COM_data)) { cam->COM_len = parms->COM_len; memcpy(cam->COM_data, parms->COM_data, parms->COM_len); } else { @@ -751,8 +748,8 @@ static int cpia2_reqbufs(struct file *file, void *fh, struct v4l2_requestbuffers { struct camera_data *cam = video_drvdata(file); - if(req->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || - req->memory != V4L2_MEMORY_MMAP) + if (req->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || + req->memory != V4L2_MEMORY_MMAP) return -EINVAL; DBG("REQBUFS requested:%d returning:%d\n", req->count, cam->num_frames); @@ -774,8 +771,8 @@ static int cpia2_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) { struct camera_data *cam = video_drvdata(file); - if(buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || - buf->index >= cam->num_frames) + if (buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || + buf->index >= cam->num_frames) return -EINVAL; buf->m.offset = cam->buffers[buf->index].data - cam->frame_buffer; @@ -783,7 +780,7 @@ static int cpia2_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) buf->memory = V4L2_MEMORY_MMAP; - if(cam->mmapped) + if (cam->mmapped) buf->flags = V4L2_BUF_FLAG_MAPPED; else buf->flags = 0; @@ -806,8 +803,8 @@ static int cpia2_querybuf(struct file *file, void *fh, struct v4l2_buffer *buf) } DBG("QUERYBUF index:%d offset:%d flags:%d seq:%d bytesused:%d\n", - buf->index, buf->m.offset, buf->flags, buf->sequence, - buf->bytesused); + buf->index, buf->m.offset, buf->flags, buf->sequence, + buf->bytesused); return 0; } @@ -824,14 +821,14 @@ static int cpia2_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) { struct camera_data *cam = video_drvdata(file); - if(buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || - buf->memory != V4L2_MEMORY_MMAP || + if (buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || + buf->memory != V4L2_MEMORY_MMAP || buf->index >= cam->num_frames) return -EINVAL; DBG("QBUF #%d\n", buf->index); - if(cam->buffers[buf->index].status == FRAME_READY) + if (cam->buffers[buf->index].status == FRAME_READY) cam->buffers[buf->index].status = FRAME_EMPTY; return 0; @@ -849,9 +846,10 @@ static int find_earliest_filled_buffer(struct camera_data *cam) { int i; int found = -1; - for (i=0; inum_frames; i++) { - if(cam->buffers[i].status == FRAME_READY) { - if(found < 0) { + + for (i = 0; i < cam->num_frames; i++) { + if (cam->buffers[i].status == FRAME_READY) { + if (found < 0) { found = i; } else { /* find which buffer is earlier */ @@ -876,22 +874,23 @@ static int cpia2_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) struct camera_data *cam = video_drvdata(file); int frame; - if(buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || - buf->memory != V4L2_MEMORY_MMAP) + if (buf->type != V4L2_BUF_TYPE_VIDEO_CAPTURE || + buf->memory != V4L2_MEMORY_MMAP) return -EINVAL; frame = find_earliest_filled_buffer(cam); - if(frame < 0 && file->f_flags&O_NONBLOCK) + if (frame < 0 && file->f_flags & O_NONBLOCK) return -EAGAIN; - if(frame < 0) { + if (frame < 0) { /* Wait for a frame to become available */ - struct framebuf *cb=cam->curbuff; + struct framebuf *cb = cam->curbuff; + mutex_unlock(&cam->v4l2_lock); wait_event_interruptible(cam->wq_stream, !video_is_registered(&cam->vdev) || - (cb=cam->curbuff)->status == FRAME_READY); + (cb = cam->curbuff)->status == FRAME_READY); mutex_lock(&cam->v4l2_lock); if (signal_pending(current)) return -ERESTARTSYS; @@ -900,7 +899,6 @@ static int cpia2_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) frame = cb->num; } - buf->index = frame; buf->bytesused = cam->buffers[buf->index].length; buf->flags = V4L2_BUF_FLAG_MAPPED | V4L2_BUF_FLAG_DONE @@ -931,7 +929,7 @@ static int cpia2_streamon(struct file *file, void *fh, enum v4l2_buf_type type) if (!cam->streaming) { ret = cpia2_usb_stream_start(cam, - cam->params.camera_state.stream_mode); + cam->params.camera_state.stream_mode); if (!ret) v4l2_ctrl_grab(cam->usb_alt, true); } @@ -969,7 +967,7 @@ static int cpia2_mmap(struct file *file, struct vm_area_struct *area) return -ERESTARTSYS; retval = cpia2_remap_buffer(cam, area); - if(!retval) + if (!retval) cam->stream_fh = file->private_data; mutex_unlock(&cam->v4l2_lock); return retval; @@ -1080,39 +1078,42 @@ int cpia2_register_camera(struct camera_data *cam) v4l2_ctrl_handler_init(hdl, 12); v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_BRIGHTNESS, - cam->params.pnp_id.device_type == DEVICE_STV_672 ? 1 : 0, - 255, 1, DEFAULT_BRIGHTNESS); + V4L2_CID_BRIGHTNESS, + cam->params.pnp_id.device_type == DEVICE_STV_672 ? 1 : 0, + 255, 1, DEFAULT_BRIGHTNESS); v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_CONTRAST, 0, 255, 1, DEFAULT_CONTRAST); + V4L2_CID_CONTRAST, 0, 255, 1, DEFAULT_CONTRAST); v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_SATURATION, 0, 255, 1, DEFAULT_SATURATION); + V4L2_CID_SATURATION, 0, 255, 1, DEFAULT_SATURATION); v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_HFLIP, 0, 1, 1, 0); + V4L2_CID_HFLIP, 0, 1, 1, 0); v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_JPEG_ACTIVE_MARKER, 0, - V4L2_JPEG_ACTIVE_MARKER_DHT, 0, - V4L2_JPEG_ACTIVE_MARKER_DHT); + V4L2_CID_JPEG_ACTIVE_MARKER, 0, + V4L2_JPEG_ACTIVE_MARKER_DHT, 0, + V4L2_JPEG_ACTIVE_MARKER_DHT); v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_JPEG_COMPRESSION_QUALITY, 1, - 100, 1, 100); + V4L2_CID_JPEG_COMPRESSION_QUALITY, 1, + 100, 1, 100); cpia2_usb_alt.def = alternate; cam->usb_alt = v4l2_ctrl_new_custom(hdl, &cpia2_usb_alt, NULL); /* VP5 Only */ if (cam->params.pnp_id.device_type != DEVICE_STV_672) v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_VFLIP, 0, 1, 1, 0); + V4L2_CID_VFLIP, 0, 1, 1, 0); /* Flicker control only valid for 672 */ if (cam->params.pnp_id.device_type == DEVICE_STV_672) v4l2_ctrl_new_std_menu(hdl, &cpia2_ctrl_ops, - V4L2_CID_POWER_LINE_FREQUENCY, - V4L2_CID_POWER_LINE_FREQUENCY_60HZ, 0, 0); + V4L2_CID_POWER_LINE_FREQUENCY, + V4L2_CID_POWER_LINE_FREQUENCY_60HZ, + 0, 0); /* Light control only valid for the QX5 Microscope */ if (cam->params.pnp_id.product == 0x151) { cam->top_light = v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_ILLUMINATORS_1, 0, 1, 1, 0); + V4L2_CID_ILLUMINATORS_1, + 0, 1, 1, 0); cam->bottom_light = v4l2_ctrl_new_std(hdl, &cpia2_ctrl_ops, - V4L2_CID_ILLUMINATORS_2, 0, 1, 1, 0); + V4L2_CID_ILLUMINATORS_2, + 0, 1, 1, 0); v4l2_ctrl_cluster(2, &cam->top_light); } @@ -1159,28 +1160,28 @@ void cpia2_unregister_camera(struct camera_data *cam) *****************************************************************************/ static void __init check_parameters(void) { - if(buffer_size < PAGE_SIZE) { + if (buffer_size < PAGE_SIZE) { buffer_size = PAGE_SIZE; LOG("buffer_size too small, setting to %d\n", buffer_size); - } else if(buffer_size > 1024*1024) { + } else if (buffer_size > 1024 * 1024) { /* arbitrary upper limiit */ - buffer_size = 1024*1024; + buffer_size = 1024 * 1024; LOG("buffer_size ridiculously large, setting to %d\n", buffer_size); } else { - buffer_size += PAGE_SIZE-1; - buffer_size &= ~(PAGE_SIZE-1); + buffer_size += PAGE_SIZE - 1; + buffer_size &= ~(PAGE_SIZE - 1); } - if(num_buffers < 1) { + if (num_buffers < 1) { num_buffers = 1; LOG("num_buffers too small, setting to %d\n", num_buffers); - } else if(num_buffers > VIDEO_MAX_FRAME) { + } else if (num_buffers > VIDEO_MAX_FRAME) { num_buffers = VIDEO_MAX_FRAME; LOG("num_buffers too large, setting to %d\n", num_buffers); } - if(alternate < USBIF_ISO_1 || alternate > USBIF_ISO_6) { + if (alternate < USBIF_ISO_1 || alternate > USBIF_ISO_6) { alternate = DEFAULT_ALT; LOG("alternate specified is invalid, using %d\n", alternate); } @@ -1197,7 +1198,6 @@ static void __init check_parameters(void) /************ Module Stuff ***************/ - /****************************************************************************** * * cpia2_init/module_init @@ -1211,7 +1211,6 @@ static int __init cpia2_init(void) return cpia2_usb_init(); } - /****************************************************************************** * * cpia2_exit/module_exit diff --git a/drivers/media/usb/dvb-usb-v2/lmedm04.c b/drivers/media/usb/dvb-usb-v2/lmedm04.c index 1b6d4e4c52ca..fe4d886442a4 100644 --- a/drivers/media/usb/dvb-usb-v2/lmedm04.c +++ b/drivers/media/usb/dvb-usb-v2/lmedm04.c @@ -1122,11 +1122,6 @@ static int lme2510_powerup(struct dvb_usb_device *d, int onoff) return ret; } -static int lme2510_get_adapter_count(struct dvb_usb_device *d) -{ - return 1; -} - static int lme2510_identify_state(struct dvb_usb_device *d, const char **name) { struct lme2510_state *st = d->priv; @@ -1211,12 +1206,12 @@ static struct dvb_usb_device_properties lme2510_props = { .frontend_attach = dm04_lme2510_frontend_attach, .tuner_attach = dm04_lme2510_tuner, .get_stream_config = lme2510_get_stream_config, - .get_adapter_count = lme2510_get_adapter_count, .streaming_ctrl = lme2510_streaming_ctrl, .get_rc_config = lme2510_get_rc_config, .exit = lme2510_exit, + .num_adapters = 1, .adapter = { { .caps = DVB_USB_ADAP_HAS_PID_FILTER| @@ -1227,8 +1222,6 @@ static struct dvb_usb_device_properties lme2510_props = { .stream = DVB_USB_STREAM_BULK(0x86, 10, 4096), }, - { - } }, }; diff --git a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c index 97ed17a141bb..83705730e37e 100644 --- a/drivers/media/usb/dvb-usb-v2/rtl28xxu.c +++ b/drivers/media/usb/dvb-usb-v2/rtl28xxu.c @@ -612,8 +612,9 @@ static int rtl28xxu_read_config(struct dvb_usb_device *d) static int rtl28xxu_identify_state(struct dvb_usb_device *d, const char **name) { struct rtl28xxu_dev *dev = d_to_priv(d); + u8 buf[1]; int ret; - struct rtl28xxu_req req_demod_i2c = {0x0020, CMD_I2C_DA_RD, 0, NULL}; + struct rtl28xxu_req req_demod_i2c = {0x0020, CMD_I2C_DA_RD, 1, buf}; dev_dbg(&d->intf->dev, "\n"); @@ -1776,7 +1777,7 @@ static int rtl2832u_rc_query(struct dvb_usb_device *d) ir_raw_event_store_with_filter(d->rc_dev, &ev); } - /* 'flush' ir_raw_event_store_with_filter() */ + /* 'flush' ir_raw_event_store_with_filter() */ ir_raw_event_handle(d->rc_dev); exit: return ret; diff --git a/drivers/media/usb/dvb-usb/Makefile b/drivers/media/usb/dvb-usb/Makefile index 28e4806a87cd..c22514948db2 100644 --- a/drivers/media/usb/dvb-usb/Makefile +++ b/drivers/media/usb/dvb-usb/Makefile @@ -83,4 +83,4 @@ obj-$(CONFIG_DVB_USB_TECHNISAT_USB2) += dvb-usb-technisat-usb2.o ccflags-y += -I$(srctree)/drivers/media/dvb-frontends/ # due to tuner-xc3028 ccflags-y += -I$(srctree)/drivers/media/tuners -ccflags-y += -I$(srctree)/drivers/media/pci/ttpci +ccflags-y += -I$(srctree)/drivers/media/common diff --git a/drivers/media/usb/dvb-usb/cinergyT2-core.c b/drivers/media/usb/dvb-usb/cinergyT2-core.c index 969a7ec71dff..23f1093d28f8 100644 --- a/drivers/media/usb/dvb-usb/cinergyT2-core.c +++ b/drivers/media/usb/dvb-usb/cinergyT2-core.c @@ -29,10 +29,8 @@ struct cinergyt2_state { unsigned char data[64]; }; -/* We are missing a release hook with usb_device data */ -static struct dvb_usb_device *cinergyt2_usb_device; - -static struct dvb_usb_device_properties cinergyt2_properties; +/* Forward declaration */ +static const struct dvb_usb_device_properties cinergyt2_properties; static int cinergyt2_streaming_ctrl(struct dvb_usb_adapter *adap, int enable) { @@ -78,13 +76,12 @@ static int cinergyt2_frontend_attach(struct dvb_usb_adapter *adap) ret = dvb_usb_generic_rw(d, st->data, 1, st->data, 3, 0); if (ret < 0) { + if (adap->fe_adap[0].fe) + adap->fe_adap[0].fe->ops.release(adap->fe_adap[0].fe); deb_rc("cinergyt2_power_ctrl() Failed to retrieve sleep state info\n"); } mutex_unlock(&d->data_mutex); - /* Copy this pointer as we are gonna need it in the release phase */ - cinergyt2_usb_device = adap->dev; - return ret; } @@ -203,7 +200,7 @@ static struct usb_device_id cinergyt2_usb_table[] = { MODULE_DEVICE_TABLE(usb, cinergyt2_usb_table); -static struct dvb_usb_device_properties cinergyt2_properties = { +static const struct dvb_usb_device_properties cinergyt2_properties = { .size_of_priv = sizeof(struct cinergyt2_state), .num_adapters = 1, .adapter = { diff --git a/drivers/media/usb/dvb-usb/cxusb.c b/drivers/media/usb/dvb-usb/cxusb.c index 761992ad05e2..7707de7bae7c 100644 --- a/drivers/media/usb/dvb-usb/cxusb.c +++ b/drivers/media/usb/dvb-usb/cxusb.c @@ -1947,7 +1947,7 @@ static struct dvb_usb_device_properties cxusb_bluebird_lgz201_properties = { .size_of_priv = sizeof(struct cxusb_state), - .num_adapters = 2, + .num_adapters = 1, .adapter = { { .num_frontends = 1, diff --git a/drivers/media/usb/dvb-usb/dtv5100.c b/drivers/media/usb/dvb-usb/dtv5100.c index fba06932a9e0..1c13e493322c 100644 --- a/drivers/media/usb/dvb-usb/dtv5100.c +++ b/drivers/media/usb/dvb-usb/dtv5100.c @@ -26,6 +26,7 @@ static int dtv5100_i2c_msg(struct dvb_usb_device *d, u8 addr, u8 *wbuf, u16 wlen, u8 *rbuf, u16 rlen) { struct dtv5100_state *st = d->priv; + unsigned int pipe; u8 request; u8 type; u16 value; @@ -34,6 +35,7 @@ static int dtv5100_i2c_msg(struct dvb_usb_device *d, u8 addr, switch (wlen) { case 1: /* write { reg }, read { value } */ + pipe = usb_rcvctrlpipe(d->udev, 0); request = (addr == DTV5100_DEMOD_ADDR ? DTV5100_DEMOD_READ : DTV5100_TUNER_READ); type = USB_TYPE_VENDOR | USB_DIR_IN; @@ -41,6 +43,7 @@ static int dtv5100_i2c_msg(struct dvb_usb_device *d, u8 addr, break; case 2: /* write { reg, value } */ + pipe = usb_sndctrlpipe(d->udev, 0); request = (addr == DTV5100_DEMOD_ADDR ? DTV5100_DEMOD_WRITE : DTV5100_TUNER_WRITE); type = USB_TYPE_VENDOR | USB_DIR_OUT; @@ -54,7 +57,7 @@ static int dtv5100_i2c_msg(struct dvb_usb_device *d, u8 addr, memcpy(st->data, rbuf, rlen); msleep(1); /* avoid I2C errors */ - return usb_control_msg(d->udev, usb_rcvctrlpipe(d->udev, 0), request, + return usb_control_msg(d->udev, pipe, request, type, value, index, st->data, rlen, DTV5100_USB_TIMEOUT); } @@ -141,7 +144,7 @@ static int dtv5100_probe(struct usb_interface *intf, /* initialize non qt1010/zl10353 part? */ for (i = 0; dtv5100_init[i].request; i++) { - ret = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), + ret = usb_control_msg(udev, usb_sndctrlpipe(udev, 0), dtv5100_init[i].request, USB_TYPE_VENDOR | USB_DIR_OUT, dtv5100_init[i].value, diff --git a/drivers/media/usb/em28xx/em28xx-cards.c b/drivers/media/usb/em28xx/em28xx-cards.c index ba9292e2a587..c1e0dccb7408 100644 --- a/drivers/media/usb/em28xx/em28xx-cards.c +++ b/drivers/media/usb/em28xx/em28xx-cards.c @@ -4065,15 +4065,15 @@ static int em28xx_usb_probe(struct usb_interface *intf, dev->dev_next->dvb_max_pkt_size_isoc = dev->dvb_max_pkt_size_isoc_ts2; dev->dev_next->dvb_alt_isoc = dev->dvb_alt_isoc; - /* Configuare hardware to support TS2*/ + /* Configure hardware to support TS2*/ if (dev->dvb_xfer_bulk) { - /* The ep4 and ep5 are configuared for BULK */ + /* The ep4 and ep5 are configured for BULK */ em28xx_write_reg(dev, 0x0b, 0x96); mdelay(100); em28xx_write_reg(dev, 0x0b, 0x80); mdelay(100); } else { - /* The ep4 and ep5 are configuared for ISO */ + /* The ep4 and ep5 are configured for ISO */ em28xx_write_reg(dev, 0x0b, 0x96); mdelay(100); em28xx_write_reg(dev, 0x0b, 0x82); diff --git a/drivers/media/usb/em28xx/em28xx-input.c b/drivers/media/usb/em28xx/em28xx-input.c index 5aa15a7a49de..59529cbf9cd0 100644 --- a/drivers/media/usb/em28xx/em28xx-input.c +++ b/drivers/media/usb/em28xx/em28xx-input.c @@ -720,7 +720,8 @@ static int em28xx_ir_init(struct em28xx *dev) dev->board.has_ir_i2c = 0; dev_warn(&dev->intf->dev, "No i2c IR remote control device found.\n"); - return -ENODEV; + err = -ENODEV; + goto ref_put; } } @@ -735,7 +736,7 @@ static int em28xx_ir_init(struct em28xx *dev) ir = kzalloc(sizeof(*ir), GFP_KERNEL); if (!ir) - return -ENOMEM; + goto ref_put; rc = rc_allocate_device(RC_DRIVER_SCANCODE); if (!rc) goto error; @@ -839,6 +840,9 @@ static int em28xx_ir_init(struct em28xx *dev) dev->ir = NULL; rc_free_device(rc); kfree(ir); +ref_put: + em28xx_shutdown_buttons(dev); + kref_put(&dev->ref, em28xx_free_device); return err; } diff --git a/drivers/media/usb/go7007/s2250-board.c b/drivers/media/usb/go7007/s2250-board.c index b9e45124673b..c742cc88fac5 100644 --- a/drivers/media/usb/go7007/s2250-board.c +++ b/drivers/media/usb/go7007/s2250-board.c @@ -398,7 +398,7 @@ static int s2250_s_ctrl(struct v4l2_ctrl *ctrl) } static int s2250_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; diff --git a/drivers/media/usb/gspca/cpia1.c b/drivers/media/usb/gspca/cpia1.c index d93d384286c1..46ed95483e22 100644 --- a/drivers/media/usb/gspca/cpia1.c +++ b/drivers/media/usb/gspca/cpia1.c @@ -365,8 +365,9 @@ struct sd { static const struct v4l2_pix_format mode[] = { {160, 120, V4L2_PIX_FMT_CPIA1, V4L2_FIELD_NONE, /* The sizeimage is trial and error, as with low framerates - the camera will pad out usb frames, making the image - data larger then strictly necessary */ + * the camera will pad out usb frames, making the image + * data larger than strictly necessary + */ .bytesperline = 160, .sizeimage = 65536, .colorspace = V4L2_COLORSPACE_SRGB, diff --git a/drivers/media/usb/gspca/gl860/gl860.c b/drivers/media/usb/gspca/gl860/gl860.c index 2c05ea2598e7..ce4ee8bc75c8 100644 --- a/drivers/media/usb/gspca/gl860/gl860.c +++ b/drivers/media/usb/gspca/gl860/gl860.c @@ -561,8 +561,8 @@ int gl860_RTx(struct gspca_dev *gspca_dev, len, 400 + 200 * (len > 1)); memcpy(pdata, gspca_dev->usb_buf, len); } else { - r = usb_control_msg(udev, usb_rcvctrlpipe(udev, 0), - req, pref, val, index, NULL, len, 400); + gspca_err(gspca_dev, "zero-length read request\n"); + r = -EINVAL; } } diff --git a/drivers/media/usb/gspca/ov519.c b/drivers/media/usb/gspca/ov519.c index cd6776c3163b..bffa94e76da5 100644 --- a/drivers/media/usb/gspca/ov519.c +++ b/drivers/media/usb/gspca/ov519.c @@ -614,7 +614,7 @@ static const struct ov_i2c_regvals norm_3620b[] = { /* * From the datasheet: "Note that after writing to register COMH * (0x12) to change the sensor mode, registers related to the - * sensor’s cropping window will be reset back to their default + * sensor's cropping window will be reset back to their default * values." * * "wait 4096 external clock ... to make sure the sensor is diff --git a/drivers/media/usb/gspca/sq905.c b/drivers/media/usb/gspca/sq905.c index 949111070971..32504ebcfd4d 100644 --- a/drivers/media/usb/gspca/sq905.c +++ b/drivers/media/usb/gspca/sq905.c @@ -116,7 +116,7 @@ static int sq905_command(struct gspca_dev *gspca_dev, u16 index) } ret = usb_control_msg(gspca_dev->dev, - usb_sndctrlpipe(gspca_dev->dev, 0), + usb_rcvctrlpipe(gspca_dev->dev, 0), USB_REQ_SYNCH_FRAME, /* request */ USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, SQ905_PING, 0, gspca_dev->usb_buf, 1, diff --git a/drivers/media/usb/gspca/sunplus.c b/drivers/media/usb/gspca/sunplus.c index ace3da40006e..971dee0a56da 100644 --- a/drivers/media/usb/gspca/sunplus.c +++ b/drivers/media/usb/gspca/sunplus.c @@ -242,6 +242,10 @@ static void reg_r(struct gspca_dev *gspca_dev, gspca_err(gspca_dev, "reg_r: buffer overflow\n"); return; } + if (len == 0) { + gspca_err(gspca_dev, "reg_r: zero-length read\n"); + return; + } if (gspca_dev->usb_err < 0) return; ret = usb_control_msg(gspca_dev->dev, @@ -250,7 +254,7 @@ static void reg_r(struct gspca_dev *gspca_dev, USB_DIR_IN | USB_TYPE_VENDOR | USB_RECIP_DEVICE, 0, /* value */ index, - len ? gspca_dev->usb_buf : NULL, len, + gspca_dev->usb_buf, len, 500); if (ret < 0) { pr_err("reg_r err %d\n", ret); @@ -727,7 +731,7 @@ static int sd_start(struct gspca_dev *gspca_dev) case MegaImageVI: reg_w_riv(gspca_dev, 0xf0, 0, 0); spca504B_WaitCmdStatus(gspca_dev); - reg_r(gspca_dev, 0xf0, 4, 0); + reg_w_riv(gspca_dev, 0xf0, 4, 0); spca504B_WaitCmdStatus(gspca_dev); break; default: diff --git a/drivers/media/usb/hackrf/hackrf.c b/drivers/media/usb/hackrf/hackrf.c index cec841ad7495..3e535be2c520 100644 --- a/drivers/media/usb/hackrf/hackrf.c +++ b/drivers/media/usb/hackrf/hackrf.c @@ -929,7 +929,6 @@ static int hackrf_s_fmt_sdr(struct file *file, void *priv, if (vb2_is_busy(q)) return -EBUSY; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < NUM_FORMATS; i++) { if (f->fmt.sdr.pixelformat == formats[i].pixelformat) { dev->pixelformat = formats[i].pixelformat; @@ -955,7 +954,6 @@ static int hackrf_g_fmt_sdr(struct file *file, void *priv, dev_dbg(dev->dev, "pixelformat fourcc %4.4s\n", (char *)&dev->pixelformat); - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); f->fmt.sdr.pixelformat = dev->pixelformat; f->fmt.sdr.buffersize = dev->buffersize; @@ -971,7 +969,6 @@ static int hackrf_try_fmt_sdr(struct file *file, void *priv, dev_dbg(dev->dev, "pixelformat fourcc %4.4s\n", (char *)&f->fmt.sdr.pixelformat); - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < NUM_FORMATS; i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { f->fmt.sdr.buffersize = formats[i].buffersize; diff --git a/drivers/media/usb/msi2500/msi2500.c b/drivers/media/usb/msi2500/msi2500.c index 63882a5248ae..71de6b4c4e4c 100644 --- a/drivers/media/usb/msi2500/msi2500.c +++ b/drivers/media/usb/msi2500/msi2500.c @@ -912,7 +912,6 @@ static int msi2500_g_fmt_sdr_cap(struct file *file, void *priv, f->fmt.sdr.pixelformat = dev->pixelformat; f->fmt.sdr.buffersize = dev->buffersize; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); return 0; } @@ -930,7 +929,6 @@ static int msi2500_s_fmt_sdr_cap(struct file *file, void *priv, if (vb2_is_busy(q)) return -EBUSY; - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < dev->num_formats; i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { dev->pixelformat = formats[i].pixelformat; @@ -957,7 +955,6 @@ static int msi2500_try_fmt_sdr_cap(struct file *file, void *priv, dev_dbg(dev->dev, "pixelformat fourcc %4.4s\n", (char *)&f->fmt.sdr.pixelformat); - memset(f->fmt.sdr.reserved, 0, sizeof(f->fmt.sdr.reserved)); for (i = 0; i < dev->num_formats; i++) { if (formats[i].pixelformat == f->fmt.sdr.pixelformat) { f->fmt.sdr.buffersize = formats[i].buffersize; diff --git a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c index f4a727918e35..d38dee1792e4 100644 --- a/drivers/media/usb/pvrusb2/pvrusb2-hdw.c +++ b/drivers/media/usb/pvrusb2/pvrusb2-hdw.c @@ -2676,9 +2676,8 @@ void pvr2_hdw_destroy(struct pvr2_hdw *hdw) pvr2_stream_destroy(hdw->vid_stream); hdw->vid_stream = NULL; } - pvr2_i2c_core_done(hdw); v4l2_device_unregister(&hdw->v4l2_dev); - pvr2_hdw_remove_usb_stuff(hdw); + pvr2_hdw_disconnect(hdw); mutex_lock(&pvr2_unit_mtx); do { if ((hdw->unit_number >= 0) && @@ -2705,6 +2704,7 @@ void pvr2_hdw_disconnect(struct pvr2_hdw *hdw) { pvr2_trace(PVR2_TRACE_INIT,"pvr2_hdw_disconnect(hdw=%p)",hdw); LOCK_TAKE(hdw->big_lock); + pvr2_i2c_core_done(hdw); LOCK_TAKE(hdw->ctl_lock); pvr2_hdw_remove_usb_stuff(hdw); LOCK_GIVE(hdw->ctl_lock); diff --git a/drivers/media/usb/s2255/s2255drv.c b/drivers/media/usb/s2255/s2255drv.c index 4af55e2478be..3b0e4ed75d99 100644 --- a/drivers/media/usb/s2255/s2255drv.c +++ b/drivers/media/usb/s2255/s2255drv.c @@ -767,8 +767,6 @@ static int vidioc_try_fmt_vid_cap(struct file *file, void *priv, if (fmt == NULL) return -EINVAL; - field = f->fmt.pix.field; - dprintk(vc->dev, 50, "%s NTSC: %d suggested width: %d, height: %d\n", __func__, is_ntsc, f->fmt.pix.width, f->fmt.pix.height); if (is_ntsc) { diff --git a/drivers/media/usb/ttusb-dec/ttusb_dec.c b/drivers/media/usb/ttusb-dec/ttusb_dec.c index a852ee5f7ac9..bfda46a36dc5 100644 --- a/drivers/media/usb/ttusb-dec/ttusb_dec.c +++ b/drivers/media/usb/ttusb-dec/ttusb_dec.c @@ -324,10 +324,10 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command, if (!b) return -ENOMEM; - if ((result = mutex_lock_interruptible(&dec->usb_mutex))) { - kfree(b); + result = mutex_lock_interruptible(&dec->usb_mutex); + if (result) { printk("%s: Failed to lock usb mutex.\n", __func__); - return result; + goto err; } b[0] = 0xaa; @@ -349,9 +349,7 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command, if (result) { printk("%s: command bulk message failed: error %d\n", __func__, result); - mutex_unlock(&dec->usb_mutex); - kfree(b); - return result; + goto err; } result = usb_bulk_msg(dec->udev, dec->result_pipe, b, @@ -360,9 +358,7 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command, if (result) { printk("%s: result bulk message failed: error %d\n", __func__, result); - mutex_unlock(&dec->usb_mutex); - kfree(b); - return result; + goto err; } else { if (debug) { printk(KERN_DEBUG "%s: result: %*ph\n", @@ -373,12 +369,13 @@ static int ttusb_dec_send_command(struct ttusb_dec *dec, const u8 command, *result_length = b[3]; if (cmd_result && b[3] > 0) memcpy(cmd_result, &b[4], b[3]); - - mutex_unlock(&dec->usb_mutex); - - kfree(b); - return 0; } + +err: + mutex_unlock(&dec->usb_mutex); + + kfree(b); + return result; } static int ttusb_dec_get_stb_state (struct ttusb_dec *dec, unsigned int *mode, diff --git a/drivers/media/usb/uvc/uvc_video.c b/drivers/media/usb/uvc/uvc_video.c index a777b389a66e..e16464606b14 100644 --- a/drivers/media/usb/uvc/uvc_video.c +++ b/drivers/media/usb/uvc/uvc_video.c @@ -127,10 +127,37 @@ int uvc_query_ctrl(struct uvc_device *dev, u8 query, u8 unit, static void uvc_fixup_video_ctrl(struct uvc_streaming *stream, struct uvc_streaming_control *ctrl) { + static const struct usb_device_id elgato_cam_link_4k = { + USB_DEVICE(0x0fd9, 0x0066) + }; struct uvc_format *format = NULL; struct uvc_frame *frame = NULL; unsigned int i; + /* + * The response of the Elgato Cam Link 4K is incorrect: The second byte + * contains bFormatIndex (instead of being the second byte of bmHint). + * The first byte is always zero. The third byte is always 1. + * + * The UVC 1.5 class specification defines the first five bits in the + * bmHint bitfield. The remaining bits are reserved and should be zero. + * Therefore a valid bmHint will be less than 32. + * + * Latest Elgato Cam Link 4K firmware as of 2021-03-23 needs this fix. + * MCU: 20.02.19, FPGA: 67 + */ + if (usb_match_one_id(stream->dev->intf, &elgato_cam_link_4k) && + ctrl->bmHint > 255) { + u8 corrected_format_index = ctrl->bmHint >> 8; + + uvc_dbg(stream->dev, VIDEO, + "Correct USB video probe response from {bmHint: 0x%04x, bFormatIndex: %u} to {bmHint: 0x%04x, bFormatIndex: %u}\n", + ctrl->bmHint, ctrl->bFormatIndex, + 1, corrected_format_index); + ctrl->bmHint = 1; + ctrl->bFormatIndex = corrected_format_index; + } + for (i = 0; i < stream->nformats; ++i) { if (stream->format[i].index == ctrl->bFormatIndex) { format = &stream->format[i]; diff --git a/drivers/media/usb/zr364xx/zr364xx.c b/drivers/media/usb/zr364xx/zr364xx.c index 1ef611e08323..538a330046ec 100644 --- a/drivers/media/usb/zr364xx/zr364xx.c +++ b/drivers/media/usb/zr364xx/zr364xx.c @@ -1032,6 +1032,7 @@ static int zr364xx_start_readpipe(struct zr364xx_camera *cam) DBG("submitting URB %p\n", pipe_info->stream_urb); retval = usb_submit_urb(pipe_info->stream_urb, GFP_KERNEL); if (retval) { + usb_free_urb(pipe_info->stream_urb); printk(KERN_ERR KBUILD_MODNAME ": start read pipe failed\n"); return retval; } diff --git a/drivers/media/v4l2-core/Kconfig b/drivers/media/v4l2-core/Kconfig index bf49f83cb86f..02dc1787e953 100644 --- a/drivers/media/v4l2-core/Kconfig +++ b/drivers/media/v4l2-core/Kconfig @@ -62,6 +62,7 @@ config V4L2_FLASH_LED_CLASS tristate "V4L2 flash API for LED flash class devices" depends on VIDEO_V4L2 && VIDEO_V4L2_SUBDEV_API depends on LEDS_CLASS_FLASH + select V4L2_ASYNC help Say Y here to enable V4L2 flash API support for LED flash class drivers. @@ -70,6 +71,10 @@ config V4L2_FLASH_LED_CLASS config V4L2_FWNODE tristate + select V4L2_ASYNC + +config V4L2_ASYNC + tristate # Used by drivers that need Videobuf modules config VIDEOBUF_GEN diff --git a/drivers/media/v4l2-core/Makefile b/drivers/media/v4l2-core/Makefile index e4cd589b99a5..66a78c556c98 100644 --- a/drivers/media/v4l2-core/Makefile +++ b/drivers/media/v4l2-core/Makefile @@ -6,16 +6,18 @@ tuner-objs := tuner-core.o videodev-objs := v4l2-dev.o v4l2-ioctl.o v4l2-device.o v4l2-fh.o \ - v4l2-event.o v4l2-ctrls.o v4l2-subdev.o \ - v4l2-async.o v4l2-common.o + v4l2-event.o v4l2-subdev.o v4l2-common.o \ + v4l2-ctrls-core.o v4l2-ctrls-api.o \ + v4l2-ctrls-request.o v4l2-ctrls-defs.o videodev-$(CONFIG_COMPAT) += v4l2-compat-ioctl32.o videodev-$(CONFIG_TRACEPOINTS) += v4l2-trace.o videodev-$(CONFIG_MEDIA_CONTROLLER) += v4l2-mc.o videodev-$(CONFIG_SPI) += v4l2-spi.o videodev-$(CONFIG_VIDEO_V4L2_I2C) += v4l2-i2c.o -obj-$(CONFIG_V4L2_FWNODE) += v4l2-fwnode.o obj-$(CONFIG_VIDEO_V4L2) += videodev.o +obj-$(CONFIG_V4L2_FWNODE) += v4l2-fwnode.o +obj-$(CONFIG_V4L2_ASYNC) += v4l2-async.o obj-$(CONFIG_VIDEO_V4L2) += v4l2-dv-timings.o obj-$(CONFIG_VIDEO_TUNER) += tuner.o diff --git a/drivers/media/v4l2-core/v4l2-async.c b/drivers/media/v4l2-core/v4l2-async.c index e638aa8aecb7..cd9e78c63791 100644 --- a/drivers/media/v4l2-core/v4l2-async.c +++ b/drivers/media/v4l2-core/v4l2-async.c @@ -854,8 +854,27 @@ static int pending_subdevs_show(struct seq_file *s, void *data) } DEFINE_SHOW_ATTRIBUTE(pending_subdevs); -void v4l2_async_debug_init(struct dentry *debugfs_dir) +static struct dentry *v4l2_async_debugfs_dir; + +static int __init v4l2_async_init(void) { - debugfs_create_file("pending_async_subdevices", 0444, debugfs_dir, NULL, + v4l2_async_debugfs_dir = debugfs_create_dir("v4l2-async", NULL); + debugfs_create_file("pending_async_subdevices", 0444, + v4l2_async_debugfs_dir, NULL, &pending_subdevs_fops); + + return 0; } + +static void __exit v4l2_async_exit(void) +{ + debugfs_remove_recursive(v4l2_async_debugfs_dir); +} + +subsys_initcall(v4l2_async_init); +module_exit(v4l2_async_exit); + +MODULE_AUTHOR("Guennadi Liakhovetski "); +MODULE_AUTHOR("Sakari Ailus "); +MODULE_AUTHOR("Ezequiel Garcia "); +MODULE_LICENSE("GPL"); diff --git a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c index 0ca75f6784c5..47aff3b19742 100644 --- a/drivers/media/v4l2-core/v4l2-compat-ioctl32.c +++ b/drivers/media/v4l2-core/v4l2-compat-ioctl32.c @@ -1244,6 +1244,9 @@ long v4l2_compat_ioctl32(struct file *file, unsigned int cmd, unsigned long arg) if (!file->f_op->unlocked_ioctl) return ret; + if (!video_is_registered(vdev)) + return -ENODEV; + if (_IOC_TYPE(cmd) == 'V' && _IOC_NR(cmd) < BASE_VIDIOC_PRIVATE) ret = file->f_op->unlocked_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); diff --git a/drivers/media/v4l2-core/v4l2-ctrls-api.c b/drivers/media/v4l2-core/v4l2-ctrls-api.c new file mode 100644 index 000000000000..db9baa0bd05f --- /dev/null +++ b/drivers/media/v4l2-core/v4l2-ctrls-api.c @@ -0,0 +1,1225 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * V4L2 controls framework uAPI implementation: + * + * Copyright (C) 2010-2021 Hans Verkuil + */ + +#define pr_fmt(fmt) "v4l2-ctrls: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "v4l2-ctrls-priv.h" + +/* Internal temporary helper struct, one for each v4l2_ext_control */ +struct v4l2_ctrl_helper { + /* Pointer to the control reference of the master control */ + struct v4l2_ctrl_ref *mref; + /* The control ref corresponding to the v4l2_ext_control ID field. */ + struct v4l2_ctrl_ref *ref; + /* + * v4l2_ext_control index of the next control belonging to the + * same cluster, or 0 if there isn't any. + */ + u32 next; +}; + +/* + * Helper functions to copy control payload data from kernel space to + * user space and vice versa. + */ + +/* Helper function: copy the given control value back to the caller */ +static int ptr_to_user(struct v4l2_ext_control *c, + struct v4l2_ctrl *ctrl, + union v4l2_ctrl_ptr ptr) +{ + u32 len; + + if (ctrl->is_ptr && !ctrl->is_string) + return copy_to_user(c->ptr, ptr.p_const, c->size) ? + -EFAULT : 0; + + switch (ctrl->type) { + case V4L2_CTRL_TYPE_STRING: + len = strlen(ptr.p_char); + if (c->size < len + 1) { + c->size = ctrl->elem_size; + return -ENOSPC; + } + return copy_to_user(c->string, ptr.p_char, len + 1) ? + -EFAULT : 0; + case V4L2_CTRL_TYPE_INTEGER64: + c->value64 = *ptr.p_s64; + break; + default: + c->value = *ptr.p_s32; + break; + } + return 0; +} + +/* Helper function: copy the current control value back to the caller */ +static int cur_to_user(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) +{ + return ptr_to_user(c, ctrl, ctrl->p_cur); +} + +/* Helper function: copy the new control value back to the caller */ +static int new_to_user(struct v4l2_ext_control *c, + struct v4l2_ctrl *ctrl) +{ + return ptr_to_user(c, ctrl, ctrl->p_new); +} + +/* Helper function: copy the request value back to the caller */ +static int req_to_user(struct v4l2_ext_control *c, + struct v4l2_ctrl_ref *ref) +{ + return ptr_to_user(c, ref->ctrl, ref->p_req); +} + +/* Helper function: copy the initial control value back to the caller */ +static int def_to_user(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) +{ + int idx; + + for (idx = 0; idx < ctrl->elems; idx++) + ctrl->type_ops->init(ctrl, idx, ctrl->p_new); + + return ptr_to_user(c, ctrl, ctrl->p_new); +} + +/* Helper function: copy the caller-provider value to the given control value */ +static int user_to_ptr(struct v4l2_ext_control *c, + struct v4l2_ctrl *ctrl, + union v4l2_ctrl_ptr ptr) +{ + int ret; + u32 size; + + ctrl->is_new = 1; + if (ctrl->is_ptr && !ctrl->is_string) { + unsigned int idx; + + ret = copy_from_user(ptr.p, c->ptr, c->size) ? -EFAULT : 0; + if (ret || !ctrl->is_array) + return ret; + for (idx = c->size / ctrl->elem_size; idx < ctrl->elems; idx++) + ctrl->type_ops->init(ctrl, idx, ptr); + return 0; + } + + switch (ctrl->type) { + case V4L2_CTRL_TYPE_INTEGER64: + *ptr.p_s64 = c->value64; + break; + case V4L2_CTRL_TYPE_STRING: + size = c->size; + if (size == 0) + return -ERANGE; + if (size > ctrl->maximum + 1) + size = ctrl->maximum + 1; + ret = copy_from_user(ptr.p_char, c->string, size) ? -EFAULT : 0; + if (!ret) { + char last = ptr.p_char[size - 1]; + + ptr.p_char[size - 1] = 0; + /* + * If the string was longer than ctrl->maximum, + * then return an error. + */ + if (strlen(ptr.p_char) == ctrl->maximum && last) + return -ERANGE; + } + return ret; + default: + *ptr.p_s32 = c->value; + break; + } + return 0; +} + +/* Helper function: copy the caller-provider value as the new control value */ +static int user_to_new(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) +{ + return user_to_ptr(c, ctrl, ctrl->p_new); +} + +/* + * VIDIOC_G/TRY/S_EXT_CTRLS implementation + */ + +/* + * Some general notes on the atomic requirements of VIDIOC_G/TRY/S_EXT_CTRLS: + * + * It is not a fully atomic operation, just best-effort only. After all, if + * multiple controls have to be set through multiple i2c writes (for example) + * then some initial writes may succeed while others fail. Thus leaving the + * system in an inconsistent state. The question is how much effort you are + * willing to spend on trying to make something atomic that really isn't. + * + * From the point of view of an application the main requirement is that + * when you call VIDIOC_S_EXT_CTRLS and some values are invalid then an + * error should be returned without actually affecting any controls. + * + * If all the values are correct, then it is acceptable to just give up + * in case of low-level errors. + * + * It is important though that the application can tell when only a partial + * configuration was done. The way we do that is through the error_idx field + * of struct v4l2_ext_controls: if that is equal to the count field then no + * controls were affected. Otherwise all controls before that index were + * successful in performing their 'get' or 'set' operation, the control at + * the given index failed, and you don't know what happened with the controls + * after the failed one. Since if they were part of a control cluster they + * could have been successfully processed (if a cluster member was encountered + * at index < error_idx), they could have failed (if a cluster member was at + * error_idx), or they may not have been processed yet (if the first cluster + * member appeared after error_idx). + * + * It is all fairly theoretical, though. In practice all you can do is to + * bail out. If error_idx == count, then it is an application bug. If + * error_idx < count then it is only an application bug if the error code was + * EBUSY. That usually means that something started streaming just when you + * tried to set the controls. In all other cases it is a driver/hardware + * problem and all you can do is to retry or bail out. + * + * Note that these rules do not apply to VIDIOC_TRY_EXT_CTRLS: since that + * never modifies controls the error_idx is just set to whatever control + * has an invalid value. + */ + +/* + * Prepare for the extended g/s/try functions. + * Find the controls in the control array and do some basic checks. + */ +static int prepare_ext_ctrls(struct v4l2_ctrl_handler *hdl, + struct v4l2_ext_controls *cs, + struct v4l2_ctrl_helper *helpers, + struct video_device *vdev, + bool get) +{ + struct v4l2_ctrl_helper *h; + bool have_clusters = false; + u32 i; + + for (i = 0, h = helpers; i < cs->count; i++, h++) { + struct v4l2_ext_control *c = &cs->controls[i]; + struct v4l2_ctrl_ref *ref; + struct v4l2_ctrl *ctrl; + u32 id = c->id & V4L2_CTRL_ID_MASK; + + cs->error_idx = i; + + if (cs->which && + cs->which != V4L2_CTRL_WHICH_DEF_VAL && + cs->which != V4L2_CTRL_WHICH_REQUEST_VAL && + V4L2_CTRL_ID2WHICH(id) != cs->which) { + dprintk(vdev, + "invalid which 0x%x or control id 0x%x\n", + cs->which, id); + return -EINVAL; + } + + /* + * Old-style private controls are not allowed for + * extended controls. + */ + if (id >= V4L2_CID_PRIVATE_BASE) { + dprintk(vdev, + "old-style private controls not allowed\n"); + return -EINVAL; + } + ref = find_ref_lock(hdl, id); + if (!ref) { + dprintk(vdev, "cannot find control id 0x%x\n", id); + return -EINVAL; + } + h->ref = ref; + ctrl = ref->ctrl; + if (ctrl->flags & V4L2_CTRL_FLAG_DISABLED) { + dprintk(vdev, "control id 0x%x is disabled\n", id); + return -EINVAL; + } + + if (ctrl->cluster[0]->ncontrols > 1) + have_clusters = true; + if (ctrl->cluster[0] != ctrl) + ref = find_ref_lock(hdl, ctrl->cluster[0]->id); + if (ctrl->is_ptr && !ctrl->is_string) { + unsigned int tot_size = ctrl->elems * ctrl->elem_size; + + if (c->size < tot_size) { + /* + * In the get case the application first + * queries to obtain the size of the control. + */ + if (get) { + c->size = tot_size; + return -ENOSPC; + } + dprintk(vdev, + "pointer control id 0x%x size too small, %d bytes but %d bytes needed\n", + id, c->size, tot_size); + return -EFAULT; + } + c->size = tot_size; + } + /* Store the ref to the master control of the cluster */ + h->mref = ref; + /* + * Initially set next to 0, meaning that there is no other + * control in this helper array belonging to the same + * cluster. + */ + h->next = 0; + } + + /* + * We are done if there were no controls that belong to a multi- + * control cluster. + */ + if (!have_clusters) + return 0; + + /* + * The code below figures out in O(n) time which controls in the list + * belong to the same cluster. + */ + + /* This has to be done with the handler lock taken. */ + mutex_lock(hdl->lock); + + /* First zero the helper field in the master control references */ + for (i = 0; i < cs->count; i++) + helpers[i].mref->helper = NULL; + for (i = 0, h = helpers; i < cs->count; i++, h++) { + struct v4l2_ctrl_ref *mref = h->mref; + + /* + * If the mref->helper is set, then it points to an earlier + * helper that belongs to the same cluster. + */ + if (mref->helper) { + /* + * Set the next field of mref->helper to the current + * index: this means that the earlier helper now + * points to the next helper in the same cluster. + */ + mref->helper->next = i; + /* + * mref should be set only for the first helper in the + * cluster, clear the others. + */ + h->mref = NULL; + } + /* Point the mref helper to the current helper struct. */ + mref->helper = h; + } + mutex_unlock(hdl->lock); + return 0; +} + +/* + * Handles the corner case where cs->count == 0. It checks whether the + * specified control class exists. If that class ID is 0, then it checks + * whether there are any controls at all. + */ +static int class_check(struct v4l2_ctrl_handler *hdl, u32 which) +{ + if (which == 0 || which == V4L2_CTRL_WHICH_DEF_VAL || + which == V4L2_CTRL_WHICH_REQUEST_VAL) + return 0; + return find_ref_lock(hdl, which | 1) ? 0 : -EINVAL; +} + +/* + * Get extended controls. Allocates the helpers array if needed. + * + * Note that v4l2_g_ext_ctrls_common() with 'which' set to + * V4L2_CTRL_WHICH_REQUEST_VAL is only called if the request was + * completed, and in that case valid_p_req is true for all controls. + */ +int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, + struct v4l2_ext_controls *cs, + struct video_device *vdev) +{ + struct v4l2_ctrl_helper helper[4]; + struct v4l2_ctrl_helper *helpers = helper; + int ret; + int i, j; + bool is_default, is_request; + + is_default = (cs->which == V4L2_CTRL_WHICH_DEF_VAL); + is_request = (cs->which == V4L2_CTRL_WHICH_REQUEST_VAL); + + cs->error_idx = cs->count; + cs->which = V4L2_CTRL_ID2WHICH(cs->which); + + if (!hdl) + return -EINVAL; + + if (cs->count == 0) + return class_check(hdl, cs->which); + + if (cs->count > ARRAY_SIZE(helper)) { + helpers = kvmalloc_array(cs->count, sizeof(helper[0]), + GFP_KERNEL); + if (!helpers) + return -ENOMEM; + } + + ret = prepare_ext_ctrls(hdl, cs, helpers, vdev, true); + cs->error_idx = cs->count; + + for (i = 0; !ret && i < cs->count; i++) + if (helpers[i].ref->ctrl->flags & V4L2_CTRL_FLAG_WRITE_ONLY) + ret = -EACCES; + + for (i = 0; !ret && i < cs->count; i++) { + struct v4l2_ctrl *master; + bool is_volatile = false; + u32 idx = i; + + if (!helpers[i].mref) + continue; + + master = helpers[i].mref->ctrl; + cs->error_idx = i; + + v4l2_ctrl_lock(master); + + /* + * g_volatile_ctrl will update the new control values. + * This makes no sense for V4L2_CTRL_WHICH_DEF_VAL and + * V4L2_CTRL_WHICH_REQUEST_VAL. In the case of requests + * it is v4l2_ctrl_request_complete() that copies the + * volatile controls at the time of request completion + * to the request, so you don't want to do that again. + */ + if (!is_default && !is_request && + ((master->flags & V4L2_CTRL_FLAG_VOLATILE) || + (master->has_volatiles && !is_cur_manual(master)))) { + for (j = 0; j < master->ncontrols; j++) + cur_to_new(master->cluster[j]); + ret = call_op(master, g_volatile_ctrl); + is_volatile = true; + } + + if (ret) { + v4l2_ctrl_unlock(master); + break; + } + + /* + * Copy the default value (if is_default is true), the + * request value (if is_request is true and p_req is valid), + * the new volatile value (if is_volatile is true) or the + * current value. + */ + do { + struct v4l2_ctrl_ref *ref = helpers[idx].ref; + + if (is_default) + ret = def_to_user(cs->controls + idx, ref->ctrl); + else if (is_request && ref->valid_p_req) + ret = req_to_user(cs->controls + idx, ref); + else if (is_volatile) + ret = new_to_user(cs->controls + idx, ref->ctrl); + else + ret = cur_to_user(cs->controls + idx, ref->ctrl); + idx = helpers[idx].next; + } while (!ret && idx); + + v4l2_ctrl_unlock(master); + } + + if (cs->count > ARRAY_SIZE(helper)) + kvfree(helpers); + return ret; +} + +int v4l2_g_ext_ctrls(struct v4l2_ctrl_handler *hdl, struct video_device *vdev, + struct media_device *mdev, struct v4l2_ext_controls *cs) +{ + if (cs->which == V4L2_CTRL_WHICH_REQUEST_VAL) + return v4l2_g_ext_ctrls_request(hdl, vdev, mdev, cs); + + return v4l2_g_ext_ctrls_common(hdl, cs, vdev); +} +EXPORT_SYMBOL(v4l2_g_ext_ctrls); + +/* Validate controls. */ +static int validate_ctrls(struct v4l2_ext_controls *cs, + struct v4l2_ctrl_helper *helpers, + struct video_device *vdev, + bool set) +{ + unsigned int i; + int ret = 0; + + cs->error_idx = cs->count; + for (i = 0; i < cs->count; i++) { + struct v4l2_ctrl *ctrl = helpers[i].ref->ctrl; + union v4l2_ctrl_ptr p_new; + + cs->error_idx = i; + + if (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY) { + dprintk(vdev, + "control id 0x%x is read-only\n", + ctrl->id); + return -EACCES; + } + /* + * This test is also done in try_set_control_cluster() which + * is called in atomic context, so that has the final say, + * but it makes sense to do an up-front check as well. Once + * an error occurs in try_set_control_cluster() some other + * controls may have been set already and we want to do a + * best-effort to avoid that. + */ + if (set && (ctrl->flags & V4L2_CTRL_FLAG_GRABBED)) { + dprintk(vdev, + "control id 0x%x is grabbed, cannot set\n", + ctrl->id); + return -EBUSY; + } + /* + * Skip validation for now if the payload needs to be copied + * from userspace into kernelspace. We'll validate those later. + */ + if (ctrl->is_ptr) + continue; + if (ctrl->type == V4L2_CTRL_TYPE_INTEGER64) + p_new.p_s64 = &cs->controls[i].value64; + else + p_new.p_s32 = &cs->controls[i].value; + ret = validate_new(ctrl, p_new); + if (ret) + return ret; + } + return 0; +} + +/* Try or try-and-set controls */ +int try_set_ext_ctrls_common(struct v4l2_fh *fh, + struct v4l2_ctrl_handler *hdl, + struct v4l2_ext_controls *cs, + struct video_device *vdev, bool set) +{ + struct v4l2_ctrl_helper helper[4]; + struct v4l2_ctrl_helper *helpers = helper; + unsigned int i, j; + int ret; + + cs->error_idx = cs->count; + + /* Default value cannot be changed */ + if (cs->which == V4L2_CTRL_WHICH_DEF_VAL) { + dprintk(vdev, "%s: cannot change default value\n", + video_device_node_name(vdev)); + return -EINVAL; + } + + cs->which = V4L2_CTRL_ID2WHICH(cs->which); + + if (!hdl) { + dprintk(vdev, "%s: invalid null control handler\n", + video_device_node_name(vdev)); + return -EINVAL; + } + + if (cs->count == 0) + return class_check(hdl, cs->which); + + if (cs->count > ARRAY_SIZE(helper)) { + helpers = kvmalloc_array(cs->count, sizeof(helper[0]), + GFP_KERNEL); + if (!helpers) + return -ENOMEM; + } + ret = prepare_ext_ctrls(hdl, cs, helpers, vdev, false); + if (!ret) + ret = validate_ctrls(cs, helpers, vdev, set); + if (ret && set) + cs->error_idx = cs->count; + for (i = 0; !ret && i < cs->count; i++) { + struct v4l2_ctrl *master; + u32 idx = i; + + if (!helpers[i].mref) + continue; + + cs->error_idx = i; + master = helpers[i].mref->ctrl; + v4l2_ctrl_lock(master); + + /* Reset the 'is_new' flags of the cluster */ + for (j = 0; j < master->ncontrols; j++) + if (master->cluster[j]) + master->cluster[j]->is_new = 0; + + /* + * For volatile autoclusters that are currently in auto mode + * we need to discover if it will be set to manual mode. + * If so, then we have to copy the current volatile values + * first since those will become the new manual values (which + * may be overwritten by explicit new values from this set + * of controls). + */ + if (master->is_auto && master->has_volatiles && + !is_cur_manual(master)) { + /* Pick an initial non-manual value */ + s32 new_auto_val = master->manual_mode_value + 1; + u32 tmp_idx = idx; + + do { + /* + * Check if the auto control is part of the + * list, and remember the new value. + */ + if (helpers[tmp_idx].ref->ctrl == master) + new_auto_val = cs->controls[tmp_idx].value; + tmp_idx = helpers[tmp_idx].next; + } while (tmp_idx); + /* + * If the new value == the manual value, then copy + * the current volatile values. + */ + if (new_auto_val == master->manual_mode_value) + update_from_auto_cluster(master); + } + + /* + * Copy the new caller-supplied control values. + * user_to_new() sets 'is_new' to 1. + */ + do { + struct v4l2_ctrl *ctrl = helpers[idx].ref->ctrl; + + ret = user_to_new(cs->controls + idx, ctrl); + if (!ret && ctrl->is_ptr) { + ret = validate_new(ctrl, ctrl->p_new); + if (ret) + dprintk(vdev, + "failed to validate control %s (%d)\n", + v4l2_ctrl_get_name(ctrl->id), ret); + } + idx = helpers[idx].next; + } while (!ret && idx); + + if (!ret) + ret = try_or_set_cluster(fh, master, + !hdl->req_obj.req && set, 0); + if (!ret && hdl->req_obj.req && set) { + for (j = 0; j < master->ncontrols; j++) { + struct v4l2_ctrl_ref *ref = + find_ref(hdl, master->cluster[j]->id); + + new_to_req(ref); + } + } + + /* Copy the new values back to userspace. */ + if (!ret) { + idx = i; + do { + ret = new_to_user(cs->controls + idx, + helpers[idx].ref->ctrl); + idx = helpers[idx].next; + } while (!ret && idx); + } + v4l2_ctrl_unlock(master); + } + + if (cs->count > ARRAY_SIZE(helper)) + kvfree(helpers); + return ret; +} + +static int try_set_ext_ctrls(struct v4l2_fh *fh, + struct v4l2_ctrl_handler *hdl, + struct video_device *vdev, + struct media_device *mdev, + struct v4l2_ext_controls *cs, bool set) +{ + int ret; + + if (cs->which == V4L2_CTRL_WHICH_REQUEST_VAL) + return try_set_ext_ctrls_request(fh, hdl, vdev, mdev, cs, set); + + ret = try_set_ext_ctrls_common(fh, hdl, cs, vdev, set); + if (ret) + dprintk(vdev, + "%s: try_set_ext_ctrls_common failed (%d)\n", + video_device_node_name(vdev), ret); + + return ret; +} + +int v4l2_try_ext_ctrls(struct v4l2_ctrl_handler *hdl, + struct video_device *vdev, + struct media_device *mdev, + struct v4l2_ext_controls *cs) +{ + return try_set_ext_ctrls(NULL, hdl, vdev, mdev, cs, false); +} +EXPORT_SYMBOL(v4l2_try_ext_ctrls); + +int v4l2_s_ext_ctrls(struct v4l2_fh *fh, + struct v4l2_ctrl_handler *hdl, + struct video_device *vdev, + struct media_device *mdev, + struct v4l2_ext_controls *cs) +{ + return try_set_ext_ctrls(fh, hdl, vdev, mdev, cs, true); +} +EXPORT_SYMBOL(v4l2_s_ext_ctrls); + +/* + * VIDIOC_G/S_CTRL implementation + */ + +/* Helper function to get a single control */ +static int get_ctrl(struct v4l2_ctrl *ctrl, struct v4l2_ext_control *c) +{ + struct v4l2_ctrl *master = ctrl->cluster[0]; + int ret = 0; + int i; + + /* Compound controls are not supported. The new_to_user() and + * cur_to_user() calls below would need to be modified not to access + * userspace memory when called from get_ctrl(). + */ + if (!ctrl->is_int && ctrl->type != V4L2_CTRL_TYPE_INTEGER64) + return -EINVAL; + + if (ctrl->flags & V4L2_CTRL_FLAG_WRITE_ONLY) + return -EACCES; + + v4l2_ctrl_lock(master); + /* g_volatile_ctrl will update the current control values */ + if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) { + for (i = 0; i < master->ncontrols; i++) + cur_to_new(master->cluster[i]); + ret = call_op(master, g_volatile_ctrl); + new_to_user(c, ctrl); + } else { + cur_to_user(c, ctrl); + } + v4l2_ctrl_unlock(master); + return ret; +} + +int v4l2_g_ctrl(struct v4l2_ctrl_handler *hdl, struct v4l2_control *control) +{ + struct v4l2_ctrl *ctrl = v4l2_ctrl_find(hdl, control->id); + struct v4l2_ext_control c; + int ret; + + if (!ctrl || !ctrl->is_int) + return -EINVAL; + ret = get_ctrl(ctrl, &c); + control->value = c.value; + return ret; +} +EXPORT_SYMBOL(v4l2_g_ctrl); + +/* Helper function for VIDIOC_S_CTRL compatibility */ +static int set_ctrl(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 ch_flags) +{ + struct v4l2_ctrl *master = ctrl->cluster[0]; + int ret; + int i; + + /* Reset the 'is_new' flags of the cluster */ + for (i = 0; i < master->ncontrols; i++) + if (master->cluster[i]) + master->cluster[i]->is_new = 0; + + ret = validate_new(ctrl, ctrl->p_new); + if (ret) + return ret; + + /* + * For autoclusters with volatiles that are switched from auto to + * manual mode we have to update the current volatile values since + * those will become the initial manual values after such a switch. + */ + if (master->is_auto && master->has_volatiles && ctrl == master && + !is_cur_manual(master) && ctrl->val == master->manual_mode_value) + update_from_auto_cluster(master); + + ctrl->is_new = 1; + return try_or_set_cluster(fh, master, true, ch_flags); +} + +/* Helper function for VIDIOC_S_CTRL compatibility */ +static int set_ctrl_lock(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, + struct v4l2_ext_control *c) +{ + int ret; + + v4l2_ctrl_lock(ctrl); + user_to_new(c, ctrl); + ret = set_ctrl(fh, ctrl, 0); + if (!ret) + cur_to_user(c, ctrl); + v4l2_ctrl_unlock(ctrl); + return ret; +} + +int v4l2_s_ctrl(struct v4l2_fh *fh, struct v4l2_ctrl_handler *hdl, + struct v4l2_control *control) +{ + struct v4l2_ctrl *ctrl = v4l2_ctrl_find(hdl, control->id); + struct v4l2_ext_control c = { control->id }; + int ret; + + if (!ctrl || !ctrl->is_int) + return -EINVAL; + + if (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY) + return -EACCES; + + c.value = control->value; + ret = set_ctrl_lock(fh, ctrl, &c); + control->value = c.value; + return ret; +} +EXPORT_SYMBOL(v4l2_s_ctrl); + +/* + * Helper functions for drivers to get/set controls. + */ + +s32 v4l2_ctrl_g_ctrl(struct v4l2_ctrl *ctrl) +{ + struct v4l2_ext_control c; + + /* It's a driver bug if this happens. */ + if (WARN_ON(!ctrl->is_int)) + return 0; + c.value = 0; + get_ctrl(ctrl, &c); + return c.value; +} +EXPORT_SYMBOL(v4l2_ctrl_g_ctrl); + +s64 v4l2_ctrl_g_ctrl_int64(struct v4l2_ctrl *ctrl) +{ + struct v4l2_ext_control c; + + /* It's a driver bug if this happens. */ + if (WARN_ON(ctrl->is_ptr || ctrl->type != V4L2_CTRL_TYPE_INTEGER64)) + return 0; + c.value64 = 0; + get_ctrl(ctrl, &c); + return c.value64; +} +EXPORT_SYMBOL(v4l2_ctrl_g_ctrl_int64); + +int __v4l2_ctrl_s_ctrl(struct v4l2_ctrl *ctrl, s32 val) +{ + lockdep_assert_held(ctrl->handler->lock); + + /* It's a driver bug if this happens. */ + if (WARN_ON(!ctrl->is_int)) + return -EINVAL; + ctrl->val = val; + return set_ctrl(NULL, ctrl, 0); +} +EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl); + +int __v4l2_ctrl_s_ctrl_int64(struct v4l2_ctrl *ctrl, s64 val) +{ + lockdep_assert_held(ctrl->handler->lock); + + /* It's a driver bug if this happens. */ + if (WARN_ON(ctrl->is_ptr || ctrl->type != V4L2_CTRL_TYPE_INTEGER64)) + return -EINVAL; + *ctrl->p_new.p_s64 = val; + return set_ctrl(NULL, ctrl, 0); +} +EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_int64); + +int __v4l2_ctrl_s_ctrl_string(struct v4l2_ctrl *ctrl, const char *s) +{ + lockdep_assert_held(ctrl->handler->lock); + + /* It's a driver bug if this happens. */ + if (WARN_ON(ctrl->type != V4L2_CTRL_TYPE_STRING)) + return -EINVAL; + strscpy(ctrl->p_new.p_char, s, ctrl->maximum + 1); + return set_ctrl(NULL, ctrl, 0); +} +EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_string); + +int __v4l2_ctrl_s_ctrl_compound(struct v4l2_ctrl *ctrl, + enum v4l2_ctrl_type type, const void *p) +{ + lockdep_assert_held(ctrl->handler->lock); + + /* It's a driver bug if this happens. */ + if (WARN_ON(ctrl->type != type)) + return -EINVAL; + memcpy(ctrl->p_new.p, p, ctrl->elems * ctrl->elem_size); + return set_ctrl(NULL, ctrl, 0); +} +EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_compound); + +/* + * Modify the range of a control. + */ +int __v4l2_ctrl_modify_range(struct v4l2_ctrl *ctrl, + s64 min, s64 max, u64 step, s64 def) +{ + bool value_changed; + bool range_changed = false; + int ret; + + lockdep_assert_held(ctrl->handler->lock); + + switch (ctrl->type) { + case V4L2_CTRL_TYPE_INTEGER: + case V4L2_CTRL_TYPE_INTEGER64: + case V4L2_CTRL_TYPE_BOOLEAN: + case V4L2_CTRL_TYPE_MENU: + case V4L2_CTRL_TYPE_INTEGER_MENU: + case V4L2_CTRL_TYPE_BITMASK: + case V4L2_CTRL_TYPE_U8: + case V4L2_CTRL_TYPE_U16: + case V4L2_CTRL_TYPE_U32: + if (ctrl->is_array) + return -EINVAL; + ret = check_range(ctrl->type, min, max, step, def); + if (ret) + return ret; + break; + default: + return -EINVAL; + } + if (ctrl->minimum != min || ctrl->maximum != max || + ctrl->step != step || ctrl->default_value != def) { + range_changed = true; + ctrl->minimum = min; + ctrl->maximum = max; + ctrl->step = step; + ctrl->default_value = def; + } + cur_to_new(ctrl); + if (validate_new(ctrl, ctrl->p_new)) { + if (ctrl->type == V4L2_CTRL_TYPE_INTEGER64) + *ctrl->p_new.p_s64 = def; + else + *ctrl->p_new.p_s32 = def; + } + + if (ctrl->type == V4L2_CTRL_TYPE_INTEGER64) + value_changed = *ctrl->p_new.p_s64 != *ctrl->p_cur.p_s64; + else + value_changed = *ctrl->p_new.p_s32 != *ctrl->p_cur.p_s32; + if (value_changed) + ret = set_ctrl(NULL, ctrl, V4L2_EVENT_CTRL_CH_RANGE); + else if (range_changed) + send_event(NULL, ctrl, V4L2_EVENT_CTRL_CH_RANGE); + return ret; +} +EXPORT_SYMBOL(__v4l2_ctrl_modify_range); + +/* Implement VIDIOC_QUERY_EXT_CTRL */ +int v4l2_query_ext_ctrl(struct v4l2_ctrl_handler *hdl, struct v4l2_query_ext_ctrl *qc) +{ + const unsigned int next_flags = V4L2_CTRL_FLAG_NEXT_CTRL | V4L2_CTRL_FLAG_NEXT_COMPOUND; + u32 id = qc->id & V4L2_CTRL_ID_MASK; + struct v4l2_ctrl_ref *ref; + struct v4l2_ctrl *ctrl; + + if (!hdl) + return -EINVAL; + + mutex_lock(hdl->lock); + + /* Try to find it */ + ref = find_ref(hdl, id); + + if ((qc->id & next_flags) && !list_empty(&hdl->ctrl_refs)) { + bool is_compound; + /* Match any control that is not hidden */ + unsigned int mask = 1; + bool match = false; + + if ((qc->id & next_flags) == V4L2_CTRL_FLAG_NEXT_COMPOUND) { + /* Match any hidden control */ + match = true; + } else if ((qc->id & next_flags) == next_flags) { + /* Match any control, compound or not */ + mask = 0; + } + + /* Find the next control with ID > qc->id */ + + /* Did we reach the end of the control list? */ + if (id >= node2id(hdl->ctrl_refs.prev)) { + ref = NULL; /* Yes, so there is no next control */ + } else if (ref) { + /* + * We found a control with the given ID, so just get + * the next valid one in the list. + */ + list_for_each_entry_continue(ref, &hdl->ctrl_refs, node) { + is_compound = ref->ctrl->is_array || + ref->ctrl->type >= V4L2_CTRL_COMPOUND_TYPES; + if (id < ref->ctrl->id && + (is_compound & mask) == match) + break; + } + if (&ref->node == &hdl->ctrl_refs) + ref = NULL; + } else { + /* + * No control with the given ID exists, so start + * searching for the next largest ID. We know there + * is one, otherwise the first 'if' above would have + * been true. + */ + list_for_each_entry(ref, &hdl->ctrl_refs, node) { + is_compound = ref->ctrl->is_array || + ref->ctrl->type >= V4L2_CTRL_COMPOUND_TYPES; + if (id < ref->ctrl->id && + (is_compound & mask) == match) + break; + } + if (&ref->node == &hdl->ctrl_refs) + ref = NULL; + } + } + mutex_unlock(hdl->lock); + + if (!ref) + return -EINVAL; + + ctrl = ref->ctrl; + memset(qc, 0, sizeof(*qc)); + if (id >= V4L2_CID_PRIVATE_BASE) + qc->id = id; + else + qc->id = ctrl->id; + strscpy(qc->name, ctrl->name, sizeof(qc->name)); + qc->flags = user_flags(ctrl); + qc->type = ctrl->type; + qc->elem_size = ctrl->elem_size; + qc->elems = ctrl->elems; + qc->nr_of_dims = ctrl->nr_of_dims; + memcpy(qc->dims, ctrl->dims, qc->nr_of_dims * sizeof(qc->dims[0])); + qc->minimum = ctrl->minimum; + qc->maximum = ctrl->maximum; + qc->default_value = ctrl->default_value; + if (ctrl->type == V4L2_CTRL_TYPE_MENU || + ctrl->type == V4L2_CTRL_TYPE_INTEGER_MENU) + qc->step = 1; + else + qc->step = ctrl->step; + return 0; +} +EXPORT_SYMBOL(v4l2_query_ext_ctrl); + +/* Implement VIDIOC_QUERYCTRL */ +int v4l2_queryctrl(struct v4l2_ctrl_handler *hdl, struct v4l2_queryctrl *qc) +{ + struct v4l2_query_ext_ctrl qec = { qc->id }; + int rc; + + rc = v4l2_query_ext_ctrl(hdl, &qec); + if (rc) + return rc; + + qc->id = qec.id; + qc->type = qec.type; + qc->flags = qec.flags; + strscpy(qc->name, qec.name, sizeof(qc->name)); + switch (qc->type) { + case V4L2_CTRL_TYPE_INTEGER: + case V4L2_CTRL_TYPE_BOOLEAN: + case V4L2_CTRL_TYPE_MENU: + case V4L2_CTRL_TYPE_INTEGER_MENU: + case V4L2_CTRL_TYPE_STRING: + case V4L2_CTRL_TYPE_BITMASK: + qc->minimum = qec.minimum; + qc->maximum = qec.maximum; + qc->step = qec.step; + qc->default_value = qec.default_value; + break; + default: + qc->minimum = 0; + qc->maximum = 0; + qc->step = 0; + qc->default_value = 0; + break; + } + return 0; +} +EXPORT_SYMBOL(v4l2_queryctrl); + +/* Implement VIDIOC_QUERYMENU */ +int v4l2_querymenu(struct v4l2_ctrl_handler *hdl, struct v4l2_querymenu *qm) +{ + struct v4l2_ctrl *ctrl; + u32 i = qm->index; + + ctrl = v4l2_ctrl_find(hdl, qm->id); + if (!ctrl) + return -EINVAL; + + qm->reserved = 0; + /* Sanity checks */ + switch (ctrl->type) { + case V4L2_CTRL_TYPE_MENU: + if (!ctrl->qmenu) + return -EINVAL; + break; + case V4L2_CTRL_TYPE_INTEGER_MENU: + if (!ctrl->qmenu_int) + return -EINVAL; + break; + default: + return -EINVAL; + } + + if (i < ctrl->minimum || i > ctrl->maximum) + return -EINVAL; + + /* Use mask to see if this menu item should be skipped */ + if (ctrl->menu_skip_mask & (1ULL << i)) + return -EINVAL; + /* Empty menu items should also be skipped */ + if (ctrl->type == V4L2_CTRL_TYPE_MENU) { + if (!ctrl->qmenu[i] || ctrl->qmenu[i][0] == '\0') + return -EINVAL; + strscpy(qm->name, ctrl->qmenu[i], sizeof(qm->name)); + } else { + qm->value = ctrl->qmenu_int[i]; + } + return 0; +} +EXPORT_SYMBOL(v4l2_querymenu); + +/* + * VIDIOC_LOG_STATUS helpers + */ + +int v4l2_ctrl_log_status(struct file *file, void *fh) +{ + struct video_device *vfd = video_devdata(file); + struct v4l2_fh *vfh = file->private_data; + + if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) && vfd->v4l2_dev) + v4l2_ctrl_handler_log_status(vfh->ctrl_handler, + vfd->v4l2_dev->name); + return 0; +} +EXPORT_SYMBOL(v4l2_ctrl_log_status); + +int v4l2_ctrl_subdev_log_status(struct v4l2_subdev *sd) +{ + v4l2_ctrl_handler_log_status(sd->ctrl_handler, sd->name); + return 0; +} +EXPORT_SYMBOL(v4l2_ctrl_subdev_log_status); + +/* + * VIDIOC_(UN)SUBSCRIBE_EVENT implementation + */ + +static int v4l2_ctrl_add_event(struct v4l2_subscribed_event *sev, + unsigned int elems) +{ + struct v4l2_ctrl *ctrl = v4l2_ctrl_find(sev->fh->ctrl_handler, sev->id); + + if (!ctrl) + return -EINVAL; + + v4l2_ctrl_lock(ctrl); + list_add_tail(&sev->node, &ctrl->ev_subs); + if (ctrl->type != V4L2_CTRL_TYPE_CTRL_CLASS && + (sev->flags & V4L2_EVENT_SUB_FL_SEND_INITIAL)) + send_initial_event(sev->fh, ctrl); + v4l2_ctrl_unlock(ctrl); + return 0; +} + +static void v4l2_ctrl_del_event(struct v4l2_subscribed_event *sev) +{ + struct v4l2_ctrl *ctrl = v4l2_ctrl_find(sev->fh->ctrl_handler, sev->id); + + if (!ctrl) + return; + + v4l2_ctrl_lock(ctrl); + list_del(&sev->node); + v4l2_ctrl_unlock(ctrl); +} + +void v4l2_ctrl_replace(struct v4l2_event *old, const struct v4l2_event *new) +{ + u32 old_changes = old->u.ctrl.changes; + + old->u.ctrl = new->u.ctrl; + old->u.ctrl.changes |= old_changes; +} +EXPORT_SYMBOL(v4l2_ctrl_replace); + +void v4l2_ctrl_merge(const struct v4l2_event *old, struct v4l2_event *new) +{ + new->u.ctrl.changes |= old->u.ctrl.changes; +} +EXPORT_SYMBOL(v4l2_ctrl_merge); + +const struct v4l2_subscribed_event_ops v4l2_ctrl_sub_ev_ops = { + .add = v4l2_ctrl_add_event, + .del = v4l2_ctrl_del_event, + .replace = v4l2_ctrl_replace, + .merge = v4l2_ctrl_merge, +}; +EXPORT_SYMBOL(v4l2_ctrl_sub_ev_ops); + +int v4l2_ctrl_subscribe_event(struct v4l2_fh *fh, + const struct v4l2_event_subscription *sub) +{ + if (sub->type == V4L2_EVENT_CTRL) + return v4l2_event_subscribe(fh, sub, 0, &v4l2_ctrl_sub_ev_ops); + return -EINVAL; +} +EXPORT_SYMBOL(v4l2_ctrl_subscribe_event); + +int v4l2_ctrl_subdev_subscribe_event(struct v4l2_subdev *sd, struct v4l2_fh *fh, + struct v4l2_event_subscription *sub) +{ + if (!sd->ctrl_handler) + return -EINVAL; + return v4l2_ctrl_subscribe_event(fh, sub); +} +EXPORT_SYMBOL(v4l2_ctrl_subdev_subscribe_event); + +/* + * poll helper + */ +__poll_t v4l2_ctrl_poll(struct file *file, struct poll_table_struct *wait) +{ + struct v4l2_fh *fh = file->private_data; + + poll_wait(file, &fh->wait, wait); + if (v4l2_event_pending(fh)) + return EPOLLPRI; + return 0; +} +EXPORT_SYMBOL(v4l2_ctrl_poll); diff --git a/drivers/media/v4l2-core/v4l2-ctrls-core.c b/drivers/media/v4l2-core/v4l2-ctrls-core.c new file mode 100644 index 000000000000..c4b5082849b6 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2-ctrls-core.c @@ -0,0 +1,1946 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * V4L2 controls framework core implementation. + * + * Copyright (C) 2010-2021 Hans Verkuil + */ + +#include +#include +#include +#include +#include +#include + +#include "v4l2-ctrls-priv.h" + +static const union v4l2_ctrl_ptr ptr_null; + +static void fill_event(struct v4l2_event *ev, struct v4l2_ctrl *ctrl, + u32 changes) +{ + memset(ev, 0, sizeof(*ev)); + ev->type = V4L2_EVENT_CTRL; + ev->id = ctrl->id; + ev->u.ctrl.changes = changes; + ev->u.ctrl.type = ctrl->type; + ev->u.ctrl.flags = user_flags(ctrl); + if (ctrl->is_ptr) + ev->u.ctrl.value64 = 0; + else + ev->u.ctrl.value64 = *ctrl->p_cur.p_s64; + ev->u.ctrl.minimum = ctrl->minimum; + ev->u.ctrl.maximum = ctrl->maximum; + if (ctrl->type == V4L2_CTRL_TYPE_MENU + || ctrl->type == V4L2_CTRL_TYPE_INTEGER_MENU) + ev->u.ctrl.step = 1; + else + ev->u.ctrl.step = ctrl->step; + ev->u.ctrl.default_value = ctrl->default_value; +} + +void send_initial_event(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl) +{ + struct v4l2_event ev; + u32 changes = V4L2_EVENT_CTRL_CH_FLAGS; + + if (!(ctrl->flags & V4L2_CTRL_FLAG_WRITE_ONLY)) + changes |= V4L2_EVENT_CTRL_CH_VALUE; + fill_event(&ev, ctrl, changes); + v4l2_event_queue_fh(fh, &ev); +} + +void send_event(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 changes) +{ + struct v4l2_event ev; + struct v4l2_subscribed_event *sev; + + if (list_empty(&ctrl->ev_subs)) + return; + fill_event(&ev, ctrl, changes); + + list_for_each_entry(sev, &ctrl->ev_subs, node) + if (sev->fh != fh || + (sev->flags & V4L2_EVENT_SUB_FL_ALLOW_FEEDBACK)) + v4l2_event_queue_fh(sev->fh, &ev); +} + +static bool std_equal(const struct v4l2_ctrl *ctrl, u32 idx, + union v4l2_ctrl_ptr ptr1, + union v4l2_ctrl_ptr ptr2) +{ + switch (ctrl->type) { + case V4L2_CTRL_TYPE_BUTTON: + return false; + case V4L2_CTRL_TYPE_STRING: + idx *= ctrl->elem_size; + /* strings are always 0-terminated */ + return !strcmp(ptr1.p_char + idx, ptr2.p_char + idx); + case V4L2_CTRL_TYPE_INTEGER64: + return ptr1.p_s64[idx] == ptr2.p_s64[idx]; + case V4L2_CTRL_TYPE_U8: + return ptr1.p_u8[idx] == ptr2.p_u8[idx]; + case V4L2_CTRL_TYPE_U16: + return ptr1.p_u16[idx] == ptr2.p_u16[idx]; + case V4L2_CTRL_TYPE_U32: + return ptr1.p_u32[idx] == ptr2.p_u32[idx]; + default: + if (ctrl->is_int) + return ptr1.p_s32[idx] == ptr2.p_s32[idx]; + idx *= ctrl->elem_size; + return !memcmp(ptr1.p_const + idx, ptr2.p_const + idx, + ctrl->elem_size); + } +} + +/* Default intra MPEG-2 quantisation coefficients, from the specification. */ +static const u8 mpeg2_intra_quant_matrix[64] = { + 8, 16, 16, 19, 16, 19, 22, 22, + 22, 22, 22, 22, 26, 24, 26, 27, + 27, 27, 26, 26, 26, 26, 27, 27, + 27, 29, 29, 29, 34, 34, 34, 29, + 29, 29, 27, 27, 29, 29, 32, 32, + 34, 34, 37, 38, 37, 35, 35, 34, + 35, 38, 38, 40, 40, 40, 48, 48, + 46, 46, 56, 56, 58, 69, 69, 83 +}; + +static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx, + union v4l2_ctrl_ptr ptr) +{ + struct v4l2_ctrl_mpeg2_sequence *p_mpeg2_sequence; + struct v4l2_ctrl_mpeg2_picture *p_mpeg2_picture; + struct v4l2_ctrl_mpeg2_quantisation *p_mpeg2_quant; + struct v4l2_ctrl_vp8_frame *p_vp8_frame; + struct v4l2_ctrl_fwht_params *p_fwht_params; + void *p = ptr.p + idx * ctrl->elem_size; + + if (ctrl->p_def.p_const) + memcpy(p, ctrl->p_def.p_const, ctrl->elem_size); + else + memset(p, 0, ctrl->elem_size); + + switch ((u32)ctrl->type) { + case V4L2_CTRL_TYPE_MPEG2_SEQUENCE: + p_mpeg2_sequence = p; + + /* 4:2:0 */ + p_mpeg2_sequence->chroma_format = 1; + break; + case V4L2_CTRL_TYPE_MPEG2_PICTURE: + p_mpeg2_picture = p; + + /* interlaced top field */ + p_mpeg2_picture->picture_structure = V4L2_MPEG2_PIC_TOP_FIELD; + p_mpeg2_picture->picture_coding_type = + V4L2_MPEG2_PIC_CODING_TYPE_I; + break; + case V4L2_CTRL_TYPE_MPEG2_QUANTISATION: + p_mpeg2_quant = p; + + memcpy(p_mpeg2_quant->intra_quantiser_matrix, + mpeg2_intra_quant_matrix, + ARRAY_SIZE(mpeg2_intra_quant_matrix)); + /* + * The default non-intra MPEG-2 quantisation + * coefficients are all 16, as per the specification. + */ + memset(p_mpeg2_quant->non_intra_quantiser_matrix, 16, + sizeof(p_mpeg2_quant->non_intra_quantiser_matrix)); + break; + case V4L2_CTRL_TYPE_VP8_FRAME: + p_vp8_frame = p; + p_vp8_frame->num_dct_parts = 1; + break; + case V4L2_CTRL_TYPE_FWHT_PARAMS: + p_fwht_params = p; + p_fwht_params->version = V4L2_FWHT_VERSION; + p_fwht_params->width = 1280; + p_fwht_params->height = 720; + p_fwht_params->flags = V4L2_FWHT_FL_PIXENC_YUV | + (2 << V4L2_FWHT_FL_COMPONENTS_NUM_OFFSET); + break; + } +} + +static void std_init(const struct v4l2_ctrl *ctrl, u32 idx, + union v4l2_ctrl_ptr ptr) +{ + switch (ctrl->type) { + case V4L2_CTRL_TYPE_STRING: + idx *= ctrl->elem_size; + memset(ptr.p_char + idx, ' ', ctrl->minimum); + ptr.p_char[idx + ctrl->minimum] = '\0'; + break; + case V4L2_CTRL_TYPE_INTEGER64: + ptr.p_s64[idx] = ctrl->default_value; + break; + case V4L2_CTRL_TYPE_INTEGER: + case V4L2_CTRL_TYPE_INTEGER_MENU: + case V4L2_CTRL_TYPE_MENU: + case V4L2_CTRL_TYPE_BITMASK: + case V4L2_CTRL_TYPE_BOOLEAN: + ptr.p_s32[idx] = ctrl->default_value; + break; + case V4L2_CTRL_TYPE_BUTTON: + case V4L2_CTRL_TYPE_CTRL_CLASS: + ptr.p_s32[idx] = 0; + break; + case V4L2_CTRL_TYPE_U8: + ptr.p_u8[idx] = ctrl->default_value; + break; + case V4L2_CTRL_TYPE_U16: + ptr.p_u16[idx] = ctrl->default_value; + break; + case V4L2_CTRL_TYPE_U32: + ptr.p_u32[idx] = ctrl->default_value; + break; + default: + std_init_compound(ctrl, idx, ptr); + break; + } +} + +static void std_log(const struct v4l2_ctrl *ctrl) +{ + union v4l2_ctrl_ptr ptr = ctrl->p_cur; + + if (ctrl->is_array) { + unsigned i; + + for (i = 0; i < ctrl->nr_of_dims; i++) + pr_cont("[%u]", ctrl->dims[i]); + pr_cont(" "); + } + + switch (ctrl->type) { + case V4L2_CTRL_TYPE_INTEGER: + pr_cont("%d", *ptr.p_s32); + break; + case V4L2_CTRL_TYPE_BOOLEAN: + pr_cont("%s", *ptr.p_s32 ? "true" : "false"); + break; + case V4L2_CTRL_TYPE_MENU: + pr_cont("%s", ctrl->qmenu[*ptr.p_s32]); + break; + case V4L2_CTRL_TYPE_INTEGER_MENU: + pr_cont("%lld", ctrl->qmenu_int[*ptr.p_s32]); + break; + case V4L2_CTRL_TYPE_BITMASK: + pr_cont("0x%08x", *ptr.p_s32); + break; + case V4L2_CTRL_TYPE_INTEGER64: + pr_cont("%lld", *ptr.p_s64); + break; + case V4L2_CTRL_TYPE_STRING: + pr_cont("%s", ptr.p_char); + break; + case V4L2_CTRL_TYPE_U8: + pr_cont("%u", (unsigned)*ptr.p_u8); + break; + case V4L2_CTRL_TYPE_U16: + pr_cont("%u", (unsigned)*ptr.p_u16); + break; + case V4L2_CTRL_TYPE_U32: + pr_cont("%u", (unsigned)*ptr.p_u32); + break; + case V4L2_CTRL_TYPE_H264_SPS: + pr_cont("H264_SPS"); + break; + case V4L2_CTRL_TYPE_H264_PPS: + pr_cont("H264_PPS"); + break; + case V4L2_CTRL_TYPE_H264_SCALING_MATRIX: + pr_cont("H264_SCALING_MATRIX"); + break; + case V4L2_CTRL_TYPE_H264_SLICE_PARAMS: + pr_cont("H264_SLICE_PARAMS"); + break; + case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: + pr_cont("H264_DECODE_PARAMS"); + break; + case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS: + pr_cont("H264_PRED_WEIGHTS"); + break; + case V4L2_CTRL_TYPE_FWHT_PARAMS: + pr_cont("FWHT_PARAMS"); + break; + case V4L2_CTRL_TYPE_VP8_FRAME: + pr_cont("VP8_FRAME"); + break; + case V4L2_CTRL_TYPE_HDR10_CLL_INFO: + pr_cont("HDR10_CLL_INFO"); + break; + case V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY: + pr_cont("HDR10_MASTERING_DISPLAY"); + break; + case V4L2_CTRL_TYPE_MPEG2_QUANTISATION: + pr_cont("MPEG2_QUANTISATION"); + break; + case V4L2_CTRL_TYPE_MPEG2_SEQUENCE: + pr_cont("MPEG2_SEQUENCE"); + break; + case V4L2_CTRL_TYPE_MPEG2_PICTURE: + pr_cont("MPEG2_PICTURE"); + break; + default: + pr_cont("unknown type %d", ctrl->type); + break; + } +} + +/* + * Round towards the closest legal value. Be careful when we are + * close to the maximum range of the control type to prevent + * wrap-arounds. + */ +#define ROUND_TO_RANGE(val, offset_type, ctrl) \ +({ \ + offset_type offset; \ + if ((ctrl)->maximum >= 0 && \ + val >= (ctrl)->maximum - (s32)((ctrl)->step / 2)) \ + val = (ctrl)->maximum; \ + else \ + val += (s32)((ctrl)->step / 2); \ + val = clamp_t(typeof(val), val, \ + (ctrl)->minimum, (ctrl)->maximum); \ + offset = (val) - (ctrl)->minimum; \ + offset = (ctrl)->step * (offset / (u32)(ctrl)->step); \ + val = (ctrl)->minimum + offset; \ + 0; \ +}) + +/* Validate a new control */ + +#define zero_padding(s) \ + memset(&(s).padding, 0, sizeof((s).padding)) +#define zero_reserved(s) \ + memset(&(s).reserved, 0, sizeof((s).reserved)) + +/* + * Compound controls validation requires setting unused fields/flags to zero + * in order to properly detect unchanged controls with std_equal's memcmp. + */ +static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, + union v4l2_ctrl_ptr ptr) +{ + struct v4l2_ctrl_mpeg2_sequence *p_mpeg2_sequence; + struct v4l2_ctrl_mpeg2_picture *p_mpeg2_picture; + struct v4l2_ctrl_vp8_frame *p_vp8_frame; + struct v4l2_ctrl_fwht_params *p_fwht_params; + struct v4l2_ctrl_h264_sps *p_h264_sps; + struct v4l2_ctrl_h264_pps *p_h264_pps; + struct v4l2_ctrl_h264_pred_weights *p_h264_pred_weights; + struct v4l2_ctrl_h264_slice_params *p_h264_slice_params; + struct v4l2_ctrl_h264_decode_params *p_h264_dec_params; + struct v4l2_ctrl_hevc_sps *p_hevc_sps; + struct v4l2_ctrl_hevc_pps *p_hevc_pps; + struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params; + struct v4l2_ctrl_hdr10_mastering_display *p_hdr10_mastering; + struct v4l2_ctrl_hevc_decode_params *p_hevc_decode_params; + struct v4l2_area *area; + void *p = ptr.p + idx * ctrl->elem_size; + unsigned int i; + + switch ((u32)ctrl->type) { + case V4L2_CTRL_TYPE_MPEG2_SEQUENCE: + p_mpeg2_sequence = p; + + switch (p_mpeg2_sequence->chroma_format) { + case 1: /* 4:2:0 */ + case 2: /* 4:2:2 */ + case 3: /* 4:4:4 */ + break; + default: + return -EINVAL; + } + break; + + case V4L2_CTRL_TYPE_MPEG2_PICTURE: + p_mpeg2_picture = p; + + switch (p_mpeg2_picture->intra_dc_precision) { + case 0: /* 8 bits */ + case 1: /* 9 bits */ + case 2: /* 10 bits */ + case 3: /* 11 bits */ + break; + default: + return -EINVAL; + } + + switch (p_mpeg2_picture->picture_structure) { + case V4L2_MPEG2_PIC_TOP_FIELD: + case V4L2_MPEG2_PIC_BOTTOM_FIELD: + case V4L2_MPEG2_PIC_FRAME: + break; + default: + return -EINVAL; + } + + switch (p_mpeg2_picture->picture_coding_type) { + case V4L2_MPEG2_PIC_CODING_TYPE_I: + case V4L2_MPEG2_PIC_CODING_TYPE_P: + case V4L2_MPEG2_PIC_CODING_TYPE_B: + break; + default: + return -EINVAL; + } + zero_reserved(*p_mpeg2_picture); + break; + + case V4L2_CTRL_TYPE_MPEG2_QUANTISATION: + break; + + case V4L2_CTRL_TYPE_FWHT_PARAMS: + p_fwht_params = p; + if (p_fwht_params->version < V4L2_FWHT_VERSION) + return -EINVAL; + if (!p_fwht_params->width || !p_fwht_params->height) + return -EINVAL; + break; + + case V4L2_CTRL_TYPE_H264_SPS: + p_h264_sps = p; + + /* Some syntax elements are only conditionally valid */ + if (p_h264_sps->pic_order_cnt_type != 0) { + p_h264_sps->log2_max_pic_order_cnt_lsb_minus4 = 0; + } else if (p_h264_sps->pic_order_cnt_type != 1) { + p_h264_sps->num_ref_frames_in_pic_order_cnt_cycle = 0; + p_h264_sps->offset_for_non_ref_pic = 0; + p_h264_sps->offset_for_top_to_bottom_field = 0; + memset(&p_h264_sps->offset_for_ref_frame, 0, + sizeof(p_h264_sps->offset_for_ref_frame)); + } + + if (!V4L2_H264_SPS_HAS_CHROMA_FORMAT(p_h264_sps)) { + p_h264_sps->chroma_format_idc = 1; + p_h264_sps->bit_depth_luma_minus8 = 0; + p_h264_sps->bit_depth_chroma_minus8 = 0; + + p_h264_sps->flags &= + ~V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; + + if (p_h264_sps->chroma_format_idc < 3) + p_h264_sps->flags &= + ~V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; + } + + if (p_h264_sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY) + p_h264_sps->flags &= + ~V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; + + /* + * Chroma 4:2:2 format require at least High 4:2:2 profile. + * + * The H264 specification and well-known parser implementations + * use profile-idc values directly, as that is clearer and + * less ambiguous. We do the same here. + */ + if (p_h264_sps->profile_idc < 122 && + p_h264_sps->chroma_format_idc > 1) + return -EINVAL; + /* Chroma 4:4:4 format require at least High 4:2:2 profile */ + if (p_h264_sps->profile_idc < 244 && + p_h264_sps->chroma_format_idc > 2) + return -EINVAL; + if (p_h264_sps->chroma_format_idc > 3) + return -EINVAL; + + if (p_h264_sps->bit_depth_luma_minus8 > 6) + return -EINVAL; + if (p_h264_sps->bit_depth_chroma_minus8 > 6) + return -EINVAL; + if (p_h264_sps->log2_max_frame_num_minus4 > 12) + return -EINVAL; + if (p_h264_sps->pic_order_cnt_type > 2) + return -EINVAL; + if (p_h264_sps->log2_max_pic_order_cnt_lsb_minus4 > 12) + return -EINVAL; + if (p_h264_sps->max_num_ref_frames > V4L2_H264_REF_LIST_LEN) + return -EINVAL; + break; + + case V4L2_CTRL_TYPE_H264_PPS: + p_h264_pps = p; + + if (p_h264_pps->num_slice_groups_minus1 > 7) + return -EINVAL; + if (p_h264_pps->num_ref_idx_l0_default_active_minus1 > + (V4L2_H264_REF_LIST_LEN - 1)) + return -EINVAL; + if (p_h264_pps->num_ref_idx_l1_default_active_minus1 > + (V4L2_H264_REF_LIST_LEN - 1)) + return -EINVAL; + if (p_h264_pps->weighted_bipred_idc > 2) + return -EINVAL; + /* + * pic_init_qp_minus26 shall be in the range of + * -(26 + QpBdOffset_y) to +25, inclusive, + * where QpBdOffset_y is 6 * bit_depth_luma_minus8 + */ + if (p_h264_pps->pic_init_qp_minus26 < -62 || + p_h264_pps->pic_init_qp_minus26 > 25) + return -EINVAL; + if (p_h264_pps->pic_init_qs_minus26 < -26 || + p_h264_pps->pic_init_qs_minus26 > 25) + return -EINVAL; + if (p_h264_pps->chroma_qp_index_offset < -12 || + p_h264_pps->chroma_qp_index_offset > 12) + return -EINVAL; + if (p_h264_pps->second_chroma_qp_index_offset < -12 || + p_h264_pps->second_chroma_qp_index_offset > 12) + return -EINVAL; + break; + + case V4L2_CTRL_TYPE_H264_SCALING_MATRIX: + break; + + case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS: + p_h264_pred_weights = p; + + if (p_h264_pred_weights->luma_log2_weight_denom > 7) + return -EINVAL; + if (p_h264_pred_weights->chroma_log2_weight_denom > 7) + return -EINVAL; + break; + + case V4L2_CTRL_TYPE_H264_SLICE_PARAMS: + p_h264_slice_params = p; + + if (p_h264_slice_params->slice_type != V4L2_H264_SLICE_TYPE_B) + p_h264_slice_params->flags &= + ~V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED; + + if (p_h264_slice_params->colour_plane_id > 2) + return -EINVAL; + if (p_h264_slice_params->cabac_init_idc > 2) + return -EINVAL; + if (p_h264_slice_params->disable_deblocking_filter_idc > 2) + return -EINVAL; + if (p_h264_slice_params->slice_alpha_c0_offset_div2 < -6 || + p_h264_slice_params->slice_alpha_c0_offset_div2 > 6) + return -EINVAL; + if (p_h264_slice_params->slice_beta_offset_div2 < -6 || + p_h264_slice_params->slice_beta_offset_div2 > 6) + return -EINVAL; + + if (p_h264_slice_params->slice_type == V4L2_H264_SLICE_TYPE_I || + p_h264_slice_params->slice_type == V4L2_H264_SLICE_TYPE_SI) + p_h264_slice_params->num_ref_idx_l0_active_minus1 = 0; + if (p_h264_slice_params->slice_type != V4L2_H264_SLICE_TYPE_B) + p_h264_slice_params->num_ref_idx_l1_active_minus1 = 0; + + if (p_h264_slice_params->num_ref_idx_l0_active_minus1 > + (V4L2_H264_REF_LIST_LEN - 1)) + return -EINVAL; + if (p_h264_slice_params->num_ref_idx_l1_active_minus1 > + (V4L2_H264_REF_LIST_LEN - 1)) + return -EINVAL; + zero_reserved(*p_h264_slice_params); + break; + + case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: + p_h264_dec_params = p; + + if (p_h264_dec_params->nal_ref_idc > 3) + return -EINVAL; + for (i = 0; i < V4L2_H264_NUM_DPB_ENTRIES; i++) { + struct v4l2_h264_dpb_entry *dpb_entry = + &p_h264_dec_params->dpb[i]; + + zero_reserved(*dpb_entry); + } + zero_reserved(*p_h264_dec_params); + break; + + case V4L2_CTRL_TYPE_VP8_FRAME: + p_vp8_frame = p; + + switch (p_vp8_frame->num_dct_parts) { + case 1: + case 2: + case 4: + case 8: + break; + default: + return -EINVAL; + } + zero_padding(p_vp8_frame->segment); + zero_padding(p_vp8_frame->lf); + zero_padding(p_vp8_frame->quant); + zero_padding(p_vp8_frame->entropy); + zero_padding(p_vp8_frame->coder_state); + break; + + case V4L2_CTRL_TYPE_HEVC_SPS: + p_hevc_sps = p; + + if (!(p_hevc_sps->flags & V4L2_HEVC_SPS_FLAG_PCM_ENABLED)) { + p_hevc_sps->pcm_sample_bit_depth_luma_minus1 = 0; + p_hevc_sps->pcm_sample_bit_depth_chroma_minus1 = 0; + p_hevc_sps->log2_min_pcm_luma_coding_block_size_minus3 = 0; + p_hevc_sps->log2_diff_max_min_pcm_luma_coding_block_size = 0; + } + + if (!(p_hevc_sps->flags & + V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT)) + p_hevc_sps->num_long_term_ref_pics_sps = 0; + break; + + case V4L2_CTRL_TYPE_HEVC_PPS: + p_hevc_pps = p; + + if (!(p_hevc_pps->flags & + V4L2_HEVC_PPS_FLAG_CU_QP_DELTA_ENABLED)) + p_hevc_pps->diff_cu_qp_delta_depth = 0; + + if (!(p_hevc_pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED)) { + p_hevc_pps->num_tile_columns_minus1 = 0; + p_hevc_pps->num_tile_rows_minus1 = 0; + memset(&p_hevc_pps->column_width_minus1, 0, + sizeof(p_hevc_pps->column_width_minus1)); + memset(&p_hevc_pps->row_height_minus1, 0, + sizeof(p_hevc_pps->row_height_minus1)); + + p_hevc_pps->flags &= + ~V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED; + } + + if (p_hevc_pps->flags & + V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER) { + p_hevc_pps->pps_beta_offset_div2 = 0; + p_hevc_pps->pps_tc_offset_div2 = 0; + } + + zero_padding(*p_hevc_pps); + break; + + case V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS: + p_hevc_decode_params = p; + + if (p_hevc_decode_params->num_active_dpb_entries > + V4L2_HEVC_DPB_ENTRIES_NUM_MAX) + return -EINVAL; + + for (i = 0; i < p_hevc_decode_params->num_active_dpb_entries; + i++) { + struct v4l2_hevc_dpb_entry *dpb_entry = + &p_hevc_decode_params->dpb[i]; + + zero_padding(*dpb_entry); + } + break; + + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: + p_hevc_slice_params = p; + + zero_padding(p_hevc_slice_params->pred_weight_table); + zero_padding(*p_hevc_slice_params); + break; + + case V4L2_CTRL_TYPE_HDR10_CLL_INFO: + break; + + case V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY: + p_hdr10_mastering = p; + + for (i = 0; i < 3; ++i) { + if (p_hdr10_mastering->display_primaries_x[i] < + V4L2_HDR10_MASTERING_PRIMARIES_X_LOW || + p_hdr10_mastering->display_primaries_x[i] > + V4L2_HDR10_MASTERING_PRIMARIES_X_HIGH || + p_hdr10_mastering->display_primaries_y[i] < + V4L2_HDR10_MASTERING_PRIMARIES_Y_LOW || + p_hdr10_mastering->display_primaries_y[i] > + V4L2_HDR10_MASTERING_PRIMARIES_Y_HIGH) + return -EINVAL; + } + + if (p_hdr10_mastering->white_point_x < + V4L2_HDR10_MASTERING_WHITE_POINT_X_LOW || + p_hdr10_mastering->white_point_x > + V4L2_HDR10_MASTERING_WHITE_POINT_X_HIGH || + p_hdr10_mastering->white_point_y < + V4L2_HDR10_MASTERING_WHITE_POINT_Y_LOW || + p_hdr10_mastering->white_point_y > + V4L2_HDR10_MASTERING_WHITE_POINT_Y_HIGH) + return -EINVAL; + + if (p_hdr10_mastering->max_display_mastering_luminance < + V4L2_HDR10_MASTERING_MAX_LUMA_LOW || + p_hdr10_mastering->max_display_mastering_luminance > + V4L2_HDR10_MASTERING_MAX_LUMA_HIGH || + p_hdr10_mastering->min_display_mastering_luminance < + V4L2_HDR10_MASTERING_MIN_LUMA_LOW || + p_hdr10_mastering->min_display_mastering_luminance > + V4L2_HDR10_MASTERING_MIN_LUMA_HIGH) + return -EINVAL; + + /* The following restriction comes from ITU-T Rec. H.265 spec */ + if (p_hdr10_mastering->max_display_mastering_luminance == + V4L2_HDR10_MASTERING_MAX_LUMA_LOW && + p_hdr10_mastering->min_display_mastering_luminance == + V4L2_HDR10_MASTERING_MIN_LUMA_HIGH) + return -EINVAL; + + break; + + case V4L2_CTRL_TYPE_AREA: + area = p; + if (!area->width || !area->height) + return -EINVAL; + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx, + union v4l2_ctrl_ptr ptr) +{ + size_t len; + u64 offset; + s64 val; + + switch ((u32)ctrl->type) { + case V4L2_CTRL_TYPE_INTEGER: + return ROUND_TO_RANGE(ptr.p_s32[idx], u32, ctrl); + case V4L2_CTRL_TYPE_INTEGER64: + /* + * We can't use the ROUND_TO_RANGE define here due to + * the u64 divide that needs special care. + */ + val = ptr.p_s64[idx]; + if (ctrl->maximum >= 0 && val >= ctrl->maximum - (s64)(ctrl->step / 2)) + val = ctrl->maximum; + else + val += (s64)(ctrl->step / 2); + val = clamp_t(s64, val, ctrl->minimum, ctrl->maximum); + offset = val - ctrl->minimum; + do_div(offset, ctrl->step); + ptr.p_s64[idx] = ctrl->minimum + offset * ctrl->step; + return 0; + case V4L2_CTRL_TYPE_U8: + return ROUND_TO_RANGE(ptr.p_u8[idx], u8, ctrl); + case V4L2_CTRL_TYPE_U16: + return ROUND_TO_RANGE(ptr.p_u16[idx], u16, ctrl); + case V4L2_CTRL_TYPE_U32: + return ROUND_TO_RANGE(ptr.p_u32[idx], u32, ctrl); + + case V4L2_CTRL_TYPE_BOOLEAN: + ptr.p_s32[idx] = !!ptr.p_s32[idx]; + return 0; + + case V4L2_CTRL_TYPE_MENU: + case V4L2_CTRL_TYPE_INTEGER_MENU: + if (ptr.p_s32[idx] < ctrl->minimum || ptr.p_s32[idx] > ctrl->maximum) + return -ERANGE; + if (ptr.p_s32[idx] < BITS_PER_LONG_LONG && + (ctrl->menu_skip_mask & BIT_ULL(ptr.p_s32[idx]))) + return -EINVAL; + if (ctrl->type == V4L2_CTRL_TYPE_MENU && + ctrl->qmenu[ptr.p_s32[idx]][0] == '\0') + return -EINVAL; + return 0; + + case V4L2_CTRL_TYPE_BITMASK: + ptr.p_s32[idx] &= ctrl->maximum; + return 0; + + case V4L2_CTRL_TYPE_BUTTON: + case V4L2_CTRL_TYPE_CTRL_CLASS: + ptr.p_s32[idx] = 0; + return 0; + + case V4L2_CTRL_TYPE_STRING: + idx *= ctrl->elem_size; + len = strlen(ptr.p_char + idx); + if (len < ctrl->minimum) + return -ERANGE; + if ((len - (u32)ctrl->minimum) % (u32)ctrl->step) + return -ERANGE; + return 0; + + default: + return std_validate_compound(ctrl, idx, ptr); + } +} + +static const struct v4l2_ctrl_type_ops std_type_ops = { + .equal = std_equal, + .init = std_init, + .log = std_log, + .validate = std_validate, +}; + +void v4l2_ctrl_notify(struct v4l2_ctrl *ctrl, v4l2_ctrl_notify_fnc notify, void *priv) +{ + if (!ctrl) + return; + if (!notify) { + ctrl->call_notify = 0; + return; + } + if (WARN_ON(ctrl->handler->notify && ctrl->handler->notify != notify)) + return; + ctrl->handler->notify = notify; + ctrl->handler->notify_priv = priv; + ctrl->call_notify = 1; +} +EXPORT_SYMBOL(v4l2_ctrl_notify); + +/* Copy the one value to another. */ +static void ptr_to_ptr(struct v4l2_ctrl *ctrl, + union v4l2_ctrl_ptr from, union v4l2_ctrl_ptr to) +{ + if (ctrl == NULL) + return; + memcpy(to.p, from.p_const, ctrl->elems * ctrl->elem_size); +} + +/* Copy the new value to the current value. */ +void new_to_cur(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 ch_flags) +{ + bool changed; + + if (ctrl == NULL) + return; + + /* has_changed is set by cluster_changed */ + changed = ctrl->has_changed; + if (changed) + ptr_to_ptr(ctrl, ctrl->p_new, ctrl->p_cur); + + if (ch_flags & V4L2_EVENT_CTRL_CH_FLAGS) { + /* Note: CH_FLAGS is only set for auto clusters. */ + ctrl->flags &= + ~(V4L2_CTRL_FLAG_INACTIVE | V4L2_CTRL_FLAG_VOLATILE); + if (!is_cur_manual(ctrl->cluster[0])) { + ctrl->flags |= V4L2_CTRL_FLAG_INACTIVE; + if (ctrl->cluster[0]->has_volatiles) + ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE; + } + fh = NULL; + } + if (changed || ch_flags) { + /* If a control was changed that was not one of the controls + modified by the application, then send the event to all. */ + if (!ctrl->is_new) + fh = NULL; + send_event(fh, ctrl, + (changed ? V4L2_EVENT_CTRL_CH_VALUE : 0) | ch_flags); + if (ctrl->call_notify && changed && ctrl->handler->notify) + ctrl->handler->notify(ctrl, ctrl->handler->notify_priv); + } +} + +/* Copy the current value to the new value */ +void cur_to_new(struct v4l2_ctrl *ctrl) +{ + if (ctrl == NULL) + return; + ptr_to_ptr(ctrl, ctrl->p_cur, ctrl->p_new); +} + +/* Copy the new value to the request value */ +void new_to_req(struct v4l2_ctrl_ref *ref) +{ + if (!ref) + return; + ptr_to_ptr(ref->ctrl, ref->ctrl->p_new, ref->p_req); + ref->valid_p_req = true; +} + +/* Copy the current value to the request value */ +void cur_to_req(struct v4l2_ctrl_ref *ref) +{ + if (!ref) + return; + ptr_to_ptr(ref->ctrl, ref->ctrl->p_cur, ref->p_req); + ref->valid_p_req = true; +} + +/* Copy the request value to the new value */ +void req_to_new(struct v4l2_ctrl_ref *ref) +{ + if (!ref) + return; + if (ref->valid_p_req) + ptr_to_ptr(ref->ctrl, ref->p_req, ref->ctrl->p_new); + else + ptr_to_ptr(ref->ctrl, ref->ctrl->p_cur, ref->ctrl->p_new); +} + +/* Control range checking */ +int check_range(enum v4l2_ctrl_type type, + s64 min, s64 max, u64 step, s64 def) +{ + switch (type) { + case V4L2_CTRL_TYPE_BOOLEAN: + if (step != 1 || max > 1 || min < 0) + return -ERANGE; + fallthrough; + case V4L2_CTRL_TYPE_U8: + case V4L2_CTRL_TYPE_U16: + case V4L2_CTRL_TYPE_U32: + case V4L2_CTRL_TYPE_INTEGER: + case V4L2_CTRL_TYPE_INTEGER64: + if (step == 0 || min > max || def < min || def > max) + return -ERANGE; + return 0; + case V4L2_CTRL_TYPE_BITMASK: + if (step || min || !max || (def & ~max)) + return -ERANGE; + return 0; + case V4L2_CTRL_TYPE_MENU: + case V4L2_CTRL_TYPE_INTEGER_MENU: + if (min > max || def < min || def > max) + return -ERANGE; + /* Note: step == menu_skip_mask for menu controls. + So here we check if the default value is masked out. */ + if (step && ((1 << def) & step)) + return -EINVAL; + return 0; + case V4L2_CTRL_TYPE_STRING: + if (min > max || min < 0 || step < 1 || def) + return -ERANGE; + return 0; + default: + return 0; + } +} + +/* Validate a new control */ +int validate_new(const struct v4l2_ctrl *ctrl, union v4l2_ctrl_ptr p_new) +{ + unsigned idx; + int err = 0; + + for (idx = 0; !err && idx < ctrl->elems; idx++) + err = ctrl->type_ops->validate(ctrl, idx, p_new); + return err; +} + +/* Set the handler's error code if it wasn't set earlier already */ +static inline int handler_set_err(struct v4l2_ctrl_handler *hdl, int err) +{ + if (hdl->error == 0) + hdl->error = err; + return err; +} + +/* Initialize the handler */ +int v4l2_ctrl_handler_init_class(struct v4l2_ctrl_handler *hdl, + unsigned nr_of_controls_hint, + struct lock_class_key *key, const char *name) +{ + mutex_init(&hdl->_lock); + hdl->lock = &hdl->_lock; + lockdep_set_class_and_name(hdl->lock, key, name); + INIT_LIST_HEAD(&hdl->ctrls); + INIT_LIST_HEAD(&hdl->ctrl_refs); + hdl->nr_of_buckets = 1 + nr_of_controls_hint / 8; + hdl->buckets = kvmalloc_array(hdl->nr_of_buckets, + sizeof(hdl->buckets[0]), + GFP_KERNEL | __GFP_ZERO); + hdl->error = hdl->buckets ? 0 : -ENOMEM; + v4l2_ctrl_handler_init_request(hdl); + return hdl->error; +} +EXPORT_SYMBOL(v4l2_ctrl_handler_init_class); + +/* Free all controls and control refs */ +void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl) +{ + struct v4l2_ctrl_ref *ref, *next_ref; + struct v4l2_ctrl *ctrl, *next_ctrl; + struct v4l2_subscribed_event *sev, *next_sev; + + if (hdl == NULL || hdl->buckets == NULL) + return; + + v4l2_ctrl_handler_free_request(hdl); + + mutex_lock(hdl->lock); + /* Free all nodes */ + list_for_each_entry_safe(ref, next_ref, &hdl->ctrl_refs, node) { + list_del(&ref->node); + kfree(ref); + } + /* Free all controls owned by the handler */ + list_for_each_entry_safe(ctrl, next_ctrl, &hdl->ctrls, node) { + list_del(&ctrl->node); + list_for_each_entry_safe(sev, next_sev, &ctrl->ev_subs, node) + list_del(&sev->node); + kvfree(ctrl); + } + kvfree(hdl->buckets); + hdl->buckets = NULL; + hdl->cached = NULL; + hdl->error = 0; + mutex_unlock(hdl->lock); + mutex_destroy(&hdl->_lock); +} +EXPORT_SYMBOL(v4l2_ctrl_handler_free); + +/* For backwards compatibility: V4L2_CID_PRIVATE_BASE should no longer + be used except in G_CTRL, S_CTRL, QUERYCTRL and QUERYMENU when dealing + with applications that do not use the NEXT_CTRL flag. + + We just find the n-th private user control. It's O(N), but that should not + be an issue in this particular case. */ +static struct v4l2_ctrl_ref *find_private_ref( + struct v4l2_ctrl_handler *hdl, u32 id) +{ + struct v4l2_ctrl_ref *ref; + + id -= V4L2_CID_PRIVATE_BASE; + list_for_each_entry(ref, &hdl->ctrl_refs, node) { + /* Search for private user controls that are compatible with + VIDIOC_G/S_CTRL. */ + if (V4L2_CTRL_ID2WHICH(ref->ctrl->id) == V4L2_CTRL_CLASS_USER && + V4L2_CTRL_DRIVER_PRIV(ref->ctrl->id)) { + if (!ref->ctrl->is_int) + continue; + if (id == 0) + return ref; + id--; + } + } + return NULL; +} + +/* Find a control with the given ID. */ +struct v4l2_ctrl_ref *find_ref(struct v4l2_ctrl_handler *hdl, u32 id) +{ + struct v4l2_ctrl_ref *ref; + int bucket; + + id &= V4L2_CTRL_ID_MASK; + + /* Old-style private controls need special handling */ + if (id >= V4L2_CID_PRIVATE_BASE) + return find_private_ref(hdl, id); + bucket = id % hdl->nr_of_buckets; + + /* Simple optimization: cache the last control found */ + if (hdl->cached && hdl->cached->ctrl->id == id) + return hdl->cached; + + /* Not in cache, search the hash */ + ref = hdl->buckets ? hdl->buckets[bucket] : NULL; + while (ref && ref->ctrl->id != id) + ref = ref->next; + + if (ref) + hdl->cached = ref; /* cache it! */ + return ref; +} + +/* Find a control with the given ID. Take the handler's lock first. */ +struct v4l2_ctrl_ref *find_ref_lock(struct v4l2_ctrl_handler *hdl, u32 id) +{ + struct v4l2_ctrl_ref *ref = NULL; + + if (hdl) { + mutex_lock(hdl->lock); + ref = find_ref(hdl, id); + mutex_unlock(hdl->lock); + } + return ref; +} + +/* Find a control with the given ID. */ +struct v4l2_ctrl *v4l2_ctrl_find(struct v4l2_ctrl_handler *hdl, u32 id) +{ + struct v4l2_ctrl_ref *ref = find_ref_lock(hdl, id); + + return ref ? ref->ctrl : NULL; +} +EXPORT_SYMBOL(v4l2_ctrl_find); + +/* Allocate a new v4l2_ctrl_ref and hook it into the handler. */ +int handler_new_ref(struct v4l2_ctrl_handler *hdl, + struct v4l2_ctrl *ctrl, + struct v4l2_ctrl_ref **ctrl_ref, + bool from_other_dev, bool allocate_req) +{ + struct v4l2_ctrl_ref *ref; + struct v4l2_ctrl_ref *new_ref; + u32 id = ctrl->id; + u32 class_ctrl = V4L2_CTRL_ID2WHICH(id) | 1; + int bucket = id % hdl->nr_of_buckets; /* which bucket to use */ + unsigned int size_extra_req = 0; + + if (ctrl_ref) + *ctrl_ref = NULL; + + /* + * Automatically add the control class if it is not yet present and + * the new control is not a compound control. + */ + if (ctrl->type < V4L2_CTRL_COMPOUND_TYPES && + id != class_ctrl && find_ref_lock(hdl, class_ctrl) == NULL) + if (!v4l2_ctrl_new_std(hdl, NULL, class_ctrl, 0, 0, 0, 0)) + return hdl->error; + + if (hdl->error) + return hdl->error; + + if (allocate_req) + size_extra_req = ctrl->elems * ctrl->elem_size; + new_ref = kzalloc(sizeof(*new_ref) + size_extra_req, GFP_KERNEL); + if (!new_ref) + return handler_set_err(hdl, -ENOMEM); + new_ref->ctrl = ctrl; + new_ref->from_other_dev = from_other_dev; + if (size_extra_req) + new_ref->p_req.p = &new_ref[1]; + + INIT_LIST_HEAD(&new_ref->node); + + mutex_lock(hdl->lock); + + /* Add immediately at the end of the list if the list is empty, or if + the last element in the list has a lower ID. + This ensures that when elements are added in ascending order the + insertion is an O(1) operation. */ + if (list_empty(&hdl->ctrl_refs) || id > node2id(hdl->ctrl_refs.prev)) { + list_add_tail(&new_ref->node, &hdl->ctrl_refs); + goto insert_in_hash; + } + + /* Find insert position in sorted list */ + list_for_each_entry(ref, &hdl->ctrl_refs, node) { + if (ref->ctrl->id < id) + continue; + /* Don't add duplicates */ + if (ref->ctrl->id == id) { + kfree(new_ref); + goto unlock; + } + list_add(&new_ref->node, ref->node.prev); + break; + } + +insert_in_hash: + /* Insert the control node in the hash */ + new_ref->next = hdl->buckets[bucket]; + hdl->buckets[bucket] = new_ref; + if (ctrl_ref) + *ctrl_ref = new_ref; + if (ctrl->handler == hdl) { + /* By default each control starts in a cluster of its own. + * new_ref->ctrl is basically a cluster array with one + * element, so that's perfect to use as the cluster pointer. + * But only do this for the handler that owns the control. + */ + ctrl->cluster = &new_ref->ctrl; + ctrl->ncontrols = 1; + } + +unlock: + mutex_unlock(hdl->lock); + return 0; +} + +/* Add a new control */ +static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_ops *ops, + const struct v4l2_ctrl_type_ops *type_ops, + u32 id, const char *name, enum v4l2_ctrl_type type, + s64 min, s64 max, u64 step, s64 def, + const u32 dims[V4L2_CTRL_MAX_DIMS], u32 elem_size, + u32 flags, const char * const *qmenu, + const s64 *qmenu_int, const union v4l2_ctrl_ptr p_def, + void *priv) +{ + struct v4l2_ctrl *ctrl; + unsigned sz_extra; + unsigned nr_of_dims = 0; + unsigned elems = 1; + bool is_array; + unsigned tot_ctrl_size; + unsigned idx; + void *data; + int err; + + if (hdl->error) + return NULL; + + while (dims && dims[nr_of_dims]) { + elems *= dims[nr_of_dims]; + nr_of_dims++; + if (nr_of_dims == V4L2_CTRL_MAX_DIMS) + break; + } + is_array = nr_of_dims > 0; + + /* Prefill elem_size for all types handled by std_type_ops */ + switch ((u32)type) { + case V4L2_CTRL_TYPE_INTEGER64: + elem_size = sizeof(s64); + break; + case V4L2_CTRL_TYPE_STRING: + elem_size = max + 1; + break; + case V4L2_CTRL_TYPE_U8: + elem_size = sizeof(u8); + break; + case V4L2_CTRL_TYPE_U16: + elem_size = sizeof(u16); + break; + case V4L2_CTRL_TYPE_U32: + elem_size = sizeof(u32); + break; + case V4L2_CTRL_TYPE_MPEG2_SEQUENCE: + elem_size = sizeof(struct v4l2_ctrl_mpeg2_sequence); + break; + case V4L2_CTRL_TYPE_MPEG2_PICTURE: + elem_size = sizeof(struct v4l2_ctrl_mpeg2_picture); + break; + case V4L2_CTRL_TYPE_MPEG2_QUANTISATION: + elem_size = sizeof(struct v4l2_ctrl_mpeg2_quantisation); + break; + case V4L2_CTRL_TYPE_FWHT_PARAMS: + elem_size = sizeof(struct v4l2_ctrl_fwht_params); + break; + case V4L2_CTRL_TYPE_H264_SPS: + elem_size = sizeof(struct v4l2_ctrl_h264_sps); + break; + case V4L2_CTRL_TYPE_H264_PPS: + elem_size = sizeof(struct v4l2_ctrl_h264_pps); + break; + case V4L2_CTRL_TYPE_H264_SCALING_MATRIX: + elem_size = sizeof(struct v4l2_ctrl_h264_scaling_matrix); + break; + case V4L2_CTRL_TYPE_H264_SLICE_PARAMS: + elem_size = sizeof(struct v4l2_ctrl_h264_slice_params); + break; + case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: + elem_size = sizeof(struct v4l2_ctrl_h264_decode_params); + break; + case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS: + elem_size = sizeof(struct v4l2_ctrl_h264_pred_weights); + break; + case V4L2_CTRL_TYPE_VP8_FRAME: + elem_size = sizeof(struct v4l2_ctrl_vp8_frame); + break; + case V4L2_CTRL_TYPE_HEVC_SPS: + elem_size = sizeof(struct v4l2_ctrl_hevc_sps); + break; + case V4L2_CTRL_TYPE_HEVC_PPS: + elem_size = sizeof(struct v4l2_ctrl_hevc_pps); + break; + case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: + elem_size = sizeof(struct v4l2_ctrl_hevc_slice_params); + break; + case V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS: + elem_size = sizeof(struct v4l2_ctrl_hevc_decode_params); + break; + case V4L2_CTRL_TYPE_HDR10_CLL_INFO: + elem_size = sizeof(struct v4l2_ctrl_hdr10_cll_info); + break; + case V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY: + elem_size = sizeof(struct v4l2_ctrl_hdr10_mastering_display); + break; + case V4L2_CTRL_TYPE_AREA: + elem_size = sizeof(struct v4l2_area); + break; + default: + if (type < V4L2_CTRL_COMPOUND_TYPES) + elem_size = sizeof(s32); + break; + } + tot_ctrl_size = elem_size * elems; + + /* Sanity checks */ + if (id == 0 || name == NULL || !elem_size || + id >= V4L2_CID_PRIVATE_BASE || + (type == V4L2_CTRL_TYPE_MENU && qmenu == NULL) || + (type == V4L2_CTRL_TYPE_INTEGER_MENU && qmenu_int == NULL)) { + handler_set_err(hdl, -ERANGE); + return NULL; + } + err = check_range(type, min, max, step, def); + if (err) { + handler_set_err(hdl, err); + return NULL; + } + if (is_array && + (type == V4L2_CTRL_TYPE_BUTTON || + type == V4L2_CTRL_TYPE_CTRL_CLASS)) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + + sz_extra = 0; + if (type == V4L2_CTRL_TYPE_BUTTON) + flags |= V4L2_CTRL_FLAG_WRITE_ONLY | + V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; + else if (type == V4L2_CTRL_TYPE_CTRL_CLASS) + flags |= V4L2_CTRL_FLAG_READ_ONLY; + else if (type == V4L2_CTRL_TYPE_INTEGER64 || + type == V4L2_CTRL_TYPE_STRING || + type >= V4L2_CTRL_COMPOUND_TYPES || + is_array) + sz_extra += 2 * tot_ctrl_size; + + if (type >= V4L2_CTRL_COMPOUND_TYPES && p_def.p_const) + sz_extra += elem_size; + + ctrl = kvzalloc(sizeof(*ctrl) + sz_extra, GFP_KERNEL); + if (ctrl == NULL) { + handler_set_err(hdl, -ENOMEM); + return NULL; + } + + INIT_LIST_HEAD(&ctrl->node); + INIT_LIST_HEAD(&ctrl->ev_subs); + ctrl->handler = hdl; + ctrl->ops = ops; + ctrl->type_ops = type_ops ? type_ops : &std_type_ops; + ctrl->id = id; + ctrl->name = name; + ctrl->type = type; + ctrl->flags = flags; + ctrl->minimum = min; + ctrl->maximum = max; + ctrl->step = step; + ctrl->default_value = def; + ctrl->is_string = !is_array && type == V4L2_CTRL_TYPE_STRING; + ctrl->is_ptr = is_array || type >= V4L2_CTRL_COMPOUND_TYPES || ctrl->is_string; + ctrl->is_int = !ctrl->is_ptr && type != V4L2_CTRL_TYPE_INTEGER64; + ctrl->is_array = is_array; + ctrl->elems = elems; + ctrl->nr_of_dims = nr_of_dims; + if (nr_of_dims) + memcpy(ctrl->dims, dims, nr_of_dims * sizeof(dims[0])); + ctrl->elem_size = elem_size; + if (type == V4L2_CTRL_TYPE_MENU) + ctrl->qmenu = qmenu; + else if (type == V4L2_CTRL_TYPE_INTEGER_MENU) + ctrl->qmenu_int = qmenu_int; + ctrl->priv = priv; + ctrl->cur.val = ctrl->val = def; + data = &ctrl[1]; + + if (!ctrl->is_int) { + ctrl->p_new.p = data; + ctrl->p_cur.p = data + tot_ctrl_size; + } else { + ctrl->p_new.p = &ctrl->val; + ctrl->p_cur.p = &ctrl->cur.val; + } + + if (type >= V4L2_CTRL_COMPOUND_TYPES && p_def.p_const) { + ctrl->p_def.p = ctrl->p_cur.p + tot_ctrl_size; + memcpy(ctrl->p_def.p, p_def.p_const, elem_size); + } + + for (idx = 0; idx < elems; idx++) { + ctrl->type_ops->init(ctrl, idx, ctrl->p_cur); + ctrl->type_ops->init(ctrl, idx, ctrl->p_new); + } + + if (handler_new_ref(hdl, ctrl, NULL, false, false)) { + kvfree(ctrl); + return NULL; + } + mutex_lock(hdl->lock); + list_add_tail(&ctrl->node, &hdl->ctrls); + mutex_unlock(hdl->lock); + return ctrl; +} + +struct v4l2_ctrl *v4l2_ctrl_new_custom(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_config *cfg, void *priv) +{ + bool is_menu; + struct v4l2_ctrl *ctrl; + const char *name = cfg->name; + const char * const *qmenu = cfg->qmenu; + const s64 *qmenu_int = cfg->qmenu_int; + enum v4l2_ctrl_type type = cfg->type; + u32 flags = cfg->flags; + s64 min = cfg->min; + s64 max = cfg->max; + u64 step = cfg->step; + s64 def = cfg->def; + + if (name == NULL) + v4l2_ctrl_fill(cfg->id, &name, &type, &min, &max, &step, + &def, &flags); + + is_menu = (type == V4L2_CTRL_TYPE_MENU || + type == V4L2_CTRL_TYPE_INTEGER_MENU); + if (is_menu) + WARN_ON(step); + else + WARN_ON(cfg->menu_skip_mask); + if (type == V4L2_CTRL_TYPE_MENU && !qmenu) { + qmenu = v4l2_ctrl_get_menu(cfg->id); + } else if (type == V4L2_CTRL_TYPE_INTEGER_MENU && !qmenu_int) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + + ctrl = v4l2_ctrl_new(hdl, cfg->ops, cfg->type_ops, cfg->id, name, + type, min, max, + is_menu ? cfg->menu_skip_mask : step, def, + cfg->dims, cfg->elem_size, + flags, qmenu, qmenu_int, cfg->p_def, priv); + if (ctrl) + ctrl->is_private = cfg->is_private; + return ctrl; +} +EXPORT_SYMBOL(v4l2_ctrl_new_custom); + +/* Helper function for standard non-menu controls */ +struct v4l2_ctrl *v4l2_ctrl_new_std(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_ops *ops, + u32 id, s64 min, s64 max, u64 step, s64 def) +{ + const char *name; + enum v4l2_ctrl_type type; + u32 flags; + + v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); + if (type == V4L2_CTRL_TYPE_MENU || + type == V4L2_CTRL_TYPE_INTEGER_MENU || + type >= V4L2_CTRL_COMPOUND_TYPES) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, + min, max, step, def, NULL, 0, + flags, NULL, NULL, ptr_null, NULL); +} +EXPORT_SYMBOL(v4l2_ctrl_new_std); + +/* Helper function for standard menu controls */ +struct v4l2_ctrl *v4l2_ctrl_new_std_menu(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_ops *ops, + u32 id, u8 _max, u64 mask, u8 _def) +{ + const char * const *qmenu = NULL; + const s64 *qmenu_int = NULL; + unsigned int qmenu_int_len = 0; + const char *name; + enum v4l2_ctrl_type type; + s64 min; + s64 max = _max; + s64 def = _def; + u64 step; + u32 flags; + + v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); + + if (type == V4L2_CTRL_TYPE_MENU) + qmenu = v4l2_ctrl_get_menu(id); + else if (type == V4L2_CTRL_TYPE_INTEGER_MENU) + qmenu_int = v4l2_ctrl_get_int_menu(id, &qmenu_int_len); + + if ((!qmenu && !qmenu_int) || (qmenu_int && max > qmenu_int_len)) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, + 0, max, mask, def, NULL, 0, + flags, qmenu, qmenu_int, ptr_null, NULL); +} +EXPORT_SYMBOL(v4l2_ctrl_new_std_menu); + +/* Helper function for standard menu controls with driver defined menu */ +struct v4l2_ctrl *v4l2_ctrl_new_std_menu_items(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_ops *ops, u32 id, u8 _max, + u64 mask, u8 _def, const char * const *qmenu) +{ + enum v4l2_ctrl_type type; + const char *name; + u32 flags; + u64 step; + s64 min; + s64 max = _max; + s64 def = _def; + + /* v4l2_ctrl_new_std_menu_items() should only be called for + * standard controls without a standard menu. + */ + if (v4l2_ctrl_get_menu(id)) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + + v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); + if (type != V4L2_CTRL_TYPE_MENU || qmenu == NULL) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, + 0, max, mask, def, NULL, 0, + flags, qmenu, NULL, ptr_null, NULL); + +} +EXPORT_SYMBOL(v4l2_ctrl_new_std_menu_items); + +/* Helper function for standard compound controls */ +struct v4l2_ctrl *v4l2_ctrl_new_std_compound(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_ops *ops, u32 id, + const union v4l2_ctrl_ptr p_def) +{ + const char *name; + enum v4l2_ctrl_type type; + u32 flags; + s64 min, max, step, def; + + v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); + if (type < V4L2_CTRL_COMPOUND_TYPES) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, + min, max, step, def, NULL, 0, + flags, NULL, NULL, p_def, NULL); +} +EXPORT_SYMBOL(v4l2_ctrl_new_std_compound); + +/* Helper function for standard integer menu controls */ +struct v4l2_ctrl *v4l2_ctrl_new_int_menu(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_ops *ops, + u32 id, u8 _max, u8 _def, const s64 *qmenu_int) +{ + const char *name; + enum v4l2_ctrl_type type; + s64 min; + u64 step; + s64 max = _max; + s64 def = _def; + u32 flags; + + v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); + if (type != V4L2_CTRL_TYPE_INTEGER_MENU) { + handler_set_err(hdl, -EINVAL); + return NULL; + } + return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, + 0, max, 0, def, NULL, 0, + flags, NULL, qmenu_int, ptr_null, NULL); +} +EXPORT_SYMBOL(v4l2_ctrl_new_int_menu); + +/* Add the controls from another handler to our own. */ +int v4l2_ctrl_add_handler(struct v4l2_ctrl_handler *hdl, + struct v4l2_ctrl_handler *add, + bool (*filter)(const struct v4l2_ctrl *ctrl), + bool from_other_dev) +{ + struct v4l2_ctrl_ref *ref; + int ret = 0; + + /* Do nothing if either handler is NULL or if they are the same */ + if (!hdl || !add || hdl == add) + return 0; + if (hdl->error) + return hdl->error; + mutex_lock(add->lock); + list_for_each_entry(ref, &add->ctrl_refs, node) { + struct v4l2_ctrl *ctrl = ref->ctrl; + + /* Skip handler-private controls. */ + if (ctrl->is_private) + continue; + /* And control classes */ + if (ctrl->type == V4L2_CTRL_TYPE_CTRL_CLASS) + continue; + /* Filter any unwanted controls */ + if (filter && !filter(ctrl)) + continue; + ret = handler_new_ref(hdl, ctrl, NULL, from_other_dev, false); + if (ret) + break; + } + mutex_unlock(add->lock); + return ret; +} +EXPORT_SYMBOL(v4l2_ctrl_add_handler); + +bool v4l2_ctrl_radio_filter(const struct v4l2_ctrl *ctrl) +{ + if (V4L2_CTRL_ID2WHICH(ctrl->id) == V4L2_CTRL_CLASS_FM_TX) + return true; + if (V4L2_CTRL_ID2WHICH(ctrl->id) == V4L2_CTRL_CLASS_FM_RX) + return true; + switch (ctrl->id) { + case V4L2_CID_AUDIO_MUTE: + case V4L2_CID_AUDIO_VOLUME: + case V4L2_CID_AUDIO_BALANCE: + case V4L2_CID_AUDIO_BASS: + case V4L2_CID_AUDIO_TREBLE: + case V4L2_CID_AUDIO_LOUDNESS: + return true; + default: + break; + } + return false; +} +EXPORT_SYMBOL(v4l2_ctrl_radio_filter); + +/* Cluster controls */ +void v4l2_ctrl_cluster(unsigned ncontrols, struct v4l2_ctrl **controls) +{ + bool has_volatiles = false; + int i; + + /* The first control is the master control and it must not be NULL */ + if (WARN_ON(ncontrols == 0 || controls[0] == NULL)) + return; + + for (i = 0; i < ncontrols; i++) { + if (controls[i]) { + controls[i]->cluster = controls; + controls[i]->ncontrols = ncontrols; + if (controls[i]->flags & V4L2_CTRL_FLAG_VOLATILE) + has_volatiles = true; + } + } + controls[0]->has_volatiles = has_volatiles; +} +EXPORT_SYMBOL(v4l2_ctrl_cluster); + +void v4l2_ctrl_auto_cluster(unsigned ncontrols, struct v4l2_ctrl **controls, + u8 manual_val, bool set_volatile) +{ + struct v4l2_ctrl *master = controls[0]; + u32 flag = 0; + int i; + + v4l2_ctrl_cluster(ncontrols, controls); + WARN_ON(ncontrols <= 1); + WARN_ON(manual_val < master->minimum || manual_val > master->maximum); + WARN_ON(set_volatile && !has_op(master, g_volatile_ctrl)); + master->is_auto = true; + master->has_volatiles = set_volatile; + master->manual_mode_value = manual_val; + master->flags |= V4L2_CTRL_FLAG_UPDATE; + + if (!is_cur_manual(master)) + flag = V4L2_CTRL_FLAG_INACTIVE | + (set_volatile ? V4L2_CTRL_FLAG_VOLATILE : 0); + + for (i = 1; i < ncontrols; i++) + if (controls[i]) + controls[i]->flags |= flag; +} +EXPORT_SYMBOL(v4l2_ctrl_auto_cluster); + +/* + * Obtain the current volatile values of an autocluster and mark them + * as new. + */ +void update_from_auto_cluster(struct v4l2_ctrl *master) +{ + int i; + + for (i = 1; i < master->ncontrols; i++) + cur_to_new(master->cluster[i]); + if (!call_op(master, g_volatile_ctrl)) + for (i = 1; i < master->ncontrols; i++) + if (master->cluster[i]) + master->cluster[i]->is_new = 1; +} + +/* + * Return non-zero if one or more of the controls in the cluster has a new + * value that differs from the current value. + */ +static int cluster_changed(struct v4l2_ctrl *master) +{ + bool changed = false; + unsigned int idx; + int i; + + for (i = 0; i < master->ncontrols; i++) { + struct v4l2_ctrl *ctrl = master->cluster[i]; + bool ctrl_changed = false; + + if (!ctrl) + continue; + + if (ctrl->flags & V4L2_CTRL_FLAG_EXECUTE_ON_WRITE) { + changed = true; + ctrl_changed = true; + } + + /* + * Set has_changed to false to avoid generating + * the event V4L2_EVENT_CTRL_CH_VALUE + */ + if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) { + ctrl->has_changed = false; + continue; + } + + for (idx = 0; !ctrl_changed && idx < ctrl->elems; idx++) + ctrl_changed = !ctrl->type_ops->equal(ctrl, idx, + ctrl->p_cur, ctrl->p_new); + ctrl->has_changed = ctrl_changed; + changed |= ctrl->has_changed; + } + return changed; +} + +/* + * Core function that calls try/s_ctrl and ensures that the new value is + * copied to the current value on a set. + * Must be called with ctrl->handler->lock held. + */ +int try_or_set_cluster(struct v4l2_fh *fh, struct v4l2_ctrl *master, + bool set, u32 ch_flags) +{ + bool update_flag; + int ret; + int i; + + /* + * Go through the cluster and either validate the new value or + * (if no new value was set), copy the current value to the new + * value, ensuring a consistent view for the control ops when + * called. + */ + for (i = 0; i < master->ncontrols; i++) { + struct v4l2_ctrl *ctrl = master->cluster[i]; + + if (!ctrl) + continue; + + if (!ctrl->is_new) { + cur_to_new(ctrl); + continue; + } + /* + * Check again: it may have changed since the + * previous check in try_or_set_ext_ctrls(). + */ + if (set && (ctrl->flags & V4L2_CTRL_FLAG_GRABBED)) + return -EBUSY; + } + + ret = call_op(master, try_ctrl); + + /* Don't set if there is no change */ + if (ret || !set || !cluster_changed(master)) + return ret; + ret = call_op(master, s_ctrl); + if (ret) + return ret; + + /* If OK, then make the new values permanent. */ + update_flag = is_cur_manual(master) != is_new_manual(master); + + for (i = 0; i < master->ncontrols; i++) { + /* + * If we switch from auto to manual mode, and this cluster + * contains volatile controls, then all non-master controls + * have to be marked as changed. The 'new' value contains + * the volatile value (obtained by update_from_auto_cluster), + * which now has to become the current value. + */ + if (i && update_flag && is_new_manual(master) && + master->has_volatiles && master->cluster[i]) + master->cluster[i]->has_changed = true; + + new_to_cur(fh, master->cluster[i], ch_flags | + ((update_flag && i > 0) ? V4L2_EVENT_CTRL_CH_FLAGS : 0)); + } + return 0; +} + +/* Activate/deactivate a control. */ +void v4l2_ctrl_activate(struct v4l2_ctrl *ctrl, bool active) +{ + /* invert since the actual flag is called 'inactive' */ + bool inactive = !active; + bool old; + + if (ctrl == NULL) + return; + + if (inactive) + /* set V4L2_CTRL_FLAG_INACTIVE */ + old = test_and_set_bit(4, &ctrl->flags); + else + /* clear V4L2_CTRL_FLAG_INACTIVE */ + old = test_and_clear_bit(4, &ctrl->flags); + if (old != inactive) + send_event(NULL, ctrl, V4L2_EVENT_CTRL_CH_FLAGS); +} +EXPORT_SYMBOL(v4l2_ctrl_activate); + +void __v4l2_ctrl_grab(struct v4l2_ctrl *ctrl, bool grabbed) +{ + bool old; + + if (ctrl == NULL) + return; + + lockdep_assert_held(ctrl->handler->lock); + + if (grabbed) + /* set V4L2_CTRL_FLAG_GRABBED */ + old = test_and_set_bit(1, &ctrl->flags); + else + /* clear V4L2_CTRL_FLAG_GRABBED */ + old = test_and_clear_bit(1, &ctrl->flags); + if (old != grabbed) + send_event(NULL, ctrl, V4L2_EVENT_CTRL_CH_FLAGS); +} +EXPORT_SYMBOL(__v4l2_ctrl_grab); + +/* Call s_ctrl for all controls owned by the handler */ +int __v4l2_ctrl_handler_setup(struct v4l2_ctrl_handler *hdl) +{ + struct v4l2_ctrl *ctrl; + int ret = 0; + + if (hdl == NULL) + return 0; + + lockdep_assert_held(hdl->lock); + + list_for_each_entry(ctrl, &hdl->ctrls, node) + ctrl->done = false; + + list_for_each_entry(ctrl, &hdl->ctrls, node) { + struct v4l2_ctrl *master = ctrl->cluster[0]; + int i; + + /* Skip if this control was already handled by a cluster. */ + /* Skip button controls and read-only controls. */ + if (ctrl->done || ctrl->type == V4L2_CTRL_TYPE_BUTTON || + (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY)) + continue; + + for (i = 0; i < master->ncontrols; i++) { + if (master->cluster[i]) { + cur_to_new(master->cluster[i]); + master->cluster[i]->is_new = 1; + master->cluster[i]->done = true; + } + } + ret = call_op(master, s_ctrl); + if (ret) + break; + } + + return ret; +} +EXPORT_SYMBOL_GPL(__v4l2_ctrl_handler_setup); + +int v4l2_ctrl_handler_setup(struct v4l2_ctrl_handler *hdl) +{ + int ret; + + if (hdl == NULL) + return 0; + + mutex_lock(hdl->lock); + ret = __v4l2_ctrl_handler_setup(hdl); + mutex_unlock(hdl->lock); + + return ret; +} +EXPORT_SYMBOL(v4l2_ctrl_handler_setup); + +/* Log the control name and value */ +static void log_ctrl(const struct v4l2_ctrl *ctrl, + const char *prefix, const char *colon) +{ + if (ctrl->flags & (V4L2_CTRL_FLAG_DISABLED | V4L2_CTRL_FLAG_WRITE_ONLY)) + return; + if (ctrl->type == V4L2_CTRL_TYPE_CTRL_CLASS) + return; + + pr_info("%s%s%s: ", prefix, colon, ctrl->name); + + ctrl->type_ops->log(ctrl); + + if (ctrl->flags & (V4L2_CTRL_FLAG_INACTIVE | + V4L2_CTRL_FLAG_GRABBED | + V4L2_CTRL_FLAG_VOLATILE)) { + if (ctrl->flags & V4L2_CTRL_FLAG_INACTIVE) + pr_cont(" inactive"); + if (ctrl->flags & V4L2_CTRL_FLAG_GRABBED) + pr_cont(" grabbed"); + if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) + pr_cont(" volatile"); + } + pr_cont("\n"); +} + +/* Log all controls owned by the handler */ +void v4l2_ctrl_handler_log_status(struct v4l2_ctrl_handler *hdl, + const char *prefix) +{ + struct v4l2_ctrl *ctrl; + const char *colon = ""; + int len; + + if (!hdl) + return; + if (!prefix) + prefix = ""; + len = strlen(prefix); + if (len && prefix[len - 1] != ' ') + colon = ": "; + mutex_lock(hdl->lock); + list_for_each_entry(ctrl, &hdl->ctrls, node) + if (!(ctrl->flags & V4L2_CTRL_FLAG_DISABLED)) + log_ctrl(ctrl, prefix, colon); + mutex_unlock(hdl->lock); +} +EXPORT_SYMBOL(v4l2_ctrl_handler_log_status); + +int v4l2_ctrl_new_fwnode_properties(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_ops *ctrl_ops, + const struct v4l2_fwnode_device_properties *p) +{ + if (p->orientation != V4L2_FWNODE_PROPERTY_UNSET) { + u32 orientation_ctrl; + + switch (p->orientation) { + case V4L2_FWNODE_ORIENTATION_FRONT: + orientation_ctrl = V4L2_CAMERA_ORIENTATION_FRONT; + break; + case V4L2_FWNODE_ORIENTATION_BACK: + orientation_ctrl = V4L2_CAMERA_ORIENTATION_BACK; + break; + case V4L2_FWNODE_ORIENTATION_EXTERNAL: + orientation_ctrl = V4L2_CAMERA_ORIENTATION_EXTERNAL; + break; + default: + return -EINVAL; + } + if (!v4l2_ctrl_new_std_menu(hdl, ctrl_ops, + V4L2_CID_CAMERA_ORIENTATION, + V4L2_CAMERA_ORIENTATION_EXTERNAL, 0, + orientation_ctrl)) + return hdl->error; + } + + if (p->rotation != V4L2_FWNODE_PROPERTY_UNSET) { + if (!v4l2_ctrl_new_std(hdl, ctrl_ops, + V4L2_CID_CAMERA_SENSOR_ROTATION, + p->rotation, p->rotation, 1, + p->rotation)) + return hdl->error; + } + + return hdl->error; +} +EXPORT_SYMBOL(v4l2_ctrl_new_fwnode_properties); diff --git a/drivers/media/v4l2-core/v4l2-ctrls-defs.c b/drivers/media/v4l2-core/v4l2-ctrls-defs.c new file mode 100644 index 000000000000..b6344bbf1e00 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2-ctrls-defs.c @@ -0,0 +1,1579 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * V4L2 controls framework control definitions. + * + * Copyright (C) 2010-2021 Hans Verkuil + */ + +#include +#include + +/* + * Returns NULL or a character pointer array containing the menu for + * the given control ID. The pointer array ends with a NULL pointer. + * An empty string signifies a menu entry that is invalid. This allows + * drivers to disable certain options if it is not supported. + */ +const char * const *v4l2_ctrl_get_menu(u32 id) +{ + static const char * const mpeg_audio_sampling_freq[] = { + "44.1 kHz", + "48 kHz", + "32 kHz", + NULL + }; + static const char * const mpeg_audio_encoding[] = { + "MPEG-1/2 Layer I", + "MPEG-1/2 Layer II", + "MPEG-1/2 Layer III", + "MPEG-2/4 AAC", + "AC-3", + NULL + }; + static const char * const mpeg_audio_l1_bitrate[] = { + "32 kbps", + "64 kbps", + "96 kbps", + "128 kbps", + "160 kbps", + "192 kbps", + "224 kbps", + "256 kbps", + "288 kbps", + "320 kbps", + "352 kbps", + "384 kbps", + "416 kbps", + "448 kbps", + NULL + }; + static const char * const mpeg_audio_l2_bitrate[] = { + "32 kbps", + "48 kbps", + "56 kbps", + "64 kbps", + "80 kbps", + "96 kbps", + "112 kbps", + "128 kbps", + "160 kbps", + "192 kbps", + "224 kbps", + "256 kbps", + "320 kbps", + "384 kbps", + NULL + }; + static const char * const mpeg_audio_l3_bitrate[] = { + "32 kbps", + "40 kbps", + "48 kbps", + "56 kbps", + "64 kbps", + "80 kbps", + "96 kbps", + "112 kbps", + "128 kbps", + "160 kbps", + "192 kbps", + "224 kbps", + "256 kbps", + "320 kbps", + NULL + }; + static const char * const mpeg_audio_ac3_bitrate[] = { + "32 kbps", + "40 kbps", + "48 kbps", + "56 kbps", + "64 kbps", + "80 kbps", + "96 kbps", + "112 kbps", + "128 kbps", + "160 kbps", + "192 kbps", + "224 kbps", + "256 kbps", + "320 kbps", + "384 kbps", + "448 kbps", + "512 kbps", + "576 kbps", + "640 kbps", + NULL + }; + static const char * const mpeg_audio_mode[] = { + "Stereo", + "Joint Stereo", + "Dual", + "Mono", + NULL + }; + static const char * const mpeg_audio_mode_extension[] = { + "Bound 4", + "Bound 8", + "Bound 12", + "Bound 16", + NULL + }; + static const char * const mpeg_audio_emphasis[] = { + "No Emphasis", + "50/15 us", + "CCITT J17", + NULL + }; + static const char * const mpeg_audio_crc[] = { + "No CRC", + "16-bit CRC", + NULL + }; + static const char * const mpeg_audio_dec_playback[] = { + "Auto", + "Stereo", + "Left", + "Right", + "Mono", + "Swapped Stereo", + NULL + }; + static const char * const mpeg_video_encoding[] = { + "MPEG-1", + "MPEG-2", + "MPEG-4 AVC", + NULL + }; + static const char * const mpeg_video_aspect[] = { + "1x1", + "4x3", + "16x9", + "2.21x1", + NULL + }; + static const char * const mpeg_video_bitrate_mode[] = { + "Variable Bitrate", + "Constant Bitrate", + "Constant Quality", + NULL + }; + static const char * const mpeg_stream_type[] = { + "MPEG-2 Program Stream", + "MPEG-2 Transport Stream", + "MPEG-1 System Stream", + "MPEG-2 DVD-compatible Stream", + "MPEG-1 VCD-compatible Stream", + "MPEG-2 SVCD-compatible Stream", + NULL + }; + static const char * const mpeg_stream_vbi_fmt[] = { + "No VBI", + "Private Packet, IVTV Format", + NULL + }; + static const char * const camera_power_line_frequency[] = { + "Disabled", + "50 Hz", + "60 Hz", + "Auto", + NULL + }; + static const char * const camera_exposure_auto[] = { + "Auto Mode", + "Manual Mode", + "Shutter Priority Mode", + "Aperture Priority Mode", + NULL + }; + static const char * const camera_exposure_metering[] = { + "Average", + "Center Weighted", + "Spot", + "Matrix", + NULL + }; + static const char * const camera_auto_focus_range[] = { + "Auto", + "Normal", + "Macro", + "Infinity", + NULL + }; + static const char * const colorfx[] = { + "None", + "Black & White", + "Sepia", + "Negative", + "Emboss", + "Sketch", + "Sky Blue", + "Grass Green", + "Skin Whiten", + "Vivid", + "Aqua", + "Art Freeze", + "Silhouette", + "Solarization", + "Antique", + "Set Cb/Cr", + NULL + }; + static const char * const auto_n_preset_white_balance[] = { + "Manual", + "Auto", + "Incandescent", + "Fluorescent", + "Fluorescent H", + "Horizon", + "Daylight", + "Flash", + "Cloudy", + "Shade", + NULL, + }; + static const char * const camera_iso_sensitivity_auto[] = { + "Manual", + "Auto", + NULL + }; + static const char * const scene_mode[] = { + "None", + "Backlight", + "Beach/Snow", + "Candle Light", + "Dusk/Dawn", + "Fall Colors", + "Fireworks", + "Landscape", + "Night", + "Party/Indoor", + "Portrait", + "Sports", + "Sunset", + "Text", + NULL + }; + static const char * const tune_emphasis[] = { + "None", + "50 Microseconds", + "75 Microseconds", + NULL, + }; + static const char * const header_mode[] = { + "Separate Buffer", + "Joined With 1st Frame", + NULL, + }; + static const char * const multi_slice[] = { + "Single", + "Max Macroblocks", + "Max Bytes", + NULL, + }; + static const char * const entropy_mode[] = { + "CAVLC", + "CABAC", + NULL, + }; + static const char * const mpeg_h264_level[] = { + "1", + "1b", + "1.1", + "1.2", + "1.3", + "2", + "2.1", + "2.2", + "3", + "3.1", + "3.2", + "4", + "4.1", + "4.2", + "5", + "5.1", + "5.2", + "6.0", + "6.1", + "6.2", + NULL, + }; + static const char * const h264_loop_filter[] = { + "Enabled", + "Disabled", + "Disabled at Slice Boundary", + NULL, + }; + static const char * const h264_profile[] = { + "Baseline", + "Constrained Baseline", + "Main", + "Extended", + "High", + "High 10", + "High 422", + "High 444 Predictive", + "High 10 Intra", + "High 422 Intra", + "High 444 Intra", + "CAVLC 444 Intra", + "Scalable Baseline", + "Scalable High", + "Scalable High Intra", + "Stereo High", + "Multiview High", + "Constrained High", + NULL, + }; + static const char * const vui_sar_idc[] = { + "Unspecified", + "1:1", + "12:11", + "10:11", + "16:11", + "40:33", + "24:11", + "20:11", + "32:11", + "80:33", + "18:11", + "15:11", + "64:33", + "160:99", + "4:3", + "3:2", + "2:1", + "Extended SAR", + NULL, + }; + static const char * const h264_fp_arrangement_type[] = { + "Checkerboard", + "Column", + "Row", + "Side by Side", + "Top Bottom", + "Temporal", + NULL, + }; + static const char * const h264_fmo_map_type[] = { + "Interleaved Slices", + "Scattered Slices", + "Foreground with Leftover", + "Box Out", + "Raster Scan", + "Wipe Scan", + "Explicit", + NULL, + }; + static const char * const h264_decode_mode[] = { + "Slice-Based", + "Frame-Based", + NULL, + }; + static const char * const h264_start_code[] = { + "No Start Code", + "Annex B Start Code", + NULL, + }; + static const char * const h264_hierarchical_coding_type[] = { + "Hier Coding B", + "Hier Coding P", + NULL, + }; + static const char * const mpeg_mpeg2_level[] = { + "Low", + "Main", + "High 1440", + "High", + NULL, + }; + static const char * const mpeg2_profile[] = { + "Simple", + "Main", + "SNR Scalable", + "Spatially Scalable", + "High", + NULL, + }; + static const char * const mpeg_mpeg4_level[] = { + "0", + "0b", + "1", + "2", + "3", + "3b", + "4", + "5", + NULL, + }; + static const char * const mpeg4_profile[] = { + "Simple", + "Advanced Simple", + "Core", + "Simple Scalable", + "Advanced Coding Efficiency", + NULL, + }; + + static const char * const vpx_golden_frame_sel[] = { + "Use Previous Frame", + "Use Previous Specific Frame", + NULL, + }; + static const char * const vp8_profile[] = { + "0", + "1", + "2", + "3", + NULL, + }; + static const char * const vp9_profile[] = { + "0", + "1", + "2", + "3", + NULL, + }; + static const char * const vp9_level[] = { + "1", + "1.1", + "2", + "2.1", + "3", + "3.1", + "4", + "4.1", + "5", + "5.1", + "5.2", + "6", + "6.1", + "6.2", + NULL, + }; + + static const char * const flash_led_mode[] = { + "Off", + "Flash", + "Torch", + NULL, + }; + static const char * const flash_strobe_source[] = { + "Software", + "External", + NULL, + }; + + static const char * const jpeg_chroma_subsampling[] = { + "4:4:4", + "4:2:2", + "4:2:0", + "4:1:1", + "4:1:0", + "Gray", + NULL, + }; + static const char * const dv_tx_mode[] = { + "DVI-D", + "HDMI", + NULL, + }; + static const char * const dv_rgb_range[] = { + "Automatic", + "RGB Limited Range (16-235)", + "RGB Full Range (0-255)", + NULL, + }; + static const char * const dv_it_content_type[] = { + "Graphics", + "Photo", + "Cinema", + "Game", + "No IT Content", + NULL, + }; + static const char * const detect_md_mode[] = { + "Disabled", + "Global", + "Threshold Grid", + "Region Grid", + NULL, + }; + + static const char * const hevc_profile[] = { + "Main", + "Main Still Picture", + "Main 10", + NULL, + }; + static const char * const hevc_level[] = { + "1", + "2", + "2.1", + "3", + "3.1", + "4", + "4.1", + "5", + "5.1", + "5.2", + "6", + "6.1", + "6.2", + NULL, + }; + static const char * const hevc_hierarchial_coding_type[] = { + "B", + "P", + NULL, + }; + static const char * const hevc_refresh_type[] = { + "None", + "CRA", + "IDR", + NULL, + }; + static const char * const hevc_size_of_length_field[] = { + "0", + "1", + "2", + "4", + NULL, + }; + static const char * const hevc_tier[] = { + "Main", + "High", + NULL, + }; + static const char * const hevc_loop_filter_mode[] = { + "Disabled", + "Enabled", + "Disabled at slice boundary", + "NULL", + }; + static const char * const hevc_decode_mode[] = { + "Slice-Based", + "Frame-Based", + NULL, + }; + static const char * const hevc_start_code[] = { + "No Start Code", + "Annex B Start Code", + NULL, + }; + static const char * const camera_orientation[] = { + "Front", + "Back", + "External", + NULL, + }; + static const char * const mpeg_video_frame_skip[] = { + "Disabled", + "Level Limit", + "VBV/CPB Limit", + NULL, + }; + + switch (id) { + case V4L2_CID_MPEG_AUDIO_SAMPLING_FREQ: + return mpeg_audio_sampling_freq; + case V4L2_CID_MPEG_AUDIO_ENCODING: + return mpeg_audio_encoding; + case V4L2_CID_MPEG_AUDIO_L1_BITRATE: + return mpeg_audio_l1_bitrate; + case V4L2_CID_MPEG_AUDIO_L2_BITRATE: + return mpeg_audio_l2_bitrate; + case V4L2_CID_MPEG_AUDIO_L3_BITRATE: + return mpeg_audio_l3_bitrate; + case V4L2_CID_MPEG_AUDIO_AC3_BITRATE: + return mpeg_audio_ac3_bitrate; + case V4L2_CID_MPEG_AUDIO_MODE: + return mpeg_audio_mode; + case V4L2_CID_MPEG_AUDIO_MODE_EXTENSION: + return mpeg_audio_mode_extension; + case V4L2_CID_MPEG_AUDIO_EMPHASIS: + return mpeg_audio_emphasis; + case V4L2_CID_MPEG_AUDIO_CRC: + return mpeg_audio_crc; + case V4L2_CID_MPEG_AUDIO_DEC_PLAYBACK: + case V4L2_CID_MPEG_AUDIO_DEC_MULTILINGUAL_PLAYBACK: + return mpeg_audio_dec_playback; + case V4L2_CID_MPEG_VIDEO_ENCODING: + return mpeg_video_encoding; + case V4L2_CID_MPEG_VIDEO_ASPECT: + return mpeg_video_aspect; + case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: + return mpeg_video_bitrate_mode; + case V4L2_CID_MPEG_STREAM_TYPE: + return mpeg_stream_type; + case V4L2_CID_MPEG_STREAM_VBI_FMT: + return mpeg_stream_vbi_fmt; + case V4L2_CID_POWER_LINE_FREQUENCY: + return camera_power_line_frequency; + case V4L2_CID_EXPOSURE_AUTO: + return camera_exposure_auto; + case V4L2_CID_EXPOSURE_METERING: + return camera_exposure_metering; + case V4L2_CID_AUTO_FOCUS_RANGE: + return camera_auto_focus_range; + case V4L2_CID_COLORFX: + return colorfx; + case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE: + return auto_n_preset_white_balance; + case V4L2_CID_ISO_SENSITIVITY_AUTO: + return camera_iso_sensitivity_auto; + case V4L2_CID_SCENE_MODE: + return scene_mode; + case V4L2_CID_TUNE_PREEMPHASIS: + return tune_emphasis; + case V4L2_CID_TUNE_DEEMPHASIS: + return tune_emphasis; + case V4L2_CID_FLASH_LED_MODE: + return flash_led_mode; + case V4L2_CID_FLASH_STROBE_SOURCE: + return flash_strobe_source; + case V4L2_CID_MPEG_VIDEO_HEADER_MODE: + return header_mode; + case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE: + return mpeg_video_frame_skip; + case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE: + return multi_slice; + case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE: + return entropy_mode; + case V4L2_CID_MPEG_VIDEO_H264_LEVEL: + return mpeg_h264_level; + case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE: + return h264_loop_filter; + case V4L2_CID_MPEG_VIDEO_H264_PROFILE: + return h264_profile; + case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC: + return vui_sar_idc; + case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE: + return h264_fp_arrangement_type; + case V4L2_CID_MPEG_VIDEO_H264_FMO_MAP_TYPE: + return h264_fmo_map_type; + case V4L2_CID_STATELESS_H264_DECODE_MODE: + return h264_decode_mode; + case V4L2_CID_STATELESS_H264_START_CODE: + return h264_start_code; + case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_TYPE: + return h264_hierarchical_coding_type; + case V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL: + return mpeg_mpeg2_level; + case V4L2_CID_MPEG_VIDEO_MPEG2_PROFILE: + return mpeg2_profile; + case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL: + return mpeg_mpeg4_level; + case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE: + return mpeg4_profile; + case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_SEL: + return vpx_golden_frame_sel; + case V4L2_CID_MPEG_VIDEO_VP8_PROFILE: + return vp8_profile; + case V4L2_CID_MPEG_VIDEO_VP9_PROFILE: + return vp9_profile; + case V4L2_CID_MPEG_VIDEO_VP9_LEVEL: + return vp9_level; + case V4L2_CID_JPEG_CHROMA_SUBSAMPLING: + return jpeg_chroma_subsampling; + case V4L2_CID_DV_TX_MODE: + return dv_tx_mode; + case V4L2_CID_DV_TX_RGB_RANGE: + case V4L2_CID_DV_RX_RGB_RANGE: + return dv_rgb_range; + case V4L2_CID_DV_TX_IT_CONTENT_TYPE: + case V4L2_CID_DV_RX_IT_CONTENT_TYPE: + return dv_it_content_type; + case V4L2_CID_DETECT_MD_MODE: + return detect_md_mode; + case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: + return hevc_profile; + case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: + return hevc_level; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_TYPE: + return hevc_hierarchial_coding_type; + case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_TYPE: + return hevc_refresh_type; + case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: + return hevc_size_of_length_field; + case V4L2_CID_MPEG_VIDEO_HEVC_TIER: + return hevc_tier; + case V4L2_CID_MPEG_VIDEO_HEVC_LOOP_FILTER_MODE: + return hevc_loop_filter_mode; + case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE: + return hevc_decode_mode; + case V4L2_CID_MPEG_VIDEO_HEVC_START_CODE: + return hevc_start_code; + case V4L2_CID_CAMERA_ORIENTATION: + return camera_orientation; + default: + return NULL; + } +} +EXPORT_SYMBOL(v4l2_ctrl_get_menu); + +#define __v4l2_qmenu_int_len(arr, len) ({ *(len) = ARRAY_SIZE(arr); (arr); }) +/* + * Returns NULL or an s64 type array containing the menu for given + * control ID. The total number of the menu items is returned in @len. + */ +const s64 *v4l2_ctrl_get_int_menu(u32 id, u32 *len) +{ + static const s64 qmenu_int_vpx_num_partitions[] = { + 1, 2, 4, 8, + }; + + static const s64 qmenu_int_vpx_num_ref_frames[] = { + 1, 2, 3, + }; + + switch (id) { + case V4L2_CID_MPEG_VIDEO_VPX_NUM_PARTITIONS: + return __v4l2_qmenu_int_len(qmenu_int_vpx_num_partitions, len); + case V4L2_CID_MPEG_VIDEO_VPX_NUM_REF_FRAMES: + return __v4l2_qmenu_int_len(qmenu_int_vpx_num_ref_frames, len); + default: + *len = 0; + return NULL; + } +} +EXPORT_SYMBOL(v4l2_ctrl_get_int_menu); + +/* Return the control name. */ +const char *v4l2_ctrl_get_name(u32 id) +{ + switch (id) { + /* USER controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_USER_CLASS: return "User Controls"; + case V4L2_CID_BRIGHTNESS: return "Brightness"; + case V4L2_CID_CONTRAST: return "Contrast"; + case V4L2_CID_SATURATION: return "Saturation"; + case V4L2_CID_HUE: return "Hue"; + case V4L2_CID_AUDIO_VOLUME: return "Volume"; + case V4L2_CID_AUDIO_BALANCE: return "Balance"; + case V4L2_CID_AUDIO_BASS: return "Bass"; + case V4L2_CID_AUDIO_TREBLE: return "Treble"; + case V4L2_CID_AUDIO_MUTE: return "Mute"; + case V4L2_CID_AUDIO_LOUDNESS: return "Loudness"; + case V4L2_CID_BLACK_LEVEL: return "Black Level"; + case V4L2_CID_AUTO_WHITE_BALANCE: return "White Balance, Automatic"; + case V4L2_CID_DO_WHITE_BALANCE: return "Do White Balance"; + case V4L2_CID_RED_BALANCE: return "Red Balance"; + case V4L2_CID_BLUE_BALANCE: return "Blue Balance"; + case V4L2_CID_GAMMA: return "Gamma"; + case V4L2_CID_EXPOSURE: return "Exposure"; + case V4L2_CID_AUTOGAIN: return "Gain, Automatic"; + case V4L2_CID_GAIN: return "Gain"; + case V4L2_CID_HFLIP: return "Horizontal Flip"; + case V4L2_CID_VFLIP: return "Vertical Flip"; + case V4L2_CID_POWER_LINE_FREQUENCY: return "Power Line Frequency"; + case V4L2_CID_HUE_AUTO: return "Hue, Automatic"; + case V4L2_CID_WHITE_BALANCE_TEMPERATURE: return "White Balance Temperature"; + case V4L2_CID_SHARPNESS: return "Sharpness"; + case V4L2_CID_BACKLIGHT_COMPENSATION: return "Backlight Compensation"; + case V4L2_CID_CHROMA_AGC: return "Chroma AGC"; + case V4L2_CID_COLOR_KILLER: return "Color Killer"; + case V4L2_CID_COLORFX: return "Color Effects"; + case V4L2_CID_AUTOBRIGHTNESS: return "Brightness, Automatic"; + case V4L2_CID_BAND_STOP_FILTER: return "Band-Stop Filter"; + case V4L2_CID_ROTATE: return "Rotate"; + case V4L2_CID_BG_COLOR: return "Background Color"; + case V4L2_CID_CHROMA_GAIN: return "Chroma Gain"; + case V4L2_CID_ILLUMINATORS_1: return "Illuminator 1"; + case V4L2_CID_ILLUMINATORS_2: return "Illuminator 2"; + case V4L2_CID_MIN_BUFFERS_FOR_CAPTURE: return "Min Number of Capture Buffers"; + case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT: return "Min Number of Output Buffers"; + case V4L2_CID_ALPHA_COMPONENT: return "Alpha Component"; + case V4L2_CID_COLORFX_CBCR: return "Color Effects, CbCr"; + + /* + * Codec controls + * + * The MPEG controls are applicable to all codec controls + * and the 'MPEG' part of the define is historical. + * + * Keep the order of the 'case's the same as in videodev2.h! + */ + case V4L2_CID_CODEC_CLASS: return "Codec Controls"; + case V4L2_CID_MPEG_STREAM_TYPE: return "Stream Type"; + case V4L2_CID_MPEG_STREAM_PID_PMT: return "Stream PMT Program ID"; + case V4L2_CID_MPEG_STREAM_PID_AUDIO: return "Stream Audio Program ID"; + case V4L2_CID_MPEG_STREAM_PID_VIDEO: return "Stream Video Program ID"; + case V4L2_CID_MPEG_STREAM_PID_PCR: return "Stream PCR Program ID"; + case V4L2_CID_MPEG_STREAM_PES_ID_AUDIO: return "Stream PES Audio ID"; + case V4L2_CID_MPEG_STREAM_PES_ID_VIDEO: return "Stream PES Video ID"; + case V4L2_CID_MPEG_STREAM_VBI_FMT: return "Stream VBI Format"; + case V4L2_CID_MPEG_AUDIO_SAMPLING_FREQ: return "Audio Sampling Frequency"; + case V4L2_CID_MPEG_AUDIO_ENCODING: return "Audio Encoding"; + case V4L2_CID_MPEG_AUDIO_L1_BITRATE: return "Audio Layer I Bitrate"; + case V4L2_CID_MPEG_AUDIO_L2_BITRATE: return "Audio Layer II Bitrate"; + case V4L2_CID_MPEG_AUDIO_L3_BITRATE: return "Audio Layer III Bitrate"; + case V4L2_CID_MPEG_AUDIO_MODE: return "Audio Stereo Mode"; + case V4L2_CID_MPEG_AUDIO_MODE_EXTENSION: return "Audio Stereo Mode Extension"; + case V4L2_CID_MPEG_AUDIO_EMPHASIS: return "Audio Emphasis"; + case V4L2_CID_MPEG_AUDIO_CRC: return "Audio CRC"; + case V4L2_CID_MPEG_AUDIO_MUTE: return "Audio Mute"; + case V4L2_CID_MPEG_AUDIO_AAC_BITRATE: return "Audio AAC Bitrate"; + case V4L2_CID_MPEG_AUDIO_AC3_BITRATE: return "Audio AC-3 Bitrate"; + case V4L2_CID_MPEG_AUDIO_DEC_PLAYBACK: return "Audio Playback"; + case V4L2_CID_MPEG_AUDIO_DEC_MULTILINGUAL_PLAYBACK: return "Audio Multilingual Playback"; + case V4L2_CID_MPEG_VIDEO_ENCODING: return "Video Encoding"; + case V4L2_CID_MPEG_VIDEO_ASPECT: return "Video Aspect"; + case V4L2_CID_MPEG_VIDEO_B_FRAMES: return "Video B Frames"; + case V4L2_CID_MPEG_VIDEO_GOP_SIZE: return "Video GOP Size"; + case V4L2_CID_MPEG_VIDEO_GOP_CLOSURE: return "Video GOP Closure"; + case V4L2_CID_MPEG_VIDEO_PULLDOWN: return "Video Pulldown"; + case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: return "Video Bitrate Mode"; + case V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY: return "Constant Quality"; + case V4L2_CID_MPEG_VIDEO_BITRATE: return "Video Bitrate"; + case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: return "Video Peak Bitrate"; + case V4L2_CID_MPEG_VIDEO_TEMPORAL_DECIMATION: return "Video Temporal Decimation"; + case V4L2_CID_MPEG_VIDEO_MUTE: return "Video Mute"; + case V4L2_CID_MPEG_VIDEO_MUTE_YUV: return "Video Mute YUV"; + case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE: return "Decoder Slice Interface"; + case V4L2_CID_MPEG_VIDEO_DECODER_MPEG4_DEBLOCK_FILTER: return "MPEG4 Loop Filter Enable"; + case V4L2_CID_MPEG_VIDEO_CYCLIC_INTRA_REFRESH_MB: return "Number of Intra Refresh MBs"; + case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE: return "Frame Level Rate Control Enable"; + case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE: return "H264 MB Level Rate Control"; + case V4L2_CID_MPEG_VIDEO_HEADER_MODE: return "Sequence Header Mode"; + case V4L2_CID_MPEG_VIDEO_MAX_REF_PIC: return "Max Number of Reference Pics"; + case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE: return "Frame Skip Mode"; + case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY: return "Display Delay"; + case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE: return "Display Delay Enable"; + case V4L2_CID_MPEG_VIDEO_AU_DELIMITER: return "Generate Access Unit Delimiters"; + case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP: return "H263 I-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP: return "H263 P-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP: return "H263 B-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_H263_MIN_QP: return "H263 Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_H263_MAX_QP: return "H263 Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_QP: return "H264 I-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_QP: return "H264 P-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP: return "H264 B-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_MAX_QP: return "H264 Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_MIN_QP: return "H264 Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM: return "H264 8x8 Transform Enable"; + case V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE: return "H264 CPB Buffer Size"; + case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE: return "H264 Entropy Mode"; + case V4L2_CID_MPEG_VIDEO_H264_I_PERIOD: return "H264 I-Frame Period"; + case V4L2_CID_MPEG_VIDEO_H264_LEVEL: return "H264 Level"; + case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_ALPHA: return "H264 Loop Filter Alpha Offset"; + case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_BETA: return "H264 Loop Filter Beta Offset"; + case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE: return "H264 Loop Filter Mode"; + case V4L2_CID_MPEG_VIDEO_H264_PROFILE: return "H264 Profile"; + case V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_HEIGHT: return "Vertical Size of SAR"; + case V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_WIDTH: return "Horizontal Size of SAR"; + case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE: return "Aspect Ratio VUI Enable"; + case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC: return "VUI Aspect Ratio IDC"; + case V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING: return "H264 Enable Frame Packing SEI"; + case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_CURRENT_FRAME_0: return "H264 Set Curr. Frame as Frame0"; + case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE: return "H264 FP Arrangement Type"; + case V4L2_CID_MPEG_VIDEO_H264_FMO: return "H264 Flexible MB Ordering"; + case V4L2_CID_MPEG_VIDEO_H264_FMO_MAP_TYPE: return "H264 Map Type for FMO"; + case V4L2_CID_MPEG_VIDEO_H264_FMO_SLICE_GROUP: return "H264 FMO Number of Slice Groups"; + case V4L2_CID_MPEG_VIDEO_H264_FMO_CHANGE_DIRECTION: return "H264 FMO Direction of Change"; + case V4L2_CID_MPEG_VIDEO_H264_FMO_CHANGE_RATE: return "H264 FMO Size of 1st Slice Grp"; + case V4L2_CID_MPEG_VIDEO_H264_FMO_RUN_LENGTH: return "H264 FMO No. of Consecutive MBs"; + case V4L2_CID_MPEG_VIDEO_H264_ASO: return "H264 Arbitrary Slice Ordering"; + case V4L2_CID_MPEG_VIDEO_H264_ASO_SLICE_ORDER: return "H264 ASO Slice Order"; + case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING: return "Enable H264 Hierarchical Coding"; + case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_TYPE: return "H264 Hierarchical Coding Type"; + case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER:return "H264 Number of HC Layers"; + case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER_QP: + return "H264 Set QP Value for HC Layers"; + case V4L2_CID_MPEG_VIDEO_H264_CONSTRAINED_INTRA_PREDICTION: + return "H264 Constrained Intra Pred"; + case V4L2_CID_MPEG_VIDEO_H264_CHROMA_QP_INDEX_OFFSET: return "H264 Chroma QP Index Offset"; + case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MIN_QP: return "H264 I-Frame Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MAX_QP: return "H264 I-Frame Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MIN_QP: return "H264 P-Frame Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MAX_QP: return "H264 P-Frame Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_MIN_QP: return "H264 B-Frame Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_MAX_QP: return "H264 B-Frame Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L0_BR: return "H264 Hierarchical Lay 0 Bitrate"; + case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L1_BR: return "H264 Hierarchical Lay 1 Bitrate"; + case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L2_BR: return "H264 Hierarchical Lay 2 Bitrate"; + case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L3_BR: return "H264 Hierarchical Lay 3 Bitrate"; + case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L4_BR: return "H264 Hierarchical Lay 4 Bitrate"; + case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L5_BR: return "H264 Hierarchical Lay 5 Bitrate"; + case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L6_BR: return "H264 Hierarchical Lay 6 Bitrate"; + case V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL: return "MPEG2 Level"; + case V4L2_CID_MPEG_VIDEO_MPEG2_PROFILE: return "MPEG2 Profile"; + case V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP: return "MPEG4 I-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP: return "MPEG4 P-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_MPEG4_B_FRAME_QP: return "MPEG4 B-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_MPEG4_MIN_QP: return "MPEG4 Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_MPEG4_MAX_QP: return "MPEG4 Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL: return "MPEG4 Level"; + case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE: return "MPEG4 Profile"; + case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL: return "Quarter Pixel Search Enable"; + case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MAX_BYTES: return "Maximum Bytes in a Slice"; + case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MAX_MB: return "Number of MBs in a Slice"; + case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE: return "Slice Partitioning Method"; + case V4L2_CID_MPEG_VIDEO_VBV_SIZE: return "VBV Buffer Size"; + case V4L2_CID_MPEG_VIDEO_DEC_PTS: return "Video Decoder PTS"; + case V4L2_CID_MPEG_VIDEO_DEC_FRAME: return "Video Decoder Frame Count"; + case V4L2_CID_MPEG_VIDEO_DEC_CONCEAL_COLOR: return "Video Decoder Conceal Color"; + case V4L2_CID_MPEG_VIDEO_VBV_DELAY: return "Initial Delay for VBV Control"; + case V4L2_CID_MPEG_VIDEO_MV_H_SEARCH_RANGE: return "Horizontal MV Search Range"; + case V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE: return "Vertical MV Search Range"; + case V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER: return "Repeat Sequence Header"; + case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: return "Force Key Frame"; + case V4L2_CID_MPEG_VIDEO_BASELAYER_PRIORITY_ID: return "Base Layer Priority ID"; + case V4L2_CID_MPEG_VIDEO_LTR_COUNT: return "LTR Count"; + case V4L2_CID_MPEG_VIDEO_FRAME_LTR_INDEX: return "Frame LTR Index"; + case V4L2_CID_MPEG_VIDEO_USE_LTR_FRAMES: return "Use LTR Frames"; + case V4L2_CID_FWHT_I_FRAME_QP: return "FWHT I-Frame QP Value"; + case V4L2_CID_FWHT_P_FRAME_QP: return "FWHT P-Frame QP Value"; + + /* VPX controls */ + case V4L2_CID_MPEG_VIDEO_VPX_NUM_PARTITIONS: return "VPX Number of Partitions"; + case V4L2_CID_MPEG_VIDEO_VPX_IMD_DISABLE_4X4: return "VPX Intra Mode Decision Disable"; + case V4L2_CID_MPEG_VIDEO_VPX_NUM_REF_FRAMES: return "VPX No. of Refs for P Frame"; + case V4L2_CID_MPEG_VIDEO_VPX_FILTER_LEVEL: return "VPX Loop Filter Level Range"; + case V4L2_CID_MPEG_VIDEO_VPX_FILTER_SHARPNESS: return "VPX Deblocking Effect Control"; + case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_REF_PERIOD: return "VPX Golden Frame Refresh Period"; + case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_SEL: return "VPX Golden Frame Indicator"; + case V4L2_CID_MPEG_VIDEO_VPX_MIN_QP: return "VPX Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_VPX_MAX_QP: return "VPX Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_VPX_I_FRAME_QP: return "VPX I-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_VPX_P_FRAME_QP: return "VPX P-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_VP8_PROFILE: return "VP8 Profile"; + case V4L2_CID_MPEG_VIDEO_VP9_PROFILE: return "VP9 Profile"; + case V4L2_CID_MPEG_VIDEO_VP9_LEVEL: return "VP9 Level"; + + /* HEVC controls */ + case V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_QP: return "HEVC I-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_QP: return "HEVC P-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_QP: return "HEVC B-Frame QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP: return "HEVC Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP: return "HEVC Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_MIN_QP: return "HEVC I-Frame Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_MAX_QP: return "HEVC I-Frame Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_MIN_QP: return "HEVC P-Frame Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_MAX_QP: return "HEVC P-Frame Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_MIN_QP: return "HEVC B-Frame Minimum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_MAX_QP: return "HEVC B-Frame Maximum QP Value"; + case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: return "HEVC Profile"; + case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: return "HEVC Level"; + case V4L2_CID_MPEG_VIDEO_HEVC_TIER: return "HEVC Tier"; + case V4L2_CID_MPEG_VIDEO_HEVC_FRAME_RATE_RESOLUTION: return "HEVC Frame Rate Resolution"; + case V4L2_CID_MPEG_VIDEO_HEVC_MAX_PARTITION_DEPTH: return "HEVC Maximum Coding Unit Depth"; + case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_TYPE: return "HEVC Refresh Type"; + case V4L2_CID_MPEG_VIDEO_HEVC_CONST_INTRA_PRED: return "HEVC Constant Intra Prediction"; + case V4L2_CID_MPEG_VIDEO_HEVC_LOSSLESS_CU: return "HEVC Lossless Encoding"; + case V4L2_CID_MPEG_VIDEO_HEVC_WAVEFRONT: return "HEVC Wavefront"; + case V4L2_CID_MPEG_VIDEO_HEVC_LOOP_FILTER_MODE: return "HEVC Loop Filter"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_QP: return "HEVC QP Values"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_TYPE: return "HEVC Hierarchical Coding Type"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_LAYER: return "HEVC Hierarchical Coding Layer"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L0_QP: return "HEVC Hierarchical Layer 0 QP"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L1_QP: return "HEVC Hierarchical Layer 1 QP"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L2_QP: return "HEVC Hierarchical Layer 2 QP"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L3_QP: return "HEVC Hierarchical Layer 3 QP"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L4_QP: return "HEVC Hierarchical Layer 4 QP"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L5_QP: return "HEVC Hierarchical Layer 5 QP"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_QP: return "HEVC Hierarchical Layer 6 QP"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L0_BR: return "HEVC Hierarchical Lay 0 BitRate"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L1_BR: return "HEVC Hierarchical Lay 1 BitRate"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L2_BR: return "HEVC Hierarchical Lay 2 BitRate"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L3_BR: return "HEVC Hierarchical Lay 3 BitRate"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L4_BR: return "HEVC Hierarchical Lay 4 BitRate"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L5_BR: return "HEVC Hierarchical Lay 5 BitRate"; + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_BR: return "HEVC Hierarchical Lay 6 BitRate"; + case V4L2_CID_MPEG_VIDEO_HEVC_GENERAL_PB: return "HEVC General PB"; + case V4L2_CID_MPEG_VIDEO_HEVC_TEMPORAL_ID: return "HEVC Temporal ID"; + case V4L2_CID_MPEG_VIDEO_HEVC_STRONG_SMOOTHING: return "HEVC Strong Intra Smoothing"; + case V4L2_CID_MPEG_VIDEO_HEVC_INTRA_PU_SPLIT: return "HEVC Intra PU Split"; + case V4L2_CID_MPEG_VIDEO_HEVC_TMV_PREDICTION: return "HEVC TMV Prediction"; + case V4L2_CID_MPEG_VIDEO_HEVC_MAX_NUM_MERGE_MV_MINUS1: return "HEVC Max Num of Candidate MVs"; + case V4L2_CID_MPEG_VIDEO_HEVC_WITHOUT_STARTCODE: return "HEVC ENC Without Startcode"; + case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_PERIOD: return "HEVC Num of I-Frame b/w 2 IDR"; + case V4L2_CID_MPEG_VIDEO_HEVC_LF_BETA_OFFSET_DIV2: return "HEVC Loop Filter Beta Offset"; + case V4L2_CID_MPEG_VIDEO_HEVC_LF_TC_OFFSET_DIV2: return "HEVC Loop Filter TC Offset"; + case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: return "HEVC Size of Length Field"; + case V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES: return "Reference Frames for a P-Frame"; + case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR: return "Prepend SPS and PPS to IDR"; + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: return "HEVC Sequence Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: return "HEVC Picture Parameter Set"; + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: return "HEVC Slice Parameters"; + case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS: return "HEVC Decode Parameters"; + case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE: return "HEVC Decode Mode"; + case V4L2_CID_MPEG_VIDEO_HEVC_START_CODE: return "HEVC Start Code"; + + /* CAMERA controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_CAMERA_CLASS: return "Camera Controls"; + case V4L2_CID_EXPOSURE_AUTO: return "Auto Exposure"; + case V4L2_CID_EXPOSURE_ABSOLUTE: return "Exposure Time, Absolute"; + case V4L2_CID_EXPOSURE_AUTO_PRIORITY: return "Exposure, Dynamic Framerate"; + case V4L2_CID_PAN_RELATIVE: return "Pan, Relative"; + case V4L2_CID_TILT_RELATIVE: return "Tilt, Relative"; + case V4L2_CID_PAN_RESET: return "Pan, Reset"; + case V4L2_CID_TILT_RESET: return "Tilt, Reset"; + case V4L2_CID_PAN_ABSOLUTE: return "Pan, Absolute"; + case V4L2_CID_TILT_ABSOLUTE: return "Tilt, Absolute"; + case V4L2_CID_FOCUS_ABSOLUTE: return "Focus, Absolute"; + case V4L2_CID_FOCUS_RELATIVE: return "Focus, Relative"; + case V4L2_CID_FOCUS_AUTO: return "Focus, Automatic Continuous"; + case V4L2_CID_ZOOM_ABSOLUTE: return "Zoom, Absolute"; + case V4L2_CID_ZOOM_RELATIVE: return "Zoom, Relative"; + case V4L2_CID_ZOOM_CONTINUOUS: return "Zoom, Continuous"; + case V4L2_CID_PRIVACY: return "Privacy"; + case V4L2_CID_IRIS_ABSOLUTE: return "Iris, Absolute"; + case V4L2_CID_IRIS_RELATIVE: return "Iris, Relative"; + case V4L2_CID_AUTO_EXPOSURE_BIAS: return "Auto Exposure, Bias"; + case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE: return "White Balance, Auto & Preset"; + case V4L2_CID_WIDE_DYNAMIC_RANGE: return "Wide Dynamic Range"; + case V4L2_CID_IMAGE_STABILIZATION: return "Image Stabilization"; + case V4L2_CID_ISO_SENSITIVITY: return "ISO Sensitivity"; + case V4L2_CID_ISO_SENSITIVITY_AUTO: return "ISO Sensitivity, Auto"; + case V4L2_CID_EXPOSURE_METERING: return "Exposure, Metering Mode"; + case V4L2_CID_SCENE_MODE: return "Scene Mode"; + case V4L2_CID_3A_LOCK: return "3A Lock"; + case V4L2_CID_AUTO_FOCUS_START: return "Auto Focus, Start"; + case V4L2_CID_AUTO_FOCUS_STOP: return "Auto Focus, Stop"; + case V4L2_CID_AUTO_FOCUS_STATUS: return "Auto Focus, Status"; + case V4L2_CID_AUTO_FOCUS_RANGE: return "Auto Focus, Range"; + case V4L2_CID_PAN_SPEED: return "Pan, Speed"; + case V4L2_CID_TILT_SPEED: return "Tilt, Speed"; + case V4L2_CID_UNIT_CELL_SIZE: return "Unit Cell Size"; + case V4L2_CID_CAMERA_ORIENTATION: return "Camera Orientation"; + case V4L2_CID_CAMERA_SENSOR_ROTATION: return "Camera Sensor Rotation"; + + /* FM Radio Modulator controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_FM_TX_CLASS: return "FM Radio Modulator Controls"; + case V4L2_CID_RDS_TX_DEVIATION: return "RDS Signal Deviation"; + case V4L2_CID_RDS_TX_PI: return "RDS Program ID"; + case V4L2_CID_RDS_TX_PTY: return "RDS Program Type"; + case V4L2_CID_RDS_TX_PS_NAME: return "RDS PS Name"; + case V4L2_CID_RDS_TX_RADIO_TEXT: return "RDS Radio Text"; + case V4L2_CID_RDS_TX_MONO_STEREO: return "RDS Stereo"; + case V4L2_CID_RDS_TX_ARTIFICIAL_HEAD: return "RDS Artificial Head"; + case V4L2_CID_RDS_TX_COMPRESSED: return "RDS Compressed"; + case V4L2_CID_RDS_TX_DYNAMIC_PTY: return "RDS Dynamic PTY"; + case V4L2_CID_RDS_TX_TRAFFIC_ANNOUNCEMENT: return "RDS Traffic Announcement"; + case V4L2_CID_RDS_TX_TRAFFIC_PROGRAM: return "RDS Traffic Program"; + case V4L2_CID_RDS_TX_MUSIC_SPEECH: return "RDS Music"; + case V4L2_CID_RDS_TX_ALT_FREQS_ENABLE: return "RDS Enable Alt Frequencies"; + case V4L2_CID_RDS_TX_ALT_FREQS: return "RDS Alternate Frequencies"; + case V4L2_CID_AUDIO_LIMITER_ENABLED: return "Audio Limiter Feature Enabled"; + case V4L2_CID_AUDIO_LIMITER_RELEASE_TIME: return "Audio Limiter Release Time"; + case V4L2_CID_AUDIO_LIMITER_DEVIATION: return "Audio Limiter Deviation"; + case V4L2_CID_AUDIO_COMPRESSION_ENABLED: return "Audio Compression Enabled"; + case V4L2_CID_AUDIO_COMPRESSION_GAIN: return "Audio Compression Gain"; + case V4L2_CID_AUDIO_COMPRESSION_THRESHOLD: return "Audio Compression Threshold"; + case V4L2_CID_AUDIO_COMPRESSION_ATTACK_TIME: return "Audio Compression Attack Time"; + case V4L2_CID_AUDIO_COMPRESSION_RELEASE_TIME: return "Audio Compression Release Time"; + case V4L2_CID_PILOT_TONE_ENABLED: return "Pilot Tone Feature Enabled"; + case V4L2_CID_PILOT_TONE_DEVIATION: return "Pilot Tone Deviation"; + case V4L2_CID_PILOT_TONE_FREQUENCY: return "Pilot Tone Frequency"; + case V4L2_CID_TUNE_PREEMPHASIS: return "Pre-Emphasis"; + case V4L2_CID_TUNE_POWER_LEVEL: return "Tune Power Level"; + case V4L2_CID_TUNE_ANTENNA_CAPACITOR: return "Tune Antenna Capacitor"; + + /* Flash controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_FLASH_CLASS: return "Flash Controls"; + case V4L2_CID_FLASH_LED_MODE: return "LED Mode"; + case V4L2_CID_FLASH_STROBE_SOURCE: return "Strobe Source"; + case V4L2_CID_FLASH_STROBE: return "Strobe"; + case V4L2_CID_FLASH_STROBE_STOP: return "Stop Strobe"; + case V4L2_CID_FLASH_STROBE_STATUS: return "Strobe Status"; + case V4L2_CID_FLASH_TIMEOUT: return "Strobe Timeout"; + case V4L2_CID_FLASH_INTENSITY: return "Intensity, Flash Mode"; + case V4L2_CID_FLASH_TORCH_INTENSITY: return "Intensity, Torch Mode"; + case V4L2_CID_FLASH_INDICATOR_INTENSITY: return "Intensity, Indicator"; + case V4L2_CID_FLASH_FAULT: return "Faults"; + case V4L2_CID_FLASH_CHARGE: return "Charge"; + case V4L2_CID_FLASH_READY: return "Ready to Strobe"; + + /* JPEG encoder controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_JPEG_CLASS: return "JPEG Compression Controls"; + case V4L2_CID_JPEG_CHROMA_SUBSAMPLING: return "Chroma Subsampling"; + case V4L2_CID_JPEG_RESTART_INTERVAL: return "Restart Interval"; + case V4L2_CID_JPEG_COMPRESSION_QUALITY: return "Compression Quality"; + case V4L2_CID_JPEG_ACTIVE_MARKER: return "Active Markers"; + + /* Image source controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_IMAGE_SOURCE_CLASS: return "Image Source Controls"; + case V4L2_CID_VBLANK: return "Vertical Blanking"; + case V4L2_CID_HBLANK: return "Horizontal Blanking"; + case V4L2_CID_ANALOGUE_GAIN: return "Analogue Gain"; + case V4L2_CID_TEST_PATTERN_RED: return "Red Pixel Value"; + case V4L2_CID_TEST_PATTERN_GREENR: return "Green (Red) Pixel Value"; + case V4L2_CID_TEST_PATTERN_BLUE: return "Blue Pixel Value"; + case V4L2_CID_TEST_PATTERN_GREENB: return "Green (Blue) Pixel Value"; + + /* Image processing controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_IMAGE_PROC_CLASS: return "Image Processing Controls"; + case V4L2_CID_LINK_FREQ: return "Link Frequency"; + case V4L2_CID_PIXEL_RATE: return "Pixel Rate"; + case V4L2_CID_TEST_PATTERN: return "Test Pattern"; + case V4L2_CID_DEINTERLACING_MODE: return "Deinterlacing Mode"; + case V4L2_CID_DIGITAL_GAIN: return "Digital Gain"; + + /* DV controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_DV_CLASS: return "Digital Video Controls"; + case V4L2_CID_DV_TX_HOTPLUG: return "Hotplug Present"; + case V4L2_CID_DV_TX_RXSENSE: return "RxSense Present"; + case V4L2_CID_DV_TX_EDID_PRESENT: return "EDID Present"; + case V4L2_CID_DV_TX_MODE: return "Transmit Mode"; + case V4L2_CID_DV_TX_RGB_RANGE: return "Tx RGB Quantization Range"; + case V4L2_CID_DV_TX_IT_CONTENT_TYPE: return "Tx IT Content Type"; + case V4L2_CID_DV_RX_POWER_PRESENT: return "Power Present"; + case V4L2_CID_DV_RX_RGB_RANGE: return "Rx RGB Quantization Range"; + case V4L2_CID_DV_RX_IT_CONTENT_TYPE: return "Rx IT Content Type"; + + case V4L2_CID_FM_RX_CLASS: return "FM Radio Receiver Controls"; + case V4L2_CID_TUNE_DEEMPHASIS: return "De-Emphasis"; + case V4L2_CID_RDS_RECEPTION: return "RDS Reception"; + case V4L2_CID_RF_TUNER_CLASS: return "RF Tuner Controls"; + case V4L2_CID_RF_TUNER_RF_GAIN: return "RF Gain"; + case V4L2_CID_RF_TUNER_LNA_GAIN_AUTO: return "LNA Gain, Auto"; + case V4L2_CID_RF_TUNER_LNA_GAIN: return "LNA Gain"; + case V4L2_CID_RF_TUNER_MIXER_GAIN_AUTO: return "Mixer Gain, Auto"; + case V4L2_CID_RF_TUNER_MIXER_GAIN: return "Mixer Gain"; + case V4L2_CID_RF_TUNER_IF_GAIN_AUTO: return "IF Gain, Auto"; + case V4L2_CID_RF_TUNER_IF_GAIN: return "IF Gain"; + case V4L2_CID_RF_TUNER_BANDWIDTH_AUTO: return "Bandwidth, Auto"; + case V4L2_CID_RF_TUNER_BANDWIDTH: return "Bandwidth"; + case V4L2_CID_RF_TUNER_PLL_LOCK: return "PLL Lock"; + case V4L2_CID_RDS_RX_PTY: return "RDS Program Type"; + case V4L2_CID_RDS_RX_PS_NAME: return "RDS PS Name"; + case V4L2_CID_RDS_RX_RADIO_TEXT: return "RDS Radio Text"; + case V4L2_CID_RDS_RX_TRAFFIC_ANNOUNCEMENT: return "RDS Traffic Announcement"; + case V4L2_CID_RDS_RX_TRAFFIC_PROGRAM: return "RDS Traffic Program"; + case V4L2_CID_RDS_RX_MUSIC_SPEECH: return "RDS Music"; + + /* Detection controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_DETECT_CLASS: return "Detection Controls"; + case V4L2_CID_DETECT_MD_MODE: return "Motion Detection Mode"; + case V4L2_CID_DETECT_MD_GLOBAL_THRESHOLD: return "MD Global Threshold"; + case V4L2_CID_DETECT_MD_THRESHOLD_GRID: return "MD Threshold Grid"; + case V4L2_CID_DETECT_MD_REGION_GRID: return "MD Region Grid"; + + /* Stateless Codec controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_CODEC_STATELESS_CLASS: return "Stateless Codec Controls"; + case V4L2_CID_STATELESS_H264_DECODE_MODE: return "H264 Decode Mode"; + case V4L2_CID_STATELESS_H264_START_CODE: return "H264 Start Code"; + case V4L2_CID_STATELESS_H264_SPS: return "H264 Sequence Parameter Set"; + case V4L2_CID_STATELESS_H264_PPS: return "H264 Picture Parameter Set"; + case V4L2_CID_STATELESS_H264_SCALING_MATRIX: return "H264 Scaling Matrix"; + case V4L2_CID_STATELESS_H264_PRED_WEIGHTS: return "H264 Prediction Weight Table"; + case V4L2_CID_STATELESS_H264_SLICE_PARAMS: return "H264 Slice Parameters"; + case V4L2_CID_STATELESS_H264_DECODE_PARAMS: return "H264 Decode Parameters"; + case V4L2_CID_STATELESS_FWHT_PARAMS: return "FWHT Stateless Parameters"; + case V4L2_CID_STATELESS_VP8_FRAME: return "VP8 Frame Parameters"; + case V4L2_CID_STATELESS_MPEG2_SEQUENCE: return "MPEG-2 Sequence Header"; + case V4L2_CID_STATELESS_MPEG2_PICTURE: return "MPEG-2 Picture Header"; + case V4L2_CID_STATELESS_MPEG2_QUANTISATION: return "MPEG-2 Quantisation Matrices"; + + /* Colorimetry controls */ + /* Keep the order of the 'case's the same as in v4l2-controls.h! */ + case V4L2_CID_COLORIMETRY_CLASS: return "Colorimetry Controls"; + case V4L2_CID_COLORIMETRY_HDR10_CLL_INFO: return "HDR10 Content Light Info"; + case V4L2_CID_COLORIMETRY_HDR10_MASTERING_DISPLAY: return "HDR10 Mastering Display"; + default: + return NULL; + } +} +EXPORT_SYMBOL(v4l2_ctrl_get_name); + +void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type, + s64 *min, s64 *max, u64 *step, s64 *def, u32 *flags) +{ + *name = v4l2_ctrl_get_name(id); + *flags = 0; + + switch (id) { + case V4L2_CID_AUDIO_MUTE: + case V4L2_CID_AUDIO_LOUDNESS: + case V4L2_CID_AUTO_WHITE_BALANCE: + case V4L2_CID_AUTOGAIN: + case V4L2_CID_HFLIP: + case V4L2_CID_VFLIP: + case V4L2_CID_HUE_AUTO: + case V4L2_CID_CHROMA_AGC: + case V4L2_CID_COLOR_KILLER: + case V4L2_CID_AUTOBRIGHTNESS: + case V4L2_CID_MPEG_AUDIO_MUTE: + case V4L2_CID_MPEG_VIDEO_MUTE: + case V4L2_CID_MPEG_VIDEO_GOP_CLOSURE: + case V4L2_CID_MPEG_VIDEO_PULLDOWN: + case V4L2_CID_EXPOSURE_AUTO_PRIORITY: + case V4L2_CID_FOCUS_AUTO: + case V4L2_CID_PRIVACY: + case V4L2_CID_AUDIO_LIMITER_ENABLED: + case V4L2_CID_AUDIO_COMPRESSION_ENABLED: + case V4L2_CID_PILOT_TONE_ENABLED: + case V4L2_CID_ILLUMINATORS_1: + case V4L2_CID_ILLUMINATORS_2: + case V4L2_CID_FLASH_STROBE_STATUS: + case V4L2_CID_FLASH_CHARGE: + case V4L2_CID_FLASH_READY: + case V4L2_CID_MPEG_VIDEO_DECODER_MPEG4_DEBLOCK_FILTER: + case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE: + case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE: + case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE: + case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE: + case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM: + case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE: + case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL: + case V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER: + case V4L2_CID_MPEG_VIDEO_AU_DELIMITER: + case V4L2_CID_WIDE_DYNAMIC_RANGE: + case V4L2_CID_IMAGE_STABILIZATION: + case V4L2_CID_RDS_RECEPTION: + case V4L2_CID_RF_TUNER_LNA_GAIN_AUTO: + case V4L2_CID_RF_TUNER_MIXER_GAIN_AUTO: + case V4L2_CID_RF_TUNER_IF_GAIN_AUTO: + case V4L2_CID_RF_TUNER_BANDWIDTH_AUTO: + case V4L2_CID_RF_TUNER_PLL_LOCK: + case V4L2_CID_RDS_TX_MONO_STEREO: + case V4L2_CID_RDS_TX_ARTIFICIAL_HEAD: + case V4L2_CID_RDS_TX_COMPRESSED: + case V4L2_CID_RDS_TX_DYNAMIC_PTY: + case V4L2_CID_RDS_TX_TRAFFIC_ANNOUNCEMENT: + case V4L2_CID_RDS_TX_TRAFFIC_PROGRAM: + case V4L2_CID_RDS_TX_MUSIC_SPEECH: + case V4L2_CID_RDS_TX_ALT_FREQS_ENABLE: + case V4L2_CID_RDS_RX_TRAFFIC_ANNOUNCEMENT: + case V4L2_CID_RDS_RX_TRAFFIC_PROGRAM: + case V4L2_CID_RDS_RX_MUSIC_SPEECH: + *type = V4L2_CTRL_TYPE_BOOLEAN; + *min = 0; + *max = *step = 1; + break; + case V4L2_CID_ROTATE: + *type = V4L2_CTRL_TYPE_INTEGER; + *flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT; + break; + case V4L2_CID_MPEG_VIDEO_MV_H_SEARCH_RANGE: + case V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE: + case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY: + *type = V4L2_CTRL_TYPE_INTEGER; + break; + case V4L2_CID_MPEG_VIDEO_LTR_COUNT: + *type = V4L2_CTRL_TYPE_INTEGER; + break; + case V4L2_CID_MPEG_VIDEO_FRAME_LTR_INDEX: + *type = V4L2_CTRL_TYPE_INTEGER; + *flags |= V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; + break; + case V4L2_CID_MPEG_VIDEO_USE_LTR_FRAMES: + *type = V4L2_CTRL_TYPE_BITMASK; + *flags |= V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; + break; + case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: + case V4L2_CID_PAN_RESET: + case V4L2_CID_TILT_RESET: + case V4L2_CID_FLASH_STROBE: + case V4L2_CID_FLASH_STROBE_STOP: + case V4L2_CID_AUTO_FOCUS_START: + case V4L2_CID_AUTO_FOCUS_STOP: + case V4L2_CID_DO_WHITE_BALANCE: + *type = V4L2_CTRL_TYPE_BUTTON; + *flags |= V4L2_CTRL_FLAG_WRITE_ONLY | + V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; + *min = *max = *step = *def = 0; + break; + case V4L2_CID_POWER_LINE_FREQUENCY: + case V4L2_CID_MPEG_AUDIO_SAMPLING_FREQ: + case V4L2_CID_MPEG_AUDIO_ENCODING: + case V4L2_CID_MPEG_AUDIO_L1_BITRATE: + case V4L2_CID_MPEG_AUDIO_L2_BITRATE: + case V4L2_CID_MPEG_AUDIO_L3_BITRATE: + case V4L2_CID_MPEG_AUDIO_AC3_BITRATE: + case V4L2_CID_MPEG_AUDIO_MODE: + case V4L2_CID_MPEG_AUDIO_MODE_EXTENSION: + case V4L2_CID_MPEG_AUDIO_EMPHASIS: + case V4L2_CID_MPEG_AUDIO_CRC: + case V4L2_CID_MPEG_AUDIO_DEC_PLAYBACK: + case V4L2_CID_MPEG_AUDIO_DEC_MULTILINGUAL_PLAYBACK: + case V4L2_CID_MPEG_VIDEO_ENCODING: + case V4L2_CID_MPEG_VIDEO_ASPECT: + case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: + case V4L2_CID_MPEG_STREAM_TYPE: + case V4L2_CID_MPEG_STREAM_VBI_FMT: + case V4L2_CID_EXPOSURE_AUTO: + case V4L2_CID_AUTO_FOCUS_RANGE: + case V4L2_CID_COLORFX: + case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE: + case V4L2_CID_TUNE_PREEMPHASIS: + case V4L2_CID_FLASH_LED_MODE: + case V4L2_CID_FLASH_STROBE_SOURCE: + case V4L2_CID_MPEG_VIDEO_HEADER_MODE: + case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE: + case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE: + case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE: + case V4L2_CID_MPEG_VIDEO_H264_LEVEL: + case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE: + case V4L2_CID_MPEG_VIDEO_H264_PROFILE: + case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC: + case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE: + case V4L2_CID_MPEG_VIDEO_H264_FMO_MAP_TYPE: + case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_TYPE: + case V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL: + case V4L2_CID_MPEG_VIDEO_MPEG2_PROFILE: + case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL: + case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE: + case V4L2_CID_JPEG_CHROMA_SUBSAMPLING: + case V4L2_CID_ISO_SENSITIVITY_AUTO: + case V4L2_CID_EXPOSURE_METERING: + case V4L2_CID_SCENE_MODE: + case V4L2_CID_DV_TX_MODE: + case V4L2_CID_DV_TX_RGB_RANGE: + case V4L2_CID_DV_TX_IT_CONTENT_TYPE: + case V4L2_CID_DV_RX_RGB_RANGE: + case V4L2_CID_DV_RX_IT_CONTENT_TYPE: + case V4L2_CID_TEST_PATTERN: + case V4L2_CID_DEINTERLACING_MODE: + case V4L2_CID_TUNE_DEEMPHASIS: + case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_SEL: + case V4L2_CID_MPEG_VIDEO_VP8_PROFILE: + case V4L2_CID_MPEG_VIDEO_VP9_PROFILE: + case V4L2_CID_MPEG_VIDEO_VP9_LEVEL: + case V4L2_CID_DETECT_MD_MODE: + case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: + case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: + case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_TYPE: + case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_TYPE: + case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: + case V4L2_CID_MPEG_VIDEO_HEVC_TIER: + case V4L2_CID_MPEG_VIDEO_HEVC_LOOP_FILTER_MODE: + case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE: + case V4L2_CID_MPEG_VIDEO_HEVC_START_CODE: + case V4L2_CID_STATELESS_H264_DECODE_MODE: + case V4L2_CID_STATELESS_H264_START_CODE: + case V4L2_CID_CAMERA_ORIENTATION: + *type = V4L2_CTRL_TYPE_MENU; + break; + case V4L2_CID_LINK_FREQ: + *type = V4L2_CTRL_TYPE_INTEGER_MENU; + break; + case V4L2_CID_RDS_TX_PS_NAME: + case V4L2_CID_RDS_TX_RADIO_TEXT: + case V4L2_CID_RDS_RX_PS_NAME: + case V4L2_CID_RDS_RX_RADIO_TEXT: + *type = V4L2_CTRL_TYPE_STRING; + break; + case V4L2_CID_ISO_SENSITIVITY: + case V4L2_CID_AUTO_EXPOSURE_BIAS: + case V4L2_CID_MPEG_VIDEO_VPX_NUM_PARTITIONS: + case V4L2_CID_MPEG_VIDEO_VPX_NUM_REF_FRAMES: + *type = V4L2_CTRL_TYPE_INTEGER_MENU; + break; + case V4L2_CID_USER_CLASS: + case V4L2_CID_CAMERA_CLASS: + case V4L2_CID_CODEC_CLASS: + case V4L2_CID_FM_TX_CLASS: + case V4L2_CID_FLASH_CLASS: + case V4L2_CID_JPEG_CLASS: + case V4L2_CID_IMAGE_SOURCE_CLASS: + case V4L2_CID_IMAGE_PROC_CLASS: + case V4L2_CID_DV_CLASS: + case V4L2_CID_FM_RX_CLASS: + case V4L2_CID_RF_TUNER_CLASS: + case V4L2_CID_DETECT_CLASS: + case V4L2_CID_CODEC_STATELESS_CLASS: + case V4L2_CID_COLORIMETRY_CLASS: + *type = V4L2_CTRL_TYPE_CTRL_CLASS; + /* You can neither read nor write these */ + *flags |= V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_WRITE_ONLY; + *min = *max = *step = *def = 0; + break; + case V4L2_CID_BG_COLOR: + *type = V4L2_CTRL_TYPE_INTEGER; + *step = 1; + *min = 0; + /* Max is calculated as RGB888 that is 2^24 */ + *max = 0xFFFFFF; + break; + case V4L2_CID_FLASH_FAULT: + case V4L2_CID_JPEG_ACTIVE_MARKER: + case V4L2_CID_3A_LOCK: + case V4L2_CID_AUTO_FOCUS_STATUS: + case V4L2_CID_DV_TX_HOTPLUG: + case V4L2_CID_DV_TX_RXSENSE: + case V4L2_CID_DV_TX_EDID_PRESENT: + case V4L2_CID_DV_RX_POWER_PRESENT: + *type = V4L2_CTRL_TYPE_BITMASK; + break; + case V4L2_CID_MIN_BUFFERS_FOR_CAPTURE: + case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT: + *type = V4L2_CTRL_TYPE_INTEGER; + *flags |= V4L2_CTRL_FLAG_READ_ONLY; + break; + case V4L2_CID_MPEG_VIDEO_DEC_PTS: + *type = V4L2_CTRL_TYPE_INTEGER64; + *flags |= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_READ_ONLY; + *min = *def = 0; + *max = 0x1ffffffffLL; + *step = 1; + break; + case V4L2_CID_MPEG_VIDEO_DEC_FRAME: + *type = V4L2_CTRL_TYPE_INTEGER64; + *flags |= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_READ_ONLY; + *min = *def = 0; + *max = 0x7fffffffffffffffLL; + *step = 1; + break; + case V4L2_CID_MPEG_VIDEO_DEC_CONCEAL_COLOR: + *type = V4L2_CTRL_TYPE_INTEGER64; + *min = 0; + /* default for 8 bit black, luma is 16, chroma is 128 */ + *def = 0x8000800010LL; + *max = 0xffffffffffffLL; + *step = 1; + break; + case V4L2_CID_PIXEL_RATE: + *type = V4L2_CTRL_TYPE_INTEGER64; + *flags |= V4L2_CTRL_FLAG_READ_ONLY; + break; + case V4L2_CID_DETECT_MD_REGION_GRID: + *type = V4L2_CTRL_TYPE_U8; + break; + case V4L2_CID_DETECT_MD_THRESHOLD_GRID: + *type = V4L2_CTRL_TYPE_U16; + break; + case V4L2_CID_RDS_TX_ALT_FREQS: + *type = V4L2_CTRL_TYPE_U32; + break; + case V4L2_CID_STATELESS_MPEG2_SEQUENCE: + *type = V4L2_CTRL_TYPE_MPEG2_SEQUENCE; + break; + case V4L2_CID_STATELESS_MPEG2_PICTURE: + *type = V4L2_CTRL_TYPE_MPEG2_PICTURE; + break; + case V4L2_CID_STATELESS_MPEG2_QUANTISATION: + *type = V4L2_CTRL_TYPE_MPEG2_QUANTISATION; + break; + case V4L2_CID_STATELESS_FWHT_PARAMS: + *type = V4L2_CTRL_TYPE_FWHT_PARAMS; + break; + case V4L2_CID_STATELESS_H264_SPS: + *type = V4L2_CTRL_TYPE_H264_SPS; + break; + case V4L2_CID_STATELESS_H264_PPS: + *type = V4L2_CTRL_TYPE_H264_PPS; + break; + case V4L2_CID_STATELESS_H264_SCALING_MATRIX: + *type = V4L2_CTRL_TYPE_H264_SCALING_MATRIX; + break; + case V4L2_CID_STATELESS_H264_SLICE_PARAMS: + *type = V4L2_CTRL_TYPE_H264_SLICE_PARAMS; + break; + case V4L2_CID_STATELESS_H264_DECODE_PARAMS: + *type = V4L2_CTRL_TYPE_H264_DECODE_PARAMS; + break; + case V4L2_CID_STATELESS_H264_PRED_WEIGHTS: + *type = V4L2_CTRL_TYPE_H264_PRED_WEIGHTS; + break; + case V4L2_CID_STATELESS_VP8_FRAME: + *type = V4L2_CTRL_TYPE_VP8_FRAME; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_SPS: + *type = V4L2_CTRL_TYPE_HEVC_SPS; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_PPS: + *type = V4L2_CTRL_TYPE_HEVC_PPS; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: + *type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS; + break; + case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS: + *type = V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS; + break; + case V4L2_CID_UNIT_CELL_SIZE: + *type = V4L2_CTRL_TYPE_AREA; + *flags |= V4L2_CTRL_FLAG_READ_ONLY; + break; + case V4L2_CID_COLORIMETRY_HDR10_CLL_INFO: + *type = V4L2_CTRL_TYPE_HDR10_CLL_INFO; + break; + case V4L2_CID_COLORIMETRY_HDR10_MASTERING_DISPLAY: + *type = V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY; + break; + default: + *type = V4L2_CTRL_TYPE_INTEGER; + break; + } + switch (id) { + case V4L2_CID_MPEG_AUDIO_ENCODING: + case V4L2_CID_MPEG_AUDIO_MODE: + case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: + case V4L2_CID_MPEG_VIDEO_B_FRAMES: + case V4L2_CID_MPEG_STREAM_TYPE: + *flags |= V4L2_CTRL_FLAG_UPDATE; + break; + case V4L2_CID_AUDIO_VOLUME: + case V4L2_CID_AUDIO_BALANCE: + case V4L2_CID_AUDIO_BASS: + case V4L2_CID_AUDIO_TREBLE: + case V4L2_CID_BRIGHTNESS: + case V4L2_CID_CONTRAST: + case V4L2_CID_SATURATION: + case V4L2_CID_HUE: + case V4L2_CID_RED_BALANCE: + case V4L2_CID_BLUE_BALANCE: + case V4L2_CID_GAMMA: + case V4L2_CID_SHARPNESS: + case V4L2_CID_CHROMA_GAIN: + case V4L2_CID_RDS_TX_DEVIATION: + case V4L2_CID_AUDIO_LIMITER_RELEASE_TIME: + case V4L2_CID_AUDIO_LIMITER_DEVIATION: + case V4L2_CID_AUDIO_COMPRESSION_GAIN: + case V4L2_CID_AUDIO_COMPRESSION_THRESHOLD: + case V4L2_CID_AUDIO_COMPRESSION_ATTACK_TIME: + case V4L2_CID_AUDIO_COMPRESSION_RELEASE_TIME: + case V4L2_CID_PILOT_TONE_DEVIATION: + case V4L2_CID_PILOT_TONE_FREQUENCY: + case V4L2_CID_TUNE_POWER_LEVEL: + case V4L2_CID_TUNE_ANTENNA_CAPACITOR: + case V4L2_CID_RF_TUNER_RF_GAIN: + case V4L2_CID_RF_TUNER_LNA_GAIN: + case V4L2_CID_RF_TUNER_MIXER_GAIN: + case V4L2_CID_RF_TUNER_IF_GAIN: + case V4L2_CID_RF_TUNER_BANDWIDTH: + case V4L2_CID_DETECT_MD_GLOBAL_THRESHOLD: + *flags |= V4L2_CTRL_FLAG_SLIDER; + break; + case V4L2_CID_PAN_RELATIVE: + case V4L2_CID_TILT_RELATIVE: + case V4L2_CID_FOCUS_RELATIVE: + case V4L2_CID_IRIS_RELATIVE: + case V4L2_CID_ZOOM_RELATIVE: + *flags |= V4L2_CTRL_FLAG_WRITE_ONLY | + V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; + break; + case V4L2_CID_FLASH_STROBE_STATUS: + case V4L2_CID_AUTO_FOCUS_STATUS: + case V4L2_CID_FLASH_READY: + case V4L2_CID_DV_TX_HOTPLUG: + case V4L2_CID_DV_TX_RXSENSE: + case V4L2_CID_DV_TX_EDID_PRESENT: + case V4L2_CID_DV_RX_POWER_PRESENT: + case V4L2_CID_DV_RX_IT_CONTENT_TYPE: + case V4L2_CID_RDS_RX_PTY: + case V4L2_CID_RDS_RX_PS_NAME: + case V4L2_CID_RDS_RX_RADIO_TEXT: + case V4L2_CID_RDS_RX_TRAFFIC_ANNOUNCEMENT: + case V4L2_CID_RDS_RX_TRAFFIC_PROGRAM: + case V4L2_CID_RDS_RX_MUSIC_SPEECH: + case V4L2_CID_CAMERA_ORIENTATION: + case V4L2_CID_CAMERA_SENSOR_ROTATION: + *flags |= V4L2_CTRL_FLAG_READ_ONLY; + break; + case V4L2_CID_RF_TUNER_PLL_LOCK: + *flags |= V4L2_CTRL_FLAG_VOLATILE; + break; + } +} +EXPORT_SYMBOL(v4l2_ctrl_fill); diff --git a/drivers/media/v4l2-core/v4l2-ctrls-priv.h b/drivers/media/v4l2-core/v4l2-ctrls-priv.h new file mode 100644 index 000000000000..d4bf2c716f97 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2-ctrls-priv.h @@ -0,0 +1,96 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +/* + * V4L2 controls framework private header. + * + * Copyright (C) 2010-2021 Hans Verkuil + */ + +#ifndef _V4L2_CTRLS_PRIV_H_ +#define _V4L2_CTRLS_PRIV_H_ + +#define dprintk(vdev, fmt, arg...) do { \ + if (!WARN_ON(!(vdev)) && ((vdev)->dev_debug & V4L2_DEV_DEBUG_CTRL)) \ + printk(KERN_DEBUG pr_fmt("%s: %s: " fmt), \ + __func__, video_device_node_name(vdev), ##arg); \ +} while (0) + +#define has_op(master, op) \ + ((master)->ops && (master)->ops->op) +#define call_op(master, op) \ + (has_op(master, op) ? (master)->ops->op(master) : 0) + +static inline u32 node2id(struct list_head *node) +{ + return list_entry(node, struct v4l2_ctrl_ref, node)->ctrl->id; +} + +/* + * Small helper function to determine if the autocluster is set to manual + * mode. + */ +static inline bool is_cur_manual(const struct v4l2_ctrl *master) +{ + return master->is_auto && master->cur.val == master->manual_mode_value; +} + +/* + * Small helper function to determine if the autocluster will be set to manual + * mode. + */ +static inline bool is_new_manual(const struct v4l2_ctrl *master) +{ + return master->is_auto && master->val == master->manual_mode_value; +} + +static inline u32 user_flags(const struct v4l2_ctrl *ctrl) +{ + u32 flags = ctrl->flags; + + if (ctrl->is_ptr) + flags |= V4L2_CTRL_FLAG_HAS_PAYLOAD; + + return flags; +} + +/* v4l2-ctrls-core.c */ +void cur_to_new(struct v4l2_ctrl *ctrl); +void cur_to_req(struct v4l2_ctrl_ref *ref); +void new_to_cur(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 ch_flags); +void new_to_req(struct v4l2_ctrl_ref *ref); +void req_to_new(struct v4l2_ctrl_ref *ref); +void send_initial_event(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl); +void send_event(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 changes); +int validate_new(const struct v4l2_ctrl *ctrl, union v4l2_ctrl_ptr p_new); +int handler_new_ref(struct v4l2_ctrl_handler *hdl, + struct v4l2_ctrl *ctrl, + struct v4l2_ctrl_ref **ctrl_ref, + bool from_other_dev, bool allocate_req); +struct v4l2_ctrl_ref *find_ref(struct v4l2_ctrl_handler *hdl, u32 id); +struct v4l2_ctrl_ref *find_ref_lock(struct v4l2_ctrl_handler *hdl, u32 id); +int check_range(enum v4l2_ctrl_type type, + s64 min, s64 max, u64 step, s64 def); +void update_from_auto_cluster(struct v4l2_ctrl *master); +int try_or_set_cluster(struct v4l2_fh *fh, struct v4l2_ctrl *master, + bool set, u32 ch_flags); + +/* v4l2-ctrls-api.c */ +int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, + struct v4l2_ext_controls *cs, + struct video_device *vdev); +int try_set_ext_ctrls_common(struct v4l2_fh *fh, + struct v4l2_ctrl_handler *hdl, + struct v4l2_ext_controls *cs, + struct video_device *vdev, bool set); + +/* v4l2-ctrls-request.c */ +void v4l2_ctrl_handler_init_request(struct v4l2_ctrl_handler *hdl); +void v4l2_ctrl_handler_free_request(struct v4l2_ctrl_handler *hdl); +int v4l2_g_ext_ctrls_request(struct v4l2_ctrl_handler *hdl, struct video_device *vdev, + struct media_device *mdev, struct v4l2_ext_controls *cs); +int try_set_ext_ctrls_request(struct v4l2_fh *fh, + struct v4l2_ctrl_handler *hdl, + struct video_device *vdev, + struct media_device *mdev, + struct v4l2_ext_controls *cs, bool set); + +#endif diff --git a/drivers/media/v4l2-core/v4l2-ctrls-request.c b/drivers/media/v4l2-core/v4l2-ctrls-request.c new file mode 100644 index 000000000000..7d098f287fd9 --- /dev/null +++ b/drivers/media/v4l2-core/v4l2-ctrls-request.c @@ -0,0 +1,496 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * V4L2 controls framework Request API implementation. + * + * Copyright (C) 2018-2021 Hans Verkuil + */ + +#define pr_fmt(fmt) "v4l2-ctrls: " fmt + +#include +#include +#include +#include +#include + +#include "v4l2-ctrls-priv.h" + +/* Initialize the request-related fields in a control handler */ +void v4l2_ctrl_handler_init_request(struct v4l2_ctrl_handler *hdl) +{ + INIT_LIST_HEAD(&hdl->requests); + INIT_LIST_HEAD(&hdl->requests_queued); + hdl->request_is_queued = false; + media_request_object_init(&hdl->req_obj); +} + +/* Free the request-related fields in a control handler */ +void v4l2_ctrl_handler_free_request(struct v4l2_ctrl_handler *hdl) +{ + struct v4l2_ctrl_handler *req, *next_req; + + /* + * Do nothing if this isn't the main handler or the main + * handler is not used in any request. + * + * The main handler can be identified by having a NULL ops pointer in + * the request object. + */ + if (hdl->req_obj.ops || list_empty(&hdl->requests)) + return; + + /* + * If the main handler is freed and it is used by handler objects in + * outstanding requests, then unbind and put those objects before + * freeing the main handler. + */ + list_for_each_entry_safe(req, next_req, &hdl->requests, requests) { + media_request_object_unbind(&req->req_obj); + media_request_object_put(&req->req_obj); + } +} + +static int v4l2_ctrl_request_clone(struct v4l2_ctrl_handler *hdl, + const struct v4l2_ctrl_handler *from) +{ + struct v4l2_ctrl_ref *ref; + int err = 0; + + if (WARN_ON(!hdl || hdl == from)) + return -EINVAL; + + if (hdl->error) + return hdl->error; + + WARN_ON(hdl->lock != &hdl->_lock); + + mutex_lock(from->lock); + list_for_each_entry(ref, &from->ctrl_refs, node) { + struct v4l2_ctrl *ctrl = ref->ctrl; + struct v4l2_ctrl_ref *new_ref; + + /* Skip refs inherited from other devices */ + if (ref->from_other_dev) + continue; + err = handler_new_ref(hdl, ctrl, &new_ref, false, true); + if (err) + break; + } + mutex_unlock(from->lock); + return err; +} + +static void v4l2_ctrl_request_queue(struct media_request_object *obj) +{ + struct v4l2_ctrl_handler *hdl = + container_of(obj, struct v4l2_ctrl_handler, req_obj); + struct v4l2_ctrl_handler *main_hdl = obj->priv; + + mutex_lock(main_hdl->lock); + list_add_tail(&hdl->requests_queued, &main_hdl->requests_queued); + hdl->request_is_queued = true; + mutex_unlock(main_hdl->lock); +} + +static void v4l2_ctrl_request_unbind(struct media_request_object *obj) +{ + struct v4l2_ctrl_handler *hdl = + container_of(obj, struct v4l2_ctrl_handler, req_obj); + struct v4l2_ctrl_handler *main_hdl = obj->priv; + + mutex_lock(main_hdl->lock); + list_del_init(&hdl->requests); + if (hdl->request_is_queued) { + list_del_init(&hdl->requests_queued); + hdl->request_is_queued = false; + } + mutex_unlock(main_hdl->lock); +} + +static void v4l2_ctrl_request_release(struct media_request_object *obj) +{ + struct v4l2_ctrl_handler *hdl = + container_of(obj, struct v4l2_ctrl_handler, req_obj); + + v4l2_ctrl_handler_free(hdl); + kfree(hdl); +} + +static const struct media_request_object_ops req_ops = { + .queue = v4l2_ctrl_request_queue, + .unbind = v4l2_ctrl_request_unbind, + .release = v4l2_ctrl_request_release, +}; + +struct v4l2_ctrl_handler *v4l2_ctrl_request_hdl_find(struct media_request *req, + struct v4l2_ctrl_handler *parent) +{ + struct media_request_object *obj; + + if (WARN_ON(req->state != MEDIA_REQUEST_STATE_VALIDATING && + req->state != MEDIA_REQUEST_STATE_QUEUED)) + return NULL; + + obj = media_request_object_find(req, &req_ops, parent); + if (obj) + return container_of(obj, struct v4l2_ctrl_handler, req_obj); + return NULL; +} +EXPORT_SYMBOL_GPL(v4l2_ctrl_request_hdl_find); + +struct v4l2_ctrl * +v4l2_ctrl_request_hdl_ctrl_find(struct v4l2_ctrl_handler *hdl, u32 id) +{ + struct v4l2_ctrl_ref *ref = find_ref_lock(hdl, id); + + return (ref && ref->valid_p_req) ? ref->ctrl : NULL; +} +EXPORT_SYMBOL_GPL(v4l2_ctrl_request_hdl_ctrl_find); + +static int v4l2_ctrl_request_bind(struct media_request *req, + struct v4l2_ctrl_handler *hdl, + struct v4l2_ctrl_handler *from) +{ + int ret; + + ret = v4l2_ctrl_request_clone(hdl, from); + + if (!ret) { + ret = media_request_object_bind(req, &req_ops, + from, false, &hdl->req_obj); + if (!ret) { + mutex_lock(from->lock); + list_add_tail(&hdl->requests, &from->requests); + mutex_unlock(from->lock); + } + } + return ret; +} + +static struct media_request_object * +v4l2_ctrls_find_req_obj(struct v4l2_ctrl_handler *hdl, + struct media_request *req, bool set) +{ + struct media_request_object *obj; + struct v4l2_ctrl_handler *new_hdl; + int ret; + + if (IS_ERR(req)) + return ERR_CAST(req); + + if (set && WARN_ON(req->state != MEDIA_REQUEST_STATE_UPDATING)) + return ERR_PTR(-EBUSY); + + obj = media_request_object_find(req, &req_ops, hdl); + if (obj) + return obj; + /* + * If there are no controls in this completed request, + * then that can only happen if: + * + * 1) no controls were present in the queued request, and + * 2) v4l2_ctrl_request_complete() could not allocate a + * control handler object to store the completed state in. + * + * So return ENOMEM to indicate that there was an out-of-memory + * error. + */ + if (!set) + return ERR_PTR(-ENOMEM); + + new_hdl = kzalloc(sizeof(*new_hdl), GFP_KERNEL); + if (!new_hdl) + return ERR_PTR(-ENOMEM); + + obj = &new_hdl->req_obj; + ret = v4l2_ctrl_handler_init(new_hdl, (hdl->nr_of_buckets - 1) * 8); + if (!ret) + ret = v4l2_ctrl_request_bind(req, new_hdl, hdl); + if (ret) { + v4l2_ctrl_handler_free(new_hdl); + kfree(new_hdl); + return ERR_PTR(ret); + } + + media_request_object_get(obj); + return obj; +} + +int v4l2_g_ext_ctrls_request(struct v4l2_ctrl_handler *hdl, struct video_device *vdev, + struct media_device *mdev, struct v4l2_ext_controls *cs) +{ + struct media_request_object *obj = NULL; + struct media_request *req = NULL; + int ret; + + if (!mdev || cs->request_fd < 0) + return -EINVAL; + + req = media_request_get_by_fd(mdev, cs->request_fd); + if (IS_ERR(req)) + return PTR_ERR(req); + + if (req->state != MEDIA_REQUEST_STATE_COMPLETE) { + media_request_put(req); + return -EACCES; + } + + ret = media_request_lock_for_access(req); + if (ret) { + media_request_put(req); + return ret; + } + + obj = v4l2_ctrls_find_req_obj(hdl, req, false); + if (IS_ERR(obj)) { + media_request_unlock_for_access(req); + media_request_put(req); + return PTR_ERR(obj); + } + + hdl = container_of(obj, struct v4l2_ctrl_handler, + req_obj); + ret = v4l2_g_ext_ctrls_common(hdl, cs, vdev); + + media_request_unlock_for_access(req); + media_request_object_put(obj); + media_request_put(req); + return ret; +} + +int try_set_ext_ctrls_request(struct v4l2_fh *fh, + struct v4l2_ctrl_handler *hdl, + struct video_device *vdev, + struct media_device *mdev, + struct v4l2_ext_controls *cs, bool set) +{ + struct media_request_object *obj = NULL; + struct media_request *req = NULL; + int ret; + + if (!mdev) { + dprintk(vdev, "%s: missing media device\n", + video_device_node_name(vdev)); + return -EINVAL; + } + + if (cs->request_fd < 0) { + dprintk(vdev, "%s: invalid request fd %d\n", + video_device_node_name(vdev), cs->request_fd); + return -EINVAL; + } + + req = media_request_get_by_fd(mdev, cs->request_fd); + if (IS_ERR(req)) { + dprintk(vdev, "%s: cannot find request fd %d\n", + video_device_node_name(vdev), cs->request_fd); + return PTR_ERR(req); + } + + ret = media_request_lock_for_update(req); + if (ret) { + dprintk(vdev, "%s: cannot lock request fd %d\n", + video_device_node_name(vdev), cs->request_fd); + media_request_put(req); + return ret; + } + + obj = v4l2_ctrls_find_req_obj(hdl, req, set); + if (IS_ERR(obj)) { + dprintk(vdev, + "%s: cannot find request object for request fd %d\n", + video_device_node_name(vdev), + cs->request_fd); + media_request_unlock_for_update(req); + media_request_put(req); + return PTR_ERR(obj); + } + + hdl = container_of(obj, struct v4l2_ctrl_handler, + req_obj); + ret = try_set_ext_ctrls_common(fh, hdl, cs, vdev, set); + if (ret) + dprintk(vdev, + "%s: try_set_ext_ctrls_common failed (%d)\n", + video_device_node_name(vdev), ret); + + media_request_unlock_for_update(req); + media_request_object_put(obj); + media_request_put(req); + + return ret; +} + +void v4l2_ctrl_request_complete(struct media_request *req, + struct v4l2_ctrl_handler *main_hdl) +{ + struct media_request_object *obj; + struct v4l2_ctrl_handler *hdl; + struct v4l2_ctrl_ref *ref; + + if (!req || !main_hdl) + return; + + /* + * Note that it is valid if nothing was found. It means + * that this request doesn't have any controls and so just + * wants to leave the controls unchanged. + */ + obj = media_request_object_find(req, &req_ops, main_hdl); + if (!obj) { + int ret; + + /* Create a new request so the driver can return controls */ + hdl = kzalloc(sizeof(*hdl), GFP_KERNEL); + if (!hdl) + return; + + ret = v4l2_ctrl_handler_init(hdl, (main_hdl->nr_of_buckets - 1) * 8); + if (!ret) + ret = v4l2_ctrl_request_bind(req, hdl, main_hdl); + if (ret) { + v4l2_ctrl_handler_free(hdl); + kfree(hdl); + return; + } + hdl->request_is_queued = true; + obj = media_request_object_find(req, &req_ops, main_hdl); + } + hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj); + + list_for_each_entry(ref, &hdl->ctrl_refs, node) { + struct v4l2_ctrl *ctrl = ref->ctrl; + struct v4l2_ctrl *master = ctrl->cluster[0]; + unsigned int i; + + if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) { + v4l2_ctrl_lock(master); + /* g_volatile_ctrl will update the current control values */ + for (i = 0; i < master->ncontrols; i++) + cur_to_new(master->cluster[i]); + call_op(master, g_volatile_ctrl); + new_to_req(ref); + v4l2_ctrl_unlock(master); + continue; + } + if (ref->valid_p_req) + continue; + + /* Copy the current control value into the request */ + v4l2_ctrl_lock(ctrl); + cur_to_req(ref); + v4l2_ctrl_unlock(ctrl); + } + + mutex_lock(main_hdl->lock); + WARN_ON(!hdl->request_is_queued); + list_del_init(&hdl->requests_queued); + hdl->request_is_queued = false; + mutex_unlock(main_hdl->lock); + media_request_object_complete(obj); + media_request_object_put(obj); +} +EXPORT_SYMBOL(v4l2_ctrl_request_complete); + +int v4l2_ctrl_request_setup(struct media_request *req, + struct v4l2_ctrl_handler *main_hdl) +{ + struct media_request_object *obj; + struct v4l2_ctrl_handler *hdl; + struct v4l2_ctrl_ref *ref; + int ret = 0; + + if (!req || !main_hdl) + return 0; + + if (WARN_ON(req->state != MEDIA_REQUEST_STATE_QUEUED)) + return -EBUSY; + + /* + * Note that it is valid if nothing was found. It means + * that this request doesn't have any controls and so just + * wants to leave the controls unchanged. + */ + obj = media_request_object_find(req, &req_ops, main_hdl); + if (!obj) + return 0; + if (obj->completed) { + media_request_object_put(obj); + return -EBUSY; + } + hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj); + + list_for_each_entry(ref, &hdl->ctrl_refs, node) + ref->req_done = false; + + list_for_each_entry(ref, &hdl->ctrl_refs, node) { + struct v4l2_ctrl *ctrl = ref->ctrl; + struct v4l2_ctrl *master = ctrl->cluster[0]; + bool have_new_data = false; + int i; + + /* + * Skip if this control was already handled by a cluster. + * Skip button controls and read-only controls. + */ + if (ref->req_done || (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY)) + continue; + + v4l2_ctrl_lock(master); + for (i = 0; i < master->ncontrols; i++) { + if (master->cluster[i]) { + struct v4l2_ctrl_ref *r = + find_ref(hdl, master->cluster[i]->id); + + if (r->valid_p_req) { + have_new_data = true; + break; + } + } + } + if (!have_new_data) { + v4l2_ctrl_unlock(master); + continue; + } + + for (i = 0; i < master->ncontrols; i++) { + if (master->cluster[i]) { + struct v4l2_ctrl_ref *r = + find_ref(hdl, master->cluster[i]->id); + + req_to_new(r); + master->cluster[i]->is_new = 1; + r->req_done = true; + } + } + /* + * For volatile autoclusters that are currently in auto mode + * we need to discover if it will be set to manual mode. + * If so, then we have to copy the current volatile values + * first since those will become the new manual values (which + * may be overwritten by explicit new values from this set + * of controls). + */ + if (master->is_auto && master->has_volatiles && + !is_cur_manual(master)) { + s32 new_auto_val = *master->p_new.p_s32; + + /* + * If the new value == the manual value, then copy + * the current volatile values. + */ + if (new_auto_val == master->manual_mode_value) + update_from_auto_cluster(master); + } + + ret = try_or_set_cluster(NULL, master, true, 0); + v4l2_ctrl_unlock(master); + + if (ret) + break; + } + + media_request_object_put(obj); + return ret; +} +EXPORT_SYMBOL(v4l2_ctrl_request_setup); diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c deleted file mode 100644 index 0d7fe1bd975a..000000000000 --- a/drivers/media/v4l2-core/v4l2-ctrls.c +++ /dev/null @@ -1,5035 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - V4L2 controls framework implementation. - - Copyright (C) 2010 Hans Verkuil - - */ - -#define pr_fmt(fmt) "v4l2-ctrls: " fmt - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define dprintk(vdev, fmt, arg...) do { \ - if (!WARN_ON(!(vdev)) && ((vdev)->dev_debug & V4L2_DEV_DEBUG_CTRL)) \ - printk(KERN_DEBUG pr_fmt("%s: %s: " fmt), \ - __func__, video_device_node_name(vdev), ##arg); \ -} while (0) - -#define has_op(master, op) \ - (master->ops && master->ops->op) -#define call_op(master, op) \ - (has_op(master, op) ? master->ops->op(master) : 0) - -static const union v4l2_ctrl_ptr ptr_null; - -/* Internal temporary helper struct, one for each v4l2_ext_control */ -struct v4l2_ctrl_helper { - /* Pointer to the control reference of the master control */ - struct v4l2_ctrl_ref *mref; - /* The control ref corresponding to the v4l2_ext_control ID field. */ - struct v4l2_ctrl_ref *ref; - /* v4l2_ext_control index of the next control belonging to the - same cluster, or 0 if there isn't any. */ - u32 next; -}; - -/* Small helper function to determine if the autocluster is set to manual - mode. */ -static bool is_cur_manual(const struct v4l2_ctrl *master) -{ - return master->is_auto && master->cur.val == master->manual_mode_value; -} - -/* Same as above, but this checks the against the new value instead of the - current value. */ -static bool is_new_manual(const struct v4l2_ctrl *master) -{ - return master->is_auto && master->val == master->manual_mode_value; -} - -/* Returns NULL or a character pointer array containing the menu for - the given control ID. The pointer array ends with a NULL pointer. - An empty string signifies a menu entry that is invalid. This allows - drivers to disable certain options if it is not supported. */ -const char * const *v4l2_ctrl_get_menu(u32 id) -{ - static const char * const mpeg_audio_sampling_freq[] = { - "44.1 kHz", - "48 kHz", - "32 kHz", - NULL - }; - static const char * const mpeg_audio_encoding[] = { - "MPEG-1/2 Layer I", - "MPEG-1/2 Layer II", - "MPEG-1/2 Layer III", - "MPEG-2/4 AAC", - "AC-3", - NULL - }; - static const char * const mpeg_audio_l1_bitrate[] = { - "32 kbps", - "64 kbps", - "96 kbps", - "128 kbps", - "160 kbps", - "192 kbps", - "224 kbps", - "256 kbps", - "288 kbps", - "320 kbps", - "352 kbps", - "384 kbps", - "416 kbps", - "448 kbps", - NULL - }; - static const char * const mpeg_audio_l2_bitrate[] = { - "32 kbps", - "48 kbps", - "56 kbps", - "64 kbps", - "80 kbps", - "96 kbps", - "112 kbps", - "128 kbps", - "160 kbps", - "192 kbps", - "224 kbps", - "256 kbps", - "320 kbps", - "384 kbps", - NULL - }; - static const char * const mpeg_audio_l3_bitrate[] = { - "32 kbps", - "40 kbps", - "48 kbps", - "56 kbps", - "64 kbps", - "80 kbps", - "96 kbps", - "112 kbps", - "128 kbps", - "160 kbps", - "192 kbps", - "224 kbps", - "256 kbps", - "320 kbps", - NULL - }; - static const char * const mpeg_audio_ac3_bitrate[] = { - "32 kbps", - "40 kbps", - "48 kbps", - "56 kbps", - "64 kbps", - "80 kbps", - "96 kbps", - "112 kbps", - "128 kbps", - "160 kbps", - "192 kbps", - "224 kbps", - "256 kbps", - "320 kbps", - "384 kbps", - "448 kbps", - "512 kbps", - "576 kbps", - "640 kbps", - NULL - }; - static const char * const mpeg_audio_mode[] = { - "Stereo", - "Joint Stereo", - "Dual", - "Mono", - NULL - }; - static const char * const mpeg_audio_mode_extension[] = { - "Bound 4", - "Bound 8", - "Bound 12", - "Bound 16", - NULL - }; - static const char * const mpeg_audio_emphasis[] = { - "No Emphasis", - "50/15 us", - "CCITT J17", - NULL - }; - static const char * const mpeg_audio_crc[] = { - "No CRC", - "16-bit CRC", - NULL - }; - static const char * const mpeg_audio_dec_playback[] = { - "Auto", - "Stereo", - "Left", - "Right", - "Mono", - "Swapped Stereo", - NULL - }; - static const char * const mpeg_video_encoding[] = { - "MPEG-1", - "MPEG-2", - "MPEG-4 AVC", - NULL - }; - static const char * const mpeg_video_aspect[] = { - "1x1", - "4x3", - "16x9", - "2.21x1", - NULL - }; - static const char * const mpeg_video_bitrate_mode[] = { - "Variable Bitrate", - "Constant Bitrate", - "Constant Quality", - NULL - }; - static const char * const mpeg_stream_type[] = { - "MPEG-2 Program Stream", - "MPEG-2 Transport Stream", - "MPEG-1 System Stream", - "MPEG-2 DVD-compatible Stream", - "MPEG-1 VCD-compatible Stream", - "MPEG-2 SVCD-compatible Stream", - NULL - }; - static const char * const mpeg_stream_vbi_fmt[] = { - "No VBI", - "Private Packet, IVTV Format", - NULL - }; - static const char * const camera_power_line_frequency[] = { - "Disabled", - "50 Hz", - "60 Hz", - "Auto", - NULL - }; - static const char * const camera_exposure_auto[] = { - "Auto Mode", - "Manual Mode", - "Shutter Priority Mode", - "Aperture Priority Mode", - NULL - }; - static const char * const camera_exposure_metering[] = { - "Average", - "Center Weighted", - "Spot", - "Matrix", - NULL - }; - static const char * const camera_auto_focus_range[] = { - "Auto", - "Normal", - "Macro", - "Infinity", - NULL - }; - static const char * const colorfx[] = { - "None", - "Black & White", - "Sepia", - "Negative", - "Emboss", - "Sketch", - "Sky Blue", - "Grass Green", - "Skin Whiten", - "Vivid", - "Aqua", - "Art Freeze", - "Silhouette", - "Solarization", - "Antique", - "Set Cb/Cr", - NULL - }; - static const char * const auto_n_preset_white_balance[] = { - "Manual", - "Auto", - "Incandescent", - "Fluorescent", - "Fluorescent H", - "Horizon", - "Daylight", - "Flash", - "Cloudy", - "Shade", - NULL, - }; - static const char * const camera_iso_sensitivity_auto[] = { - "Manual", - "Auto", - NULL - }; - static const char * const scene_mode[] = { - "None", - "Backlight", - "Beach/Snow", - "Candle Light", - "Dusk/Dawn", - "Fall Colors", - "Fireworks", - "Landscape", - "Night", - "Party/Indoor", - "Portrait", - "Sports", - "Sunset", - "Text", - NULL - }; - static const char * const tune_emphasis[] = { - "None", - "50 Microseconds", - "75 Microseconds", - NULL, - }; - static const char * const header_mode[] = { - "Separate Buffer", - "Joined With 1st Frame", - NULL, - }; - static const char * const multi_slice[] = { - "Single", - "Max Macroblocks", - "Max Bytes", - NULL, - }; - static const char * const entropy_mode[] = { - "CAVLC", - "CABAC", - NULL, - }; - static const char * const mpeg_h264_level[] = { - "1", - "1b", - "1.1", - "1.2", - "1.3", - "2", - "2.1", - "2.2", - "3", - "3.1", - "3.2", - "4", - "4.1", - "4.2", - "5", - "5.1", - "5.2", - "6.0", - "6.1", - "6.2", - NULL, - }; - static const char * const h264_loop_filter[] = { - "Enabled", - "Disabled", - "Disabled at Slice Boundary", - NULL, - }; - static const char * const h264_profile[] = { - "Baseline", - "Constrained Baseline", - "Main", - "Extended", - "High", - "High 10", - "High 422", - "High 444 Predictive", - "High 10 Intra", - "High 422 Intra", - "High 444 Intra", - "CAVLC 444 Intra", - "Scalable Baseline", - "Scalable High", - "Scalable High Intra", - "Stereo High", - "Multiview High", - "Constrained High", - NULL, - }; - static const char * const vui_sar_idc[] = { - "Unspecified", - "1:1", - "12:11", - "10:11", - "16:11", - "40:33", - "24:11", - "20:11", - "32:11", - "80:33", - "18:11", - "15:11", - "64:33", - "160:99", - "4:3", - "3:2", - "2:1", - "Extended SAR", - NULL, - }; - static const char * const h264_fp_arrangement_type[] = { - "Checkerboard", - "Column", - "Row", - "Side by Side", - "Top Bottom", - "Temporal", - NULL, - }; - static const char * const h264_fmo_map_type[] = { - "Interleaved Slices", - "Scattered Slices", - "Foreground with Leftover", - "Box Out", - "Raster Scan", - "Wipe Scan", - "Explicit", - NULL, - }; - static const char * const h264_decode_mode[] = { - "Slice-Based", - "Frame-Based", - NULL, - }; - static const char * const h264_start_code[] = { - "No Start Code", - "Annex B Start Code", - NULL, - }; - static const char * const h264_hierarchical_coding_type[] = { - "Hier Coding B", - "Hier Coding P", - NULL, - }; - static const char * const mpeg_mpeg2_level[] = { - "Low", - "Main", - "High 1440", - "High", - NULL, - }; - static const char * const mpeg2_profile[] = { - "Simple", - "Main", - "SNR Scalable", - "Spatially Scalable", - "High", - NULL, - }; - static const char * const mpeg_mpeg4_level[] = { - "0", - "0b", - "1", - "2", - "3", - "3b", - "4", - "5", - NULL, - }; - static const char * const mpeg4_profile[] = { - "Simple", - "Advanced Simple", - "Core", - "Simple Scalable", - "Advanced Coding Efficiency", - NULL, - }; - - static const char * const vpx_golden_frame_sel[] = { - "Use Previous Frame", - "Use Previous Specific Frame", - NULL, - }; - static const char * const vp8_profile[] = { - "0", - "1", - "2", - "3", - NULL, - }; - static const char * const vp9_profile[] = { - "0", - "1", - "2", - "3", - NULL, - }; - static const char * const vp9_level[] = { - "1", - "1.1", - "2", - "2.1", - "3", - "3.1", - "4", - "4.1", - "5", - "5.1", - "5.2", - "6", - "6.1", - "6.2", - NULL, - }; - - static const char * const flash_led_mode[] = { - "Off", - "Flash", - "Torch", - NULL, - }; - static const char * const flash_strobe_source[] = { - "Software", - "External", - NULL, - }; - - static const char * const jpeg_chroma_subsampling[] = { - "4:4:4", - "4:2:2", - "4:2:0", - "4:1:1", - "4:1:0", - "Gray", - NULL, - }; - static const char * const dv_tx_mode[] = { - "DVI-D", - "HDMI", - NULL, - }; - static const char * const dv_rgb_range[] = { - "Automatic", - "RGB Limited Range (16-235)", - "RGB Full Range (0-255)", - NULL, - }; - static const char * const dv_it_content_type[] = { - "Graphics", - "Photo", - "Cinema", - "Game", - "No IT Content", - NULL, - }; - static const char * const detect_md_mode[] = { - "Disabled", - "Global", - "Threshold Grid", - "Region Grid", - NULL, - }; - - static const char * const hevc_profile[] = { - "Main", - "Main Still Picture", - "Main 10", - NULL, - }; - static const char * const hevc_level[] = { - "1", - "2", - "2.1", - "3", - "3.1", - "4", - "4.1", - "5", - "5.1", - "5.2", - "6", - "6.1", - "6.2", - NULL, - }; - static const char * const hevc_hierarchial_coding_type[] = { - "B", - "P", - NULL, - }; - static const char * const hevc_refresh_type[] = { - "None", - "CRA", - "IDR", - NULL, - }; - static const char * const hevc_size_of_length_field[] = { - "0", - "1", - "2", - "4", - NULL, - }; - static const char * const hevc_tier[] = { - "Main", - "High", - NULL, - }; - static const char * const hevc_loop_filter_mode[] = { - "Disabled", - "Enabled", - "Disabled at slice boundary", - "NULL", - }; - static const char * const hevc_decode_mode[] = { - "Slice-Based", - "Frame-Based", - NULL, - }; - static const char * const hevc_start_code[] = { - "No Start Code", - "Annex B Start Code", - NULL, - }; - static const char * const camera_orientation[] = { - "Front", - "Back", - "External", - NULL, - }; - static const char * const mpeg_video_frame_skip[] = { - "Disabled", - "Level Limit", - "VBV/CPB Limit", - NULL, - }; - - switch (id) { - case V4L2_CID_MPEG_AUDIO_SAMPLING_FREQ: - return mpeg_audio_sampling_freq; - case V4L2_CID_MPEG_AUDIO_ENCODING: - return mpeg_audio_encoding; - case V4L2_CID_MPEG_AUDIO_L1_BITRATE: - return mpeg_audio_l1_bitrate; - case V4L2_CID_MPEG_AUDIO_L2_BITRATE: - return mpeg_audio_l2_bitrate; - case V4L2_CID_MPEG_AUDIO_L3_BITRATE: - return mpeg_audio_l3_bitrate; - case V4L2_CID_MPEG_AUDIO_AC3_BITRATE: - return mpeg_audio_ac3_bitrate; - case V4L2_CID_MPEG_AUDIO_MODE: - return mpeg_audio_mode; - case V4L2_CID_MPEG_AUDIO_MODE_EXTENSION: - return mpeg_audio_mode_extension; - case V4L2_CID_MPEG_AUDIO_EMPHASIS: - return mpeg_audio_emphasis; - case V4L2_CID_MPEG_AUDIO_CRC: - return mpeg_audio_crc; - case V4L2_CID_MPEG_AUDIO_DEC_PLAYBACK: - case V4L2_CID_MPEG_AUDIO_DEC_MULTILINGUAL_PLAYBACK: - return mpeg_audio_dec_playback; - case V4L2_CID_MPEG_VIDEO_ENCODING: - return mpeg_video_encoding; - case V4L2_CID_MPEG_VIDEO_ASPECT: - return mpeg_video_aspect; - case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: - return mpeg_video_bitrate_mode; - case V4L2_CID_MPEG_STREAM_TYPE: - return mpeg_stream_type; - case V4L2_CID_MPEG_STREAM_VBI_FMT: - return mpeg_stream_vbi_fmt; - case V4L2_CID_POWER_LINE_FREQUENCY: - return camera_power_line_frequency; - case V4L2_CID_EXPOSURE_AUTO: - return camera_exposure_auto; - case V4L2_CID_EXPOSURE_METERING: - return camera_exposure_metering; - case V4L2_CID_AUTO_FOCUS_RANGE: - return camera_auto_focus_range; - case V4L2_CID_COLORFX: - return colorfx; - case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE: - return auto_n_preset_white_balance; - case V4L2_CID_ISO_SENSITIVITY_AUTO: - return camera_iso_sensitivity_auto; - case V4L2_CID_SCENE_MODE: - return scene_mode; - case V4L2_CID_TUNE_PREEMPHASIS: - return tune_emphasis; - case V4L2_CID_TUNE_DEEMPHASIS: - return tune_emphasis; - case V4L2_CID_FLASH_LED_MODE: - return flash_led_mode; - case V4L2_CID_FLASH_STROBE_SOURCE: - return flash_strobe_source; - case V4L2_CID_MPEG_VIDEO_HEADER_MODE: - return header_mode; - case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE: - return mpeg_video_frame_skip; - case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE: - return multi_slice; - case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE: - return entropy_mode; - case V4L2_CID_MPEG_VIDEO_H264_LEVEL: - return mpeg_h264_level; - case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE: - return h264_loop_filter; - case V4L2_CID_MPEG_VIDEO_H264_PROFILE: - return h264_profile; - case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC: - return vui_sar_idc; - case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE: - return h264_fp_arrangement_type; - case V4L2_CID_MPEG_VIDEO_H264_FMO_MAP_TYPE: - return h264_fmo_map_type; - case V4L2_CID_STATELESS_H264_DECODE_MODE: - return h264_decode_mode; - case V4L2_CID_STATELESS_H264_START_CODE: - return h264_start_code; - case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_TYPE: - return h264_hierarchical_coding_type; - case V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL: - return mpeg_mpeg2_level; - case V4L2_CID_MPEG_VIDEO_MPEG2_PROFILE: - return mpeg2_profile; - case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL: - return mpeg_mpeg4_level; - case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE: - return mpeg4_profile; - case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_SEL: - return vpx_golden_frame_sel; - case V4L2_CID_MPEG_VIDEO_VP8_PROFILE: - return vp8_profile; - case V4L2_CID_MPEG_VIDEO_VP9_PROFILE: - return vp9_profile; - case V4L2_CID_MPEG_VIDEO_VP9_LEVEL: - return vp9_level; - case V4L2_CID_JPEG_CHROMA_SUBSAMPLING: - return jpeg_chroma_subsampling; - case V4L2_CID_DV_TX_MODE: - return dv_tx_mode; - case V4L2_CID_DV_TX_RGB_RANGE: - case V4L2_CID_DV_RX_RGB_RANGE: - return dv_rgb_range; - case V4L2_CID_DV_TX_IT_CONTENT_TYPE: - case V4L2_CID_DV_RX_IT_CONTENT_TYPE: - return dv_it_content_type; - case V4L2_CID_DETECT_MD_MODE: - return detect_md_mode; - case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: - return hevc_profile; - case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: - return hevc_level; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_TYPE: - return hevc_hierarchial_coding_type; - case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_TYPE: - return hevc_refresh_type; - case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: - return hevc_size_of_length_field; - case V4L2_CID_MPEG_VIDEO_HEVC_TIER: - return hevc_tier; - case V4L2_CID_MPEG_VIDEO_HEVC_LOOP_FILTER_MODE: - return hevc_loop_filter_mode; - case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE: - return hevc_decode_mode; - case V4L2_CID_MPEG_VIDEO_HEVC_START_CODE: - return hevc_start_code; - case V4L2_CID_CAMERA_ORIENTATION: - return camera_orientation; - default: - return NULL; - } -} -EXPORT_SYMBOL(v4l2_ctrl_get_menu); - -#define __v4l2_qmenu_int_len(arr, len) ({ *(len) = ARRAY_SIZE(arr); arr; }) -/* - * Returns NULL or an s64 type array containing the menu for given - * control ID. The total number of the menu items is returned in @len. - */ -const s64 *v4l2_ctrl_get_int_menu(u32 id, u32 *len) -{ - static const s64 qmenu_int_vpx_num_partitions[] = { - 1, 2, 4, 8, - }; - - static const s64 qmenu_int_vpx_num_ref_frames[] = { - 1, 2, 3, - }; - - switch (id) { - case V4L2_CID_MPEG_VIDEO_VPX_NUM_PARTITIONS: - return __v4l2_qmenu_int_len(qmenu_int_vpx_num_partitions, len); - case V4L2_CID_MPEG_VIDEO_VPX_NUM_REF_FRAMES: - return __v4l2_qmenu_int_len(qmenu_int_vpx_num_ref_frames, len); - default: - *len = 0; - return NULL; - } -} -EXPORT_SYMBOL(v4l2_ctrl_get_int_menu); - -/* Return the control name. */ -const char *v4l2_ctrl_get_name(u32 id) -{ - switch (id) { - /* USER controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_USER_CLASS: return "User Controls"; - case V4L2_CID_BRIGHTNESS: return "Brightness"; - case V4L2_CID_CONTRAST: return "Contrast"; - case V4L2_CID_SATURATION: return "Saturation"; - case V4L2_CID_HUE: return "Hue"; - case V4L2_CID_AUDIO_VOLUME: return "Volume"; - case V4L2_CID_AUDIO_BALANCE: return "Balance"; - case V4L2_CID_AUDIO_BASS: return "Bass"; - case V4L2_CID_AUDIO_TREBLE: return "Treble"; - case V4L2_CID_AUDIO_MUTE: return "Mute"; - case V4L2_CID_AUDIO_LOUDNESS: return "Loudness"; - case V4L2_CID_BLACK_LEVEL: return "Black Level"; - case V4L2_CID_AUTO_WHITE_BALANCE: return "White Balance, Automatic"; - case V4L2_CID_DO_WHITE_BALANCE: return "Do White Balance"; - case V4L2_CID_RED_BALANCE: return "Red Balance"; - case V4L2_CID_BLUE_BALANCE: return "Blue Balance"; - case V4L2_CID_GAMMA: return "Gamma"; - case V4L2_CID_EXPOSURE: return "Exposure"; - case V4L2_CID_AUTOGAIN: return "Gain, Automatic"; - case V4L2_CID_GAIN: return "Gain"; - case V4L2_CID_HFLIP: return "Horizontal Flip"; - case V4L2_CID_VFLIP: return "Vertical Flip"; - case V4L2_CID_POWER_LINE_FREQUENCY: return "Power Line Frequency"; - case V4L2_CID_HUE_AUTO: return "Hue, Automatic"; - case V4L2_CID_WHITE_BALANCE_TEMPERATURE: return "White Balance Temperature"; - case V4L2_CID_SHARPNESS: return "Sharpness"; - case V4L2_CID_BACKLIGHT_COMPENSATION: return "Backlight Compensation"; - case V4L2_CID_CHROMA_AGC: return "Chroma AGC"; - case V4L2_CID_COLOR_KILLER: return "Color Killer"; - case V4L2_CID_COLORFX: return "Color Effects"; - case V4L2_CID_AUTOBRIGHTNESS: return "Brightness, Automatic"; - case V4L2_CID_BAND_STOP_FILTER: return "Band-Stop Filter"; - case V4L2_CID_ROTATE: return "Rotate"; - case V4L2_CID_BG_COLOR: return "Background Color"; - case V4L2_CID_CHROMA_GAIN: return "Chroma Gain"; - case V4L2_CID_ILLUMINATORS_1: return "Illuminator 1"; - case V4L2_CID_ILLUMINATORS_2: return "Illuminator 2"; - case V4L2_CID_MIN_BUFFERS_FOR_CAPTURE: return "Min Number of Capture Buffers"; - case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT: return "Min Number of Output Buffers"; - case V4L2_CID_ALPHA_COMPONENT: return "Alpha Component"; - case V4L2_CID_COLORFX_CBCR: return "Color Effects, CbCr"; - - /* Codec controls */ - /* The MPEG controls are applicable to all codec controls - * and the 'MPEG' part of the define is historical */ - /* Keep the order of the 'case's the same as in videodev2.h! */ - case V4L2_CID_CODEC_CLASS: return "Codec Controls"; - case V4L2_CID_MPEG_STREAM_TYPE: return "Stream Type"; - case V4L2_CID_MPEG_STREAM_PID_PMT: return "Stream PMT Program ID"; - case V4L2_CID_MPEG_STREAM_PID_AUDIO: return "Stream Audio Program ID"; - case V4L2_CID_MPEG_STREAM_PID_VIDEO: return "Stream Video Program ID"; - case V4L2_CID_MPEG_STREAM_PID_PCR: return "Stream PCR Program ID"; - case V4L2_CID_MPEG_STREAM_PES_ID_AUDIO: return "Stream PES Audio ID"; - case V4L2_CID_MPEG_STREAM_PES_ID_VIDEO: return "Stream PES Video ID"; - case V4L2_CID_MPEG_STREAM_VBI_FMT: return "Stream VBI Format"; - case V4L2_CID_MPEG_AUDIO_SAMPLING_FREQ: return "Audio Sampling Frequency"; - case V4L2_CID_MPEG_AUDIO_ENCODING: return "Audio Encoding"; - case V4L2_CID_MPEG_AUDIO_L1_BITRATE: return "Audio Layer I Bitrate"; - case V4L2_CID_MPEG_AUDIO_L2_BITRATE: return "Audio Layer II Bitrate"; - case V4L2_CID_MPEG_AUDIO_L3_BITRATE: return "Audio Layer III Bitrate"; - case V4L2_CID_MPEG_AUDIO_MODE: return "Audio Stereo Mode"; - case V4L2_CID_MPEG_AUDIO_MODE_EXTENSION: return "Audio Stereo Mode Extension"; - case V4L2_CID_MPEG_AUDIO_EMPHASIS: return "Audio Emphasis"; - case V4L2_CID_MPEG_AUDIO_CRC: return "Audio CRC"; - case V4L2_CID_MPEG_AUDIO_MUTE: return "Audio Mute"; - case V4L2_CID_MPEG_AUDIO_AAC_BITRATE: return "Audio AAC Bitrate"; - case V4L2_CID_MPEG_AUDIO_AC3_BITRATE: return "Audio AC-3 Bitrate"; - case V4L2_CID_MPEG_AUDIO_DEC_PLAYBACK: return "Audio Playback"; - case V4L2_CID_MPEG_AUDIO_DEC_MULTILINGUAL_PLAYBACK: return "Audio Multilingual Playback"; - case V4L2_CID_MPEG_VIDEO_ENCODING: return "Video Encoding"; - case V4L2_CID_MPEG_VIDEO_ASPECT: return "Video Aspect"; - case V4L2_CID_MPEG_VIDEO_B_FRAMES: return "Video B Frames"; - case V4L2_CID_MPEG_VIDEO_GOP_SIZE: return "Video GOP Size"; - case V4L2_CID_MPEG_VIDEO_GOP_CLOSURE: return "Video GOP Closure"; - case V4L2_CID_MPEG_VIDEO_PULLDOWN: return "Video Pulldown"; - case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: return "Video Bitrate Mode"; - case V4L2_CID_MPEG_VIDEO_CONSTANT_QUALITY: return "Constant Quality"; - case V4L2_CID_MPEG_VIDEO_BITRATE: return "Video Bitrate"; - case V4L2_CID_MPEG_VIDEO_BITRATE_PEAK: return "Video Peak Bitrate"; - case V4L2_CID_MPEG_VIDEO_TEMPORAL_DECIMATION: return "Video Temporal Decimation"; - case V4L2_CID_MPEG_VIDEO_MUTE: return "Video Mute"; - case V4L2_CID_MPEG_VIDEO_MUTE_YUV: return "Video Mute YUV"; - case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE: return "Decoder Slice Interface"; - case V4L2_CID_MPEG_VIDEO_DECODER_MPEG4_DEBLOCK_FILTER: return "MPEG4 Loop Filter Enable"; - case V4L2_CID_MPEG_VIDEO_CYCLIC_INTRA_REFRESH_MB: return "Number of Intra Refresh MBs"; - case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE: return "Frame Level Rate Control Enable"; - case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE: return "H264 MB Level Rate Control"; - case V4L2_CID_MPEG_VIDEO_HEADER_MODE: return "Sequence Header Mode"; - case V4L2_CID_MPEG_VIDEO_MAX_REF_PIC: return "Max Number of Reference Pics"; - case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE: return "Frame Skip Mode"; - case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY: return "Display Delay"; - case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE: return "Display Delay Enable"; - case V4L2_CID_MPEG_VIDEO_AU_DELIMITER: return "Generate Access Unit Delimiters"; - case V4L2_CID_MPEG_VIDEO_H263_I_FRAME_QP: return "H263 I-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_H263_P_FRAME_QP: return "H263 P-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_H263_B_FRAME_QP: return "H263 B-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_H263_MIN_QP: return "H263 Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_H263_MAX_QP: return "H263 Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_QP: return "H264 I-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_QP: return "H264 P-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_QP: return "H264 B-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_MAX_QP: return "H264 Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_MIN_QP: return "H264 Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM: return "H264 8x8 Transform Enable"; - case V4L2_CID_MPEG_VIDEO_H264_CPB_SIZE: return "H264 CPB Buffer Size"; - case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE: return "H264 Entropy Mode"; - case V4L2_CID_MPEG_VIDEO_H264_I_PERIOD: return "H264 I-Frame Period"; - case V4L2_CID_MPEG_VIDEO_H264_LEVEL: return "H264 Level"; - case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_ALPHA: return "H264 Loop Filter Alpha Offset"; - case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_BETA: return "H264 Loop Filter Beta Offset"; - case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE: return "H264 Loop Filter Mode"; - case V4L2_CID_MPEG_VIDEO_H264_PROFILE: return "H264 Profile"; - case V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_HEIGHT: return "Vertical Size of SAR"; - case V4L2_CID_MPEG_VIDEO_H264_VUI_EXT_SAR_WIDTH: return "Horizontal Size of SAR"; - case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE: return "Aspect Ratio VUI Enable"; - case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC: return "VUI Aspect Ratio IDC"; - case V4L2_CID_MPEG_VIDEO_H264_SEI_FRAME_PACKING: return "H264 Enable Frame Packing SEI"; - case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_CURRENT_FRAME_0: return "H264 Set Curr. Frame as Frame0"; - case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE: return "H264 FP Arrangement Type"; - case V4L2_CID_MPEG_VIDEO_H264_FMO: return "H264 Flexible MB Ordering"; - case V4L2_CID_MPEG_VIDEO_H264_FMO_MAP_TYPE: return "H264 Map Type for FMO"; - case V4L2_CID_MPEG_VIDEO_H264_FMO_SLICE_GROUP: return "H264 FMO Number of Slice Groups"; - case V4L2_CID_MPEG_VIDEO_H264_FMO_CHANGE_DIRECTION: return "H264 FMO Direction of Change"; - case V4L2_CID_MPEG_VIDEO_H264_FMO_CHANGE_RATE: return "H264 FMO Size of 1st Slice Grp"; - case V4L2_CID_MPEG_VIDEO_H264_FMO_RUN_LENGTH: return "H264 FMO No. of Consecutive MBs"; - case V4L2_CID_MPEG_VIDEO_H264_ASO: return "H264 Arbitrary Slice Ordering"; - case V4L2_CID_MPEG_VIDEO_H264_ASO_SLICE_ORDER: return "H264 ASO Slice Order"; - case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING: return "Enable H264 Hierarchical Coding"; - case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_TYPE: return "H264 Hierarchical Coding Type"; - case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER:return "H264 Number of HC Layers"; - case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_LAYER_QP: - return "H264 Set QP Value for HC Layers"; - case V4L2_CID_MPEG_VIDEO_H264_CONSTRAINED_INTRA_PREDICTION: - return "H264 Constrained Intra Pred"; - case V4L2_CID_MPEG_VIDEO_H264_CHROMA_QP_INDEX_OFFSET: return "H264 Chroma QP Index Offset"; - case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MIN_QP: return "H264 I-Frame Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_I_FRAME_MAX_QP: return "H264 I-Frame Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MIN_QP: return "H264 P-Frame Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_P_FRAME_MAX_QP: return "H264 P-Frame Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_MIN_QP: return "H264 B-Frame Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_B_FRAME_MAX_QP: return "H264 B-Frame Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L0_BR: return "H264 Hierarchical Lay 0 Bitrate"; - case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L1_BR: return "H264 Hierarchical Lay 1 Bitrate"; - case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L2_BR: return "H264 Hierarchical Lay 2 Bitrate"; - case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L3_BR: return "H264 Hierarchical Lay 3 Bitrate"; - case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L4_BR: return "H264 Hierarchical Lay 4 Bitrate"; - case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L5_BR: return "H264 Hierarchical Lay 5 Bitrate"; - case V4L2_CID_MPEG_VIDEO_H264_HIER_CODING_L6_BR: return "H264 Hierarchical Lay 6 Bitrate"; - case V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL: return "MPEG2 Level"; - case V4L2_CID_MPEG_VIDEO_MPEG2_PROFILE: return "MPEG2 Profile"; - case V4L2_CID_MPEG_VIDEO_MPEG4_I_FRAME_QP: return "MPEG4 I-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_MPEG4_P_FRAME_QP: return "MPEG4 P-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_MPEG4_B_FRAME_QP: return "MPEG4 B-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_MPEG4_MIN_QP: return "MPEG4 Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_MPEG4_MAX_QP: return "MPEG4 Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL: return "MPEG4 Level"; - case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE: return "MPEG4 Profile"; - case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL: return "Quarter Pixel Search Enable"; - case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MAX_BYTES: return "Maximum Bytes in a Slice"; - case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MAX_MB: return "Number of MBs in a Slice"; - case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE: return "Slice Partitioning Method"; - case V4L2_CID_MPEG_VIDEO_VBV_SIZE: return "VBV Buffer Size"; - case V4L2_CID_MPEG_VIDEO_DEC_PTS: return "Video Decoder PTS"; - case V4L2_CID_MPEG_VIDEO_DEC_FRAME: return "Video Decoder Frame Count"; - case V4L2_CID_MPEG_VIDEO_DEC_CONCEAL_COLOR: return "Video Decoder Conceal Color"; - case V4L2_CID_MPEG_VIDEO_VBV_DELAY: return "Initial Delay for VBV Control"; - case V4L2_CID_MPEG_VIDEO_MV_H_SEARCH_RANGE: return "Horizontal MV Search Range"; - case V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE: return "Vertical MV Search Range"; - case V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER: return "Repeat Sequence Header"; - case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: return "Force Key Frame"; - case V4L2_CID_MPEG_VIDEO_BASELAYER_PRIORITY_ID: return "Base Layer Priority ID"; - case V4L2_CID_MPEG_VIDEO_LTR_COUNT: return "LTR Count"; - case V4L2_CID_MPEG_VIDEO_FRAME_LTR_INDEX: return "Frame LTR Index"; - case V4L2_CID_MPEG_VIDEO_USE_LTR_FRAMES: return "Use LTR Frames"; - case V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS: return "MPEG-2 Slice Parameters"; - case V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION: return "MPEG-2 Quantization Matrices"; - case V4L2_CID_FWHT_I_FRAME_QP: return "FWHT I-Frame QP Value"; - case V4L2_CID_FWHT_P_FRAME_QP: return "FWHT P-Frame QP Value"; - - /* VPX controls */ - case V4L2_CID_MPEG_VIDEO_VPX_NUM_PARTITIONS: return "VPX Number of Partitions"; - case V4L2_CID_MPEG_VIDEO_VPX_IMD_DISABLE_4X4: return "VPX Intra Mode Decision Disable"; - case V4L2_CID_MPEG_VIDEO_VPX_NUM_REF_FRAMES: return "VPX No. of Refs for P Frame"; - case V4L2_CID_MPEG_VIDEO_VPX_FILTER_LEVEL: return "VPX Loop Filter Level Range"; - case V4L2_CID_MPEG_VIDEO_VPX_FILTER_SHARPNESS: return "VPX Deblocking Effect Control"; - case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_REF_PERIOD: return "VPX Golden Frame Refresh Period"; - case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_SEL: return "VPX Golden Frame Indicator"; - case V4L2_CID_MPEG_VIDEO_VPX_MIN_QP: return "VPX Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_VPX_MAX_QP: return "VPX Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_VPX_I_FRAME_QP: return "VPX I-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_VPX_P_FRAME_QP: return "VPX P-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_VP8_PROFILE: return "VP8 Profile"; - case V4L2_CID_MPEG_VIDEO_VP9_PROFILE: return "VP9 Profile"; - case V4L2_CID_MPEG_VIDEO_VP9_LEVEL: return "VP9 Level"; - - /* HEVC controls */ - case V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_QP: return "HEVC I-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_QP: return "HEVC P-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_QP: return "HEVC B-Frame QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_MIN_QP: return "HEVC Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_MAX_QP: return "HEVC Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_MIN_QP: return "HEVC I-Frame Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_I_FRAME_MAX_QP: return "HEVC I-Frame Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_MIN_QP: return "HEVC P-Frame Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_P_FRAME_MAX_QP: return "HEVC P-Frame Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_MIN_QP: return "HEVC B-Frame Minimum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_B_FRAME_MAX_QP: return "HEVC B-Frame Maximum QP Value"; - case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: return "HEVC Profile"; - case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: return "HEVC Level"; - case V4L2_CID_MPEG_VIDEO_HEVC_TIER: return "HEVC Tier"; - case V4L2_CID_MPEG_VIDEO_HEVC_FRAME_RATE_RESOLUTION: return "HEVC Frame Rate Resolution"; - case V4L2_CID_MPEG_VIDEO_HEVC_MAX_PARTITION_DEPTH: return "HEVC Maximum Coding Unit Depth"; - case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_TYPE: return "HEVC Refresh Type"; - case V4L2_CID_MPEG_VIDEO_HEVC_CONST_INTRA_PRED: return "HEVC Constant Intra Prediction"; - case V4L2_CID_MPEG_VIDEO_HEVC_LOSSLESS_CU: return "HEVC Lossless Encoding"; - case V4L2_CID_MPEG_VIDEO_HEVC_WAVEFRONT: return "HEVC Wavefront"; - case V4L2_CID_MPEG_VIDEO_HEVC_LOOP_FILTER_MODE: return "HEVC Loop Filter"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_QP: return "HEVC QP Values"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_TYPE: return "HEVC Hierarchical Coding Type"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_LAYER: return "HEVC Hierarchical Coding Layer"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L0_QP: return "HEVC Hierarchical Layer 0 QP"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L1_QP: return "HEVC Hierarchical Layer 1 QP"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L2_QP: return "HEVC Hierarchical Layer 2 QP"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L3_QP: return "HEVC Hierarchical Layer 3 QP"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L4_QP: return "HEVC Hierarchical Layer 4 QP"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L5_QP: return "HEVC Hierarchical Layer 5 QP"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_QP: return "HEVC Hierarchical Layer 6 QP"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L0_BR: return "HEVC Hierarchical Lay 0 BitRate"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L1_BR: return "HEVC Hierarchical Lay 1 BitRate"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L2_BR: return "HEVC Hierarchical Lay 2 BitRate"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L3_BR: return "HEVC Hierarchical Lay 3 BitRate"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L4_BR: return "HEVC Hierarchical Lay 4 BitRate"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L5_BR: return "HEVC Hierarchical Lay 5 BitRate"; - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_L6_BR: return "HEVC Hierarchical Lay 6 BitRate"; - case V4L2_CID_MPEG_VIDEO_HEVC_GENERAL_PB: return "HEVC General PB"; - case V4L2_CID_MPEG_VIDEO_HEVC_TEMPORAL_ID: return "HEVC Temporal ID"; - case V4L2_CID_MPEG_VIDEO_HEVC_STRONG_SMOOTHING: return "HEVC Strong Intra Smoothing"; - case V4L2_CID_MPEG_VIDEO_HEVC_INTRA_PU_SPLIT: return "HEVC Intra PU Split"; - case V4L2_CID_MPEG_VIDEO_HEVC_TMV_PREDICTION: return "HEVC TMV Prediction"; - case V4L2_CID_MPEG_VIDEO_HEVC_MAX_NUM_MERGE_MV_MINUS1: return "HEVC Max Num of Candidate MVs"; - case V4L2_CID_MPEG_VIDEO_HEVC_WITHOUT_STARTCODE: return "HEVC ENC Without Startcode"; - case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_PERIOD: return "HEVC Num of I-Frame b/w 2 IDR"; - case V4L2_CID_MPEG_VIDEO_HEVC_LF_BETA_OFFSET_DIV2: return "HEVC Loop Filter Beta Offset"; - case V4L2_CID_MPEG_VIDEO_HEVC_LF_TC_OFFSET_DIV2: return "HEVC Loop Filter TC Offset"; - case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: return "HEVC Size of Length Field"; - case V4L2_CID_MPEG_VIDEO_REF_NUMBER_FOR_PFRAMES: return "Reference Frames for a P-Frame"; - case V4L2_CID_MPEG_VIDEO_PREPEND_SPSPPS_TO_IDR: return "Prepend SPS and PPS to IDR"; - case V4L2_CID_MPEG_VIDEO_HEVC_SPS: return "HEVC Sequence Parameter Set"; - case V4L2_CID_MPEG_VIDEO_HEVC_PPS: return "HEVC Picture Parameter Set"; - case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: return "HEVC Slice Parameters"; - case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE: return "HEVC Decode Mode"; - case V4L2_CID_MPEG_VIDEO_HEVC_START_CODE: return "HEVC Start Code"; - - /* CAMERA controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_CAMERA_CLASS: return "Camera Controls"; - case V4L2_CID_EXPOSURE_AUTO: return "Auto Exposure"; - case V4L2_CID_EXPOSURE_ABSOLUTE: return "Exposure Time, Absolute"; - case V4L2_CID_EXPOSURE_AUTO_PRIORITY: return "Exposure, Dynamic Framerate"; - case V4L2_CID_PAN_RELATIVE: return "Pan, Relative"; - case V4L2_CID_TILT_RELATIVE: return "Tilt, Relative"; - case V4L2_CID_PAN_RESET: return "Pan, Reset"; - case V4L2_CID_TILT_RESET: return "Tilt, Reset"; - case V4L2_CID_PAN_ABSOLUTE: return "Pan, Absolute"; - case V4L2_CID_TILT_ABSOLUTE: return "Tilt, Absolute"; - case V4L2_CID_FOCUS_ABSOLUTE: return "Focus, Absolute"; - case V4L2_CID_FOCUS_RELATIVE: return "Focus, Relative"; - case V4L2_CID_FOCUS_AUTO: return "Focus, Automatic Continuous"; - case V4L2_CID_ZOOM_ABSOLUTE: return "Zoom, Absolute"; - case V4L2_CID_ZOOM_RELATIVE: return "Zoom, Relative"; - case V4L2_CID_ZOOM_CONTINUOUS: return "Zoom, Continuous"; - case V4L2_CID_PRIVACY: return "Privacy"; - case V4L2_CID_IRIS_ABSOLUTE: return "Iris, Absolute"; - case V4L2_CID_IRIS_RELATIVE: return "Iris, Relative"; - case V4L2_CID_AUTO_EXPOSURE_BIAS: return "Auto Exposure, Bias"; - case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE: return "White Balance, Auto & Preset"; - case V4L2_CID_WIDE_DYNAMIC_RANGE: return "Wide Dynamic Range"; - case V4L2_CID_IMAGE_STABILIZATION: return "Image Stabilization"; - case V4L2_CID_ISO_SENSITIVITY: return "ISO Sensitivity"; - case V4L2_CID_ISO_SENSITIVITY_AUTO: return "ISO Sensitivity, Auto"; - case V4L2_CID_EXPOSURE_METERING: return "Exposure, Metering Mode"; - case V4L2_CID_SCENE_MODE: return "Scene Mode"; - case V4L2_CID_3A_LOCK: return "3A Lock"; - case V4L2_CID_AUTO_FOCUS_START: return "Auto Focus, Start"; - case V4L2_CID_AUTO_FOCUS_STOP: return "Auto Focus, Stop"; - case V4L2_CID_AUTO_FOCUS_STATUS: return "Auto Focus, Status"; - case V4L2_CID_AUTO_FOCUS_RANGE: return "Auto Focus, Range"; - case V4L2_CID_PAN_SPEED: return "Pan, Speed"; - case V4L2_CID_TILT_SPEED: return "Tilt, Speed"; - case V4L2_CID_UNIT_CELL_SIZE: return "Unit Cell Size"; - case V4L2_CID_CAMERA_ORIENTATION: return "Camera Orientation"; - case V4L2_CID_CAMERA_SENSOR_ROTATION: return "Camera Sensor Rotation"; - - /* FM Radio Modulator controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_FM_TX_CLASS: return "FM Radio Modulator Controls"; - case V4L2_CID_RDS_TX_DEVIATION: return "RDS Signal Deviation"; - case V4L2_CID_RDS_TX_PI: return "RDS Program ID"; - case V4L2_CID_RDS_TX_PTY: return "RDS Program Type"; - case V4L2_CID_RDS_TX_PS_NAME: return "RDS PS Name"; - case V4L2_CID_RDS_TX_RADIO_TEXT: return "RDS Radio Text"; - case V4L2_CID_RDS_TX_MONO_STEREO: return "RDS Stereo"; - case V4L2_CID_RDS_TX_ARTIFICIAL_HEAD: return "RDS Artificial Head"; - case V4L2_CID_RDS_TX_COMPRESSED: return "RDS Compressed"; - case V4L2_CID_RDS_TX_DYNAMIC_PTY: return "RDS Dynamic PTY"; - case V4L2_CID_RDS_TX_TRAFFIC_ANNOUNCEMENT: return "RDS Traffic Announcement"; - case V4L2_CID_RDS_TX_TRAFFIC_PROGRAM: return "RDS Traffic Program"; - case V4L2_CID_RDS_TX_MUSIC_SPEECH: return "RDS Music"; - case V4L2_CID_RDS_TX_ALT_FREQS_ENABLE: return "RDS Enable Alt Frequencies"; - case V4L2_CID_RDS_TX_ALT_FREQS: return "RDS Alternate Frequencies"; - case V4L2_CID_AUDIO_LIMITER_ENABLED: return "Audio Limiter Feature Enabled"; - case V4L2_CID_AUDIO_LIMITER_RELEASE_TIME: return "Audio Limiter Release Time"; - case V4L2_CID_AUDIO_LIMITER_DEVIATION: return "Audio Limiter Deviation"; - case V4L2_CID_AUDIO_COMPRESSION_ENABLED: return "Audio Compression Enabled"; - case V4L2_CID_AUDIO_COMPRESSION_GAIN: return "Audio Compression Gain"; - case V4L2_CID_AUDIO_COMPRESSION_THRESHOLD: return "Audio Compression Threshold"; - case V4L2_CID_AUDIO_COMPRESSION_ATTACK_TIME: return "Audio Compression Attack Time"; - case V4L2_CID_AUDIO_COMPRESSION_RELEASE_TIME: return "Audio Compression Release Time"; - case V4L2_CID_PILOT_TONE_ENABLED: return "Pilot Tone Feature Enabled"; - case V4L2_CID_PILOT_TONE_DEVIATION: return "Pilot Tone Deviation"; - case V4L2_CID_PILOT_TONE_FREQUENCY: return "Pilot Tone Frequency"; - case V4L2_CID_TUNE_PREEMPHASIS: return "Pre-Emphasis"; - case V4L2_CID_TUNE_POWER_LEVEL: return "Tune Power Level"; - case V4L2_CID_TUNE_ANTENNA_CAPACITOR: return "Tune Antenna Capacitor"; - - /* Flash controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_FLASH_CLASS: return "Flash Controls"; - case V4L2_CID_FLASH_LED_MODE: return "LED Mode"; - case V4L2_CID_FLASH_STROBE_SOURCE: return "Strobe Source"; - case V4L2_CID_FLASH_STROBE: return "Strobe"; - case V4L2_CID_FLASH_STROBE_STOP: return "Stop Strobe"; - case V4L2_CID_FLASH_STROBE_STATUS: return "Strobe Status"; - case V4L2_CID_FLASH_TIMEOUT: return "Strobe Timeout"; - case V4L2_CID_FLASH_INTENSITY: return "Intensity, Flash Mode"; - case V4L2_CID_FLASH_TORCH_INTENSITY: return "Intensity, Torch Mode"; - case V4L2_CID_FLASH_INDICATOR_INTENSITY: return "Intensity, Indicator"; - case V4L2_CID_FLASH_FAULT: return "Faults"; - case V4L2_CID_FLASH_CHARGE: return "Charge"; - case V4L2_CID_FLASH_READY: return "Ready to Strobe"; - - /* JPEG encoder controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_JPEG_CLASS: return "JPEG Compression Controls"; - case V4L2_CID_JPEG_CHROMA_SUBSAMPLING: return "Chroma Subsampling"; - case V4L2_CID_JPEG_RESTART_INTERVAL: return "Restart Interval"; - case V4L2_CID_JPEG_COMPRESSION_QUALITY: return "Compression Quality"; - case V4L2_CID_JPEG_ACTIVE_MARKER: return "Active Markers"; - - /* Image source controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_IMAGE_SOURCE_CLASS: return "Image Source Controls"; - case V4L2_CID_VBLANK: return "Vertical Blanking"; - case V4L2_CID_HBLANK: return "Horizontal Blanking"; - case V4L2_CID_ANALOGUE_GAIN: return "Analogue Gain"; - case V4L2_CID_TEST_PATTERN_RED: return "Red Pixel Value"; - case V4L2_CID_TEST_PATTERN_GREENR: return "Green (Red) Pixel Value"; - case V4L2_CID_TEST_PATTERN_BLUE: return "Blue Pixel Value"; - case V4L2_CID_TEST_PATTERN_GREENB: return "Green (Blue) Pixel Value"; - - /* Image processing controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_IMAGE_PROC_CLASS: return "Image Processing Controls"; - case V4L2_CID_LINK_FREQ: return "Link Frequency"; - case V4L2_CID_PIXEL_RATE: return "Pixel Rate"; - case V4L2_CID_TEST_PATTERN: return "Test Pattern"; - case V4L2_CID_DEINTERLACING_MODE: return "Deinterlacing Mode"; - case V4L2_CID_DIGITAL_GAIN: return "Digital Gain"; - - /* DV controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_DV_CLASS: return "Digital Video Controls"; - case V4L2_CID_DV_TX_HOTPLUG: return "Hotplug Present"; - case V4L2_CID_DV_TX_RXSENSE: return "RxSense Present"; - case V4L2_CID_DV_TX_EDID_PRESENT: return "EDID Present"; - case V4L2_CID_DV_TX_MODE: return "Transmit Mode"; - case V4L2_CID_DV_TX_RGB_RANGE: return "Tx RGB Quantization Range"; - case V4L2_CID_DV_TX_IT_CONTENT_TYPE: return "Tx IT Content Type"; - case V4L2_CID_DV_RX_POWER_PRESENT: return "Power Present"; - case V4L2_CID_DV_RX_RGB_RANGE: return "Rx RGB Quantization Range"; - case V4L2_CID_DV_RX_IT_CONTENT_TYPE: return "Rx IT Content Type"; - - case V4L2_CID_FM_RX_CLASS: return "FM Radio Receiver Controls"; - case V4L2_CID_TUNE_DEEMPHASIS: return "De-Emphasis"; - case V4L2_CID_RDS_RECEPTION: return "RDS Reception"; - case V4L2_CID_RF_TUNER_CLASS: return "RF Tuner Controls"; - case V4L2_CID_RF_TUNER_RF_GAIN: return "RF Gain"; - case V4L2_CID_RF_TUNER_LNA_GAIN_AUTO: return "LNA Gain, Auto"; - case V4L2_CID_RF_TUNER_LNA_GAIN: return "LNA Gain"; - case V4L2_CID_RF_TUNER_MIXER_GAIN_AUTO: return "Mixer Gain, Auto"; - case V4L2_CID_RF_TUNER_MIXER_GAIN: return "Mixer Gain"; - case V4L2_CID_RF_TUNER_IF_GAIN_AUTO: return "IF Gain, Auto"; - case V4L2_CID_RF_TUNER_IF_GAIN: return "IF Gain"; - case V4L2_CID_RF_TUNER_BANDWIDTH_AUTO: return "Bandwidth, Auto"; - case V4L2_CID_RF_TUNER_BANDWIDTH: return "Bandwidth"; - case V4L2_CID_RF_TUNER_PLL_LOCK: return "PLL Lock"; - case V4L2_CID_RDS_RX_PTY: return "RDS Program Type"; - case V4L2_CID_RDS_RX_PS_NAME: return "RDS PS Name"; - case V4L2_CID_RDS_RX_RADIO_TEXT: return "RDS Radio Text"; - case V4L2_CID_RDS_RX_TRAFFIC_ANNOUNCEMENT: return "RDS Traffic Announcement"; - case V4L2_CID_RDS_RX_TRAFFIC_PROGRAM: return "RDS Traffic Program"; - case V4L2_CID_RDS_RX_MUSIC_SPEECH: return "RDS Music"; - - /* Detection controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_DETECT_CLASS: return "Detection Controls"; - case V4L2_CID_DETECT_MD_MODE: return "Motion Detection Mode"; - case V4L2_CID_DETECT_MD_GLOBAL_THRESHOLD: return "MD Global Threshold"; - case V4L2_CID_DETECT_MD_THRESHOLD_GRID: return "MD Threshold Grid"; - case V4L2_CID_DETECT_MD_REGION_GRID: return "MD Region Grid"; - - /* Stateless Codec controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_CODEC_STATELESS_CLASS: return "Stateless Codec Controls"; - case V4L2_CID_STATELESS_H264_DECODE_MODE: return "H264 Decode Mode"; - case V4L2_CID_STATELESS_H264_START_CODE: return "H264 Start Code"; - case V4L2_CID_STATELESS_H264_SPS: return "H264 Sequence Parameter Set"; - case V4L2_CID_STATELESS_H264_PPS: return "H264 Picture Parameter Set"; - case V4L2_CID_STATELESS_H264_SCALING_MATRIX: return "H264 Scaling Matrix"; - case V4L2_CID_STATELESS_H264_PRED_WEIGHTS: return "H264 Prediction Weight Table"; - case V4L2_CID_STATELESS_H264_SLICE_PARAMS: return "H264 Slice Parameters"; - case V4L2_CID_STATELESS_H264_DECODE_PARAMS: return "H264 Decode Parameters"; - case V4L2_CID_STATELESS_FWHT_PARAMS: return "FWHT Stateless Parameters"; - case V4L2_CID_STATELESS_VP8_FRAME: return "VP8 Frame Parameters"; - - /* Colorimetry controls */ - /* Keep the order of the 'case's the same as in v4l2-controls.h! */ - case V4L2_CID_COLORIMETRY_CLASS: return "Colorimetry Controls"; - case V4L2_CID_COLORIMETRY_HDR10_CLL_INFO: return "HDR10 Content Light Info"; - case V4L2_CID_COLORIMETRY_HDR10_MASTERING_DISPLAY: return "HDR10 Mastering Display"; - default: - return NULL; - } -} -EXPORT_SYMBOL(v4l2_ctrl_get_name); - -void v4l2_ctrl_fill(u32 id, const char **name, enum v4l2_ctrl_type *type, - s64 *min, s64 *max, u64 *step, s64 *def, u32 *flags) -{ - *name = v4l2_ctrl_get_name(id); - *flags = 0; - - switch (id) { - case V4L2_CID_AUDIO_MUTE: - case V4L2_CID_AUDIO_LOUDNESS: - case V4L2_CID_AUTO_WHITE_BALANCE: - case V4L2_CID_AUTOGAIN: - case V4L2_CID_HFLIP: - case V4L2_CID_VFLIP: - case V4L2_CID_HUE_AUTO: - case V4L2_CID_CHROMA_AGC: - case V4L2_CID_COLOR_KILLER: - case V4L2_CID_AUTOBRIGHTNESS: - case V4L2_CID_MPEG_AUDIO_MUTE: - case V4L2_CID_MPEG_VIDEO_MUTE: - case V4L2_CID_MPEG_VIDEO_GOP_CLOSURE: - case V4L2_CID_MPEG_VIDEO_PULLDOWN: - case V4L2_CID_EXPOSURE_AUTO_PRIORITY: - case V4L2_CID_FOCUS_AUTO: - case V4L2_CID_PRIVACY: - case V4L2_CID_AUDIO_LIMITER_ENABLED: - case V4L2_CID_AUDIO_COMPRESSION_ENABLED: - case V4L2_CID_PILOT_TONE_ENABLED: - case V4L2_CID_ILLUMINATORS_1: - case V4L2_CID_ILLUMINATORS_2: - case V4L2_CID_FLASH_STROBE_STATUS: - case V4L2_CID_FLASH_CHARGE: - case V4L2_CID_FLASH_READY: - case V4L2_CID_MPEG_VIDEO_DECODER_MPEG4_DEBLOCK_FILTER: - case V4L2_CID_MPEG_VIDEO_DECODER_SLICE_INTERFACE: - case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY_ENABLE: - case V4L2_CID_MPEG_VIDEO_FRAME_RC_ENABLE: - case V4L2_CID_MPEG_VIDEO_MB_RC_ENABLE: - case V4L2_CID_MPEG_VIDEO_H264_8X8_TRANSFORM: - case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_ENABLE: - case V4L2_CID_MPEG_VIDEO_MPEG4_QPEL: - case V4L2_CID_MPEG_VIDEO_REPEAT_SEQ_HEADER: - case V4L2_CID_MPEG_VIDEO_AU_DELIMITER: - case V4L2_CID_WIDE_DYNAMIC_RANGE: - case V4L2_CID_IMAGE_STABILIZATION: - case V4L2_CID_RDS_RECEPTION: - case V4L2_CID_RF_TUNER_LNA_GAIN_AUTO: - case V4L2_CID_RF_TUNER_MIXER_GAIN_AUTO: - case V4L2_CID_RF_TUNER_IF_GAIN_AUTO: - case V4L2_CID_RF_TUNER_BANDWIDTH_AUTO: - case V4L2_CID_RF_TUNER_PLL_LOCK: - case V4L2_CID_RDS_TX_MONO_STEREO: - case V4L2_CID_RDS_TX_ARTIFICIAL_HEAD: - case V4L2_CID_RDS_TX_COMPRESSED: - case V4L2_CID_RDS_TX_DYNAMIC_PTY: - case V4L2_CID_RDS_TX_TRAFFIC_ANNOUNCEMENT: - case V4L2_CID_RDS_TX_TRAFFIC_PROGRAM: - case V4L2_CID_RDS_TX_MUSIC_SPEECH: - case V4L2_CID_RDS_TX_ALT_FREQS_ENABLE: - case V4L2_CID_RDS_RX_TRAFFIC_ANNOUNCEMENT: - case V4L2_CID_RDS_RX_TRAFFIC_PROGRAM: - case V4L2_CID_RDS_RX_MUSIC_SPEECH: - *type = V4L2_CTRL_TYPE_BOOLEAN; - *min = 0; - *max = *step = 1; - break; - case V4L2_CID_ROTATE: - *type = V4L2_CTRL_TYPE_INTEGER; - *flags |= V4L2_CTRL_FLAG_MODIFY_LAYOUT; - break; - case V4L2_CID_MPEG_VIDEO_MV_H_SEARCH_RANGE: - case V4L2_CID_MPEG_VIDEO_MV_V_SEARCH_RANGE: - case V4L2_CID_MPEG_VIDEO_DEC_DISPLAY_DELAY: - *type = V4L2_CTRL_TYPE_INTEGER; - break; - case V4L2_CID_MPEG_VIDEO_LTR_COUNT: - *type = V4L2_CTRL_TYPE_INTEGER; - break; - case V4L2_CID_MPEG_VIDEO_FRAME_LTR_INDEX: - *type = V4L2_CTRL_TYPE_INTEGER; - *flags |= V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; - break; - case V4L2_CID_MPEG_VIDEO_USE_LTR_FRAMES: - *type = V4L2_CTRL_TYPE_BITMASK; - *flags |= V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; - break; - case V4L2_CID_MPEG_VIDEO_FORCE_KEY_FRAME: - case V4L2_CID_PAN_RESET: - case V4L2_CID_TILT_RESET: - case V4L2_CID_FLASH_STROBE: - case V4L2_CID_FLASH_STROBE_STOP: - case V4L2_CID_AUTO_FOCUS_START: - case V4L2_CID_AUTO_FOCUS_STOP: - case V4L2_CID_DO_WHITE_BALANCE: - *type = V4L2_CTRL_TYPE_BUTTON; - *flags |= V4L2_CTRL_FLAG_WRITE_ONLY | - V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; - *min = *max = *step = *def = 0; - break; - case V4L2_CID_POWER_LINE_FREQUENCY: - case V4L2_CID_MPEG_AUDIO_SAMPLING_FREQ: - case V4L2_CID_MPEG_AUDIO_ENCODING: - case V4L2_CID_MPEG_AUDIO_L1_BITRATE: - case V4L2_CID_MPEG_AUDIO_L2_BITRATE: - case V4L2_CID_MPEG_AUDIO_L3_BITRATE: - case V4L2_CID_MPEG_AUDIO_AC3_BITRATE: - case V4L2_CID_MPEG_AUDIO_MODE: - case V4L2_CID_MPEG_AUDIO_MODE_EXTENSION: - case V4L2_CID_MPEG_AUDIO_EMPHASIS: - case V4L2_CID_MPEG_AUDIO_CRC: - case V4L2_CID_MPEG_AUDIO_DEC_PLAYBACK: - case V4L2_CID_MPEG_AUDIO_DEC_MULTILINGUAL_PLAYBACK: - case V4L2_CID_MPEG_VIDEO_ENCODING: - case V4L2_CID_MPEG_VIDEO_ASPECT: - case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: - case V4L2_CID_MPEG_STREAM_TYPE: - case V4L2_CID_MPEG_STREAM_VBI_FMT: - case V4L2_CID_EXPOSURE_AUTO: - case V4L2_CID_AUTO_FOCUS_RANGE: - case V4L2_CID_COLORFX: - case V4L2_CID_AUTO_N_PRESET_WHITE_BALANCE: - case V4L2_CID_TUNE_PREEMPHASIS: - case V4L2_CID_FLASH_LED_MODE: - case V4L2_CID_FLASH_STROBE_SOURCE: - case V4L2_CID_MPEG_VIDEO_HEADER_MODE: - case V4L2_CID_MPEG_VIDEO_FRAME_SKIP_MODE: - case V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE: - case V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE: - case V4L2_CID_MPEG_VIDEO_H264_LEVEL: - case V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE: - case V4L2_CID_MPEG_VIDEO_H264_PROFILE: - case V4L2_CID_MPEG_VIDEO_H264_VUI_SAR_IDC: - case V4L2_CID_MPEG_VIDEO_H264_SEI_FP_ARRANGEMENT_TYPE: - case V4L2_CID_MPEG_VIDEO_H264_FMO_MAP_TYPE: - case V4L2_CID_MPEG_VIDEO_H264_HIERARCHICAL_CODING_TYPE: - case V4L2_CID_MPEG_VIDEO_MPEG2_LEVEL: - case V4L2_CID_MPEG_VIDEO_MPEG2_PROFILE: - case V4L2_CID_MPEG_VIDEO_MPEG4_LEVEL: - case V4L2_CID_MPEG_VIDEO_MPEG4_PROFILE: - case V4L2_CID_JPEG_CHROMA_SUBSAMPLING: - case V4L2_CID_ISO_SENSITIVITY_AUTO: - case V4L2_CID_EXPOSURE_METERING: - case V4L2_CID_SCENE_MODE: - case V4L2_CID_DV_TX_MODE: - case V4L2_CID_DV_TX_RGB_RANGE: - case V4L2_CID_DV_TX_IT_CONTENT_TYPE: - case V4L2_CID_DV_RX_RGB_RANGE: - case V4L2_CID_DV_RX_IT_CONTENT_TYPE: - case V4L2_CID_TEST_PATTERN: - case V4L2_CID_DEINTERLACING_MODE: - case V4L2_CID_TUNE_DEEMPHASIS: - case V4L2_CID_MPEG_VIDEO_VPX_GOLDEN_FRAME_SEL: - case V4L2_CID_MPEG_VIDEO_VP8_PROFILE: - case V4L2_CID_MPEG_VIDEO_VP9_PROFILE: - case V4L2_CID_MPEG_VIDEO_VP9_LEVEL: - case V4L2_CID_DETECT_MD_MODE: - case V4L2_CID_MPEG_VIDEO_HEVC_PROFILE: - case V4L2_CID_MPEG_VIDEO_HEVC_LEVEL: - case V4L2_CID_MPEG_VIDEO_HEVC_HIER_CODING_TYPE: - case V4L2_CID_MPEG_VIDEO_HEVC_REFRESH_TYPE: - case V4L2_CID_MPEG_VIDEO_HEVC_SIZE_OF_LENGTH_FIELD: - case V4L2_CID_MPEG_VIDEO_HEVC_TIER: - case V4L2_CID_MPEG_VIDEO_HEVC_LOOP_FILTER_MODE: - case V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE: - case V4L2_CID_MPEG_VIDEO_HEVC_START_CODE: - case V4L2_CID_STATELESS_H264_DECODE_MODE: - case V4L2_CID_STATELESS_H264_START_CODE: - case V4L2_CID_CAMERA_ORIENTATION: - *type = V4L2_CTRL_TYPE_MENU; - break; - case V4L2_CID_LINK_FREQ: - *type = V4L2_CTRL_TYPE_INTEGER_MENU; - break; - case V4L2_CID_RDS_TX_PS_NAME: - case V4L2_CID_RDS_TX_RADIO_TEXT: - case V4L2_CID_RDS_RX_PS_NAME: - case V4L2_CID_RDS_RX_RADIO_TEXT: - *type = V4L2_CTRL_TYPE_STRING; - break; - case V4L2_CID_ISO_SENSITIVITY: - case V4L2_CID_AUTO_EXPOSURE_BIAS: - case V4L2_CID_MPEG_VIDEO_VPX_NUM_PARTITIONS: - case V4L2_CID_MPEG_VIDEO_VPX_NUM_REF_FRAMES: - *type = V4L2_CTRL_TYPE_INTEGER_MENU; - break; - case V4L2_CID_USER_CLASS: - case V4L2_CID_CAMERA_CLASS: - case V4L2_CID_CODEC_CLASS: - case V4L2_CID_FM_TX_CLASS: - case V4L2_CID_FLASH_CLASS: - case V4L2_CID_JPEG_CLASS: - case V4L2_CID_IMAGE_SOURCE_CLASS: - case V4L2_CID_IMAGE_PROC_CLASS: - case V4L2_CID_DV_CLASS: - case V4L2_CID_FM_RX_CLASS: - case V4L2_CID_RF_TUNER_CLASS: - case V4L2_CID_DETECT_CLASS: - case V4L2_CID_CODEC_STATELESS_CLASS: - case V4L2_CID_COLORIMETRY_CLASS: - *type = V4L2_CTRL_TYPE_CTRL_CLASS; - /* You can neither read nor write these */ - *flags |= V4L2_CTRL_FLAG_READ_ONLY | V4L2_CTRL_FLAG_WRITE_ONLY; - *min = *max = *step = *def = 0; - break; - case V4L2_CID_BG_COLOR: - *type = V4L2_CTRL_TYPE_INTEGER; - *step = 1; - *min = 0; - /* Max is calculated as RGB888 that is 2^24 */ - *max = 0xFFFFFF; - break; - case V4L2_CID_FLASH_FAULT: - case V4L2_CID_JPEG_ACTIVE_MARKER: - case V4L2_CID_3A_LOCK: - case V4L2_CID_AUTO_FOCUS_STATUS: - case V4L2_CID_DV_TX_HOTPLUG: - case V4L2_CID_DV_TX_RXSENSE: - case V4L2_CID_DV_TX_EDID_PRESENT: - case V4L2_CID_DV_RX_POWER_PRESENT: - *type = V4L2_CTRL_TYPE_BITMASK; - break; - case V4L2_CID_MIN_BUFFERS_FOR_CAPTURE: - case V4L2_CID_MIN_BUFFERS_FOR_OUTPUT: - *type = V4L2_CTRL_TYPE_INTEGER; - *flags |= V4L2_CTRL_FLAG_READ_ONLY; - break; - case V4L2_CID_MPEG_VIDEO_DEC_PTS: - *type = V4L2_CTRL_TYPE_INTEGER64; - *flags |= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_READ_ONLY; - *min = *def = 0; - *max = 0x1ffffffffLL; - *step = 1; - break; - case V4L2_CID_MPEG_VIDEO_DEC_FRAME: - *type = V4L2_CTRL_TYPE_INTEGER64; - *flags |= V4L2_CTRL_FLAG_VOLATILE | V4L2_CTRL_FLAG_READ_ONLY; - *min = *def = 0; - *max = 0x7fffffffffffffffLL; - *step = 1; - break; - case V4L2_CID_MPEG_VIDEO_DEC_CONCEAL_COLOR: - *type = V4L2_CTRL_TYPE_INTEGER64; - *min = 0; - /* default for 8 bit black, luma is 16, chroma is 128 */ - *def = 0x8000800010LL; - *max = 0xffffffffffffLL; - *step = 1; - break; - case V4L2_CID_PIXEL_RATE: - *type = V4L2_CTRL_TYPE_INTEGER64; - *flags |= V4L2_CTRL_FLAG_READ_ONLY; - break; - case V4L2_CID_DETECT_MD_REGION_GRID: - *type = V4L2_CTRL_TYPE_U8; - break; - case V4L2_CID_DETECT_MD_THRESHOLD_GRID: - *type = V4L2_CTRL_TYPE_U16; - break; - case V4L2_CID_RDS_TX_ALT_FREQS: - *type = V4L2_CTRL_TYPE_U32; - break; - case V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS: - *type = V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS; - break; - case V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION: - *type = V4L2_CTRL_TYPE_MPEG2_QUANTIZATION; - break; - case V4L2_CID_STATELESS_FWHT_PARAMS: - *type = V4L2_CTRL_TYPE_FWHT_PARAMS; - break; - case V4L2_CID_STATELESS_H264_SPS: - *type = V4L2_CTRL_TYPE_H264_SPS; - break; - case V4L2_CID_STATELESS_H264_PPS: - *type = V4L2_CTRL_TYPE_H264_PPS; - break; - case V4L2_CID_STATELESS_H264_SCALING_MATRIX: - *type = V4L2_CTRL_TYPE_H264_SCALING_MATRIX; - break; - case V4L2_CID_STATELESS_H264_SLICE_PARAMS: - *type = V4L2_CTRL_TYPE_H264_SLICE_PARAMS; - break; - case V4L2_CID_STATELESS_H264_DECODE_PARAMS: - *type = V4L2_CTRL_TYPE_H264_DECODE_PARAMS; - break; - case V4L2_CID_STATELESS_H264_PRED_WEIGHTS: - *type = V4L2_CTRL_TYPE_H264_PRED_WEIGHTS; - break; - case V4L2_CID_STATELESS_VP8_FRAME: - *type = V4L2_CTRL_TYPE_VP8_FRAME; - break; - case V4L2_CID_MPEG_VIDEO_HEVC_SPS: - *type = V4L2_CTRL_TYPE_HEVC_SPS; - break; - case V4L2_CID_MPEG_VIDEO_HEVC_PPS: - *type = V4L2_CTRL_TYPE_HEVC_PPS; - break; - case V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS: - *type = V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS; - break; - case V4L2_CID_UNIT_CELL_SIZE: - *type = V4L2_CTRL_TYPE_AREA; - *flags |= V4L2_CTRL_FLAG_READ_ONLY; - break; - case V4L2_CID_COLORIMETRY_HDR10_CLL_INFO: - *type = V4L2_CTRL_TYPE_HDR10_CLL_INFO; - break; - case V4L2_CID_COLORIMETRY_HDR10_MASTERING_DISPLAY: - *type = V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY; - break; - default: - *type = V4L2_CTRL_TYPE_INTEGER; - break; - } - switch (id) { - case V4L2_CID_MPEG_AUDIO_ENCODING: - case V4L2_CID_MPEG_AUDIO_MODE: - case V4L2_CID_MPEG_VIDEO_BITRATE_MODE: - case V4L2_CID_MPEG_VIDEO_B_FRAMES: - case V4L2_CID_MPEG_STREAM_TYPE: - *flags |= V4L2_CTRL_FLAG_UPDATE; - break; - case V4L2_CID_AUDIO_VOLUME: - case V4L2_CID_AUDIO_BALANCE: - case V4L2_CID_AUDIO_BASS: - case V4L2_CID_AUDIO_TREBLE: - case V4L2_CID_BRIGHTNESS: - case V4L2_CID_CONTRAST: - case V4L2_CID_SATURATION: - case V4L2_CID_HUE: - case V4L2_CID_RED_BALANCE: - case V4L2_CID_BLUE_BALANCE: - case V4L2_CID_GAMMA: - case V4L2_CID_SHARPNESS: - case V4L2_CID_CHROMA_GAIN: - case V4L2_CID_RDS_TX_DEVIATION: - case V4L2_CID_AUDIO_LIMITER_RELEASE_TIME: - case V4L2_CID_AUDIO_LIMITER_DEVIATION: - case V4L2_CID_AUDIO_COMPRESSION_GAIN: - case V4L2_CID_AUDIO_COMPRESSION_THRESHOLD: - case V4L2_CID_AUDIO_COMPRESSION_ATTACK_TIME: - case V4L2_CID_AUDIO_COMPRESSION_RELEASE_TIME: - case V4L2_CID_PILOT_TONE_DEVIATION: - case V4L2_CID_PILOT_TONE_FREQUENCY: - case V4L2_CID_TUNE_POWER_LEVEL: - case V4L2_CID_TUNE_ANTENNA_CAPACITOR: - case V4L2_CID_RF_TUNER_RF_GAIN: - case V4L2_CID_RF_TUNER_LNA_GAIN: - case V4L2_CID_RF_TUNER_MIXER_GAIN: - case V4L2_CID_RF_TUNER_IF_GAIN: - case V4L2_CID_RF_TUNER_BANDWIDTH: - case V4L2_CID_DETECT_MD_GLOBAL_THRESHOLD: - *flags |= V4L2_CTRL_FLAG_SLIDER; - break; - case V4L2_CID_PAN_RELATIVE: - case V4L2_CID_TILT_RELATIVE: - case V4L2_CID_FOCUS_RELATIVE: - case V4L2_CID_IRIS_RELATIVE: - case V4L2_CID_ZOOM_RELATIVE: - *flags |= V4L2_CTRL_FLAG_WRITE_ONLY | - V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; - break; - case V4L2_CID_FLASH_STROBE_STATUS: - case V4L2_CID_AUTO_FOCUS_STATUS: - case V4L2_CID_FLASH_READY: - case V4L2_CID_DV_TX_HOTPLUG: - case V4L2_CID_DV_TX_RXSENSE: - case V4L2_CID_DV_TX_EDID_PRESENT: - case V4L2_CID_DV_RX_POWER_PRESENT: - case V4L2_CID_DV_RX_IT_CONTENT_TYPE: - case V4L2_CID_RDS_RX_PTY: - case V4L2_CID_RDS_RX_PS_NAME: - case V4L2_CID_RDS_RX_RADIO_TEXT: - case V4L2_CID_RDS_RX_TRAFFIC_ANNOUNCEMENT: - case V4L2_CID_RDS_RX_TRAFFIC_PROGRAM: - case V4L2_CID_RDS_RX_MUSIC_SPEECH: - case V4L2_CID_CAMERA_ORIENTATION: - case V4L2_CID_CAMERA_SENSOR_ROTATION: - *flags |= V4L2_CTRL_FLAG_READ_ONLY; - break; - case V4L2_CID_RF_TUNER_PLL_LOCK: - *flags |= V4L2_CTRL_FLAG_VOLATILE; - break; - } -} -EXPORT_SYMBOL(v4l2_ctrl_fill); - -static u32 user_flags(const struct v4l2_ctrl *ctrl) -{ - u32 flags = ctrl->flags; - - if (ctrl->is_ptr) - flags |= V4L2_CTRL_FLAG_HAS_PAYLOAD; - - return flags; -} - -static void fill_event(struct v4l2_event *ev, struct v4l2_ctrl *ctrl, u32 changes) -{ - memset(ev, 0, sizeof(*ev)); - ev->type = V4L2_EVENT_CTRL; - ev->id = ctrl->id; - ev->u.ctrl.changes = changes; - ev->u.ctrl.type = ctrl->type; - ev->u.ctrl.flags = user_flags(ctrl); - if (ctrl->is_ptr) - ev->u.ctrl.value64 = 0; - else - ev->u.ctrl.value64 = *ctrl->p_cur.p_s64; - ev->u.ctrl.minimum = ctrl->minimum; - ev->u.ctrl.maximum = ctrl->maximum; - if (ctrl->type == V4L2_CTRL_TYPE_MENU - || ctrl->type == V4L2_CTRL_TYPE_INTEGER_MENU) - ev->u.ctrl.step = 1; - else - ev->u.ctrl.step = ctrl->step; - ev->u.ctrl.default_value = ctrl->default_value; -} - -static void send_event(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 changes) -{ - struct v4l2_event ev; - struct v4l2_subscribed_event *sev; - - if (list_empty(&ctrl->ev_subs)) - return; - fill_event(&ev, ctrl, changes); - - list_for_each_entry(sev, &ctrl->ev_subs, node) - if (sev->fh != fh || - (sev->flags & V4L2_EVENT_SUB_FL_ALLOW_FEEDBACK)) - v4l2_event_queue_fh(sev->fh, &ev); -} - -static bool std_equal(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr1, - union v4l2_ctrl_ptr ptr2) -{ - switch (ctrl->type) { - case V4L2_CTRL_TYPE_BUTTON: - return false; - case V4L2_CTRL_TYPE_STRING: - idx *= ctrl->elem_size; - /* strings are always 0-terminated */ - return !strcmp(ptr1.p_char + idx, ptr2.p_char + idx); - case V4L2_CTRL_TYPE_INTEGER64: - return ptr1.p_s64[idx] == ptr2.p_s64[idx]; - case V4L2_CTRL_TYPE_U8: - return ptr1.p_u8[idx] == ptr2.p_u8[idx]; - case V4L2_CTRL_TYPE_U16: - return ptr1.p_u16[idx] == ptr2.p_u16[idx]; - case V4L2_CTRL_TYPE_U32: - return ptr1.p_u32[idx] == ptr2.p_u32[idx]; - default: - if (ctrl->is_int) - return ptr1.p_s32[idx] == ptr2.p_s32[idx]; - idx *= ctrl->elem_size; - return !memcmp(ptr1.p_const + idx, ptr2.p_const + idx, - ctrl->elem_size); - } -} - -static void std_init_compound(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr) -{ - struct v4l2_ctrl_mpeg2_slice_params *p_mpeg2_slice_params; - struct v4l2_ctrl_vp8_frame *p_vp8_frame; - struct v4l2_ctrl_fwht_params *p_fwht_params; - void *p = ptr.p + idx * ctrl->elem_size; - - if (ctrl->p_def.p_const) - memcpy(p, ctrl->p_def.p_const, ctrl->elem_size); - else - memset(p, 0, ctrl->elem_size); - - /* - * The cast is needed to get rid of a gcc warning complaining that - * V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS is not part of the - * v4l2_ctrl_type enum. - */ - switch ((u32)ctrl->type) { - case V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS: - p_mpeg2_slice_params = p; - /* 4:2:0 */ - p_mpeg2_slice_params->sequence.chroma_format = 1; - /* interlaced top field */ - p_mpeg2_slice_params->picture.picture_structure = 1; - p_mpeg2_slice_params->picture.picture_coding_type = - V4L2_MPEG2_PICTURE_CODING_TYPE_I; - break; - case V4L2_CTRL_TYPE_VP8_FRAME: - p_vp8_frame = p; - p_vp8_frame->num_dct_parts = 1; - break; - case V4L2_CTRL_TYPE_FWHT_PARAMS: - p_fwht_params = p; - p_fwht_params->version = V4L2_FWHT_VERSION; - p_fwht_params->width = 1280; - p_fwht_params->height = 720; - p_fwht_params->flags = V4L2_FWHT_FL_PIXENC_YUV | - (2 << V4L2_FWHT_FL_COMPONENTS_NUM_OFFSET); - break; - } -} - -static void std_init(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr) -{ - switch (ctrl->type) { - case V4L2_CTRL_TYPE_STRING: - idx *= ctrl->elem_size; - memset(ptr.p_char + idx, ' ', ctrl->minimum); - ptr.p_char[idx + ctrl->minimum] = '\0'; - break; - case V4L2_CTRL_TYPE_INTEGER64: - ptr.p_s64[idx] = ctrl->default_value; - break; - case V4L2_CTRL_TYPE_INTEGER: - case V4L2_CTRL_TYPE_INTEGER_MENU: - case V4L2_CTRL_TYPE_MENU: - case V4L2_CTRL_TYPE_BITMASK: - case V4L2_CTRL_TYPE_BOOLEAN: - ptr.p_s32[idx] = ctrl->default_value; - break; - case V4L2_CTRL_TYPE_BUTTON: - case V4L2_CTRL_TYPE_CTRL_CLASS: - ptr.p_s32[idx] = 0; - break; - case V4L2_CTRL_TYPE_U8: - ptr.p_u8[idx] = ctrl->default_value; - break; - case V4L2_CTRL_TYPE_U16: - ptr.p_u16[idx] = ctrl->default_value; - break; - case V4L2_CTRL_TYPE_U32: - ptr.p_u32[idx] = ctrl->default_value; - break; - default: - std_init_compound(ctrl, idx, ptr); - break; - } -} - -static void std_log(const struct v4l2_ctrl *ctrl) -{ - union v4l2_ctrl_ptr ptr = ctrl->p_cur; - - if (ctrl->is_array) { - unsigned i; - - for (i = 0; i < ctrl->nr_of_dims; i++) - pr_cont("[%u]", ctrl->dims[i]); - pr_cont(" "); - } - - switch (ctrl->type) { - case V4L2_CTRL_TYPE_INTEGER: - pr_cont("%d", *ptr.p_s32); - break; - case V4L2_CTRL_TYPE_BOOLEAN: - pr_cont("%s", *ptr.p_s32 ? "true" : "false"); - break; - case V4L2_CTRL_TYPE_MENU: - pr_cont("%s", ctrl->qmenu[*ptr.p_s32]); - break; - case V4L2_CTRL_TYPE_INTEGER_MENU: - pr_cont("%lld", ctrl->qmenu_int[*ptr.p_s32]); - break; - case V4L2_CTRL_TYPE_BITMASK: - pr_cont("0x%08x", *ptr.p_s32); - break; - case V4L2_CTRL_TYPE_INTEGER64: - pr_cont("%lld", *ptr.p_s64); - break; - case V4L2_CTRL_TYPE_STRING: - pr_cont("%s", ptr.p_char); - break; - case V4L2_CTRL_TYPE_U8: - pr_cont("%u", (unsigned)*ptr.p_u8); - break; - case V4L2_CTRL_TYPE_U16: - pr_cont("%u", (unsigned)*ptr.p_u16); - break; - case V4L2_CTRL_TYPE_U32: - pr_cont("%u", (unsigned)*ptr.p_u32); - break; - case V4L2_CTRL_TYPE_H264_SPS: - pr_cont("H264_SPS"); - break; - case V4L2_CTRL_TYPE_H264_PPS: - pr_cont("H264_PPS"); - break; - case V4L2_CTRL_TYPE_H264_SCALING_MATRIX: - pr_cont("H264_SCALING_MATRIX"); - break; - case V4L2_CTRL_TYPE_H264_SLICE_PARAMS: - pr_cont("H264_SLICE_PARAMS"); - break; - case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: - pr_cont("H264_DECODE_PARAMS"); - break; - case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS: - pr_cont("H264_PRED_WEIGHTS"); - break; - case V4L2_CTRL_TYPE_FWHT_PARAMS: - pr_cont("FWHT_PARAMS"); - break; - case V4L2_CTRL_TYPE_VP8_FRAME: - pr_cont("VP8_FRAME"); - break; - case V4L2_CTRL_TYPE_HDR10_CLL_INFO: - pr_cont("HDR10_CLL_INFO"); - break; - case V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY: - pr_cont("HDR10_MASTERING_DISPLAY"); - break; - default: - pr_cont("unknown type %d", ctrl->type); - break; - } -} - -/* - * Round towards the closest legal value. Be careful when we are - * close to the maximum range of the control type to prevent - * wrap-arounds. - */ -#define ROUND_TO_RANGE(val, offset_type, ctrl) \ -({ \ - offset_type offset; \ - if ((ctrl)->maximum >= 0 && \ - val >= (ctrl)->maximum - (s32)((ctrl)->step / 2)) \ - val = (ctrl)->maximum; \ - else \ - val += (s32)((ctrl)->step / 2); \ - val = clamp_t(typeof(val), val, \ - (ctrl)->minimum, (ctrl)->maximum); \ - offset = (val) - (ctrl)->minimum; \ - offset = (ctrl)->step * (offset / (u32)(ctrl)->step); \ - val = (ctrl)->minimum + offset; \ - 0; \ -}) - -/* Validate a new control */ - -#define zero_padding(s) \ - memset(&(s).padding, 0, sizeof((s).padding)) -#define zero_reserved(s) \ - memset(&(s).reserved, 0, sizeof((s).reserved)) - -/* - * Compound controls validation requires setting unused fields/flags to zero - * in order to properly detect unchanged controls with std_equal's memcmp. - */ -static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr) -{ - struct v4l2_ctrl_mpeg2_slice_params *p_mpeg2_slice_params; - struct v4l2_ctrl_vp8_frame *p_vp8_frame; - struct v4l2_ctrl_fwht_params *p_fwht_params; - struct v4l2_ctrl_h264_sps *p_h264_sps; - struct v4l2_ctrl_h264_pps *p_h264_pps; - struct v4l2_ctrl_h264_pred_weights *p_h264_pred_weights; - struct v4l2_ctrl_h264_slice_params *p_h264_slice_params; - struct v4l2_ctrl_h264_decode_params *p_h264_dec_params; - struct v4l2_ctrl_hevc_sps *p_hevc_sps; - struct v4l2_ctrl_hevc_pps *p_hevc_pps; - struct v4l2_ctrl_hevc_slice_params *p_hevc_slice_params; - struct v4l2_ctrl_hdr10_mastering_display *p_hdr10_mastering; - struct v4l2_area *area; - void *p = ptr.p + idx * ctrl->elem_size; - unsigned int i; - - switch ((u32)ctrl->type) { - case V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS: - p_mpeg2_slice_params = p; - - switch (p_mpeg2_slice_params->sequence.chroma_format) { - case 1: /* 4:2:0 */ - case 2: /* 4:2:2 */ - case 3: /* 4:4:4 */ - break; - default: - return -EINVAL; - } - - switch (p_mpeg2_slice_params->picture.intra_dc_precision) { - case 0: /* 8 bits */ - case 1: /* 9 bits */ - case 2: /* 10 bits */ - case 3: /* 11 bits */ - break; - default: - return -EINVAL; - } - - switch (p_mpeg2_slice_params->picture.picture_structure) { - case 1: /* interlaced top field */ - case 2: /* interlaced bottom field */ - case 3: /* progressive */ - break; - default: - return -EINVAL; - } - - switch (p_mpeg2_slice_params->picture.picture_coding_type) { - case V4L2_MPEG2_PICTURE_CODING_TYPE_I: - case V4L2_MPEG2_PICTURE_CODING_TYPE_P: - case V4L2_MPEG2_PICTURE_CODING_TYPE_B: - break; - default: - return -EINVAL; - } - - break; - - case V4L2_CTRL_TYPE_MPEG2_QUANTIZATION: - break; - - case V4L2_CTRL_TYPE_FWHT_PARAMS: - p_fwht_params = p; - if (p_fwht_params->version < V4L2_FWHT_VERSION) - return -EINVAL; - if (!p_fwht_params->width || !p_fwht_params->height) - return -EINVAL; - break; - - case V4L2_CTRL_TYPE_H264_SPS: - p_h264_sps = p; - - /* Some syntax elements are only conditionally valid */ - if (p_h264_sps->pic_order_cnt_type != 0) { - p_h264_sps->log2_max_pic_order_cnt_lsb_minus4 = 0; - } else if (p_h264_sps->pic_order_cnt_type != 1) { - p_h264_sps->num_ref_frames_in_pic_order_cnt_cycle = 0; - p_h264_sps->offset_for_non_ref_pic = 0; - p_h264_sps->offset_for_top_to_bottom_field = 0; - memset(&p_h264_sps->offset_for_ref_frame, 0, - sizeof(p_h264_sps->offset_for_ref_frame)); - } - - if (!V4L2_H264_SPS_HAS_CHROMA_FORMAT(p_h264_sps)) { - p_h264_sps->chroma_format_idc = 1; - p_h264_sps->bit_depth_luma_minus8 = 0; - p_h264_sps->bit_depth_chroma_minus8 = 0; - - p_h264_sps->flags &= - ~V4L2_H264_SPS_FLAG_QPPRIME_Y_ZERO_TRANSFORM_BYPASS; - - if (p_h264_sps->chroma_format_idc < 3) - p_h264_sps->flags &= - ~V4L2_H264_SPS_FLAG_SEPARATE_COLOUR_PLANE; - } - - if (p_h264_sps->flags & V4L2_H264_SPS_FLAG_FRAME_MBS_ONLY) - p_h264_sps->flags &= - ~V4L2_H264_SPS_FLAG_MB_ADAPTIVE_FRAME_FIELD; - - /* - * Chroma 4:2:2 format require at least High 4:2:2 profile. - * - * The H264 specification and well-known parser implementations - * use profile-idc values directly, as that is clearer and - * less ambiguous. We do the same here. - */ - if (p_h264_sps->profile_idc < 122 && - p_h264_sps->chroma_format_idc > 1) - return -EINVAL; - /* Chroma 4:4:4 format require at least High 4:2:2 profile */ - if (p_h264_sps->profile_idc < 244 && - p_h264_sps->chroma_format_idc > 2) - return -EINVAL; - if (p_h264_sps->chroma_format_idc > 3) - return -EINVAL; - - if (p_h264_sps->bit_depth_luma_minus8 > 6) - return -EINVAL; - if (p_h264_sps->bit_depth_chroma_minus8 > 6) - return -EINVAL; - if (p_h264_sps->log2_max_frame_num_minus4 > 12) - return -EINVAL; - if (p_h264_sps->pic_order_cnt_type > 2) - return -EINVAL; - if (p_h264_sps->log2_max_pic_order_cnt_lsb_minus4 > 12) - return -EINVAL; - if (p_h264_sps->max_num_ref_frames > V4L2_H264_REF_LIST_LEN) - return -EINVAL; - break; - - case V4L2_CTRL_TYPE_H264_PPS: - p_h264_pps = p; - - if (p_h264_pps->num_slice_groups_minus1 > 7) - return -EINVAL; - if (p_h264_pps->num_ref_idx_l0_default_active_minus1 > - (V4L2_H264_REF_LIST_LEN - 1)) - return -EINVAL; - if (p_h264_pps->num_ref_idx_l1_default_active_minus1 > - (V4L2_H264_REF_LIST_LEN - 1)) - return -EINVAL; - if (p_h264_pps->weighted_bipred_idc > 2) - return -EINVAL; - /* - * pic_init_qp_minus26 shall be in the range of - * -(26 + QpBdOffset_y) to +25, inclusive, - * where QpBdOffset_y is 6 * bit_depth_luma_minus8 - */ - if (p_h264_pps->pic_init_qp_minus26 < -62 || - p_h264_pps->pic_init_qp_minus26 > 25) - return -EINVAL; - if (p_h264_pps->pic_init_qs_minus26 < -26 || - p_h264_pps->pic_init_qs_minus26 > 25) - return -EINVAL; - if (p_h264_pps->chroma_qp_index_offset < -12 || - p_h264_pps->chroma_qp_index_offset > 12) - return -EINVAL; - if (p_h264_pps->second_chroma_qp_index_offset < -12 || - p_h264_pps->second_chroma_qp_index_offset > 12) - return -EINVAL; - break; - - case V4L2_CTRL_TYPE_H264_SCALING_MATRIX: - break; - - case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS: - p_h264_pred_weights = p; - - if (p_h264_pred_weights->luma_log2_weight_denom > 7) - return -EINVAL; - if (p_h264_pred_weights->chroma_log2_weight_denom > 7) - return -EINVAL; - break; - - case V4L2_CTRL_TYPE_H264_SLICE_PARAMS: - p_h264_slice_params = p; - - if (p_h264_slice_params->slice_type != V4L2_H264_SLICE_TYPE_B) - p_h264_slice_params->flags &= - ~V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED; - - if (p_h264_slice_params->colour_plane_id > 2) - return -EINVAL; - if (p_h264_slice_params->cabac_init_idc > 2) - return -EINVAL; - if (p_h264_slice_params->disable_deblocking_filter_idc > 2) - return -EINVAL; - if (p_h264_slice_params->slice_alpha_c0_offset_div2 < -6 || - p_h264_slice_params->slice_alpha_c0_offset_div2 > 6) - return -EINVAL; - if (p_h264_slice_params->slice_beta_offset_div2 < -6 || - p_h264_slice_params->slice_beta_offset_div2 > 6) - return -EINVAL; - - if (p_h264_slice_params->slice_type == V4L2_H264_SLICE_TYPE_I || - p_h264_slice_params->slice_type == V4L2_H264_SLICE_TYPE_SI) - p_h264_slice_params->num_ref_idx_l0_active_minus1 = 0; - if (p_h264_slice_params->slice_type != V4L2_H264_SLICE_TYPE_B) - p_h264_slice_params->num_ref_idx_l1_active_minus1 = 0; - - if (p_h264_slice_params->num_ref_idx_l0_active_minus1 > - (V4L2_H264_REF_LIST_LEN - 1)) - return -EINVAL; - if (p_h264_slice_params->num_ref_idx_l1_active_minus1 > - (V4L2_H264_REF_LIST_LEN - 1)) - return -EINVAL; - zero_reserved(*p_h264_slice_params); - break; - - case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: - p_h264_dec_params = p; - - if (p_h264_dec_params->nal_ref_idc > 3) - return -EINVAL; - for (i = 0; i < V4L2_H264_NUM_DPB_ENTRIES; i++) { - struct v4l2_h264_dpb_entry *dpb_entry = - &p_h264_dec_params->dpb[i]; - - zero_reserved(*dpb_entry); - } - zero_reserved(*p_h264_dec_params); - break; - - case V4L2_CTRL_TYPE_VP8_FRAME: - p_vp8_frame = p; - - switch (p_vp8_frame->num_dct_parts) { - case 1: - case 2: - case 4: - case 8: - break; - default: - return -EINVAL; - } - zero_padding(p_vp8_frame->segment); - zero_padding(p_vp8_frame->lf); - zero_padding(p_vp8_frame->quant); - zero_padding(p_vp8_frame->entropy); - zero_padding(p_vp8_frame->coder_state); - break; - - case V4L2_CTRL_TYPE_HEVC_SPS: - p_hevc_sps = p; - - if (!(p_hevc_sps->flags & V4L2_HEVC_SPS_FLAG_PCM_ENABLED)) { - p_hevc_sps->pcm_sample_bit_depth_luma_minus1 = 0; - p_hevc_sps->pcm_sample_bit_depth_chroma_minus1 = 0; - p_hevc_sps->log2_min_pcm_luma_coding_block_size_minus3 = 0; - p_hevc_sps->log2_diff_max_min_pcm_luma_coding_block_size = 0; - } - - if (!(p_hevc_sps->flags & - V4L2_HEVC_SPS_FLAG_LONG_TERM_REF_PICS_PRESENT)) - p_hevc_sps->num_long_term_ref_pics_sps = 0; - break; - - case V4L2_CTRL_TYPE_HEVC_PPS: - p_hevc_pps = p; - - if (!(p_hevc_pps->flags & - V4L2_HEVC_PPS_FLAG_CU_QP_DELTA_ENABLED)) - p_hevc_pps->diff_cu_qp_delta_depth = 0; - - if (!(p_hevc_pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED)) { - p_hevc_pps->num_tile_columns_minus1 = 0; - p_hevc_pps->num_tile_rows_minus1 = 0; - memset(&p_hevc_pps->column_width_minus1, 0, - sizeof(p_hevc_pps->column_width_minus1)); - memset(&p_hevc_pps->row_height_minus1, 0, - sizeof(p_hevc_pps->row_height_minus1)); - - p_hevc_pps->flags &= - ~V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED; - } - - if (p_hevc_pps->flags & - V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER) { - p_hevc_pps->pps_beta_offset_div2 = 0; - p_hevc_pps->pps_tc_offset_div2 = 0; - } - - zero_padding(*p_hevc_pps); - break; - - case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: - p_hevc_slice_params = p; - - if (p_hevc_slice_params->num_active_dpb_entries > - V4L2_HEVC_DPB_ENTRIES_NUM_MAX) - return -EINVAL; - - zero_padding(p_hevc_slice_params->pred_weight_table); - - for (i = 0; i < p_hevc_slice_params->num_active_dpb_entries; - i++) { - struct v4l2_hevc_dpb_entry *dpb_entry = - &p_hevc_slice_params->dpb[i]; - - zero_padding(*dpb_entry); - } - - zero_padding(*p_hevc_slice_params); - break; - - case V4L2_CTRL_TYPE_HDR10_CLL_INFO: - break; - - case V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY: - p_hdr10_mastering = p; - - for (i = 0; i < 3; ++i) { - if (p_hdr10_mastering->display_primaries_x[i] < - V4L2_HDR10_MASTERING_PRIMARIES_X_LOW || - p_hdr10_mastering->display_primaries_x[i] > - V4L2_HDR10_MASTERING_PRIMARIES_X_HIGH || - p_hdr10_mastering->display_primaries_y[i] < - V4L2_HDR10_MASTERING_PRIMARIES_Y_LOW || - p_hdr10_mastering->display_primaries_y[i] > - V4L2_HDR10_MASTERING_PRIMARIES_Y_HIGH) - return -EINVAL; - } - - if (p_hdr10_mastering->white_point_x < - V4L2_HDR10_MASTERING_WHITE_POINT_X_LOW || - p_hdr10_mastering->white_point_x > - V4L2_HDR10_MASTERING_WHITE_POINT_X_HIGH || - p_hdr10_mastering->white_point_y < - V4L2_HDR10_MASTERING_WHITE_POINT_Y_LOW || - p_hdr10_mastering->white_point_y > - V4L2_HDR10_MASTERING_WHITE_POINT_Y_HIGH) - return -EINVAL; - - if (p_hdr10_mastering->max_display_mastering_luminance < - V4L2_HDR10_MASTERING_MAX_LUMA_LOW || - p_hdr10_mastering->max_display_mastering_luminance > - V4L2_HDR10_MASTERING_MAX_LUMA_HIGH || - p_hdr10_mastering->min_display_mastering_luminance < - V4L2_HDR10_MASTERING_MIN_LUMA_LOW || - p_hdr10_mastering->min_display_mastering_luminance > - V4L2_HDR10_MASTERING_MIN_LUMA_HIGH) - return -EINVAL; - - /* The following restriction comes from ITU-T Rec. H.265 spec */ - if (p_hdr10_mastering->max_display_mastering_luminance == - V4L2_HDR10_MASTERING_MAX_LUMA_LOW && - p_hdr10_mastering->min_display_mastering_luminance == - V4L2_HDR10_MASTERING_MIN_LUMA_HIGH) - return -EINVAL; - - break; - - case V4L2_CTRL_TYPE_AREA: - area = p; - if (!area->width || !area->height) - return -EINVAL; - break; - - default: - return -EINVAL; - } - - return 0; -} - -static int std_validate(const struct v4l2_ctrl *ctrl, u32 idx, - union v4l2_ctrl_ptr ptr) -{ - size_t len; - u64 offset; - s64 val; - - switch ((u32)ctrl->type) { - case V4L2_CTRL_TYPE_INTEGER: - return ROUND_TO_RANGE(ptr.p_s32[idx], u32, ctrl); - case V4L2_CTRL_TYPE_INTEGER64: - /* - * We can't use the ROUND_TO_RANGE define here due to - * the u64 divide that needs special care. - */ - val = ptr.p_s64[idx]; - if (ctrl->maximum >= 0 && val >= ctrl->maximum - (s64)(ctrl->step / 2)) - val = ctrl->maximum; - else - val += (s64)(ctrl->step / 2); - val = clamp_t(s64, val, ctrl->minimum, ctrl->maximum); - offset = val - ctrl->minimum; - do_div(offset, ctrl->step); - ptr.p_s64[idx] = ctrl->minimum + offset * ctrl->step; - return 0; - case V4L2_CTRL_TYPE_U8: - return ROUND_TO_RANGE(ptr.p_u8[idx], u8, ctrl); - case V4L2_CTRL_TYPE_U16: - return ROUND_TO_RANGE(ptr.p_u16[idx], u16, ctrl); - case V4L2_CTRL_TYPE_U32: - return ROUND_TO_RANGE(ptr.p_u32[idx], u32, ctrl); - - case V4L2_CTRL_TYPE_BOOLEAN: - ptr.p_s32[idx] = !!ptr.p_s32[idx]; - return 0; - - case V4L2_CTRL_TYPE_MENU: - case V4L2_CTRL_TYPE_INTEGER_MENU: - if (ptr.p_s32[idx] < ctrl->minimum || ptr.p_s32[idx] > ctrl->maximum) - return -ERANGE; - if (ptr.p_s32[idx] < BITS_PER_LONG_LONG && - (ctrl->menu_skip_mask & BIT_ULL(ptr.p_s32[idx]))) - return -EINVAL; - if (ctrl->type == V4L2_CTRL_TYPE_MENU && - ctrl->qmenu[ptr.p_s32[idx]][0] == '\0') - return -EINVAL; - return 0; - - case V4L2_CTRL_TYPE_BITMASK: - ptr.p_s32[idx] &= ctrl->maximum; - return 0; - - case V4L2_CTRL_TYPE_BUTTON: - case V4L2_CTRL_TYPE_CTRL_CLASS: - ptr.p_s32[idx] = 0; - return 0; - - case V4L2_CTRL_TYPE_STRING: - idx *= ctrl->elem_size; - len = strlen(ptr.p_char + idx); - if (len < ctrl->minimum) - return -ERANGE; - if ((len - (u32)ctrl->minimum) % (u32)ctrl->step) - return -ERANGE; - return 0; - - default: - return std_validate_compound(ctrl, idx, ptr); - } -} - -static const struct v4l2_ctrl_type_ops std_type_ops = { - .equal = std_equal, - .init = std_init, - .log = std_log, - .validate = std_validate, -}; - -/* Helper function: copy the given control value back to the caller */ -static int ptr_to_user(struct v4l2_ext_control *c, - struct v4l2_ctrl *ctrl, - union v4l2_ctrl_ptr ptr) -{ - u32 len; - - if (ctrl->is_ptr && !ctrl->is_string) - return copy_to_user(c->ptr, ptr.p_const, c->size) ? - -EFAULT : 0; - - switch (ctrl->type) { - case V4L2_CTRL_TYPE_STRING: - len = strlen(ptr.p_char); - if (c->size < len + 1) { - c->size = ctrl->elem_size; - return -ENOSPC; - } - return copy_to_user(c->string, ptr.p_char, len + 1) ? - -EFAULT : 0; - case V4L2_CTRL_TYPE_INTEGER64: - c->value64 = *ptr.p_s64; - break; - default: - c->value = *ptr.p_s32; - break; - } - return 0; -} - -/* Helper function: copy the current control value back to the caller */ -static int cur_to_user(struct v4l2_ext_control *c, - struct v4l2_ctrl *ctrl) -{ - return ptr_to_user(c, ctrl, ctrl->p_cur); -} - -/* Helper function: copy the new control value back to the caller */ -static int new_to_user(struct v4l2_ext_control *c, - struct v4l2_ctrl *ctrl) -{ - return ptr_to_user(c, ctrl, ctrl->p_new); -} - -/* Helper function: copy the request value back to the caller */ -static int req_to_user(struct v4l2_ext_control *c, - struct v4l2_ctrl_ref *ref) -{ - return ptr_to_user(c, ref->ctrl, ref->p_req); -} - -/* Helper function: copy the initial control value back to the caller */ -static int def_to_user(struct v4l2_ext_control *c, struct v4l2_ctrl *ctrl) -{ - int idx; - - for (idx = 0; idx < ctrl->elems; idx++) - ctrl->type_ops->init(ctrl, idx, ctrl->p_new); - - return ptr_to_user(c, ctrl, ctrl->p_new); -} - -/* Helper function: copy the caller-provider value to the given control value */ -static int user_to_ptr(struct v4l2_ext_control *c, - struct v4l2_ctrl *ctrl, - union v4l2_ctrl_ptr ptr) -{ - int ret; - u32 size; - - ctrl->is_new = 1; - if (ctrl->is_ptr && !ctrl->is_string) { - unsigned idx; - - ret = copy_from_user(ptr.p, c->ptr, c->size) ? -EFAULT : 0; - if (ret || !ctrl->is_array) - return ret; - for (idx = c->size / ctrl->elem_size; idx < ctrl->elems; idx++) - ctrl->type_ops->init(ctrl, idx, ptr); - return 0; - } - - switch (ctrl->type) { - case V4L2_CTRL_TYPE_INTEGER64: - *ptr.p_s64 = c->value64; - break; - case V4L2_CTRL_TYPE_STRING: - size = c->size; - if (size == 0) - return -ERANGE; - if (size > ctrl->maximum + 1) - size = ctrl->maximum + 1; - ret = copy_from_user(ptr.p_char, c->string, size) ? -EFAULT : 0; - if (!ret) { - char last = ptr.p_char[size - 1]; - - ptr.p_char[size - 1] = 0; - /* If the string was longer than ctrl->maximum, - then return an error. */ - if (strlen(ptr.p_char) == ctrl->maximum && last) - return -ERANGE; - } - return ret; - default: - *ptr.p_s32 = c->value; - break; - } - return 0; -} - -/* Helper function: copy the caller-provider value as the new control value */ -static int user_to_new(struct v4l2_ext_control *c, - struct v4l2_ctrl *ctrl) -{ - return user_to_ptr(c, ctrl, ctrl->p_new); -} - -/* Copy the one value to another. */ -static void ptr_to_ptr(struct v4l2_ctrl *ctrl, - union v4l2_ctrl_ptr from, union v4l2_ctrl_ptr to) -{ - if (ctrl == NULL) - return; - memcpy(to.p, from.p_const, ctrl->elems * ctrl->elem_size); -} - -/* Copy the new value to the current value. */ -static void new_to_cur(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 ch_flags) -{ - bool changed; - - if (ctrl == NULL) - return; - - /* has_changed is set by cluster_changed */ - changed = ctrl->has_changed; - if (changed) - ptr_to_ptr(ctrl, ctrl->p_new, ctrl->p_cur); - - if (ch_flags & V4L2_EVENT_CTRL_CH_FLAGS) { - /* Note: CH_FLAGS is only set for auto clusters. */ - ctrl->flags &= - ~(V4L2_CTRL_FLAG_INACTIVE | V4L2_CTRL_FLAG_VOLATILE); - if (!is_cur_manual(ctrl->cluster[0])) { - ctrl->flags |= V4L2_CTRL_FLAG_INACTIVE; - if (ctrl->cluster[0]->has_volatiles) - ctrl->flags |= V4L2_CTRL_FLAG_VOLATILE; - } - fh = NULL; - } - if (changed || ch_flags) { - /* If a control was changed that was not one of the controls - modified by the application, then send the event to all. */ - if (!ctrl->is_new) - fh = NULL; - send_event(fh, ctrl, - (changed ? V4L2_EVENT_CTRL_CH_VALUE : 0) | ch_flags); - if (ctrl->call_notify && changed && ctrl->handler->notify) - ctrl->handler->notify(ctrl, ctrl->handler->notify_priv); - } -} - -/* Copy the current value to the new value */ -static void cur_to_new(struct v4l2_ctrl *ctrl) -{ - if (ctrl == NULL) - return; - ptr_to_ptr(ctrl, ctrl->p_cur, ctrl->p_new); -} - -/* Copy the new value to the request value */ -static void new_to_req(struct v4l2_ctrl_ref *ref) -{ - if (!ref) - return; - ptr_to_ptr(ref->ctrl, ref->ctrl->p_new, ref->p_req); - ref->valid_p_req = true; -} - -/* Copy the current value to the request value */ -static void cur_to_req(struct v4l2_ctrl_ref *ref) -{ - if (!ref) - return; - ptr_to_ptr(ref->ctrl, ref->ctrl->p_cur, ref->p_req); - ref->valid_p_req = true; -} - -/* Copy the request value to the new value */ -static void req_to_new(struct v4l2_ctrl_ref *ref) -{ - if (!ref) - return; - if (ref->valid_p_req) - ptr_to_ptr(ref->ctrl, ref->p_req, ref->ctrl->p_new); - else - ptr_to_ptr(ref->ctrl, ref->ctrl->p_cur, ref->ctrl->p_new); -} - -/* Return non-zero if one or more of the controls in the cluster has a new - value that differs from the current value. */ -static int cluster_changed(struct v4l2_ctrl *master) -{ - bool changed = false; - unsigned idx; - int i; - - for (i = 0; i < master->ncontrols; i++) { - struct v4l2_ctrl *ctrl = master->cluster[i]; - bool ctrl_changed = false; - - if (ctrl == NULL) - continue; - - if (ctrl->flags & V4L2_CTRL_FLAG_EXECUTE_ON_WRITE) - changed = ctrl_changed = true; - - /* - * Set has_changed to false to avoid generating - * the event V4L2_EVENT_CTRL_CH_VALUE - */ - if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) { - ctrl->has_changed = false; - continue; - } - - for (idx = 0; !ctrl_changed && idx < ctrl->elems; idx++) - ctrl_changed = !ctrl->type_ops->equal(ctrl, idx, - ctrl->p_cur, ctrl->p_new); - ctrl->has_changed = ctrl_changed; - changed |= ctrl->has_changed; - } - return changed; -} - -/* Control range checking */ -static int check_range(enum v4l2_ctrl_type type, - s64 min, s64 max, u64 step, s64 def) -{ - switch (type) { - case V4L2_CTRL_TYPE_BOOLEAN: - if (step != 1 || max > 1 || min < 0) - return -ERANGE; - fallthrough; - case V4L2_CTRL_TYPE_U8: - case V4L2_CTRL_TYPE_U16: - case V4L2_CTRL_TYPE_U32: - case V4L2_CTRL_TYPE_INTEGER: - case V4L2_CTRL_TYPE_INTEGER64: - if (step == 0 || min > max || def < min || def > max) - return -ERANGE; - return 0; - case V4L2_CTRL_TYPE_BITMASK: - if (step || min || !max || (def & ~max)) - return -ERANGE; - return 0; - case V4L2_CTRL_TYPE_MENU: - case V4L2_CTRL_TYPE_INTEGER_MENU: - if (min > max || def < min || def > max) - return -ERANGE; - /* Note: step == menu_skip_mask for menu controls. - So here we check if the default value is masked out. */ - if (step && ((1 << def) & step)) - return -EINVAL; - return 0; - case V4L2_CTRL_TYPE_STRING: - if (min > max || min < 0 || step < 1 || def) - return -ERANGE; - return 0; - default: - return 0; - } -} - -/* Validate a new control */ -static int validate_new(const struct v4l2_ctrl *ctrl, union v4l2_ctrl_ptr p_new) -{ - unsigned idx; - int err = 0; - - for (idx = 0; !err && idx < ctrl->elems; idx++) - err = ctrl->type_ops->validate(ctrl, idx, p_new); - return err; -} - -static inline u32 node2id(struct list_head *node) -{ - return list_entry(node, struct v4l2_ctrl_ref, node)->ctrl->id; -} - -/* Set the handler's error code if it wasn't set earlier already */ -static inline int handler_set_err(struct v4l2_ctrl_handler *hdl, int err) -{ - if (hdl->error == 0) - hdl->error = err; - return err; -} - -/* Initialize the handler */ -int v4l2_ctrl_handler_init_class(struct v4l2_ctrl_handler *hdl, - unsigned nr_of_controls_hint, - struct lock_class_key *key, const char *name) -{ - mutex_init(&hdl->_lock); - hdl->lock = &hdl->_lock; - lockdep_set_class_and_name(hdl->lock, key, name); - INIT_LIST_HEAD(&hdl->ctrls); - INIT_LIST_HEAD(&hdl->ctrl_refs); - INIT_LIST_HEAD(&hdl->requests); - INIT_LIST_HEAD(&hdl->requests_queued); - hdl->request_is_queued = false; - hdl->nr_of_buckets = 1 + nr_of_controls_hint / 8; - hdl->buckets = kvmalloc_array(hdl->nr_of_buckets, - sizeof(hdl->buckets[0]), - GFP_KERNEL | __GFP_ZERO); - hdl->error = hdl->buckets ? 0 : -ENOMEM; - media_request_object_init(&hdl->req_obj); - return hdl->error; -} -EXPORT_SYMBOL(v4l2_ctrl_handler_init_class); - -/* Free all controls and control refs */ -void v4l2_ctrl_handler_free(struct v4l2_ctrl_handler *hdl) -{ - struct v4l2_ctrl_ref *ref, *next_ref; - struct v4l2_ctrl *ctrl, *next_ctrl; - struct v4l2_subscribed_event *sev, *next_sev; - - if (hdl == NULL || hdl->buckets == NULL) - return; - - /* - * If the main handler is freed and it is used by handler objects in - * outstanding requests, then unbind and put those objects before - * freeing the main handler. - * - * The main handler can be identified by having a NULL ops pointer in - * the request object. - */ - if (!hdl->req_obj.ops && !list_empty(&hdl->requests)) { - struct v4l2_ctrl_handler *req, *next_req; - - list_for_each_entry_safe(req, next_req, &hdl->requests, requests) { - media_request_object_unbind(&req->req_obj); - media_request_object_put(&req->req_obj); - } - } - mutex_lock(hdl->lock); - /* Free all nodes */ - list_for_each_entry_safe(ref, next_ref, &hdl->ctrl_refs, node) { - list_del(&ref->node); - kfree(ref); - } - /* Free all controls owned by the handler */ - list_for_each_entry_safe(ctrl, next_ctrl, &hdl->ctrls, node) { - list_del(&ctrl->node); - list_for_each_entry_safe(sev, next_sev, &ctrl->ev_subs, node) - list_del(&sev->node); - kvfree(ctrl); - } - kvfree(hdl->buckets); - hdl->buckets = NULL; - hdl->cached = NULL; - hdl->error = 0; - mutex_unlock(hdl->lock); - mutex_destroy(&hdl->_lock); -} -EXPORT_SYMBOL(v4l2_ctrl_handler_free); - -/* For backwards compatibility: V4L2_CID_PRIVATE_BASE should no longer - be used except in G_CTRL, S_CTRL, QUERYCTRL and QUERYMENU when dealing - with applications that do not use the NEXT_CTRL flag. - - We just find the n-th private user control. It's O(N), but that should not - be an issue in this particular case. */ -static struct v4l2_ctrl_ref *find_private_ref( - struct v4l2_ctrl_handler *hdl, u32 id) -{ - struct v4l2_ctrl_ref *ref; - - id -= V4L2_CID_PRIVATE_BASE; - list_for_each_entry(ref, &hdl->ctrl_refs, node) { - /* Search for private user controls that are compatible with - VIDIOC_G/S_CTRL. */ - if (V4L2_CTRL_ID2WHICH(ref->ctrl->id) == V4L2_CTRL_CLASS_USER && - V4L2_CTRL_DRIVER_PRIV(ref->ctrl->id)) { - if (!ref->ctrl->is_int) - continue; - if (id == 0) - return ref; - id--; - } - } - return NULL; -} - -/* Find a control with the given ID. */ -static struct v4l2_ctrl_ref *find_ref(struct v4l2_ctrl_handler *hdl, u32 id) -{ - struct v4l2_ctrl_ref *ref; - int bucket; - - id &= V4L2_CTRL_ID_MASK; - - /* Old-style private controls need special handling */ - if (id >= V4L2_CID_PRIVATE_BASE) - return find_private_ref(hdl, id); - bucket = id % hdl->nr_of_buckets; - - /* Simple optimization: cache the last control found */ - if (hdl->cached && hdl->cached->ctrl->id == id) - return hdl->cached; - - /* Not in cache, search the hash */ - ref = hdl->buckets ? hdl->buckets[bucket] : NULL; - while (ref && ref->ctrl->id != id) - ref = ref->next; - - if (ref) - hdl->cached = ref; /* cache it! */ - return ref; -} - -/* Find a control with the given ID. Take the handler's lock first. */ -static struct v4l2_ctrl_ref *find_ref_lock( - struct v4l2_ctrl_handler *hdl, u32 id) -{ - struct v4l2_ctrl_ref *ref = NULL; - - if (hdl) { - mutex_lock(hdl->lock); - ref = find_ref(hdl, id); - mutex_unlock(hdl->lock); - } - return ref; -} - -/* Find a control with the given ID. */ -struct v4l2_ctrl *v4l2_ctrl_find(struct v4l2_ctrl_handler *hdl, u32 id) -{ - struct v4l2_ctrl_ref *ref = find_ref_lock(hdl, id); - - return ref ? ref->ctrl : NULL; -} -EXPORT_SYMBOL(v4l2_ctrl_find); - -/* Allocate a new v4l2_ctrl_ref and hook it into the handler. */ -static int handler_new_ref(struct v4l2_ctrl_handler *hdl, - struct v4l2_ctrl *ctrl, - struct v4l2_ctrl_ref **ctrl_ref, - bool from_other_dev, bool allocate_req) -{ - struct v4l2_ctrl_ref *ref; - struct v4l2_ctrl_ref *new_ref; - u32 id = ctrl->id; - u32 class_ctrl = V4L2_CTRL_ID2WHICH(id) | 1; - int bucket = id % hdl->nr_of_buckets; /* which bucket to use */ - unsigned int size_extra_req = 0; - - if (ctrl_ref) - *ctrl_ref = NULL; - - /* - * Automatically add the control class if it is not yet present and - * the new control is not a compound control. - */ - if (ctrl->type < V4L2_CTRL_COMPOUND_TYPES && - id != class_ctrl && find_ref_lock(hdl, class_ctrl) == NULL) - if (!v4l2_ctrl_new_std(hdl, NULL, class_ctrl, 0, 0, 0, 0)) - return hdl->error; - - if (hdl->error) - return hdl->error; - - if (allocate_req) - size_extra_req = ctrl->elems * ctrl->elem_size; - new_ref = kzalloc(sizeof(*new_ref) + size_extra_req, GFP_KERNEL); - if (!new_ref) - return handler_set_err(hdl, -ENOMEM); - new_ref->ctrl = ctrl; - new_ref->from_other_dev = from_other_dev; - if (size_extra_req) - new_ref->p_req.p = &new_ref[1]; - - INIT_LIST_HEAD(&new_ref->node); - - mutex_lock(hdl->lock); - - /* Add immediately at the end of the list if the list is empty, or if - the last element in the list has a lower ID. - This ensures that when elements are added in ascending order the - insertion is an O(1) operation. */ - if (list_empty(&hdl->ctrl_refs) || id > node2id(hdl->ctrl_refs.prev)) { - list_add_tail(&new_ref->node, &hdl->ctrl_refs); - goto insert_in_hash; - } - - /* Find insert position in sorted list */ - list_for_each_entry(ref, &hdl->ctrl_refs, node) { - if (ref->ctrl->id < id) - continue; - /* Don't add duplicates */ - if (ref->ctrl->id == id) { - kfree(new_ref); - goto unlock; - } - list_add(&new_ref->node, ref->node.prev); - break; - } - -insert_in_hash: - /* Insert the control node in the hash */ - new_ref->next = hdl->buckets[bucket]; - hdl->buckets[bucket] = new_ref; - if (ctrl_ref) - *ctrl_ref = new_ref; - if (ctrl->handler == hdl) { - /* By default each control starts in a cluster of its own. - * new_ref->ctrl is basically a cluster array with one - * element, so that's perfect to use as the cluster pointer. - * But only do this for the handler that owns the control. - */ - ctrl->cluster = &new_ref->ctrl; - ctrl->ncontrols = 1; - } - -unlock: - mutex_unlock(hdl->lock); - return 0; -} - -/* Add a new control */ -static struct v4l2_ctrl *v4l2_ctrl_new(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_ops *ops, - const struct v4l2_ctrl_type_ops *type_ops, - u32 id, const char *name, enum v4l2_ctrl_type type, - s64 min, s64 max, u64 step, s64 def, - const u32 dims[V4L2_CTRL_MAX_DIMS], u32 elem_size, - u32 flags, const char * const *qmenu, - const s64 *qmenu_int, const union v4l2_ctrl_ptr p_def, - void *priv) -{ - struct v4l2_ctrl *ctrl; - unsigned sz_extra; - unsigned nr_of_dims = 0; - unsigned elems = 1; - bool is_array; - unsigned tot_ctrl_size; - unsigned idx; - void *data; - int err; - - if (hdl->error) - return NULL; - - while (dims && dims[nr_of_dims]) { - elems *= dims[nr_of_dims]; - nr_of_dims++; - if (nr_of_dims == V4L2_CTRL_MAX_DIMS) - break; - } - is_array = nr_of_dims > 0; - - /* Prefill elem_size for all types handled by std_type_ops */ - switch ((u32)type) { - case V4L2_CTRL_TYPE_INTEGER64: - elem_size = sizeof(s64); - break; - case V4L2_CTRL_TYPE_STRING: - elem_size = max + 1; - break; - case V4L2_CTRL_TYPE_U8: - elem_size = sizeof(u8); - break; - case V4L2_CTRL_TYPE_U16: - elem_size = sizeof(u16); - break; - case V4L2_CTRL_TYPE_U32: - elem_size = sizeof(u32); - break; - case V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS: - elem_size = sizeof(struct v4l2_ctrl_mpeg2_slice_params); - break; - case V4L2_CTRL_TYPE_MPEG2_QUANTIZATION: - elem_size = sizeof(struct v4l2_ctrl_mpeg2_quantization); - break; - case V4L2_CTRL_TYPE_FWHT_PARAMS: - elem_size = sizeof(struct v4l2_ctrl_fwht_params); - break; - case V4L2_CTRL_TYPE_H264_SPS: - elem_size = sizeof(struct v4l2_ctrl_h264_sps); - break; - case V4L2_CTRL_TYPE_H264_PPS: - elem_size = sizeof(struct v4l2_ctrl_h264_pps); - break; - case V4L2_CTRL_TYPE_H264_SCALING_MATRIX: - elem_size = sizeof(struct v4l2_ctrl_h264_scaling_matrix); - break; - case V4L2_CTRL_TYPE_H264_SLICE_PARAMS: - elem_size = sizeof(struct v4l2_ctrl_h264_slice_params); - break; - case V4L2_CTRL_TYPE_H264_DECODE_PARAMS: - elem_size = sizeof(struct v4l2_ctrl_h264_decode_params); - break; - case V4L2_CTRL_TYPE_H264_PRED_WEIGHTS: - elem_size = sizeof(struct v4l2_ctrl_h264_pred_weights); - break; - case V4L2_CTRL_TYPE_VP8_FRAME: - elem_size = sizeof(struct v4l2_ctrl_vp8_frame); - break; - case V4L2_CTRL_TYPE_HEVC_SPS: - elem_size = sizeof(struct v4l2_ctrl_hevc_sps); - break; - case V4L2_CTRL_TYPE_HEVC_PPS: - elem_size = sizeof(struct v4l2_ctrl_hevc_pps); - break; - case V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS: - elem_size = sizeof(struct v4l2_ctrl_hevc_slice_params); - break; - case V4L2_CTRL_TYPE_HDR10_CLL_INFO: - elem_size = sizeof(struct v4l2_ctrl_hdr10_cll_info); - break; - case V4L2_CTRL_TYPE_HDR10_MASTERING_DISPLAY: - elem_size = sizeof(struct v4l2_ctrl_hdr10_mastering_display); - break; - case V4L2_CTRL_TYPE_AREA: - elem_size = sizeof(struct v4l2_area); - break; - default: - if (type < V4L2_CTRL_COMPOUND_TYPES) - elem_size = sizeof(s32); - break; - } - tot_ctrl_size = elem_size * elems; - - /* Sanity checks */ - if (id == 0 || name == NULL || !elem_size || - id >= V4L2_CID_PRIVATE_BASE || - (type == V4L2_CTRL_TYPE_MENU && qmenu == NULL) || - (type == V4L2_CTRL_TYPE_INTEGER_MENU && qmenu_int == NULL)) { - handler_set_err(hdl, -ERANGE); - return NULL; - } - err = check_range(type, min, max, step, def); - if (err) { - handler_set_err(hdl, err); - return NULL; - } - if (is_array && - (type == V4L2_CTRL_TYPE_BUTTON || - type == V4L2_CTRL_TYPE_CTRL_CLASS)) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - - sz_extra = 0; - if (type == V4L2_CTRL_TYPE_BUTTON) - flags |= V4L2_CTRL_FLAG_WRITE_ONLY | - V4L2_CTRL_FLAG_EXECUTE_ON_WRITE; - else if (type == V4L2_CTRL_TYPE_CTRL_CLASS) - flags |= V4L2_CTRL_FLAG_READ_ONLY; - else if (type == V4L2_CTRL_TYPE_INTEGER64 || - type == V4L2_CTRL_TYPE_STRING || - type >= V4L2_CTRL_COMPOUND_TYPES || - is_array) - sz_extra += 2 * tot_ctrl_size; - - if (type >= V4L2_CTRL_COMPOUND_TYPES && p_def.p_const) - sz_extra += elem_size; - - ctrl = kvzalloc(sizeof(*ctrl) + sz_extra, GFP_KERNEL); - if (ctrl == NULL) { - handler_set_err(hdl, -ENOMEM); - return NULL; - } - - INIT_LIST_HEAD(&ctrl->node); - INIT_LIST_HEAD(&ctrl->ev_subs); - ctrl->handler = hdl; - ctrl->ops = ops; - ctrl->type_ops = type_ops ? type_ops : &std_type_ops; - ctrl->id = id; - ctrl->name = name; - ctrl->type = type; - ctrl->flags = flags; - ctrl->minimum = min; - ctrl->maximum = max; - ctrl->step = step; - ctrl->default_value = def; - ctrl->is_string = !is_array && type == V4L2_CTRL_TYPE_STRING; - ctrl->is_ptr = is_array || type >= V4L2_CTRL_COMPOUND_TYPES || ctrl->is_string; - ctrl->is_int = !ctrl->is_ptr && type != V4L2_CTRL_TYPE_INTEGER64; - ctrl->is_array = is_array; - ctrl->elems = elems; - ctrl->nr_of_dims = nr_of_dims; - if (nr_of_dims) - memcpy(ctrl->dims, dims, nr_of_dims * sizeof(dims[0])); - ctrl->elem_size = elem_size; - if (type == V4L2_CTRL_TYPE_MENU) - ctrl->qmenu = qmenu; - else if (type == V4L2_CTRL_TYPE_INTEGER_MENU) - ctrl->qmenu_int = qmenu_int; - ctrl->priv = priv; - ctrl->cur.val = ctrl->val = def; - data = &ctrl[1]; - - if (!ctrl->is_int) { - ctrl->p_new.p = data; - ctrl->p_cur.p = data + tot_ctrl_size; - } else { - ctrl->p_new.p = &ctrl->val; - ctrl->p_cur.p = &ctrl->cur.val; - } - - if (type >= V4L2_CTRL_COMPOUND_TYPES && p_def.p_const) { - ctrl->p_def.p = ctrl->p_cur.p + tot_ctrl_size; - memcpy(ctrl->p_def.p, p_def.p_const, elem_size); - } - - for (idx = 0; idx < elems; idx++) { - ctrl->type_ops->init(ctrl, idx, ctrl->p_cur); - ctrl->type_ops->init(ctrl, idx, ctrl->p_new); - } - - if (handler_new_ref(hdl, ctrl, NULL, false, false)) { - kvfree(ctrl); - return NULL; - } - mutex_lock(hdl->lock); - list_add_tail(&ctrl->node, &hdl->ctrls); - mutex_unlock(hdl->lock); - return ctrl; -} - -struct v4l2_ctrl *v4l2_ctrl_new_custom(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_config *cfg, void *priv) -{ - bool is_menu; - struct v4l2_ctrl *ctrl; - const char *name = cfg->name; - const char * const *qmenu = cfg->qmenu; - const s64 *qmenu_int = cfg->qmenu_int; - enum v4l2_ctrl_type type = cfg->type; - u32 flags = cfg->flags; - s64 min = cfg->min; - s64 max = cfg->max; - u64 step = cfg->step; - s64 def = cfg->def; - - if (name == NULL) - v4l2_ctrl_fill(cfg->id, &name, &type, &min, &max, &step, - &def, &flags); - - is_menu = (type == V4L2_CTRL_TYPE_MENU || - type == V4L2_CTRL_TYPE_INTEGER_MENU); - if (is_menu) - WARN_ON(step); - else - WARN_ON(cfg->menu_skip_mask); - if (type == V4L2_CTRL_TYPE_MENU && !qmenu) { - qmenu = v4l2_ctrl_get_menu(cfg->id); - } else if (type == V4L2_CTRL_TYPE_INTEGER_MENU && !qmenu_int) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - - ctrl = v4l2_ctrl_new(hdl, cfg->ops, cfg->type_ops, cfg->id, name, - type, min, max, - is_menu ? cfg->menu_skip_mask : step, def, - cfg->dims, cfg->elem_size, - flags, qmenu, qmenu_int, cfg->p_def, priv); - if (ctrl) - ctrl->is_private = cfg->is_private; - return ctrl; -} -EXPORT_SYMBOL(v4l2_ctrl_new_custom); - -/* Helper function for standard non-menu controls */ -struct v4l2_ctrl *v4l2_ctrl_new_std(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_ops *ops, - u32 id, s64 min, s64 max, u64 step, s64 def) -{ - const char *name; - enum v4l2_ctrl_type type; - u32 flags; - - v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); - if (type == V4L2_CTRL_TYPE_MENU || - type == V4L2_CTRL_TYPE_INTEGER_MENU || - type >= V4L2_CTRL_COMPOUND_TYPES) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, - min, max, step, def, NULL, 0, - flags, NULL, NULL, ptr_null, NULL); -} -EXPORT_SYMBOL(v4l2_ctrl_new_std); - -/* Helper function for standard menu controls */ -struct v4l2_ctrl *v4l2_ctrl_new_std_menu(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_ops *ops, - u32 id, u8 _max, u64 mask, u8 _def) -{ - const char * const *qmenu = NULL; - const s64 *qmenu_int = NULL; - unsigned int qmenu_int_len = 0; - const char *name; - enum v4l2_ctrl_type type; - s64 min; - s64 max = _max; - s64 def = _def; - u64 step; - u32 flags; - - v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); - - if (type == V4L2_CTRL_TYPE_MENU) - qmenu = v4l2_ctrl_get_menu(id); - else if (type == V4L2_CTRL_TYPE_INTEGER_MENU) - qmenu_int = v4l2_ctrl_get_int_menu(id, &qmenu_int_len); - - if ((!qmenu && !qmenu_int) || (qmenu_int && max > qmenu_int_len)) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, - 0, max, mask, def, NULL, 0, - flags, qmenu, qmenu_int, ptr_null, NULL); -} -EXPORT_SYMBOL(v4l2_ctrl_new_std_menu); - -/* Helper function for standard menu controls with driver defined menu */ -struct v4l2_ctrl *v4l2_ctrl_new_std_menu_items(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_ops *ops, u32 id, u8 _max, - u64 mask, u8 _def, const char * const *qmenu) -{ - enum v4l2_ctrl_type type; - const char *name; - u32 flags; - u64 step; - s64 min; - s64 max = _max; - s64 def = _def; - - /* v4l2_ctrl_new_std_menu_items() should only be called for - * standard controls without a standard menu. - */ - if (v4l2_ctrl_get_menu(id)) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - - v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); - if (type != V4L2_CTRL_TYPE_MENU || qmenu == NULL) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, - 0, max, mask, def, NULL, 0, - flags, qmenu, NULL, ptr_null, NULL); - -} -EXPORT_SYMBOL(v4l2_ctrl_new_std_menu_items); - -/* Helper function for standard compound controls */ -struct v4l2_ctrl *v4l2_ctrl_new_std_compound(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_ops *ops, u32 id, - const union v4l2_ctrl_ptr p_def) -{ - const char *name; - enum v4l2_ctrl_type type; - u32 flags; - s64 min, max, step, def; - - v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); - if (type < V4L2_CTRL_COMPOUND_TYPES) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, - min, max, step, def, NULL, 0, - flags, NULL, NULL, p_def, NULL); -} -EXPORT_SYMBOL(v4l2_ctrl_new_std_compound); - -/* Helper function for standard integer menu controls */ -struct v4l2_ctrl *v4l2_ctrl_new_int_menu(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_ops *ops, - u32 id, u8 _max, u8 _def, const s64 *qmenu_int) -{ - const char *name; - enum v4l2_ctrl_type type; - s64 min; - u64 step; - s64 max = _max; - s64 def = _def; - u32 flags; - - v4l2_ctrl_fill(id, &name, &type, &min, &max, &step, &def, &flags); - if (type != V4L2_CTRL_TYPE_INTEGER_MENU) { - handler_set_err(hdl, -EINVAL); - return NULL; - } - return v4l2_ctrl_new(hdl, ops, NULL, id, name, type, - 0, max, 0, def, NULL, 0, - flags, NULL, qmenu_int, ptr_null, NULL); -} -EXPORT_SYMBOL(v4l2_ctrl_new_int_menu); - -/* Add the controls from another handler to our own. */ -int v4l2_ctrl_add_handler(struct v4l2_ctrl_handler *hdl, - struct v4l2_ctrl_handler *add, - bool (*filter)(const struct v4l2_ctrl *ctrl), - bool from_other_dev) -{ - struct v4l2_ctrl_ref *ref; - int ret = 0; - - /* Do nothing if either handler is NULL or if they are the same */ - if (!hdl || !add || hdl == add) - return 0; - if (hdl->error) - return hdl->error; - mutex_lock(add->lock); - list_for_each_entry(ref, &add->ctrl_refs, node) { - struct v4l2_ctrl *ctrl = ref->ctrl; - - /* Skip handler-private controls. */ - if (ctrl->is_private) - continue; - /* And control classes */ - if (ctrl->type == V4L2_CTRL_TYPE_CTRL_CLASS) - continue; - /* Filter any unwanted controls */ - if (filter && !filter(ctrl)) - continue; - ret = handler_new_ref(hdl, ctrl, NULL, from_other_dev, false); - if (ret) - break; - } - mutex_unlock(add->lock); - return ret; -} -EXPORT_SYMBOL(v4l2_ctrl_add_handler); - -bool v4l2_ctrl_radio_filter(const struct v4l2_ctrl *ctrl) -{ - if (V4L2_CTRL_ID2WHICH(ctrl->id) == V4L2_CTRL_CLASS_FM_TX) - return true; - if (V4L2_CTRL_ID2WHICH(ctrl->id) == V4L2_CTRL_CLASS_FM_RX) - return true; - switch (ctrl->id) { - case V4L2_CID_AUDIO_MUTE: - case V4L2_CID_AUDIO_VOLUME: - case V4L2_CID_AUDIO_BALANCE: - case V4L2_CID_AUDIO_BASS: - case V4L2_CID_AUDIO_TREBLE: - case V4L2_CID_AUDIO_LOUDNESS: - return true; - default: - break; - } - return false; -} -EXPORT_SYMBOL(v4l2_ctrl_radio_filter); - -/* Cluster controls */ -void v4l2_ctrl_cluster(unsigned ncontrols, struct v4l2_ctrl **controls) -{ - bool has_volatiles = false; - int i; - - /* The first control is the master control and it must not be NULL */ - if (WARN_ON(ncontrols == 0 || controls[0] == NULL)) - return; - - for (i = 0; i < ncontrols; i++) { - if (controls[i]) { - controls[i]->cluster = controls; - controls[i]->ncontrols = ncontrols; - if (controls[i]->flags & V4L2_CTRL_FLAG_VOLATILE) - has_volatiles = true; - } - } - controls[0]->has_volatiles = has_volatiles; -} -EXPORT_SYMBOL(v4l2_ctrl_cluster); - -void v4l2_ctrl_auto_cluster(unsigned ncontrols, struct v4l2_ctrl **controls, - u8 manual_val, bool set_volatile) -{ - struct v4l2_ctrl *master = controls[0]; - u32 flag = 0; - int i; - - v4l2_ctrl_cluster(ncontrols, controls); - WARN_ON(ncontrols <= 1); - WARN_ON(manual_val < master->minimum || manual_val > master->maximum); - WARN_ON(set_volatile && !has_op(master, g_volatile_ctrl)); - master->is_auto = true; - master->has_volatiles = set_volatile; - master->manual_mode_value = manual_val; - master->flags |= V4L2_CTRL_FLAG_UPDATE; - - if (!is_cur_manual(master)) - flag = V4L2_CTRL_FLAG_INACTIVE | - (set_volatile ? V4L2_CTRL_FLAG_VOLATILE : 0); - - for (i = 1; i < ncontrols; i++) - if (controls[i]) - controls[i]->flags |= flag; -} -EXPORT_SYMBOL(v4l2_ctrl_auto_cluster); - -/* Activate/deactivate a control. */ -void v4l2_ctrl_activate(struct v4l2_ctrl *ctrl, bool active) -{ - /* invert since the actual flag is called 'inactive' */ - bool inactive = !active; - bool old; - - if (ctrl == NULL) - return; - - if (inactive) - /* set V4L2_CTRL_FLAG_INACTIVE */ - old = test_and_set_bit(4, &ctrl->flags); - else - /* clear V4L2_CTRL_FLAG_INACTIVE */ - old = test_and_clear_bit(4, &ctrl->flags); - if (old != inactive) - send_event(NULL, ctrl, V4L2_EVENT_CTRL_CH_FLAGS); -} -EXPORT_SYMBOL(v4l2_ctrl_activate); - -void __v4l2_ctrl_grab(struct v4l2_ctrl *ctrl, bool grabbed) -{ - bool old; - - if (ctrl == NULL) - return; - - lockdep_assert_held(ctrl->handler->lock); - - if (grabbed) - /* set V4L2_CTRL_FLAG_GRABBED */ - old = test_and_set_bit(1, &ctrl->flags); - else - /* clear V4L2_CTRL_FLAG_GRABBED */ - old = test_and_clear_bit(1, &ctrl->flags); - if (old != grabbed) - send_event(NULL, ctrl, V4L2_EVENT_CTRL_CH_FLAGS); -} -EXPORT_SYMBOL(__v4l2_ctrl_grab); - -/* Log the control name and value */ -static void log_ctrl(const struct v4l2_ctrl *ctrl, - const char *prefix, const char *colon) -{ - if (ctrl->flags & (V4L2_CTRL_FLAG_DISABLED | V4L2_CTRL_FLAG_WRITE_ONLY)) - return; - if (ctrl->type == V4L2_CTRL_TYPE_CTRL_CLASS) - return; - - pr_info("%s%s%s: ", prefix, colon, ctrl->name); - - ctrl->type_ops->log(ctrl); - - if (ctrl->flags & (V4L2_CTRL_FLAG_INACTIVE | - V4L2_CTRL_FLAG_GRABBED | - V4L2_CTRL_FLAG_VOLATILE)) { - if (ctrl->flags & V4L2_CTRL_FLAG_INACTIVE) - pr_cont(" inactive"); - if (ctrl->flags & V4L2_CTRL_FLAG_GRABBED) - pr_cont(" grabbed"); - if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) - pr_cont(" volatile"); - } - pr_cont("\n"); -} - -/* Log all controls owned by the handler */ -void v4l2_ctrl_handler_log_status(struct v4l2_ctrl_handler *hdl, - const char *prefix) -{ - struct v4l2_ctrl *ctrl; - const char *colon = ""; - int len; - - if (hdl == NULL) - return; - if (prefix == NULL) - prefix = ""; - len = strlen(prefix); - if (len && prefix[len - 1] != ' ') - colon = ": "; - mutex_lock(hdl->lock); - list_for_each_entry(ctrl, &hdl->ctrls, node) - if (!(ctrl->flags & V4L2_CTRL_FLAG_DISABLED)) - log_ctrl(ctrl, prefix, colon); - mutex_unlock(hdl->lock); -} -EXPORT_SYMBOL(v4l2_ctrl_handler_log_status); - -int v4l2_ctrl_subdev_log_status(struct v4l2_subdev *sd) -{ - v4l2_ctrl_handler_log_status(sd->ctrl_handler, sd->name); - return 0; -} -EXPORT_SYMBOL(v4l2_ctrl_subdev_log_status); - -/* Call s_ctrl for all controls owned by the handler */ -int __v4l2_ctrl_handler_setup(struct v4l2_ctrl_handler *hdl) -{ - struct v4l2_ctrl *ctrl; - int ret = 0; - - if (hdl == NULL) - return 0; - - lockdep_assert_held(hdl->lock); - - list_for_each_entry(ctrl, &hdl->ctrls, node) - ctrl->done = false; - - list_for_each_entry(ctrl, &hdl->ctrls, node) { - struct v4l2_ctrl *master = ctrl->cluster[0]; - int i; - - /* Skip if this control was already handled by a cluster. */ - /* Skip button controls and read-only controls. */ - if (ctrl->done || ctrl->type == V4L2_CTRL_TYPE_BUTTON || - (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY)) - continue; - - for (i = 0; i < master->ncontrols; i++) { - if (master->cluster[i]) { - cur_to_new(master->cluster[i]); - master->cluster[i]->is_new = 1; - master->cluster[i]->done = true; - } - } - ret = call_op(master, s_ctrl); - if (ret) - break; - } - - return ret; -} -EXPORT_SYMBOL_GPL(__v4l2_ctrl_handler_setup); - -int v4l2_ctrl_handler_setup(struct v4l2_ctrl_handler *hdl) -{ - int ret; - - if (hdl == NULL) - return 0; - - mutex_lock(hdl->lock); - ret = __v4l2_ctrl_handler_setup(hdl); - mutex_unlock(hdl->lock); - - return ret; -} -EXPORT_SYMBOL(v4l2_ctrl_handler_setup); - -/* Implement VIDIOC_QUERY_EXT_CTRL */ -int v4l2_query_ext_ctrl(struct v4l2_ctrl_handler *hdl, struct v4l2_query_ext_ctrl *qc) -{ - const unsigned next_flags = V4L2_CTRL_FLAG_NEXT_CTRL | V4L2_CTRL_FLAG_NEXT_COMPOUND; - u32 id = qc->id & V4L2_CTRL_ID_MASK; - struct v4l2_ctrl_ref *ref; - struct v4l2_ctrl *ctrl; - - if (hdl == NULL) - return -EINVAL; - - mutex_lock(hdl->lock); - - /* Try to find it */ - ref = find_ref(hdl, id); - - if ((qc->id & next_flags) && !list_empty(&hdl->ctrl_refs)) { - bool is_compound; - /* Match any control that is not hidden */ - unsigned mask = 1; - bool match = false; - - if ((qc->id & next_flags) == V4L2_CTRL_FLAG_NEXT_COMPOUND) { - /* Match any hidden control */ - match = true; - } else if ((qc->id & next_flags) == next_flags) { - /* Match any control, compound or not */ - mask = 0; - } - - /* Find the next control with ID > qc->id */ - - /* Did we reach the end of the control list? */ - if (id >= node2id(hdl->ctrl_refs.prev)) { - ref = NULL; /* Yes, so there is no next control */ - } else if (ref) { - /* We found a control with the given ID, so just get - the next valid one in the list. */ - list_for_each_entry_continue(ref, &hdl->ctrl_refs, node) { - is_compound = ref->ctrl->is_array || - ref->ctrl->type >= V4L2_CTRL_COMPOUND_TYPES; - if (id < ref->ctrl->id && - (is_compound & mask) == match) - break; - } - if (&ref->node == &hdl->ctrl_refs) - ref = NULL; - } else { - /* No control with the given ID exists, so start - searching for the next largest ID. We know there - is one, otherwise the first 'if' above would have - been true. */ - list_for_each_entry(ref, &hdl->ctrl_refs, node) { - is_compound = ref->ctrl->is_array || - ref->ctrl->type >= V4L2_CTRL_COMPOUND_TYPES; - if (id < ref->ctrl->id && - (is_compound & mask) == match) - break; - } - if (&ref->node == &hdl->ctrl_refs) - ref = NULL; - } - } - mutex_unlock(hdl->lock); - - if (!ref) - return -EINVAL; - - ctrl = ref->ctrl; - memset(qc, 0, sizeof(*qc)); - if (id >= V4L2_CID_PRIVATE_BASE) - qc->id = id; - else - qc->id = ctrl->id; - strscpy(qc->name, ctrl->name, sizeof(qc->name)); - qc->flags = user_flags(ctrl); - qc->type = ctrl->type; - qc->elem_size = ctrl->elem_size; - qc->elems = ctrl->elems; - qc->nr_of_dims = ctrl->nr_of_dims; - memcpy(qc->dims, ctrl->dims, qc->nr_of_dims * sizeof(qc->dims[0])); - qc->minimum = ctrl->minimum; - qc->maximum = ctrl->maximum; - qc->default_value = ctrl->default_value; - if (ctrl->type == V4L2_CTRL_TYPE_MENU - || ctrl->type == V4L2_CTRL_TYPE_INTEGER_MENU) - qc->step = 1; - else - qc->step = ctrl->step; - return 0; -} -EXPORT_SYMBOL(v4l2_query_ext_ctrl); - -/* Implement VIDIOC_QUERYCTRL */ -int v4l2_queryctrl(struct v4l2_ctrl_handler *hdl, struct v4l2_queryctrl *qc) -{ - struct v4l2_query_ext_ctrl qec = { qc->id }; - int rc; - - rc = v4l2_query_ext_ctrl(hdl, &qec); - if (rc) - return rc; - - qc->id = qec.id; - qc->type = qec.type; - qc->flags = qec.flags; - strscpy(qc->name, qec.name, sizeof(qc->name)); - switch (qc->type) { - case V4L2_CTRL_TYPE_INTEGER: - case V4L2_CTRL_TYPE_BOOLEAN: - case V4L2_CTRL_TYPE_MENU: - case V4L2_CTRL_TYPE_INTEGER_MENU: - case V4L2_CTRL_TYPE_STRING: - case V4L2_CTRL_TYPE_BITMASK: - qc->minimum = qec.minimum; - qc->maximum = qec.maximum; - qc->step = qec.step; - qc->default_value = qec.default_value; - break; - default: - qc->minimum = 0; - qc->maximum = 0; - qc->step = 0; - qc->default_value = 0; - break; - } - return 0; -} -EXPORT_SYMBOL(v4l2_queryctrl); - -/* Implement VIDIOC_QUERYMENU */ -int v4l2_querymenu(struct v4l2_ctrl_handler *hdl, struct v4l2_querymenu *qm) -{ - struct v4l2_ctrl *ctrl; - u32 i = qm->index; - - ctrl = v4l2_ctrl_find(hdl, qm->id); - if (!ctrl) - return -EINVAL; - - qm->reserved = 0; - /* Sanity checks */ - switch (ctrl->type) { - case V4L2_CTRL_TYPE_MENU: - if (ctrl->qmenu == NULL) - return -EINVAL; - break; - case V4L2_CTRL_TYPE_INTEGER_MENU: - if (ctrl->qmenu_int == NULL) - return -EINVAL; - break; - default: - return -EINVAL; - } - - if (i < ctrl->minimum || i > ctrl->maximum) - return -EINVAL; - - /* Use mask to see if this menu item should be skipped */ - if (ctrl->menu_skip_mask & (1ULL << i)) - return -EINVAL; - /* Empty menu items should also be skipped */ - if (ctrl->type == V4L2_CTRL_TYPE_MENU) { - if (ctrl->qmenu[i] == NULL || ctrl->qmenu[i][0] == '\0') - return -EINVAL; - strscpy(qm->name, ctrl->qmenu[i], sizeof(qm->name)); - } else { - qm->value = ctrl->qmenu_int[i]; - } - return 0; -} -EXPORT_SYMBOL(v4l2_querymenu); - -static int v4l2_ctrl_request_clone(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_handler *from) -{ - struct v4l2_ctrl_ref *ref; - int err = 0; - - if (WARN_ON(!hdl || hdl == from)) - return -EINVAL; - - if (hdl->error) - return hdl->error; - - WARN_ON(hdl->lock != &hdl->_lock); - - mutex_lock(from->lock); - list_for_each_entry(ref, &from->ctrl_refs, node) { - struct v4l2_ctrl *ctrl = ref->ctrl; - struct v4l2_ctrl_ref *new_ref; - - /* Skip refs inherited from other devices */ - if (ref->from_other_dev) - continue; - err = handler_new_ref(hdl, ctrl, &new_ref, false, true); - if (err) - break; - } - mutex_unlock(from->lock); - return err; -} - -static void v4l2_ctrl_request_queue(struct media_request_object *obj) -{ - struct v4l2_ctrl_handler *hdl = - container_of(obj, struct v4l2_ctrl_handler, req_obj); - struct v4l2_ctrl_handler *main_hdl = obj->priv; - - mutex_lock(main_hdl->lock); - list_add_tail(&hdl->requests_queued, &main_hdl->requests_queued); - hdl->request_is_queued = true; - mutex_unlock(main_hdl->lock); -} - -static void v4l2_ctrl_request_unbind(struct media_request_object *obj) -{ - struct v4l2_ctrl_handler *hdl = - container_of(obj, struct v4l2_ctrl_handler, req_obj); - struct v4l2_ctrl_handler *main_hdl = obj->priv; - - mutex_lock(main_hdl->lock); - list_del_init(&hdl->requests); - if (hdl->request_is_queued) { - list_del_init(&hdl->requests_queued); - hdl->request_is_queued = false; - } - mutex_unlock(main_hdl->lock); -} - -static void v4l2_ctrl_request_release(struct media_request_object *obj) -{ - struct v4l2_ctrl_handler *hdl = - container_of(obj, struct v4l2_ctrl_handler, req_obj); - - v4l2_ctrl_handler_free(hdl); - kfree(hdl); -} - -static const struct media_request_object_ops req_ops = { - .queue = v4l2_ctrl_request_queue, - .unbind = v4l2_ctrl_request_unbind, - .release = v4l2_ctrl_request_release, -}; - -struct v4l2_ctrl_handler *v4l2_ctrl_request_hdl_find(struct media_request *req, - struct v4l2_ctrl_handler *parent) -{ - struct media_request_object *obj; - - if (WARN_ON(req->state != MEDIA_REQUEST_STATE_VALIDATING && - req->state != MEDIA_REQUEST_STATE_QUEUED)) - return NULL; - - obj = media_request_object_find(req, &req_ops, parent); - if (obj) - return container_of(obj, struct v4l2_ctrl_handler, req_obj); - return NULL; -} -EXPORT_SYMBOL_GPL(v4l2_ctrl_request_hdl_find); - -struct v4l2_ctrl * -v4l2_ctrl_request_hdl_ctrl_find(struct v4l2_ctrl_handler *hdl, u32 id) -{ - struct v4l2_ctrl_ref *ref = find_ref_lock(hdl, id); - - return (ref && ref->valid_p_req) ? ref->ctrl : NULL; -} -EXPORT_SYMBOL_GPL(v4l2_ctrl_request_hdl_ctrl_find); - -static int v4l2_ctrl_request_bind(struct media_request *req, - struct v4l2_ctrl_handler *hdl, - struct v4l2_ctrl_handler *from) -{ - int ret; - - ret = v4l2_ctrl_request_clone(hdl, from); - - if (!ret) { - ret = media_request_object_bind(req, &req_ops, - from, false, &hdl->req_obj); - if (!ret) { - mutex_lock(from->lock); - list_add_tail(&hdl->requests, &from->requests); - mutex_unlock(from->lock); - } - } - return ret; -} - -/* Some general notes on the atomic requirements of VIDIOC_G/TRY/S_EXT_CTRLS: - - It is not a fully atomic operation, just best-effort only. After all, if - multiple controls have to be set through multiple i2c writes (for example) - then some initial writes may succeed while others fail. Thus leaving the - system in an inconsistent state. The question is how much effort you are - willing to spend on trying to make something atomic that really isn't. - - From the point of view of an application the main requirement is that - when you call VIDIOC_S_EXT_CTRLS and some values are invalid then an - error should be returned without actually affecting any controls. - - If all the values are correct, then it is acceptable to just give up - in case of low-level errors. - - It is important though that the application can tell when only a partial - configuration was done. The way we do that is through the error_idx field - of struct v4l2_ext_controls: if that is equal to the count field then no - controls were affected. Otherwise all controls before that index were - successful in performing their 'get' or 'set' operation, the control at - the given index failed, and you don't know what happened with the controls - after the failed one. Since if they were part of a control cluster they - could have been successfully processed (if a cluster member was encountered - at index < error_idx), they could have failed (if a cluster member was at - error_idx), or they may not have been processed yet (if the first cluster - member appeared after error_idx). - - It is all fairly theoretical, though. In practice all you can do is to - bail out. If error_idx == count, then it is an application bug. If - error_idx < count then it is only an application bug if the error code was - EBUSY. That usually means that something started streaming just when you - tried to set the controls. In all other cases it is a driver/hardware - problem and all you can do is to retry or bail out. - - Note that these rules do not apply to VIDIOC_TRY_EXT_CTRLS: since that - never modifies controls the error_idx is just set to whatever control - has an invalid value. - */ - -/* Prepare for the extended g/s/try functions. - Find the controls in the control array and do some basic checks. */ -static int prepare_ext_ctrls(struct v4l2_ctrl_handler *hdl, - struct v4l2_ext_controls *cs, - struct v4l2_ctrl_helper *helpers, - struct video_device *vdev, - bool get) -{ - struct v4l2_ctrl_helper *h; - bool have_clusters = false; - u32 i; - - for (i = 0, h = helpers; i < cs->count; i++, h++) { - struct v4l2_ext_control *c = &cs->controls[i]; - struct v4l2_ctrl_ref *ref; - struct v4l2_ctrl *ctrl; - u32 id = c->id & V4L2_CTRL_ID_MASK; - - cs->error_idx = i; - - if (cs->which && - cs->which != V4L2_CTRL_WHICH_DEF_VAL && - cs->which != V4L2_CTRL_WHICH_REQUEST_VAL && - V4L2_CTRL_ID2WHICH(id) != cs->which) { - dprintk(vdev, - "invalid which 0x%x or control id 0x%x\n", - cs->which, id); - return -EINVAL; - } - - /* Old-style private controls are not allowed for - extended controls */ - if (id >= V4L2_CID_PRIVATE_BASE) { - dprintk(vdev, - "old-style private controls not allowed\n"); - return -EINVAL; - } - ref = find_ref_lock(hdl, id); - if (ref == NULL) { - dprintk(vdev, "cannot find control id 0x%x\n", id); - return -EINVAL; - } - h->ref = ref; - ctrl = ref->ctrl; - if (ctrl->flags & V4L2_CTRL_FLAG_DISABLED) { - dprintk(vdev, "control id 0x%x is disabled\n", id); - return -EINVAL; - } - - if (ctrl->cluster[0]->ncontrols > 1) - have_clusters = true; - if (ctrl->cluster[0] != ctrl) - ref = find_ref_lock(hdl, ctrl->cluster[0]->id); - if (ctrl->is_ptr && !ctrl->is_string) { - unsigned tot_size = ctrl->elems * ctrl->elem_size; - - if (c->size < tot_size) { - /* - * In the get case the application first - * queries to obtain the size of the control. - */ - if (get) { - c->size = tot_size; - return -ENOSPC; - } - dprintk(vdev, - "pointer control id 0x%x size too small, %d bytes but %d bytes needed\n", - id, c->size, tot_size); - return -EFAULT; - } - c->size = tot_size; - } - /* Store the ref to the master control of the cluster */ - h->mref = ref; - /* Initially set next to 0, meaning that there is no other - control in this helper array belonging to the same - cluster */ - h->next = 0; - } - - /* We are done if there were no controls that belong to a multi- - control cluster. */ - if (!have_clusters) - return 0; - - /* The code below figures out in O(n) time which controls in the list - belong to the same cluster. */ - - /* This has to be done with the handler lock taken. */ - mutex_lock(hdl->lock); - - /* First zero the helper field in the master control references */ - for (i = 0; i < cs->count; i++) - helpers[i].mref->helper = NULL; - for (i = 0, h = helpers; i < cs->count; i++, h++) { - struct v4l2_ctrl_ref *mref = h->mref; - - /* If the mref->helper is set, then it points to an earlier - helper that belongs to the same cluster. */ - if (mref->helper) { - /* Set the next field of mref->helper to the current - index: this means that that earlier helper now - points to the next helper in the same cluster. */ - mref->helper->next = i; - /* mref should be set only for the first helper in the - cluster, clear the others. */ - h->mref = NULL; - } - /* Point the mref helper to the current helper struct. */ - mref->helper = h; - } - mutex_unlock(hdl->lock); - return 0; -} - -/* Handles the corner case where cs->count == 0. It checks whether the - specified control class exists. If that class ID is 0, then it checks - whether there are any controls at all. */ -static int class_check(struct v4l2_ctrl_handler *hdl, u32 which) -{ - if (which == 0 || which == V4L2_CTRL_WHICH_DEF_VAL || - which == V4L2_CTRL_WHICH_REQUEST_VAL) - return 0; - return find_ref_lock(hdl, which | 1) ? 0 : -EINVAL; -} - -/* - * Get extended controls. Allocates the helpers array if needed. - * - * Note that v4l2_g_ext_ctrls_common() with 'which' set to - * V4L2_CTRL_WHICH_REQUEST_VAL is only called if the request was - * completed, and in that case valid_p_req is true for all controls. - */ -static int v4l2_g_ext_ctrls_common(struct v4l2_ctrl_handler *hdl, - struct v4l2_ext_controls *cs, - struct video_device *vdev) -{ - struct v4l2_ctrl_helper helper[4]; - struct v4l2_ctrl_helper *helpers = helper; - int ret; - int i, j; - bool is_default, is_request; - - is_default = (cs->which == V4L2_CTRL_WHICH_DEF_VAL); - is_request = (cs->which == V4L2_CTRL_WHICH_REQUEST_VAL); - - cs->error_idx = cs->count; - cs->which = V4L2_CTRL_ID2WHICH(cs->which); - - if (hdl == NULL) - return -EINVAL; - - if (cs->count == 0) - return class_check(hdl, cs->which); - - if (cs->count > ARRAY_SIZE(helper)) { - helpers = kvmalloc_array(cs->count, sizeof(helper[0]), - GFP_KERNEL); - if (helpers == NULL) - return -ENOMEM; - } - - ret = prepare_ext_ctrls(hdl, cs, helpers, vdev, true); - cs->error_idx = cs->count; - - for (i = 0; !ret && i < cs->count; i++) - if (helpers[i].ref->ctrl->flags & V4L2_CTRL_FLAG_WRITE_ONLY) - ret = -EACCES; - - for (i = 0; !ret && i < cs->count; i++) { - struct v4l2_ctrl *master; - bool is_volatile = false; - u32 idx = i; - - if (helpers[i].mref == NULL) - continue; - - master = helpers[i].mref->ctrl; - cs->error_idx = i; - - v4l2_ctrl_lock(master); - - /* - * g_volatile_ctrl will update the new control values. - * This makes no sense for V4L2_CTRL_WHICH_DEF_VAL and - * V4L2_CTRL_WHICH_REQUEST_VAL. In the case of requests - * it is v4l2_ctrl_request_complete() that copies the - * volatile controls at the time of request completion - * to the request, so you don't want to do that again. - */ - if (!is_default && !is_request && - ((master->flags & V4L2_CTRL_FLAG_VOLATILE) || - (master->has_volatiles && !is_cur_manual(master)))) { - for (j = 0; j < master->ncontrols; j++) - cur_to_new(master->cluster[j]); - ret = call_op(master, g_volatile_ctrl); - is_volatile = true; - } - - if (ret) { - v4l2_ctrl_unlock(master); - break; - } - - /* - * Copy the default value (if is_default is true), the - * request value (if is_request is true and p_req is valid), - * the new volatile value (if is_volatile is true) or the - * current value. - */ - do { - struct v4l2_ctrl_ref *ref = helpers[idx].ref; - - if (is_default) - ret = def_to_user(cs->controls + idx, ref->ctrl); - else if (is_request && ref->valid_p_req) - ret = req_to_user(cs->controls + idx, ref); - else if (is_volatile) - ret = new_to_user(cs->controls + idx, ref->ctrl); - else - ret = cur_to_user(cs->controls + idx, ref->ctrl); - idx = helpers[idx].next; - } while (!ret && idx); - - v4l2_ctrl_unlock(master); - } - - if (cs->count > ARRAY_SIZE(helper)) - kvfree(helpers); - return ret; -} - -static struct media_request_object * -v4l2_ctrls_find_req_obj(struct v4l2_ctrl_handler *hdl, - struct media_request *req, bool set) -{ - struct media_request_object *obj; - struct v4l2_ctrl_handler *new_hdl; - int ret; - - if (IS_ERR(req)) - return ERR_CAST(req); - - if (set && WARN_ON(req->state != MEDIA_REQUEST_STATE_UPDATING)) - return ERR_PTR(-EBUSY); - - obj = media_request_object_find(req, &req_ops, hdl); - if (obj) - return obj; - if (!set) - return ERR_PTR(-ENOENT); - - new_hdl = kzalloc(sizeof(*new_hdl), GFP_KERNEL); - if (!new_hdl) - return ERR_PTR(-ENOMEM); - - obj = &new_hdl->req_obj; - ret = v4l2_ctrl_handler_init(new_hdl, (hdl->nr_of_buckets - 1) * 8); - if (!ret) - ret = v4l2_ctrl_request_bind(req, new_hdl, hdl); - if (ret) { - kfree(new_hdl); - - return ERR_PTR(ret); - } - - media_request_object_get(obj); - return obj; -} - -int v4l2_g_ext_ctrls(struct v4l2_ctrl_handler *hdl, struct video_device *vdev, - struct media_device *mdev, struct v4l2_ext_controls *cs) -{ - struct media_request_object *obj = NULL; - struct media_request *req = NULL; - int ret; - - if (cs->which == V4L2_CTRL_WHICH_REQUEST_VAL) { - if (!mdev || cs->request_fd < 0) - return -EINVAL; - - req = media_request_get_by_fd(mdev, cs->request_fd); - if (IS_ERR(req)) - return PTR_ERR(req); - - if (req->state != MEDIA_REQUEST_STATE_COMPLETE) { - media_request_put(req); - return -EACCES; - } - - ret = media_request_lock_for_access(req); - if (ret) { - media_request_put(req); - return ret; - } - - obj = v4l2_ctrls_find_req_obj(hdl, req, false); - if (IS_ERR(obj)) { - media_request_unlock_for_access(req); - media_request_put(req); - return PTR_ERR(obj); - } - - hdl = container_of(obj, struct v4l2_ctrl_handler, - req_obj); - } - - ret = v4l2_g_ext_ctrls_common(hdl, cs, vdev); - - if (obj) { - media_request_unlock_for_access(req); - media_request_object_put(obj); - media_request_put(req); - } - return ret; -} -EXPORT_SYMBOL(v4l2_g_ext_ctrls); - -/* Helper function to get a single control */ -static int get_ctrl(struct v4l2_ctrl *ctrl, struct v4l2_ext_control *c) -{ - struct v4l2_ctrl *master = ctrl->cluster[0]; - int ret = 0; - int i; - - /* Compound controls are not supported. The new_to_user() and - * cur_to_user() calls below would need to be modified not to access - * userspace memory when called from get_ctrl(). - */ - if (!ctrl->is_int && ctrl->type != V4L2_CTRL_TYPE_INTEGER64) - return -EINVAL; - - if (ctrl->flags & V4L2_CTRL_FLAG_WRITE_ONLY) - return -EACCES; - - v4l2_ctrl_lock(master); - /* g_volatile_ctrl will update the current control values */ - if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) { - for (i = 0; i < master->ncontrols; i++) - cur_to_new(master->cluster[i]); - ret = call_op(master, g_volatile_ctrl); - new_to_user(c, ctrl); - } else { - cur_to_user(c, ctrl); - } - v4l2_ctrl_unlock(master); - return ret; -} - -int v4l2_g_ctrl(struct v4l2_ctrl_handler *hdl, struct v4l2_control *control) -{ - struct v4l2_ctrl *ctrl = v4l2_ctrl_find(hdl, control->id); - struct v4l2_ext_control c; - int ret; - - if (ctrl == NULL || !ctrl->is_int) - return -EINVAL; - ret = get_ctrl(ctrl, &c); - control->value = c.value; - return ret; -} -EXPORT_SYMBOL(v4l2_g_ctrl); - -s32 v4l2_ctrl_g_ctrl(struct v4l2_ctrl *ctrl) -{ - struct v4l2_ext_control c; - - /* It's a driver bug if this happens. */ - if (WARN_ON(!ctrl->is_int)) - return 0; - c.value = 0; - get_ctrl(ctrl, &c); - return c.value; -} -EXPORT_SYMBOL(v4l2_ctrl_g_ctrl); - -s64 v4l2_ctrl_g_ctrl_int64(struct v4l2_ctrl *ctrl) -{ - struct v4l2_ext_control c; - - /* It's a driver bug if this happens. */ - if (WARN_ON(ctrl->is_ptr || ctrl->type != V4L2_CTRL_TYPE_INTEGER64)) - return 0; - c.value64 = 0; - get_ctrl(ctrl, &c); - return c.value64; -} -EXPORT_SYMBOL(v4l2_ctrl_g_ctrl_int64); - - -/* Core function that calls try/s_ctrl and ensures that the new value is - copied to the current value on a set. - Must be called with ctrl->handler->lock held. */ -static int try_or_set_cluster(struct v4l2_fh *fh, struct v4l2_ctrl *master, - bool set, u32 ch_flags) -{ - bool update_flag; - int ret; - int i; - - /* Go through the cluster and either validate the new value or - (if no new value was set), copy the current value to the new - value, ensuring a consistent view for the control ops when - called. */ - for (i = 0; i < master->ncontrols; i++) { - struct v4l2_ctrl *ctrl = master->cluster[i]; - - if (ctrl == NULL) - continue; - - if (!ctrl->is_new) { - cur_to_new(ctrl); - continue; - } - /* Check again: it may have changed since the - previous check in try_or_set_ext_ctrls(). */ - if (set && (ctrl->flags & V4L2_CTRL_FLAG_GRABBED)) - return -EBUSY; - } - - ret = call_op(master, try_ctrl); - - /* Don't set if there is no change */ - if (ret || !set || !cluster_changed(master)) - return ret; - ret = call_op(master, s_ctrl); - if (ret) - return ret; - - /* If OK, then make the new values permanent. */ - update_flag = is_cur_manual(master) != is_new_manual(master); - - for (i = 0; i < master->ncontrols; i++) { - /* - * If we switch from auto to manual mode, and this cluster - * contains volatile controls, then all non-master controls - * have to be marked as changed. The 'new' value contains - * the volatile value (obtained by update_from_auto_cluster), - * which now has to become the current value. - */ - if (i && update_flag && is_new_manual(master) && - master->has_volatiles && master->cluster[i]) - master->cluster[i]->has_changed = true; - - new_to_cur(fh, master->cluster[i], ch_flags | - ((update_flag && i > 0) ? V4L2_EVENT_CTRL_CH_FLAGS : 0)); - } - return 0; -} - -/* Validate controls. */ -static int validate_ctrls(struct v4l2_ext_controls *cs, - struct v4l2_ctrl_helper *helpers, - struct video_device *vdev, - bool set) -{ - unsigned i; - int ret = 0; - - cs->error_idx = cs->count; - for (i = 0; i < cs->count; i++) { - struct v4l2_ctrl *ctrl = helpers[i].ref->ctrl; - union v4l2_ctrl_ptr p_new; - - cs->error_idx = i; - - if (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY) { - dprintk(vdev, - "control id 0x%x is read-only\n", - ctrl->id); - return -EACCES; - } - /* This test is also done in try_set_control_cluster() which - is called in atomic context, so that has the final say, - but it makes sense to do an up-front check as well. Once - an error occurs in try_set_control_cluster() some other - controls may have been set already and we want to do a - best-effort to avoid that. */ - if (set && (ctrl->flags & V4L2_CTRL_FLAG_GRABBED)) { - dprintk(vdev, - "control id 0x%x is grabbed, cannot set\n", - ctrl->id); - return -EBUSY; - } - /* - * Skip validation for now if the payload needs to be copied - * from userspace into kernelspace. We'll validate those later. - */ - if (ctrl->is_ptr) - continue; - if (ctrl->type == V4L2_CTRL_TYPE_INTEGER64) - p_new.p_s64 = &cs->controls[i].value64; - else - p_new.p_s32 = &cs->controls[i].value; - ret = validate_new(ctrl, p_new); - if (ret) - return ret; - } - return 0; -} - -/* Obtain the current volatile values of an autocluster and mark them - as new. */ -static void update_from_auto_cluster(struct v4l2_ctrl *master) -{ - int i; - - for (i = 1; i < master->ncontrols; i++) - cur_to_new(master->cluster[i]); - if (!call_op(master, g_volatile_ctrl)) - for (i = 1; i < master->ncontrols; i++) - if (master->cluster[i]) - master->cluster[i]->is_new = 1; -} - -/* Try or try-and-set controls */ -static int try_set_ext_ctrls_common(struct v4l2_fh *fh, - struct v4l2_ctrl_handler *hdl, - struct v4l2_ext_controls *cs, - struct video_device *vdev, bool set) -{ - struct v4l2_ctrl_helper helper[4]; - struct v4l2_ctrl_helper *helpers = helper; - unsigned i, j; - int ret; - - cs->error_idx = cs->count; - - /* Default value cannot be changed */ - if (cs->which == V4L2_CTRL_WHICH_DEF_VAL) { - dprintk(vdev, "%s: cannot change default value\n", - video_device_node_name(vdev)); - return -EINVAL; - } - - cs->which = V4L2_CTRL_ID2WHICH(cs->which); - - if (hdl == NULL) { - dprintk(vdev, "%s: invalid null control handler\n", - video_device_node_name(vdev)); - return -EINVAL; - } - - if (cs->count == 0) - return class_check(hdl, cs->which); - - if (cs->count > ARRAY_SIZE(helper)) { - helpers = kvmalloc_array(cs->count, sizeof(helper[0]), - GFP_KERNEL); - if (!helpers) - return -ENOMEM; - } - ret = prepare_ext_ctrls(hdl, cs, helpers, vdev, false); - if (!ret) - ret = validate_ctrls(cs, helpers, vdev, set); - if (ret && set) - cs->error_idx = cs->count; - for (i = 0; !ret && i < cs->count; i++) { - struct v4l2_ctrl *master; - u32 idx = i; - - if (helpers[i].mref == NULL) - continue; - - cs->error_idx = i; - master = helpers[i].mref->ctrl; - v4l2_ctrl_lock(master); - - /* Reset the 'is_new' flags of the cluster */ - for (j = 0; j < master->ncontrols; j++) - if (master->cluster[j]) - master->cluster[j]->is_new = 0; - - /* For volatile autoclusters that are currently in auto mode - we need to discover if it will be set to manual mode. - If so, then we have to copy the current volatile values - first since those will become the new manual values (which - may be overwritten by explicit new values from this set - of controls). */ - if (master->is_auto && master->has_volatiles && - !is_cur_manual(master)) { - /* Pick an initial non-manual value */ - s32 new_auto_val = master->manual_mode_value + 1; - u32 tmp_idx = idx; - - do { - /* Check if the auto control is part of the - list, and remember the new value. */ - if (helpers[tmp_idx].ref->ctrl == master) - new_auto_val = cs->controls[tmp_idx].value; - tmp_idx = helpers[tmp_idx].next; - } while (tmp_idx); - /* If the new value == the manual value, then copy - the current volatile values. */ - if (new_auto_val == master->manual_mode_value) - update_from_auto_cluster(master); - } - - /* Copy the new caller-supplied control values. - user_to_new() sets 'is_new' to 1. */ - do { - struct v4l2_ctrl *ctrl = helpers[idx].ref->ctrl; - - ret = user_to_new(cs->controls + idx, ctrl); - if (!ret && ctrl->is_ptr) { - ret = validate_new(ctrl, ctrl->p_new); - if (ret) - dprintk(vdev, - "failed to validate control %s (%d)\n", - v4l2_ctrl_get_name(ctrl->id), ret); - } - idx = helpers[idx].next; - } while (!ret && idx); - - if (!ret) - ret = try_or_set_cluster(fh, master, - !hdl->req_obj.req && set, 0); - if (!ret && hdl->req_obj.req && set) { - for (j = 0; j < master->ncontrols; j++) { - struct v4l2_ctrl_ref *ref = - find_ref(hdl, master->cluster[j]->id); - - new_to_req(ref); - } - } - - /* Copy the new values back to userspace. */ - if (!ret) { - idx = i; - do { - ret = new_to_user(cs->controls + idx, - helpers[idx].ref->ctrl); - idx = helpers[idx].next; - } while (!ret && idx); - } - v4l2_ctrl_unlock(master); - } - - if (cs->count > ARRAY_SIZE(helper)) - kvfree(helpers); - return ret; -} - -static int try_set_ext_ctrls(struct v4l2_fh *fh, - struct v4l2_ctrl_handler *hdl, - struct video_device *vdev, - struct media_device *mdev, - struct v4l2_ext_controls *cs, bool set) -{ - struct media_request_object *obj = NULL; - struct media_request *req = NULL; - int ret; - - if (cs->which == V4L2_CTRL_WHICH_REQUEST_VAL) { - if (!mdev) { - dprintk(vdev, "%s: missing media device\n", - video_device_node_name(vdev)); - return -EINVAL; - } - - if (cs->request_fd < 0) { - dprintk(vdev, "%s: invalid request fd %d\n", - video_device_node_name(vdev), cs->request_fd); - return -EINVAL; - } - - req = media_request_get_by_fd(mdev, cs->request_fd); - if (IS_ERR(req)) { - dprintk(vdev, "%s: cannot find request fd %d\n", - video_device_node_name(vdev), cs->request_fd); - return PTR_ERR(req); - } - - ret = media_request_lock_for_update(req); - if (ret) { - dprintk(vdev, "%s: cannot lock request fd %d\n", - video_device_node_name(vdev), cs->request_fd); - media_request_put(req); - return ret; - } - - obj = v4l2_ctrls_find_req_obj(hdl, req, set); - if (IS_ERR(obj)) { - dprintk(vdev, - "%s: cannot find request object for request fd %d\n", - video_device_node_name(vdev), - cs->request_fd); - media_request_unlock_for_update(req); - media_request_put(req); - return PTR_ERR(obj); - } - hdl = container_of(obj, struct v4l2_ctrl_handler, - req_obj); - } - - ret = try_set_ext_ctrls_common(fh, hdl, cs, vdev, set); - if (ret) - dprintk(vdev, - "%s: try_set_ext_ctrls_common failed (%d)\n", - video_device_node_name(vdev), ret); - - if (obj) { - media_request_unlock_for_update(req); - media_request_object_put(obj); - media_request_put(req); - } - - return ret; -} - -int v4l2_try_ext_ctrls(struct v4l2_ctrl_handler *hdl, - struct video_device *vdev, - struct media_device *mdev, - struct v4l2_ext_controls *cs) -{ - return try_set_ext_ctrls(NULL, hdl, vdev, mdev, cs, false); -} -EXPORT_SYMBOL(v4l2_try_ext_ctrls); - -int v4l2_s_ext_ctrls(struct v4l2_fh *fh, - struct v4l2_ctrl_handler *hdl, - struct video_device *vdev, - struct media_device *mdev, - struct v4l2_ext_controls *cs) -{ - return try_set_ext_ctrls(fh, hdl, vdev, mdev, cs, true); -} -EXPORT_SYMBOL(v4l2_s_ext_ctrls); - -/* Helper function for VIDIOC_S_CTRL compatibility */ -static int set_ctrl(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, u32 ch_flags) -{ - struct v4l2_ctrl *master = ctrl->cluster[0]; - int ret; - int i; - - /* Reset the 'is_new' flags of the cluster */ - for (i = 0; i < master->ncontrols; i++) - if (master->cluster[i]) - master->cluster[i]->is_new = 0; - - ret = validate_new(ctrl, ctrl->p_new); - if (ret) - return ret; - - /* For autoclusters with volatiles that are switched from auto to - manual mode we have to update the current volatile values since - those will become the initial manual values after such a switch. */ - if (master->is_auto && master->has_volatiles && ctrl == master && - !is_cur_manual(master) && ctrl->val == master->manual_mode_value) - update_from_auto_cluster(master); - - ctrl->is_new = 1; - return try_or_set_cluster(fh, master, true, ch_flags); -} - -/* Helper function for VIDIOC_S_CTRL compatibility */ -static int set_ctrl_lock(struct v4l2_fh *fh, struct v4l2_ctrl *ctrl, - struct v4l2_ext_control *c) -{ - int ret; - - v4l2_ctrl_lock(ctrl); - user_to_new(c, ctrl); - ret = set_ctrl(fh, ctrl, 0); - if (!ret) - cur_to_user(c, ctrl); - v4l2_ctrl_unlock(ctrl); - return ret; -} - -int v4l2_s_ctrl(struct v4l2_fh *fh, struct v4l2_ctrl_handler *hdl, - struct v4l2_control *control) -{ - struct v4l2_ctrl *ctrl = v4l2_ctrl_find(hdl, control->id); - struct v4l2_ext_control c = { control->id }; - int ret; - - if (ctrl == NULL || !ctrl->is_int) - return -EINVAL; - - if (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY) - return -EACCES; - - c.value = control->value; - ret = set_ctrl_lock(fh, ctrl, &c); - control->value = c.value; - return ret; -} -EXPORT_SYMBOL(v4l2_s_ctrl); - -int __v4l2_ctrl_s_ctrl(struct v4l2_ctrl *ctrl, s32 val) -{ - lockdep_assert_held(ctrl->handler->lock); - - /* It's a driver bug if this happens. */ - if (WARN_ON(!ctrl->is_int)) - return -EINVAL; - ctrl->val = val; - return set_ctrl(NULL, ctrl, 0); -} -EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl); - -int __v4l2_ctrl_s_ctrl_int64(struct v4l2_ctrl *ctrl, s64 val) -{ - lockdep_assert_held(ctrl->handler->lock); - - /* It's a driver bug if this happens. */ - if (WARN_ON(ctrl->is_ptr || ctrl->type != V4L2_CTRL_TYPE_INTEGER64)) - return -EINVAL; - *ctrl->p_new.p_s64 = val; - return set_ctrl(NULL, ctrl, 0); -} -EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_int64); - -int __v4l2_ctrl_s_ctrl_string(struct v4l2_ctrl *ctrl, const char *s) -{ - lockdep_assert_held(ctrl->handler->lock); - - /* It's a driver bug if this happens. */ - if (WARN_ON(ctrl->type != V4L2_CTRL_TYPE_STRING)) - return -EINVAL; - strscpy(ctrl->p_new.p_char, s, ctrl->maximum + 1); - return set_ctrl(NULL, ctrl, 0); -} -EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_string); - -int __v4l2_ctrl_s_ctrl_compound(struct v4l2_ctrl *ctrl, - enum v4l2_ctrl_type type, const void *p) -{ - lockdep_assert_held(ctrl->handler->lock); - - /* It's a driver bug if this happens. */ - if (WARN_ON(ctrl->type != type)) - return -EINVAL; - memcpy(ctrl->p_new.p, p, ctrl->elems * ctrl->elem_size); - return set_ctrl(NULL, ctrl, 0); -} -EXPORT_SYMBOL(__v4l2_ctrl_s_ctrl_compound); - -void v4l2_ctrl_request_complete(struct media_request *req, - struct v4l2_ctrl_handler *main_hdl) -{ - struct media_request_object *obj; - struct v4l2_ctrl_handler *hdl; - struct v4l2_ctrl_ref *ref; - - if (!req || !main_hdl) - return; - - /* - * Note that it is valid if nothing was found. It means - * that this request doesn't have any controls and so just - * wants to leave the controls unchanged. - */ - obj = media_request_object_find(req, &req_ops, main_hdl); - if (!obj) - return; - hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj); - - list_for_each_entry(ref, &hdl->ctrl_refs, node) { - struct v4l2_ctrl *ctrl = ref->ctrl; - struct v4l2_ctrl *master = ctrl->cluster[0]; - unsigned int i; - - if (ctrl->flags & V4L2_CTRL_FLAG_VOLATILE) { - v4l2_ctrl_lock(master); - /* g_volatile_ctrl will update the current control values */ - for (i = 0; i < master->ncontrols; i++) - cur_to_new(master->cluster[i]); - call_op(master, g_volatile_ctrl); - new_to_req(ref); - v4l2_ctrl_unlock(master); - continue; - } - if (ref->valid_p_req) - continue; - - /* Copy the current control value into the request */ - v4l2_ctrl_lock(ctrl); - cur_to_req(ref); - v4l2_ctrl_unlock(ctrl); - } - - mutex_lock(main_hdl->lock); - WARN_ON(!hdl->request_is_queued); - list_del_init(&hdl->requests_queued); - hdl->request_is_queued = false; - mutex_unlock(main_hdl->lock); - media_request_object_complete(obj); - media_request_object_put(obj); -} -EXPORT_SYMBOL(v4l2_ctrl_request_complete); - -int v4l2_ctrl_request_setup(struct media_request *req, - struct v4l2_ctrl_handler *main_hdl) -{ - struct media_request_object *obj; - struct v4l2_ctrl_handler *hdl; - struct v4l2_ctrl_ref *ref; - int ret = 0; - - if (!req || !main_hdl) - return 0; - - if (WARN_ON(req->state != MEDIA_REQUEST_STATE_QUEUED)) - return -EBUSY; - - /* - * Note that it is valid if nothing was found. It means - * that this request doesn't have any controls and so just - * wants to leave the controls unchanged. - */ - obj = media_request_object_find(req, &req_ops, main_hdl); - if (!obj) - return 0; - if (obj->completed) { - media_request_object_put(obj); - return -EBUSY; - } - hdl = container_of(obj, struct v4l2_ctrl_handler, req_obj); - - list_for_each_entry(ref, &hdl->ctrl_refs, node) - ref->req_done = false; - - list_for_each_entry(ref, &hdl->ctrl_refs, node) { - struct v4l2_ctrl *ctrl = ref->ctrl; - struct v4l2_ctrl *master = ctrl->cluster[0]; - bool have_new_data = false; - int i; - - /* - * Skip if this control was already handled by a cluster. - * Skip button controls and read-only controls. - */ - if (ref->req_done || (ctrl->flags & V4L2_CTRL_FLAG_READ_ONLY)) - continue; - - v4l2_ctrl_lock(master); - for (i = 0; i < master->ncontrols; i++) { - if (master->cluster[i]) { - struct v4l2_ctrl_ref *r = - find_ref(hdl, master->cluster[i]->id); - - if (r->valid_p_req) { - have_new_data = true; - break; - } - } - } - if (!have_new_data) { - v4l2_ctrl_unlock(master); - continue; - } - - for (i = 0; i < master->ncontrols; i++) { - if (master->cluster[i]) { - struct v4l2_ctrl_ref *r = - find_ref(hdl, master->cluster[i]->id); - - req_to_new(r); - master->cluster[i]->is_new = 1; - r->req_done = true; - } - } - /* - * For volatile autoclusters that are currently in auto mode - * we need to discover if it will be set to manual mode. - * If so, then we have to copy the current volatile values - * first since those will become the new manual values (which - * may be overwritten by explicit new values from this set - * of controls). - */ - if (master->is_auto && master->has_volatiles && - !is_cur_manual(master)) { - s32 new_auto_val = *master->p_new.p_s32; - - /* - * If the new value == the manual value, then copy - * the current volatile values. - */ - if (new_auto_val == master->manual_mode_value) - update_from_auto_cluster(master); - } - - ret = try_or_set_cluster(NULL, master, true, 0); - v4l2_ctrl_unlock(master); - - if (ret) - break; - } - - media_request_object_put(obj); - return ret; -} -EXPORT_SYMBOL(v4l2_ctrl_request_setup); - -void v4l2_ctrl_notify(struct v4l2_ctrl *ctrl, v4l2_ctrl_notify_fnc notify, void *priv) -{ - if (ctrl == NULL) - return; - if (notify == NULL) { - ctrl->call_notify = 0; - return; - } - if (WARN_ON(ctrl->handler->notify && ctrl->handler->notify != notify)) - return; - ctrl->handler->notify = notify; - ctrl->handler->notify_priv = priv; - ctrl->call_notify = 1; -} -EXPORT_SYMBOL(v4l2_ctrl_notify); - -int __v4l2_ctrl_modify_range(struct v4l2_ctrl *ctrl, - s64 min, s64 max, u64 step, s64 def) -{ - bool value_changed; - bool range_changed = false; - int ret; - - lockdep_assert_held(ctrl->handler->lock); - - switch (ctrl->type) { - case V4L2_CTRL_TYPE_INTEGER: - case V4L2_CTRL_TYPE_INTEGER64: - case V4L2_CTRL_TYPE_BOOLEAN: - case V4L2_CTRL_TYPE_MENU: - case V4L2_CTRL_TYPE_INTEGER_MENU: - case V4L2_CTRL_TYPE_BITMASK: - case V4L2_CTRL_TYPE_U8: - case V4L2_CTRL_TYPE_U16: - case V4L2_CTRL_TYPE_U32: - if (ctrl->is_array) - return -EINVAL; - ret = check_range(ctrl->type, min, max, step, def); - if (ret) - return ret; - break; - default: - return -EINVAL; - } - if ((ctrl->minimum != min) || (ctrl->maximum != max) || - (ctrl->step != step) || ctrl->default_value != def) { - range_changed = true; - ctrl->minimum = min; - ctrl->maximum = max; - ctrl->step = step; - ctrl->default_value = def; - } - cur_to_new(ctrl); - if (validate_new(ctrl, ctrl->p_new)) { - if (ctrl->type == V4L2_CTRL_TYPE_INTEGER64) - *ctrl->p_new.p_s64 = def; - else - *ctrl->p_new.p_s32 = def; - } - - if (ctrl->type == V4L2_CTRL_TYPE_INTEGER64) - value_changed = *ctrl->p_new.p_s64 != *ctrl->p_cur.p_s64; - else - value_changed = *ctrl->p_new.p_s32 != *ctrl->p_cur.p_s32; - if (value_changed) - ret = set_ctrl(NULL, ctrl, V4L2_EVENT_CTRL_CH_RANGE); - else if (range_changed) - send_event(NULL, ctrl, V4L2_EVENT_CTRL_CH_RANGE); - return ret; -} -EXPORT_SYMBOL(__v4l2_ctrl_modify_range); - -static int v4l2_ctrl_add_event(struct v4l2_subscribed_event *sev, unsigned elems) -{ - struct v4l2_ctrl *ctrl = v4l2_ctrl_find(sev->fh->ctrl_handler, sev->id); - - if (ctrl == NULL) - return -EINVAL; - - v4l2_ctrl_lock(ctrl); - list_add_tail(&sev->node, &ctrl->ev_subs); - if (ctrl->type != V4L2_CTRL_TYPE_CTRL_CLASS && - (sev->flags & V4L2_EVENT_SUB_FL_SEND_INITIAL)) { - struct v4l2_event ev; - u32 changes = V4L2_EVENT_CTRL_CH_FLAGS; - - if (!(ctrl->flags & V4L2_CTRL_FLAG_WRITE_ONLY)) - changes |= V4L2_EVENT_CTRL_CH_VALUE; - fill_event(&ev, ctrl, changes); - /* Mark the queue as active, allowing this initial - event to be accepted. */ - sev->elems = elems; - v4l2_event_queue_fh(sev->fh, &ev); - } - v4l2_ctrl_unlock(ctrl); - return 0; -} - -static void v4l2_ctrl_del_event(struct v4l2_subscribed_event *sev) -{ - struct v4l2_ctrl *ctrl = v4l2_ctrl_find(sev->fh->ctrl_handler, sev->id); - - if (ctrl == NULL) - return; - - v4l2_ctrl_lock(ctrl); - list_del(&sev->node); - v4l2_ctrl_unlock(ctrl); -} - -void v4l2_ctrl_replace(struct v4l2_event *old, const struct v4l2_event *new) -{ - u32 old_changes = old->u.ctrl.changes; - - old->u.ctrl = new->u.ctrl; - old->u.ctrl.changes |= old_changes; -} -EXPORT_SYMBOL(v4l2_ctrl_replace); - -void v4l2_ctrl_merge(const struct v4l2_event *old, struct v4l2_event *new) -{ - new->u.ctrl.changes |= old->u.ctrl.changes; -} -EXPORT_SYMBOL(v4l2_ctrl_merge); - -const struct v4l2_subscribed_event_ops v4l2_ctrl_sub_ev_ops = { - .add = v4l2_ctrl_add_event, - .del = v4l2_ctrl_del_event, - .replace = v4l2_ctrl_replace, - .merge = v4l2_ctrl_merge, -}; -EXPORT_SYMBOL(v4l2_ctrl_sub_ev_ops); - -int v4l2_ctrl_log_status(struct file *file, void *fh) -{ - struct video_device *vfd = video_devdata(file); - struct v4l2_fh *vfh = file->private_data; - - if (test_bit(V4L2_FL_USES_V4L2_FH, &vfd->flags) && vfd->v4l2_dev) - v4l2_ctrl_handler_log_status(vfh->ctrl_handler, - vfd->v4l2_dev->name); - return 0; -} -EXPORT_SYMBOL(v4l2_ctrl_log_status); - -int v4l2_ctrl_subscribe_event(struct v4l2_fh *fh, - const struct v4l2_event_subscription *sub) -{ - if (sub->type == V4L2_EVENT_CTRL) - return v4l2_event_subscribe(fh, sub, 0, &v4l2_ctrl_sub_ev_ops); - return -EINVAL; -} -EXPORT_SYMBOL(v4l2_ctrl_subscribe_event); - -int v4l2_ctrl_subdev_subscribe_event(struct v4l2_subdev *sd, struct v4l2_fh *fh, - struct v4l2_event_subscription *sub) -{ - if (!sd->ctrl_handler) - return -EINVAL; - return v4l2_ctrl_subscribe_event(fh, sub); -} -EXPORT_SYMBOL(v4l2_ctrl_subdev_subscribe_event); - -__poll_t v4l2_ctrl_poll(struct file *file, struct poll_table_struct *wait) -{ - struct v4l2_fh *fh = file->private_data; - - poll_wait(file, &fh->wait, wait); - if (v4l2_event_pending(fh)) - return EPOLLPRI; - return 0; -} -EXPORT_SYMBOL(v4l2_ctrl_poll); - -int v4l2_ctrl_new_fwnode_properties(struct v4l2_ctrl_handler *hdl, - const struct v4l2_ctrl_ops *ctrl_ops, - const struct v4l2_fwnode_device_properties *p) -{ - if (p->orientation != V4L2_FWNODE_PROPERTY_UNSET) { - u32 orientation_ctrl; - - switch (p->orientation) { - case V4L2_FWNODE_ORIENTATION_FRONT: - orientation_ctrl = V4L2_CAMERA_ORIENTATION_FRONT; - break; - case V4L2_FWNODE_ORIENTATION_BACK: - orientation_ctrl = V4L2_CAMERA_ORIENTATION_BACK; - break; - case V4L2_FWNODE_ORIENTATION_EXTERNAL: - orientation_ctrl = V4L2_CAMERA_ORIENTATION_EXTERNAL; - break; - default: - return -EINVAL; - } - if (!v4l2_ctrl_new_std_menu(hdl, ctrl_ops, - V4L2_CID_CAMERA_ORIENTATION, - V4L2_CAMERA_ORIENTATION_EXTERNAL, 0, - orientation_ctrl)) - return hdl->error; - } - - if (p->rotation != V4L2_FWNODE_PROPERTY_UNSET) { - if (!v4l2_ctrl_new_std(hdl, ctrl_ops, - V4L2_CID_CAMERA_SENSOR_ROTATION, - p->rotation, p->rotation, 1, - p->rotation)) - return hdl->error; - } - - return hdl->error; -} -EXPORT_SYMBOL(v4l2_ctrl_new_fwnode_properties); diff --git a/drivers/media/v4l2-core/v4l2-dev.c b/drivers/media/v4l2-core/v4l2-dev.c index 7d0edf3530be..d03ace324db0 100644 --- a/drivers/media/v4l2-core/v4l2-dev.c +++ b/drivers/media/v4l2-core/v4l2-dev.c @@ -39,8 +39,6 @@ __func__, ##arg); \ } while (0) -static struct dentry *v4l2_debugfs_dir; - /* * sysfs stuff */ @@ -520,9 +518,8 @@ static int get_index(struct video_device *vdev) return find_first_zero_bit(used, VIDEO_NUM_DEVICES); } -#define SET_VALID_IOCTL(ops, cmd, op) \ - if (ops->op) \ - set_bit(_IOC_NR(cmd), valid_ioctls) +#define SET_VALID_IOCTL(ops, cmd, op) \ + do { if ((ops)->op) set_bit(_IOC_NR(cmd), valid_ioctls); } while (0) /* This determines which ioctls are actually implemented in the driver. It's a one-time thing which simplifies video_ioctl2 as it can just do @@ -1121,8 +1118,6 @@ static int __init videodev_init(void) return -EIO; } - v4l2_debugfs_dir = debugfs_create_dir("video4linux", NULL); - v4l2_async_debug_init(v4l2_debugfs_dir); return 0; } @@ -1130,7 +1125,6 @@ static void __exit videodev_exit(void) { dev_t dev = MKDEV(VIDEO_MAJOR, 0); - debugfs_remove_recursive(v4l2_debugfs_dir); class_unregister(&video_class); unregister_chrdev_region(dev, VIDEO_NUM_DEVICES); } diff --git a/drivers/media/v4l2-core/v4l2-event.c b/drivers/media/v4l2-core/v4l2-event.c index caad58bde326..c5ce9f11ad7b 100644 --- a/drivers/media/v4l2-core/v4l2-event.c +++ b/drivers/media/v4l2-core/v4l2-event.c @@ -18,7 +18,7 @@ #include #include -static unsigned sev_pos(const struct v4l2_subscribed_event *sev, unsigned idx) +static unsigned int sev_pos(const struct v4l2_subscribed_event *sev, unsigned int idx) { idx += sev->first; return idx >= sev->elems ? idx - sev->elems : idx; @@ -221,12 +221,12 @@ static void __v4l2_event_unsubscribe(struct v4l2_subscribed_event *sev) } int v4l2_event_subscribe(struct v4l2_fh *fh, - const struct v4l2_event_subscription *sub, unsigned elems, + const struct v4l2_event_subscription *sub, unsigned int elems, const struct v4l2_subscribed_event_ops *ops) { struct v4l2_subscribed_event *sev, *found_ev; unsigned long flags; - unsigned i; + unsigned int i; int ret = 0; if (sub->type == V4L2_EVENT_ALL) diff --git a/drivers/media/v4l2-core/v4l2-fh.c b/drivers/media/v4l2-core/v4l2-fh.c index 684574f58e82..90eec79ee995 100644 --- a/drivers/media/v4l2-core/v4l2-fh.c +++ b/drivers/media/v4l2-core/v4l2-fh.c @@ -96,6 +96,7 @@ int v4l2_fh_release(struct file *filp) v4l2_fh_del(fh); v4l2_fh_exit(fh); kfree(fh); + filp->private_data = NULL; } return 0; } diff --git a/drivers/media/v4l2-core/v4l2-ioctl.c b/drivers/media/v4l2-core/v4l2-ioctl.c index 2673f51aafa4..05d5db3d85e5 100644 --- a/drivers/media/v4l2-core/v4l2-ioctl.c +++ b/drivers/media/v4l2-core/v4l2-ioctl.c @@ -3072,8 +3072,8 @@ static int check_array_args(unsigned int cmd, void *parg, size_t *array_size, static unsigned int video_translate_cmd(unsigned int cmd) { +#if !defined(CONFIG_64BIT) && defined(CONFIG_COMPAT_32BIT_TIME) switch (cmd) { -#ifdef CONFIG_COMPAT_32BIT_TIME case VIDIOC_DQEVENT_TIME32: return VIDIOC_DQEVENT; case VIDIOC_QUERYBUF_TIME32: @@ -3084,8 +3084,8 @@ static unsigned int video_translate_cmd(unsigned int cmd) return VIDIOC_DQBUF; case VIDIOC_PREPARE_BUF_TIME32: return VIDIOC_PREPARE_BUF; -#endif } +#endif if (in_compat_syscall()) return v4l2_compat_translate_cmd(cmd); @@ -3124,10 +3124,12 @@ static int video_get_user(void __user *arg, void *parg, if (copy_from_user(parg, (void __user *)arg, n)) err = -EFAULT; } else if (in_compat_syscall()) { + memset(parg, 0, n); err = v4l2_compat_get_user(arg, parg, cmd); } else { + memset(parg, 0, n); +#if !defined(CONFIG_64BIT) && defined(CONFIG_COMPAT_32BIT_TIME) switch (cmd) { -#ifdef CONFIG_COMPAT_32BIT_TIME case VIDIOC_QUERYBUF_TIME32: case VIDIOC_QBUF_TIME32: case VIDIOC_DQBUF_TIME32: @@ -3140,23 +3142,23 @@ static int video_get_user(void __user *arg, void *parg, *vb = (struct v4l2_buffer) { .index = vb32.index, - .type = vb32.type, - .bytesused = vb32.bytesused, - .flags = vb32.flags, - .field = vb32.field, - .timestamp.tv_sec = vb32.timestamp.tv_sec, - .timestamp.tv_usec = vb32.timestamp.tv_usec, - .timecode = vb32.timecode, - .sequence = vb32.sequence, - .memory = vb32.memory, - .m.userptr = vb32.m.userptr, - .length = vb32.length, - .request_fd = vb32.request_fd, + .type = vb32.type, + .bytesused = vb32.bytesused, + .flags = vb32.flags, + .field = vb32.field, + .timestamp.tv_sec = vb32.timestamp.tv_sec, + .timestamp.tv_usec = vb32.timestamp.tv_usec, + .timecode = vb32.timecode, + .sequence = vb32.sequence, + .memory = vb32.memory, + .m.userptr = vb32.m.userptr, + .length = vb32.length, + .request_fd = vb32.request_fd, }; break; } -#endif } +#endif } /* zero out anything we don't copy from userspace */ @@ -3181,8 +3183,8 @@ static int video_put_user(void __user *arg, void *parg, if (in_compat_syscall()) return v4l2_compat_put_user(arg, parg, cmd); +#if !defined(CONFIG_64BIT) && defined(CONFIG_COMPAT_32BIT_TIME) switch (cmd) { -#ifdef CONFIG_COMPAT_32BIT_TIME case VIDIOC_DQEVENT_TIME32: { struct v4l2_event *ev = parg; struct v4l2_event_time32 ev32; @@ -3230,8 +3232,8 @@ static int video_put_user(void __user *arg, void *parg, return -EFAULT; break; } -#endif } +#endif return 0; } diff --git a/drivers/media/v4l2-core/v4l2-subdev.c b/drivers/media/v4l2-core/v4l2-subdev.c index 956dafab43d4..5d27a27cc2f2 100644 --- a/drivers/media/v4l2-core/v4l2-subdev.c +++ b/drivers/media/v4l2-core/v4l2-subdev.c @@ -26,19 +26,21 @@ #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API) static int subdev_fh_init(struct v4l2_subdev_fh *fh, struct v4l2_subdev *sd) { - if (sd->entity.num_pads) { - fh->pad = v4l2_subdev_alloc_pad_config(sd); - if (fh->pad == NULL) - return -ENOMEM; - } + struct v4l2_subdev_state *state; + + state = v4l2_subdev_alloc_state(sd); + if (IS_ERR(state)) + return PTR_ERR(state); + + fh->state = state; return 0; } static void subdev_fh_free(struct v4l2_subdev_fh *fh) { - v4l2_subdev_free_pad_config(fh->pad); - fh->pad = NULL; + v4l2_subdev_free_state(fh->state); + fh->state = NULL; } static int subdev_open(struct file *file) @@ -146,63 +148,63 @@ static inline int check_pad(struct v4l2_subdev *sd, u32 pad) return 0; } -static int check_cfg(u32 which, struct v4l2_subdev_pad_config *cfg) +static int check_state_pads(u32 which, struct v4l2_subdev_state *state) { - if (which == V4L2_SUBDEV_FORMAT_TRY && !cfg) + if (which == V4L2_SUBDEV_FORMAT_TRY && (!state || !state->pads)) return -EINVAL; return 0; } static inline int check_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_format *format) { if (!format) return -EINVAL; return check_which(format->which) ? : check_pad(sd, format->pad) ? : - check_cfg(format->which, cfg); + check_state_pads(format->which, state); } static int call_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_format *format) { - return check_format(sd, cfg, format) ? : - sd->ops->pad->get_fmt(sd, cfg, format); + return check_format(sd, state, format) ? : + sd->ops->pad->get_fmt(sd, state, format); } static int call_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_format *format) { - return check_format(sd, cfg, format) ? : - sd->ops->pad->set_fmt(sd, cfg, format); + return check_format(sd, state, format) ? : + sd->ops->pad->set_fmt(sd, state, format); } static int call_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_mbus_code_enum *code) { if (!code) return -EINVAL; return check_which(code->which) ? : check_pad(sd, code->pad) ? : - check_cfg(code->which, cfg) ? : - sd->ops->pad->enum_mbus_code(sd, cfg, code); + check_state_pads(code->which, state) ? : + sd->ops->pad->enum_mbus_code(sd, state, code); } static int call_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_frame_size_enum *fse) { if (!fse) return -EINVAL; return check_which(fse->which) ? : check_pad(sd, fse->pad) ? : - check_cfg(fse->which, cfg) ? : - sd->ops->pad->enum_frame_size(sd, cfg, fse); + check_state_pads(fse->which, state) ? : + sd->ops->pad->enum_frame_size(sd, state, fse); } static inline int check_frame_interval(struct v4l2_subdev *sd, @@ -229,42 +231,42 @@ static int call_s_frame_interval(struct v4l2_subdev *sd, } static int call_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_frame_interval_enum *fie) { if (!fie) return -EINVAL; return check_which(fie->which) ? : check_pad(sd, fie->pad) ? : - check_cfg(fie->which, cfg) ? : - sd->ops->pad->enum_frame_interval(sd, cfg, fie); + check_state_pads(fie->which, state) ? : + sd->ops->pad->enum_frame_interval(sd, state, fie); } static inline int check_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_selection *sel) { if (!sel) return -EINVAL; return check_which(sel->which) ? : check_pad(sd, sel->pad) ? : - check_cfg(sel->which, cfg); + check_state_pads(sel->which, state); } static int call_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_selection *sel) { - return check_selection(sd, cfg, sel) ? : - sd->ops->pad->get_selection(sd, cfg, sel); + return check_selection(sd, state, sel) ? : + sd->ops->pad->get_selection(sd, state, sel); } static int call_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_selection *sel) { - return check_selection(sd, cfg, sel) ? : - sd->ops->pad->set_selection(sd, cfg, sel); + return check_selection(sd, state, sel) ? : + sd->ops->pad->set_selection(sd, state, sel); } static inline int check_edid(struct v4l2_subdev *sd, @@ -428,30 +430,6 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) return v4l2_event_dequeue(vfh, arg, file->f_flags & O_NONBLOCK); - case VIDIOC_DQEVENT_TIME32: { - struct v4l2_event_time32 *ev32 = arg; - struct v4l2_event ev = { }; - - if (!(sd->flags & V4L2_SUBDEV_FL_HAS_EVENTS)) - return -ENOIOCTLCMD; - - rval = v4l2_event_dequeue(vfh, &ev, file->f_flags & O_NONBLOCK); - - *ev32 = (struct v4l2_event_time32) { - .type = ev.type, - .pending = ev.pending, - .sequence = ev.sequence, - .timestamp.tv_sec = ev.timestamp.tv_sec, - .timestamp.tv_nsec = ev.timestamp.tv_nsec, - .id = ev.id, - }; - - memcpy(&ev32->u, &ev.u, sizeof(ev.u)); - memcpy(&ev32->reserved, &ev.reserved, sizeof(ev.reserved)); - - return rval; - } - case VIDIOC_SUBSCRIBE_EVENT: return v4l2_subdev_call(sd, core, subscribe_event, vfh, arg); @@ -506,7 +484,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) memset(format->reserved, 0, sizeof(format->reserved)); memset(format->format.reserved, 0, sizeof(format->format.reserved)); - return v4l2_subdev_call(sd, pad, get_fmt, subdev_fh->pad, format); + return v4l2_subdev_call(sd, pad, get_fmt, subdev_fh->state, format); } case VIDIOC_SUBDEV_S_FMT: { @@ -517,7 +495,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) memset(format->reserved, 0, sizeof(format->reserved)); memset(format->format.reserved, 0, sizeof(format->format.reserved)); - return v4l2_subdev_call(sd, pad, set_fmt, subdev_fh->pad, format); + return v4l2_subdev_call(sd, pad, set_fmt, subdev_fh->state, format); } case VIDIOC_SUBDEV_G_CROP: { @@ -531,7 +509,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) sel.target = V4L2_SEL_TGT_CROP; rval = v4l2_subdev_call( - sd, pad, get_selection, subdev_fh->pad, &sel); + sd, pad, get_selection, subdev_fh->state, &sel); crop->rect = sel.r; @@ -553,7 +531,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) sel.r = crop->rect; rval = v4l2_subdev_call( - sd, pad, set_selection, subdev_fh->pad, &sel); + sd, pad, set_selection, subdev_fh->state, &sel); crop->rect = sel.r; @@ -564,7 +542,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) struct v4l2_subdev_mbus_code_enum *code = arg; memset(code->reserved, 0, sizeof(code->reserved)); - return v4l2_subdev_call(sd, pad, enum_mbus_code, subdev_fh->pad, + return v4l2_subdev_call(sd, pad, enum_mbus_code, subdev_fh->state, code); } @@ -572,7 +550,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) struct v4l2_subdev_frame_size_enum *fse = arg; memset(fse->reserved, 0, sizeof(fse->reserved)); - return v4l2_subdev_call(sd, pad, enum_frame_size, subdev_fh->pad, + return v4l2_subdev_call(sd, pad, enum_frame_size, subdev_fh->state, fse); } @@ -597,7 +575,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) struct v4l2_subdev_frame_interval_enum *fie = arg; memset(fie->reserved, 0, sizeof(fie->reserved)); - return v4l2_subdev_call(sd, pad, enum_frame_interval, subdev_fh->pad, + return v4l2_subdev_call(sd, pad, enum_frame_interval, subdev_fh->state, fie); } @@ -606,7 +584,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) memset(sel->reserved, 0, sizeof(sel->reserved)); return v4l2_subdev_call( - sd, pad, get_selection, subdev_fh->pad, sel); + sd, pad, get_selection, subdev_fh->state, sel); } case VIDIOC_SUBDEV_S_SELECTION: { @@ -617,7 +595,7 @@ static long subdev_do_ioctl(struct file *file, unsigned int cmd, void *arg) memset(sel->reserved, 0, sizeof(sel->reserved)); return v4l2_subdev_call( - sd, pad, set_selection, subdev_fh->pad, sel); + sd, pad, set_selection, subdev_fh->state, sel); } case VIDIOC_G_EDID: { @@ -892,35 +870,51 @@ int v4l2_subdev_link_validate(struct media_link *link) } EXPORT_SYMBOL_GPL(v4l2_subdev_link_validate); -struct v4l2_subdev_pad_config * -v4l2_subdev_alloc_pad_config(struct v4l2_subdev *sd) +struct v4l2_subdev_state *v4l2_subdev_alloc_state(struct v4l2_subdev *sd) { - struct v4l2_subdev_pad_config *cfg; + struct v4l2_subdev_state *state; int ret; - if (!sd->entity.num_pads) - return NULL; + state = kzalloc(sizeof(*state), GFP_KERNEL); + if (!state) + return ERR_PTR(-ENOMEM); - cfg = kvmalloc_array(sd->entity.num_pads, sizeof(*cfg), - GFP_KERNEL | __GFP_ZERO); - if (!cfg) - return NULL; - - ret = v4l2_subdev_call(sd, pad, init_cfg, cfg); - if (ret < 0 && ret != -ENOIOCTLCMD) { - kvfree(cfg); - return NULL; + if (sd->entity.num_pads) { + state->pads = kvmalloc_array(sd->entity.num_pads, + sizeof(*state->pads), + GFP_KERNEL | __GFP_ZERO); + if (!state->pads) { + ret = -ENOMEM; + goto err; + } } - return cfg; -} -EXPORT_SYMBOL_GPL(v4l2_subdev_alloc_pad_config); + ret = v4l2_subdev_call(sd, pad, init_cfg, state); + if (ret < 0 && ret != -ENOIOCTLCMD) + goto err; -void v4l2_subdev_free_pad_config(struct v4l2_subdev_pad_config *cfg) -{ - kvfree(cfg); + return state; + +err: + if (state && state->pads) + kvfree(state->pads); + + kfree(state); + + return ERR_PTR(ret); } -EXPORT_SYMBOL_GPL(v4l2_subdev_free_pad_config); +EXPORT_SYMBOL_GPL(v4l2_subdev_alloc_state); + +void v4l2_subdev_free_state(struct v4l2_subdev_state *state) +{ + if (!state) + return; + + kvfree(state->pads); + kfree(state); +} +EXPORT_SYMBOL_GPL(v4l2_subdev_free_state); + #endif /* CONFIG_MEDIA_CONTROLLER */ void v4l2_subdev_init(struct v4l2_subdev *sd, const struct v4l2_subdev_ops *ops) diff --git a/drivers/media/v4l2-core/videobuf-dma-sg.c b/drivers/media/v4l2-core/videobuf-dma-sg.c index 8dd0562de287..f75e5eedeee0 100644 --- a/drivers/media/v4l2-core/videobuf-dma-sg.c +++ b/drivers/media/v4l2-core/videobuf-dma-sg.c @@ -423,7 +423,6 @@ static void videobuf_vm_close(struct vm_area_struct *vma) videobuf_queue_unlock(q); kfree(map); } - return; } /* diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c index 0e8254d0cf0b..a164896dc6d4 100644 --- a/drivers/misc/lkdtm/bugs.c +++ b/drivers/misc/lkdtm/bugs.c @@ -463,7 +463,7 @@ void lkdtm_DOUBLE_FAULT(void) #ifdef CONFIG_ARM64 static noinline void change_pac_parameters(void) { - if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH)) { + if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) { /* Reset the keys of current task */ ptrauth_thread_init_kernel(current); ptrauth_thread_switch_kernel(current); @@ -477,8 +477,8 @@ noinline void lkdtm_CORRUPT_PAC(void) #define CORRUPT_PAC_ITERATE 10 int i; - if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH)) - pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH\n"); + if (!IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL)) + pr_err("FAIL: kernel not built with CONFIG_ARM64_PTR_AUTH_KERNEL\n"); if (!system_supports_address_auth()) { pr_err("FAIL: CPU lacks pointer authentication feature\n"); diff --git a/drivers/mtd/mtdpstore.c b/drivers/mtd/mtdpstore.c index a3ae8778f6a9..e13d42c0acb0 100644 --- a/drivers/mtd/mtdpstore.c +++ b/drivers/mtd/mtdpstore.c @@ -423,13 +423,13 @@ static void mtdpstore_notify_add(struct mtd_info *mtd) longcnt = BITS_TO_LONGS(div_u64(mtd->size, mtd->erasesize)); cxt->badmap = kcalloc(longcnt, sizeof(long), GFP_KERNEL); - cxt->dev.total_size = mtd->size; /* just support dmesg right now */ cxt->dev.flags = PSTORE_FLAGS_DMESG; - cxt->dev.read = mtdpstore_read; - cxt->dev.write = mtdpstore_write; - cxt->dev.erase = mtdpstore_erase; - cxt->dev.panic_write = mtdpstore_panic_write; + cxt->dev.zone.read = mtdpstore_read; + cxt->dev.zone.write = mtdpstore_write; + cxt->dev.zone.erase = mtdpstore_erase; + cxt->dev.zone.panic_write = mtdpstore_panic_write; + cxt->dev.zone.total_size = mtd->size; ret = register_pstore_device(&cxt->dev); if (ret) { diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index aef3fccc27a9..598e5e6ace18 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -2184,6 +2184,7 @@ static int bnxt_hwrm_handler(struct bnxt *bp, struct tx_cmp *txcmp) case CMPL_BASE_TYPE_HWRM_ASYNC_EVENT: bnxt_async_event_process(bp, (struct hwrm_async_event_cmpl *)txcmp); + break; default: break; diff --git a/drivers/net/ethernet/neterion/vxge/vxge-config.c b/drivers/net/ethernet/neterion/vxge/vxge-config.c index 5162b938a1ac..b47d74743f5a 100644 --- a/drivers/net/ethernet/neterion/vxge/vxge-config.c +++ b/drivers/net/ethernet/neterion/vxge/vxge-config.c @@ -3784,6 +3784,7 @@ vxge_hw_rts_rth_data0_data1_get(u32 j, u64 *data0, u64 *data1, VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_ENTRY_EN | VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_DATA( itable[j]); + return; default: return; } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index b3cabc274121..3b8e675087de 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -103,6 +103,7 @@ nfp_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) case NFP_PORT_PF_PORT: case NFP_PORT_VF_PORT: nfp_repr_vnic_get_stats64(repr->port, stats); + break; default: break; } diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c index 08f9477d2ee8..35ec9aab3dc7 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_init.c @@ -1685,6 +1685,7 @@ netxen_process_rcv_ring(struct nx_host_sds_ring *sds_ring, int max) break; case NETXEN_NIC_RESPONSE_DESC: netxen_handle_fw_message(desc_cnt, consumer, sds_ring); + goto skip; default: goto skip; } diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c index bdf15d2a6431..af4c516a9e7c 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c @@ -1390,6 +1390,7 @@ static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int ma break; case QLCNIC_RESPONSE_DESC: qlcnic_handle_fw_message(desc_cnt, consumer, sds_ring); + goto skip; default: goto skip; } diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c index 3beafc60747e..918220ad0d53 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c @@ -3456,6 +3456,7 @@ qlcnic_fwinit_work(struct work_struct *work) adapter->fw_wait_cnt = 0; return; } + break; case QLCNIC_DEV_FAILED: break; default: diff --git a/drivers/perf/arm-cci.c b/drivers/perf/arm-cci.c index 666d8a9b557f..54aca3a62814 100644 --- a/drivers/perf/arm-cci.c +++ b/drivers/perf/arm-cci.c @@ -37,7 +37,7 @@ #define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size) #define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model)) -#define CCI_PMU_CNTR_MASK ((1ULL << 32) -1) +#define CCI_PMU_CNTR_MASK ((1ULL << 32) - 1) #define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1) #define CCI_PMU_MAX_HW_CNTRS(model) \ @@ -806,7 +806,7 @@ static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *ev return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event); /* Generic code to find an unused idx from the mask */ - for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) + for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) if (!test_and_set_bit(idx, hw->used_mask)) return idx; diff --git a/drivers/perf/arm-ccn.c b/drivers/perf/arm-ccn.c index 96d47cb302dd..a96c31604545 100644 --- a/drivers/perf/arm-ccn.c +++ b/drivers/perf/arm-ccn.c @@ -1211,7 +1211,7 @@ static int arm_ccn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) perf_pmu_migrate_context(&dt->pmu, cpu, target); dt->cpu = target; if (ccn->irq) - WARN_ON(irq_set_affinity_hint(ccn->irq, cpumask_of(dt->cpu))); + WARN_ON(irq_set_affinity(ccn->irq, cpumask_of(dt->cpu))); return 0; } @@ -1291,7 +1291,7 @@ static int arm_ccn_pmu_init(struct arm_ccn *ccn) /* Also make sure that the overflow interrupt is handled by this CPU */ if (ccn->irq) { - err = irq_set_affinity_hint(ccn->irq, cpumask_of(ccn->dt.cpu)); + err = irq_set_affinity(ccn->irq, cpumask_of(ccn->dt.cpu)); if (err) { dev_err(ccn->dev, "Failed to set interrupt affinity!\n"); goto error_set_affinity; @@ -1325,8 +1325,6 @@ static void arm_ccn_pmu_cleanup(struct arm_ccn *ccn) cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_CCN_ONLINE, &ccn->dt.node); - if (ccn->irq) - irq_set_affinity_hint(ccn->irq, NULL); for (i = 0; i < ccn->num_xps; i++) writel(0, ccn->xp[i].base + CCN_XP_DT_CONTROL); writel(0, ccn->dt.base + CCN_DT_PMCR); diff --git a/drivers/perf/arm-cmn.c b/drivers/perf/arm-cmn.c index 56a5c355701d..bc3cba5f8c5d 100644 --- a/drivers/perf/arm-cmn.c +++ b/drivers/perf/arm-cmn.c @@ -31,7 +31,7 @@ #define CMN_CI_CHILD_COUNT GENMASK_ULL(15, 0) #define CMN_CI_CHILD_PTR_OFFSET GENMASK_ULL(31, 16) -#define CMN_CHILD_NODE_ADDR GENMASK(27,0) +#define CMN_CHILD_NODE_ADDR GENMASK(27, 0) #define CMN_CHILD_NODE_EXTERNAL BIT(31) #define CMN_ADDR_NODE_PTR GENMASK(27, 14) @@ -1162,7 +1162,7 @@ static int arm_cmn_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) perf_pmu_migrate_context(&cmn->pmu, cpu, target); for (i = 0; i < cmn->num_dtcs; i++) - irq_set_affinity_hint(cmn->dtc[i].irq, cpumask_of(target)); + irq_set_affinity(cmn->dtc[i].irq, cpumask_of(target)); cmn->cpu = target; return 0; } @@ -1212,7 +1212,7 @@ static int arm_cmn_init_irqs(struct arm_cmn *cmn) irq = cmn->dtc[i].irq; for (j = i; j--; ) { if (cmn->dtc[j].irq == irq) { - cmn->dtc[j].irq_friend = j - i; + cmn->dtc[j].irq_friend = i - j; goto next; } } @@ -1222,7 +1222,7 @@ static int arm_cmn_init_irqs(struct arm_cmn *cmn) if (err) return err; - err = irq_set_affinity_hint(irq, cpumask_of(cmn->cpu)); + err = irq_set_affinity(irq, cpumask_of(cmn->cpu)); if (err) return err; next: @@ -1568,16 +1568,11 @@ static int arm_cmn_probe(struct platform_device *pdev) static int arm_cmn_remove(struct platform_device *pdev) { struct arm_cmn *cmn = platform_get_drvdata(pdev); - int i; writel_relaxed(0, cmn->dtc[0].base + CMN_DT_DTC_CTL); perf_pmu_unregister(&cmn->pmu); cpuhp_state_remove_instance(arm_cmn_hp_state, &cmn->cpuhp_node); - - for (i = 0; i < cmn->num_dtcs; i++) - irq_set_affinity_hint(cmn->dtc[i].irq, NULL); - return 0; } diff --git a/drivers/perf/arm_dmc620_pmu.c b/drivers/perf/arm_dmc620_pmu.c index b6c2511d59af..280a6ae3e27c 100644 --- a/drivers/perf/arm_dmc620_pmu.c +++ b/drivers/perf/arm_dmc620_pmu.c @@ -421,7 +421,7 @@ static struct dmc620_pmu_irq *__dmc620_pmu_get_irq(int irq_num) if (ret) goto out_free_aff; - ret = irq_set_affinity_hint(irq_num, cpumask_of(irq->cpu)); + ret = irq_set_affinity(irq_num, cpumask_of(irq->cpu)); if (ret) goto out_free_irq; @@ -475,7 +475,6 @@ static void dmc620_pmu_put_irq(struct dmc620_pmu *dmc620_pmu) list_del(&irq->irqs_node); mutex_unlock(&dmc620_pmu_irqs_lock); - WARN_ON(irq_set_affinity_hint(irq->irq_num, NULL)); free_irq(irq->irq_num, irq); cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &irq->node); kfree(irq); @@ -622,7 +621,7 @@ static int dmc620_pmu_cpu_teardown(unsigned int cpu, perf_pmu_migrate_context(&dmc620_pmu->pmu, irq->cpu, target); mutex_unlock(&dmc620_pmu_irqs_lock); - WARN_ON(irq_set_affinity_hint(irq->irq_num, cpumask_of(target))); + WARN_ON(irq_set_affinity(irq->irq_num, cpumask_of(target))); irq->cpu = target; return 0; diff --git a/drivers/perf/arm_dsu_pmu.c b/drivers/perf/arm_dsu_pmu.c index 196faea074d0..a36698a90d2f 100644 --- a/drivers/perf/arm_dsu_pmu.c +++ b/drivers/perf/arm_dsu_pmu.c @@ -687,7 +687,7 @@ static void dsu_pmu_probe_pmu(struct dsu_pmu *dsu_pmu) static void dsu_pmu_set_active_cpu(int cpu, struct dsu_pmu *dsu_pmu) { cpumask_set_cpu(cpu, &dsu_pmu->active_cpu); - if (irq_set_affinity_hint(dsu_pmu->irq, &dsu_pmu->active_cpu)) + if (irq_set_affinity(dsu_pmu->irq, &dsu_pmu->active_cpu)) pr_warn("Failed to set irq affinity to %d\n", cpu); } @@ -769,7 +769,6 @@ static int dsu_pmu_device_probe(struct platform_device *pdev) if (rc) { cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node); - irq_set_affinity_hint(dsu_pmu->irq, NULL); } return rc; @@ -781,7 +780,6 @@ static int dsu_pmu_device_remove(struct platform_device *pdev) perf_pmu_unregister(&dsu_pmu->pmu); cpuhp_state_remove_instance(dsu_pmu_cpuhp_state, &dsu_pmu->cpuhp_node); - irq_set_affinity_hint(dsu_pmu->irq, NULL); return 0; } @@ -840,10 +838,8 @@ static int dsu_pmu_cpu_teardown(unsigned int cpu, struct hlist_node *node) dst = dsu_pmu_get_online_cpu_any_but(dsu_pmu, cpu); /* If there are no active CPUs in the DSU, leave IRQ disabled */ - if (dst >= nr_cpu_ids) { - irq_set_affinity_hint(dsu_pmu->irq, NULL); + if (dst >= nr_cpu_ids) return 0; - } perf_pmu_migrate_context(&dsu_pmu->pmu, cpu, dst); dsu_pmu_set_active_cpu(dst, dsu_pmu); diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c index d4f7f1f9cc77..3cbc3baf087f 100644 --- a/drivers/perf/arm_pmu.c +++ b/drivers/perf/arm_pmu.c @@ -563,14 +563,14 @@ static int armpmu_filter_match(struct perf_event *event) return ret; } -static ssize_t armpmu_cpumask_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t cpus_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct arm_pmu *armpmu = to_arm_pmu(dev_get_drvdata(dev)); return cpumap_print_to_pagebuf(true, buf, &armpmu->supported_cpus); } -static DEVICE_ATTR(cpus, S_IRUGO, armpmu_cpumask_show, NULL); +static DEVICE_ATTR_RO(cpus); static struct attribute *armpmu_common_attrs[] = { &dev_attr_cpus.attr, @@ -644,11 +644,9 @@ int armpmu_request_irq(int irq, int cpu) } irq_flags = IRQF_PERCPU | - IRQF_NOBALANCING | + IRQF_NOBALANCING | IRQF_NO_AUTOEN | IRQF_NO_THREAD; - irq_set_status_flags(irq, IRQ_NOAUTOEN); - err = request_nmi(irq, handler, irq_flags, "arm-pmu", per_cpu_ptr(&cpu_armpmu, cpu)); @@ -670,7 +668,7 @@ int armpmu_request_irq(int irq, int cpu) &cpu_armpmu); irq_ops = &percpu_pmuirq_ops; } else { - has_nmi= true; + has_nmi = true; irq_ops = &percpu_pmunmi_ops; } } else { @@ -869,10 +867,8 @@ static struct arm_pmu *__armpmu_alloc(gfp_t flags) int cpu; pmu = kzalloc(sizeof(*pmu), flags); - if (!pmu) { - pr_info("failed to allocate PMU device!\n"); + if (!pmu) goto out; - } pmu->hw_events = alloc_percpu_gfp(struct pmu_hw_events, flags); if (!pmu->hw_events) { diff --git a/drivers/perf/arm_smmuv3_pmu.c b/drivers/perf/arm_smmuv3_pmu.c index ff6fab4bae30..226348822ab3 100644 --- a/drivers/perf/arm_smmuv3_pmu.c +++ b/drivers/perf/arm_smmuv3_pmu.c @@ -277,7 +277,7 @@ static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, struct perf_event *event, int idx) { u32 span, sid; - unsigned int num_ctrs = smmu_pmu->num_counters; + unsigned int cur_idx, num_ctrs = smmu_pmu->num_counters; bool filter_en = !!get_filter_enable(event); span = filter_en ? get_filter_span(event) : @@ -285,17 +285,19 @@ static int smmu_pmu_apply_event_filter(struct smmu_pmu *smmu_pmu, sid = filter_en ? get_filter_stream_id(event) : SMMU_PMCG_DEFAULT_FILTER_SID; - /* Support individual filter settings */ - if (!smmu_pmu->global_filter) { + cur_idx = find_first_bit(smmu_pmu->used_counters, num_ctrs); + /* + * Per-counter filtering, or scheduling the first globally-filtered + * event into an empty PMU so idx == 0 and it works out equivalent. + */ + if (!smmu_pmu->global_filter || cur_idx == num_ctrs) { smmu_pmu_set_event_filter(event, idx, span, sid); return 0; } - /* Requested settings same as current global settings*/ - idx = find_first_bit(smmu_pmu->used_counters, num_ctrs); - if (idx == num_ctrs || - smmu_pmu_check_global_filter(smmu_pmu->events[idx], event)) { - smmu_pmu_set_event_filter(event, 0, span, sid); + /* Otherwise, must match whatever's currently scheduled */ + if (smmu_pmu_check_global_filter(smmu_pmu->events[cur_idx], event)) { + smmu_pmu_set_evtyper(smmu_pmu, idx, get_event(event)); return 0; } @@ -509,11 +511,8 @@ static ssize_t smmu_pmu_event_show(struct device *dev, return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); } -#define SMMU_EVENT_ATTR(name, config) \ - (&((struct perf_pmu_events_attr) { \ - .attr = __ATTR(name, 0444, smmu_pmu_event_show, NULL), \ - .id = config, \ - }).attr.attr) +#define SMMU_EVENT_ATTR(name, config) \ + PMU_EVENT_ATTR_ID(name, smmu_pmu_event_show, config) static struct attribute *smmu_pmu_events[] = { SMMU_EVENT_ATTR(cycles, 0), @@ -628,7 +627,7 @@ static int smmu_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) perf_pmu_migrate_context(&smmu_pmu->pmu, cpu, target); smmu_pmu->on_cpu = target; - WARN_ON(irq_set_affinity_hint(smmu_pmu->irq, cpumask_of(target))); + WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(target))); return 0; } @@ -839,15 +838,14 @@ static int smmu_pmu_probe(struct platform_device *pdev) /* Pick one CPU to be the preferred one to use */ smmu_pmu->on_cpu = raw_smp_processor_id(); - WARN_ON(irq_set_affinity_hint(smmu_pmu->irq, - cpumask_of(smmu_pmu->on_cpu))); + WARN_ON(irq_set_affinity(smmu_pmu->irq, cpumask_of(smmu_pmu->on_cpu))); err = cpuhp_state_add_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); if (err) { dev_err(dev, "Error %d registering hotplug, PMU @%pa\n", err, &res_0->start); - goto out_clear_affinity; + return err; } err = perf_pmu_register(&smmu_pmu->pmu, name, -1); @@ -866,8 +864,6 @@ static int smmu_pmu_probe(struct platform_device *pdev) out_unregister: cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); -out_clear_affinity: - irq_set_affinity_hint(smmu_pmu->irq, NULL); return err; } @@ -877,7 +873,6 @@ static int smmu_pmu_remove(struct platform_device *pdev) perf_pmu_unregister(&smmu_pmu->pmu); cpuhp_state_remove_instance_nocalls(cpuhp_state_num, &smmu_pmu->node); - irq_set_affinity_hint(smmu_pmu->irq, NULL); return 0; } diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index 8a1e86ab2d8e..d44bcc29d99c 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -231,15 +231,14 @@ static const struct attribute_group arm_spe_pmu_format_group = { .attrs = arm_spe_pmu_formats_attr, }; -static ssize_t arm_spe_pmu_get_attr_cpumask(struct device *dev, - struct device_attribute *attr, - char *buf) +static ssize_t cpumask_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct arm_spe_pmu *spe_pmu = dev_get_drvdata(dev); return cpumap_print_to_pagebuf(true, buf, &spe_pmu->supported_cpus); } -static DEVICE_ATTR(cpumask, S_IRUGO, arm_spe_pmu_get_attr_cpumask, NULL); +static DEVICE_ATTR_RO(cpumask); static struct attribute *arm_spe_pmu_attrs[] = { &dev_attr_cpumask.attr, @@ -1044,7 +1043,6 @@ static void __arm_spe_pmu_dev_probe(void *info) spe_pmu->max_record_sz, spe_pmu->align, spe_pmu->features); spe_pmu->features |= SPE_PMU_FEAT_DEV_PROBED; - return; } static void __arm_spe_pmu_reset_local(void) @@ -1190,10 +1188,8 @@ static int arm_spe_pmu_device_probe(struct platform_device *pdev) } spe_pmu = devm_kzalloc(dev, sizeof(*spe_pmu), GFP_KERNEL); - if (!spe_pmu) { - dev_err(dev, "failed to allocate spe_pmu\n"); + if (!spe_pmu) return -ENOMEM; - } spe_pmu->handle = alloc_percpu(typeof(*spe_pmu->handle)); if (!spe_pmu->handle) diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c index 2bbb93188064..94ebc1ecace7 100644 --- a/drivers/perf/fsl_imx8_ddr_perf.c +++ b/drivers/perf/fsl_imx8_ddr_perf.c @@ -222,11 +222,8 @@ ddr_pmu_event_show(struct device *dev, struct device_attribute *attr, return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); } -#define IMX8_DDR_PMU_EVENT_ATTR(_name, _id) \ - (&((struct perf_pmu_events_attr[]) { \ - { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\ - .id = _id, } \ - })[0].attr.attr) +#define IMX8_DDR_PMU_EVENT_ATTR(_name, _id) \ + PMU_EVENT_ATTR_ID(_name, ddr_pmu_event_show, _id) static struct attribute *ddr_perf_events_attrs[] = { IMX8_DDR_PMU_EVENT_ATTR(cycles, EVENT_CYCLES_ID), @@ -674,7 +671,7 @@ static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node) perf_pmu_migrate_context(&pmu->pmu, cpu, target); pmu->cpu = target; - WARN_ON(irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu))); + WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu))); return 0; } @@ -705,8 +702,10 @@ static int ddr_perf_probe(struct platform_device *pdev) name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", num); - if (!name) - return -ENOMEM; + if (!name) { + ret = -ENOMEM; + goto cpuhp_state_err; + } pmu->devtype_data = of_device_get_match_data(&pdev->dev); @@ -749,7 +748,7 @@ static int ddr_perf_probe(struct platform_device *pdev) } pmu->irq = irq; - ret = irq_set_affinity_hint(pmu->irq, cpumask_of(pmu->cpu)); + ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)); if (ret) { dev_err(pmu->dev, "Failed to set interrupt affinity!\n"); goto ddr_perf_err; @@ -777,7 +776,6 @@ static int ddr_perf_remove(struct platform_device *pdev) cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node); cpuhp_remove_multi_state(pmu->cpuhp_state); - irq_set_affinity_hint(pmu->irq, NULL); perf_pmu_unregister(&pmu->pmu); diff --git a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c index 7c8a4bc21db4..62299ab5a9be 100644 --- a/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_ddrc_pmu.c @@ -2,7 +2,7 @@ /* * HiSilicon SoC DDRC uncore Hardware event counters support * - * Copyright (C) 2017 Hisilicon Limited + * Copyright (C) 2017 HiSilicon Limited * Author: Shaokun Zhang * Anurup M * @@ -537,7 +537,6 @@ static int hisi_ddrc_pmu_probe(struct platform_device *pdev) dev_err(ddrc_pmu->dev, "DDRC PMU register failed!\n"); cpuhp_state_remove_instance_nocalls( CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, &ddrc_pmu->node); - irq_set_affinity_hint(ddrc_pmu->irq, NULL); } return ret; @@ -550,8 +549,6 @@ static int hisi_ddrc_pmu_remove(struct platform_device *pdev) perf_pmu_unregister(&ddrc_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_DDRC_ONLINE, &ddrc_pmu->node); - irq_set_affinity_hint(ddrc_pmu->irq, NULL); - return 0; } diff --git a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c index 0316fabe32f1..393513150106 100644 --- a/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_hha_pmu.c @@ -2,7 +2,7 @@ /* * HiSilicon SoC HHA uncore Hardware event counters support * - * Copyright (C) 2017 Hisilicon Limited + * Copyright (C) 2017 HiSilicon Limited * Author: Shaokun Zhang * Anurup M * @@ -90,7 +90,7 @@ static void hisi_hha_pmu_config_ds(struct perf_event *event) val = readl(hha_pmu->base + HHA_DATSRC_CTRL); val |= HHA_DATSRC_SKT_EN; - writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL); + writel(val, hha_pmu->base + HHA_DATSRC_CTRL); } } @@ -104,7 +104,7 @@ static void hisi_hha_pmu_clear_ds(struct perf_event *event) val = readl(hha_pmu->base + HHA_DATSRC_CTRL); val &= ~HHA_DATSRC_SKT_EN; - writel(ds_skt, hha_pmu->base + HHA_DATSRC_CTRL); + writel(val, hha_pmu->base + HHA_DATSRC_CTRL); } } @@ -540,7 +540,6 @@ static int hisi_hha_pmu_probe(struct platform_device *pdev) dev_err(hha_pmu->dev, "HHA PMU register failed!\n"); cpuhp_state_remove_instance_nocalls( CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE, &hha_pmu->node); - irq_set_affinity_hint(hha_pmu->irq, NULL); } return ret; @@ -553,8 +552,6 @@ static int hisi_hha_pmu_remove(struct platform_device *pdev) perf_pmu_unregister(&hha_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_HHA_ONLINE, &hha_pmu->node); - irq_set_affinity_hint(hha_pmu->irq, NULL); - return 0; } diff --git a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c index bf9f7772cac9..560ab964c8b5 100644 --- a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c @@ -2,7 +2,7 @@ /* * HiSilicon SoC L3C uncore Hardware event counters support * - * Copyright (C) 2017 Hisilicon Limited + * Copyright (C) 2017 HiSilicon Limited * Author: Anurup M * Shaokun Zhang * @@ -578,7 +578,6 @@ static int hisi_l3c_pmu_probe(struct platform_device *pdev) dev_err(l3c_pmu->dev, "L3C PMU register failed!\n"); cpuhp_state_remove_instance_nocalls( CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, &l3c_pmu->node); - irq_set_affinity_hint(l3c_pmu->irq, NULL); } return ret; @@ -591,8 +590,6 @@ static int hisi_l3c_pmu_remove(struct platform_device *pdev) perf_pmu_unregister(&l3c_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_L3_ONLINE, &l3c_pmu->node); - irq_set_affinity_hint(l3c_pmu->irq, NULL); - return 0; } diff --git a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c index 14f23eb31248..83264ec0a957 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_pa_pmu.c @@ -333,7 +333,7 @@ static struct attribute *hisi_pa_pmu_identifier_attrs[] = { NULL }; -static struct attribute_group hisi_pa_pmu_identifier_group = { +static const struct attribute_group hisi_pa_pmu_identifier_group = { .attrs = hisi_pa_pmu_identifier_attrs, }; @@ -436,7 +436,6 @@ static int hisi_pa_pmu_probe(struct platform_device *pdev) dev_err(pa_pmu->dev, "PMU register failed, ret = %d\n", ret); cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE, &pa_pmu->node); - irq_set_affinity_hint(pa_pmu->irq, NULL); return ret; } @@ -451,8 +450,6 @@ static int hisi_pa_pmu_remove(struct platform_device *pdev) perf_pmu_unregister(&pa_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_PA_ONLINE, &pa_pmu->node); - irq_set_affinity_hint(pa_pmu->irq, NULL); - return 0; } diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.c b/drivers/perf/hisilicon/hisi_uncore_pmu.c index 13c68b5e39c4..a738aeab5c04 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_pmu.c @@ -2,7 +2,7 @@ /* * HiSilicon SoC Hardware event counters support * - * Copyright (C) 2017 Hisilicon Limited + * Copyright (C) 2017 HiSilicon Limited * Author: Anurup M * Shaokun Zhang * @@ -488,7 +488,7 @@ int hisi_uncore_pmu_online_cpu(unsigned int cpu, struct hlist_node *node) hisi_pmu->on_cpu = cpu; /* Overflow interrupt also should use the same CPU */ - WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(cpu))); + WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(cpu))); return 0; } @@ -521,7 +521,7 @@ int hisi_uncore_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node) perf_pmu_migrate_context(&hisi_pmu->pmu, cpu, target); /* Use this CPU for event counting */ hisi_pmu->on_cpu = target; - WARN_ON(irq_set_affinity_hint(hisi_pmu->irq, cpumask_of(target))); + WARN_ON(irq_set_affinity(hisi_pmu->irq, cpumask_of(target))); return 0; } diff --git a/drivers/perf/hisilicon/hisi_uncore_pmu.h b/drivers/perf/hisilicon/hisi_uncore_pmu.h index ea9d89bbc1ea..7f5841d6f592 100644 --- a/drivers/perf/hisilicon/hisi_uncore_pmu.h +++ b/drivers/perf/hisilicon/hisi_uncore_pmu.h @@ -2,7 +2,7 @@ /* * HiSilicon SoC Hardware event counters support * - * Copyright (C) 2017 Hisilicon Limited + * Copyright (C) 2017 HiSilicon Limited * Author: Anurup M * Shaokun Zhang * diff --git a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c index 46be312fa126..6aedc303ff56 100644 --- a/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_sllc_pmu.c @@ -2,7 +2,7 @@ /* * HiSilicon SLLC uncore Hardware event counters support * - * Copyright (C) 2020 Hisilicon Limited + * Copyright (C) 2020 HiSilicon Limited * Author: Shaokun Zhang * * This code is based on the uncore PMUs like arm-cci and arm-ccn. @@ -366,7 +366,7 @@ static struct attribute *hisi_sllc_pmu_identifier_attrs[] = { NULL }; -static struct attribute_group hisi_sllc_pmu_identifier_group = { +static const struct attribute_group hisi_sllc_pmu_identifier_group = { .attrs = hisi_sllc_pmu_identifier_attrs, }; @@ -465,7 +465,6 @@ static int hisi_sllc_pmu_probe(struct platform_device *pdev) dev_err(sllc_pmu->dev, "PMU register failed, ret = %d\n", ret); cpuhp_state_remove_instance(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, &sllc_pmu->node); - irq_set_affinity_hint(sllc_pmu->irq, NULL); return ret; } @@ -481,8 +480,6 @@ static int hisi_sllc_pmu_remove(struct platform_device *pdev) perf_pmu_unregister(&sllc_pmu->pmu); cpuhp_state_remove_instance_nocalls(CPUHP_AP_PERF_ARM_HISI_SLLC_ONLINE, &sllc_pmu->node); - irq_set_affinity_hint(sllc_pmu->irq, NULL); - return 0; } diff --git a/drivers/perf/qcom_l2_pmu.c b/drivers/perf/qcom_l2_pmu.c index fc54a80f9c5c..5b093badd0f6 100644 --- a/drivers/perf/qcom_l2_pmu.c +++ b/drivers/perf/qcom_l2_pmu.c @@ -679,11 +679,8 @@ static ssize_t l2cache_pmu_event_show(struct device *dev, return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id); } -#define L2CACHE_EVENT_ATTR(_name, _id) \ - (&((struct perf_pmu_events_attr[]) { \ - { .attr = __ATTR(_name, 0444, l2cache_pmu_event_show, NULL), \ - .id = _id, } \ - })[0].attr.attr) +#define L2CACHE_EVENT_ATTR(_name, _id) \ + PMU_EVENT_ATTR_ID(_name, l2cache_pmu_event_show, _id) static struct attribute *l2_cache_pmu_events[] = { L2CACHE_EVENT_ATTR(cycles, L2_EVENT_CYCLES), @@ -869,14 +866,14 @@ static int l2_cache_pmu_probe_cluster(struct device *dev, void *data) irq = platform_get_irq(sdev, 0); if (irq < 0) return irq; - irq_set_status_flags(irq, IRQ_NOAUTOEN); cluster->irq = irq; cluster->l2cache_pmu = l2cache_pmu; cluster->on_cpu = -1; err = devm_request_irq(&pdev->dev, irq, l2_cache_handle_irq, - IRQF_NOBALANCING | IRQF_NO_THREAD, + IRQF_NOBALANCING | IRQF_NO_THREAD | + IRQF_NO_AUTOEN, "l2-cache-pmu", cluster); if (err) { dev_err(&pdev->dev, diff --git a/drivers/perf/qcom_l3_pmu.c b/drivers/perf/qcom_l3_pmu.c index bba078077c93..1ff2ff6582bf 100644 --- a/drivers/perf/qcom_l3_pmu.c +++ b/drivers/perf/qcom_l3_pmu.c @@ -647,10 +647,7 @@ static ssize_t l3cache_pmu_event_show(struct device *dev, } #define L3CACHE_EVENT_ATTR(_name, _id) \ - (&((struct perf_pmu_events_attr[]) { \ - { .attr = __ATTR(_name, 0444, l3cache_pmu_event_show, NULL), \ - .id = _id, } \ - })[0].attr.attr) + PMU_EVENT_ATTR_ID(_name, l3cache_pmu_event_show, _id) static struct attribute *qcom_l3_cache_pmu_events[] = { L3CACHE_EVENT_ATTR(cycles, L3_EVENT_CYCLES), @@ -670,15 +667,15 @@ static const struct attribute_group qcom_l3_cache_pmu_events_group = { /* cpumask */ -static ssize_t qcom_l3_cache_pmu_cpumask_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t cpumask_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct l3cache_pmu *l3pmu = to_l3cache_pmu(dev_get_drvdata(dev)); return cpumap_print_to_pagebuf(true, buf, &l3pmu->cpumask); } -static DEVICE_ATTR(cpumask, 0444, qcom_l3_cache_pmu_cpumask_show, NULL); +static DEVICE_ATTR_RO(cpumask); static struct attribute *qcom_l3_cache_pmu_cpumask_attrs[] = { &dev_attr_cpumask.attr, @@ -767,10 +764,8 @@ static int qcom_l3_cache_pmu_probe(struct platform_device *pdev) memrc = platform_get_resource(pdev, IORESOURCE_MEM, 0); l3pmu->regs = devm_ioremap_resource(&pdev->dev, memrc); - if (IS_ERR(l3pmu->regs)) { - dev_err(&pdev->dev, "Can't map PMU @%pa\n", &memrc->start); + if (IS_ERR(l3pmu->regs)) return PTR_ERR(l3pmu->regs); - } qcom_l3_cache__init(l3pmu); diff --git a/drivers/perf/thunderx2_pmu.c b/drivers/perf/thunderx2_pmu.c index 06a6d569b0b5..fc1a376ee906 100644 --- a/drivers/perf/thunderx2_pmu.c +++ b/drivers/perf/thunderx2_pmu.c @@ -817,10 +817,8 @@ static struct tx2_uncore_pmu *tx2_uncore_pmu_init_dev(struct device *dev, } base = devm_ioremap_resource(dev, &res); - if (IS_ERR(base)) { - dev_err(dev, "PMU type %d: Fail to map resource\n", type); + if (IS_ERR(base)) return NULL; - } tx2_pmu = devm_kzalloc(dev, sizeof(*tx2_pmu), GFP_KERNEL); if (!tx2_pmu) diff --git a/drivers/perf/xgene_pmu.c b/drivers/perf/xgene_pmu.c index ffe3bdeec845..2b6d476bd213 100644 --- a/drivers/perf/xgene_pmu.c +++ b/drivers/perf/xgene_pmu.c @@ -278,17 +278,14 @@ static const struct attribute_group mc_pmu_v3_format_attr_group = { static ssize_t xgene_pmu_event_show(struct device *dev, struct device_attribute *attr, char *buf) { - struct dev_ext_attribute *eattr; + struct perf_pmu_events_attr *pmu_attr = + container_of(attr, struct perf_pmu_events_attr, attr); - eattr = container_of(attr, struct dev_ext_attribute, attr); - return sysfs_emit(buf, "config=0x%lx\n", (unsigned long) eattr->var); + return sysfs_emit(buf, "config=0x%llx\n", pmu_attr->id); } #define XGENE_PMU_EVENT_ATTR(_name, _config) \ - (&((struct dev_ext_attribute[]) { \ - { .attr = __ATTR(_name, S_IRUGO, xgene_pmu_event_show, NULL), \ - .var = (void *) _config, } \ - })[0].attr.attr) + PMU_EVENT_ATTR_ID(_name, xgene_pmu_event_show, _config) static struct attribute *l3c_pmu_events_attrs[] = { XGENE_PMU_EVENT_ATTR(cycle-count, 0x00), @@ -604,15 +601,15 @@ static const struct attribute_group mc_pmu_v3_events_attr_group = { /* * sysfs cpumask attributes */ -static ssize_t xgene_pmu_cpumask_show(struct device *dev, - struct device_attribute *attr, char *buf) +static ssize_t cpumask_show(struct device *dev, + struct device_attribute *attr, char *buf) { struct xgene_pmu_dev *pmu_dev = to_pmu_dev(dev_get_drvdata(dev)); return cpumap_print_to_pagebuf(true, buf, &pmu_dev->parent->cpu); } -static DEVICE_ATTR(cpumask, S_IRUGO, xgene_pmu_cpumask_show, NULL); +static DEVICE_ATTR_RO(cpumask); static struct attribute *xgene_pmu_cpumask_attrs[] = { &dev_attr_cpumask.attr, diff --git a/drivers/soc/ixp4xx/ixp4xx-npe.c b/drivers/soc/ixp4xx/ixp4xx-npe.c index ec90b44fa0cd..3c158251a58b 100644 --- a/drivers/soc/ixp4xx/ixp4xx-npe.c +++ b/drivers/soc/ixp4xx/ixp4xx-npe.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -679,6 +680,7 @@ static int ixp4xx_npe_probe(struct platform_device *pdev) { int i, found = 0; struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; struct resource *res; for (i = 0; i < NPE_COUNT; i++) { @@ -711,6 +713,11 @@ static int ixp4xx_npe_probe(struct platform_device *pdev) if (!found) return -ENODEV; + + /* Spawn crypto subdevice if using device tree */ + if (IS_ENABLED(CONFIG_OF) && np) + devm_of_platform_populate(dev); + return 0; } diff --git a/drivers/staging/media/Kconfig b/drivers/staging/media/Kconfig index ca59986b20f8..e3aaae920847 100644 --- a/drivers/staging/media/Kconfig +++ b/drivers/staging/media/Kconfig @@ -42,4 +42,6 @@ source "drivers/staging/media/tegra-video/Kconfig" source "drivers/staging/media/ipu3/Kconfig" +source "drivers/staging/media/av7110/Kconfig" + endif diff --git a/drivers/staging/media/Makefile b/drivers/staging/media/Makefile index 716929a1a313..5b5afc5b03a0 100644 --- a/drivers/staging/media/Makefile +++ b/drivers/staging/media/Makefile @@ -10,3 +10,4 @@ obj-$(CONFIG_TEGRA_VDE) += tegra-vde/ obj-$(CONFIG_VIDEO_HANTRO) += hantro/ obj-$(CONFIG_VIDEO_IPU3_IMGU) += ipu3/ obj-$(CONFIG_VIDEO_ZORAN) += zoran/ +obj-$(CONFIG_DVB_AV7110) += av7110/ diff --git a/drivers/staging/media/atomisp/Makefile b/drivers/staging/media/atomisp/Makefile index 51498b2e85b8..606b7754fdfd 100644 --- a/drivers/staging/media/atomisp/Makefile +++ b/drivers/staging/media/atomisp/Makefile @@ -16,7 +16,6 @@ atomisp-objs += \ pci/atomisp_acc.o \ pci/atomisp_cmd.o \ pci/atomisp_compat_css20.o \ - pci/atomisp_compat_ioctl32.o \ pci/atomisp_csi2.o \ pci/atomisp_drvfs.o \ pci/atomisp_file.o \ diff --git a/drivers/staging/media/atomisp/TODO b/drivers/staging/media/atomisp/TODO index 6987bb2d32cf..2d1ef9eb262a 100644 --- a/drivers/staging/media/atomisp/TODO +++ b/drivers/staging/media/atomisp/TODO @@ -120,6 +120,11 @@ TODO for this driver until the other work is done, as there will be a lot of code churn until this driver becomes functional again. +16. Fix private ioctls to not need a compat_ioctl handler for running + 32-bit tasks. The compat code has been removed because of bugs, + and should not be needed for modern drivers. Fixing this properly + unfortunately means an incompatible ABI change. + Limitations =========== diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c index d170d0adfea4..687888d643df 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c @@ -300,7 +300,7 @@ static int gc0310_get_intg_factor(struct i2c_client *client, /* pixel clock calculattion */ dev->vt_pix_clk_freq_mhz = 14400000; // 16.8MHz buf->vt_pix_clk_freq_mhz = dev->vt_pix_clk_freq_mhz; - pr_info("vt_pix_clk_freq_mhz=%d\n", buf->vt_pix_clk_freq_mhz); + dev_dbg(&client->dev, "vt_pix_clk_freq_mhz=%d\n", buf->vt_pix_clk_freq_mhz); /* get integration time */ buf->coarse_integration_time_min = GC0310_COARSE_INTG_TIME_MIN; @@ -326,7 +326,7 @@ static int gc0310_get_intg_factor(struct i2c_client *client, if (ret) return ret; buf->crop_horizontal_start = val | (reg_val & 0xFF); - pr_info("crop_horizontal_start=%d\n", buf->crop_horizontal_start); + dev_dbg(&client->dev, "crop_horizontal_start=%d\n", buf->crop_horizontal_start); /* Getting crop_vertical_start */ ret = gc0310_read_reg(client, GC0310_8BIT, @@ -339,7 +339,7 @@ static int gc0310_get_intg_factor(struct i2c_client *client, if (ret) return ret; buf->crop_vertical_start = val | (reg_val & 0xFF); - pr_info("crop_vertical_start=%d\n", buf->crop_vertical_start); + dev_dbg(&client->dev, "crop_vertical_start=%d\n", buf->crop_vertical_start); /* Getting output_width */ ret = gc0310_read_reg(client, GC0310_8BIT, @@ -352,7 +352,7 @@ static int gc0310_get_intg_factor(struct i2c_client *client, if (ret) return ret; buf->output_width = val | (reg_val & 0xFF); - pr_info("output_width=%d\n", buf->output_width); + dev_dbg(&client->dev, "output_width=%d\n", buf->output_width); /* Getting output_height */ ret = gc0310_read_reg(client, GC0310_8BIT, @@ -365,12 +365,12 @@ static int gc0310_get_intg_factor(struct i2c_client *client, if (ret) return ret; buf->output_height = val | (reg_val & 0xFF); - pr_info("output_height=%d\n", buf->output_height); + dev_dbg(&client->dev, "output_height=%d\n", buf->output_height); buf->crop_horizontal_end = buf->crop_horizontal_start + buf->output_width - 1; buf->crop_vertical_end = buf->crop_vertical_start + buf->output_height - 1; - pr_info("crop_horizontal_end=%d\n", buf->crop_horizontal_end); - pr_info("crop_vertical_end=%d\n", buf->crop_vertical_end); + dev_dbg(&client->dev, "crop_horizontal_end=%d\n", buf->crop_horizontal_end); + dev_dbg(&client->dev, "crop_vertical_end=%d\n", buf->crop_vertical_end); /* Getting line_length_pck */ ret = gc0310_read_reg(client, GC0310_8BIT, @@ -389,7 +389,7 @@ static int gc0310_get_intg_factor(struct i2c_client *client, return ret; sh_delay = reg_val; buf->line_length_pck = buf->output_width + hori_blanking + sh_delay + 4; - pr_info("hori_blanking=%d sh_delay=%d line_length_pck=%d\n", hori_blanking, + dev_dbg(&client->dev, "hori_blanking=%d sh_delay=%d line_length_pck=%d\n", hori_blanking, sh_delay, buf->line_length_pck); /* Getting frame_length_lines */ @@ -404,7 +404,7 @@ static int gc0310_get_intg_factor(struct i2c_client *client, return ret; vert_blanking = val | (reg_val & 0xFF); buf->frame_length_lines = buf->output_height + vert_blanking; - pr_info("vert_blanking=%d frame_length_lines=%d\n", vert_blanking, + dev_dbg(&client->dev, "vert_blanking=%d frame_length_lines=%d\n", vert_blanking, buf->frame_length_lines); buf->binning_factor_x = res->bin_factor_x ? @@ -434,7 +434,7 @@ static int gc0310_set_gain(struct v4l2_subdev *sd, int gain) dgain = gain / 2; } - pr_info("gain=0x%x again=0x%x dgain=0x%x\n", gain, again, dgain); + dev_dbg(&client->dev, "gain=0x%x again=0x%x dgain=0x%x\n", gain, again, dgain); /* set analog gain */ ret = gc0310_write_reg(client, GC0310_8BIT, @@ -458,7 +458,7 @@ static int __gc0310_set_exposure(struct v4l2_subdev *sd, int coarse_itg, struct i2c_client *client = v4l2_get_subdevdata(sd); int ret; - pr_info("coarse_itg=%d gain=%d digitgain=%d\n", coarse_itg, gain, digitgain); + dev_dbg(&client->dev, "coarse_itg=%d gain=%d digitgain=%d\n", coarse_itg, gain, digitgain); /* set exposure */ ret = gc0310_write_reg(client, GC0310_8BIT, @@ -718,7 +718,6 @@ static int gc0310_init(struct v4l2_subdev *sd) struct i2c_client *client = v4l2_get_subdevdata(sd); struct gc0310_device *dev = to_gc0310_sensor(sd); - pr_info("%s S\n", __func__); mutex_lock(&dev->input_lock); /* set initial registers */ @@ -730,7 +729,6 @@ static int gc0310_init(struct v4l2_subdev *sd) mutex_unlock(&dev->input_lock); - pr_info("%s E\n", __func__); return ret; } @@ -796,7 +794,6 @@ static int power_up(struct v4l2_subdev *sd) struct i2c_client *client = v4l2_get_subdevdata(sd); int ret; - pr_info("%s S\n", __func__); if (!dev->platform_data) { dev_err(&client->dev, "no camera_sensor_platform_data"); @@ -823,7 +820,6 @@ static int power_up(struct v4l2_subdev *sd) msleep(100); - pr_info("%s E\n", __func__); return 0; fail_gpio: @@ -959,20 +955,17 @@ static int startup(struct v4l2_subdev *sd) struct i2c_client *client = v4l2_get_subdevdata(sd); int ret = 0; - pr_info("%s S\n", __func__); - ret = gc0310_write_reg_array(client, gc0310_res[dev->fmt_idx].regs); if (ret) { dev_err(&client->dev, "gc0310 write register err.\n"); return ret; } - pr_info("%s E\n", __func__); return ret; } static int gc0310_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -982,8 +975,6 @@ static int gc0310_set_fmt(struct v4l2_subdev *sd, int ret = 0; int idx = 0; - pr_info("%s S\n", __func__); - if (format->pad) return -EINVAL; @@ -1008,7 +999,7 @@ static int gc0310_set_fmt(struct v4l2_subdev *sd, fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; mutex_unlock(&dev->input_lock); return 0; } @@ -1020,8 +1011,8 @@ static int gc0310_set_fmt(struct v4l2_subdev *sd, return -EINVAL; } - printk("%s: before gc0310_write_reg_array %s\n", __func__, - gc0310_res[dev->fmt_idx].desc); + dev_dbg(&client->dev, "%s: before gc0310_write_reg_array %s\n", + __func__, gc0310_res[dev->fmt_idx].desc); ret = startup(sd); if (ret) { dev_err(&client->dev, "gc0310 startup err\n"); @@ -1035,14 +1026,13 @@ static int gc0310_set_fmt(struct v4l2_subdev *sd, goto err; } - pr_info("%s E\n", __func__); err: mutex_unlock(&dev->input_lock); return ret; } static int gc0310_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -1068,7 +1058,6 @@ static int gc0310_detect(struct i2c_client *client) int ret; u16 id; - pr_info("%s S\n", __func__); if (!i2c_check_functionality(adapter, I2C_FUNC_I2C)) return -ENODEV; @@ -1085,7 +1074,7 @@ static int gc0310_detect(struct i2c_client *client) return -ENODEV; } id = ((((u16)high) << 8) | (u16)low); - pr_info("sensor ID = 0x%x\n", id); + dev_dbg(&client->dev, "sensor ID = 0x%x\n", id); if (id != GC0310_ID) { dev_err(&client->dev, "sensor ID error, read id = 0x%x, target id = 0x%x\n", id, @@ -1095,8 +1084,6 @@ static int gc0310_detect(struct i2c_client *client) dev_dbg(&client->dev, "detect gc0310 success\n"); - pr_info("%s E\n", __func__); - return 0; } @@ -1106,7 +1093,7 @@ static int gc0310_s_stream(struct v4l2_subdev *sd, int enable) struct i2c_client *client = v4l2_get_subdevdata(sd); int ret; - pr_info("%s S enable=%d\n", __func__, enable); + dev_dbg(&client->dev, "%s S enable=%d\n", __func__, enable); mutex_lock(&dev->input_lock); if (enable) { @@ -1142,7 +1129,6 @@ static int gc0310_s_stream(struct v4l2_subdev *sd, int enable) } mutex_unlock(&dev->input_lock); - pr_info("%s E\n", __func__); return ret; } @@ -1153,7 +1139,6 @@ static int gc0310_s_config(struct v4l2_subdev *sd, struct i2c_client *client = v4l2_get_subdevdata(sd); int ret = 0; - pr_info("%s S\n", __func__); if (!platform_data) return -ENODEV; @@ -1196,7 +1181,6 @@ static int gc0310_s_config(struct v4l2_subdev *sd, } mutex_unlock(&dev->input_lock); - pr_info("%s E\n", __func__); return 0; fail_csi_cfg: @@ -1221,7 +1205,7 @@ static int gc0310_g_frame_interval(struct v4l2_subdev *sd, } static int gc0310_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= MAX_FMTS) @@ -1232,7 +1216,7 @@ static int gc0310_enum_mbus_code(struct v4l2_subdev *sd, } static int gc0310_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int index = fse->index; @@ -1365,7 +1349,6 @@ static int gc0310_probe(struct i2c_client *client) if (ret) gc0310_remove(client); - pr_info("%s E\n", __func__); return ret; out_free: v4l2_device_unregister_subdev(&dev->sd); diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c index 78147ffb6099..9363c1a52ae9 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c @@ -171,8 +171,8 @@ static int __gc2235_buf_reg_array(struct i2c_client *client, } static int __gc2235_write_reg_is_consecutive(struct i2c_client *client, - struct gc2235_write_ctrl *ctrl, - const struct gc2235_reg *next) + struct gc2235_write_ctrl *ctrl, + const struct gc2235_reg *next) { if (ctrl->index == 0) return 1; @@ -228,7 +228,7 @@ static int gc2235_g_focal(struct v4l2_subdev *sd, s32 *val) static int gc2235_g_fnumber(struct v4l2_subdev *sd, s32 *val) { - /*const f number for imx*/ + /* const f number for imx */ *val = (GC2235_F_NUMBER_DEFAULT_NUM << 16) | GC2235_F_NUMBER_DEM; return 0; } @@ -427,7 +427,8 @@ static long gc2235_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg) return 0; } -/* This returns the exposure time being used. This should only be used +/* + * This returns the exposure time being used. This should only be used * for filling in EXIF data, not for actual image processing. */ static int gc2235_q_exposure(struct v4l2_subdev *sd, s32 *value) @@ -658,9 +659,9 @@ static int gc2235_s_power(struct v4l2_subdev *sd, int on) { int ret; - if (on == 0) + if (on == 0) { ret = power_down(sd); - else { + } else { ret = power_up(sd); if (!ret) ret = __gc2235_init(sd); @@ -746,11 +747,12 @@ static int startup(struct v4l2_subdev *sd) int ret = 0; if (is_init == 0) { - /* force gc2235 to do a reset in res change, otherwise it - * can not output normal after switching res. and it is not - * necessary for first time run up after power on, for the sack - * of performance - */ + /* + * force gc2235 to do a reset in res change, otherwise it + * can not output normal after switching res. and it is not + * necessary for first time run up after power on, for the sack + * of performance + */ power_down(sd); power_up(sd); gc2235_write_reg_array(client, gc2235_init_settings); @@ -767,7 +769,7 @@ static int startup(struct v4l2_subdev *sd) } static int gc2235_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -796,7 +798,7 @@ static int gc2235_set_fmt(struct v4l2_subdev *sd, } fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; mutex_unlock(&dev->input_lock); return 0; } @@ -825,7 +827,7 @@ static int gc2235_set_fmt(struct v4l2_subdev *sd, } static int gc2235_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -904,7 +906,8 @@ static int gc2235_s_config(struct v4l2_subdev *sd, (struct camera_sensor_platform_data *)platform_data; mutex_lock(&dev->input_lock); - /* power off the module, then power on it in future + /* + * power off the module, then power on it in future * as first power on by board may not fulfill the * power on sequqence needed by the module */ @@ -963,7 +966,7 @@ static int gc2235_g_frame_interval(struct v4l2_subdev *sd, } static int gc2235_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= MAX_FMTS) @@ -974,7 +977,7 @@ static int gc2235_enum_mbus_code(struct v4l2_subdev *sd, } static int gc2235_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int index = fse->index; diff --git a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c b/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c index b93c80471f22..7a20d918a9d5 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c @@ -50,14 +50,16 @@ struct tbd_data_record_header { static int set_msr_configuration(struct i2c_client *client, uint8_t *bufptr, unsigned int size) { - /* The configuration data contains any number of sequences where + /* + * The configuration data contains any number of sequences where * the first byte (that is, uint8_t) that marks the number of bytes * in the sequence to follow, is indeed followed by the indicated * number of bytes of actual data to be written to sensor. * By convention, the first two bytes of actual data should be * understood as an address in the sensor address space (hibyte * followed by lobyte) where the remaining data in the sequence - * will be written. */ + * will be written. + */ u8 *ptr = bufptr; diff --git a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c index f5de81132177..11196180a206 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c @@ -475,10 +475,12 @@ static int gpio_ctrl(struct v4l2_subdev *sd, bool flag) if (!dev || !dev->platform_data) return -ENODEV; - /* Note: current modules wire only one GPIO signal (RESET#), + /* + * Note: current modules wire only one GPIO signal (RESET#), * but the schematic wires up two to the connector. BIOS * versions have been unfortunately inconsistent with which - * ACPI index RESET# is on, so hit both */ + * ACPI index RESET# is on, so hit both + */ if (flag) { ret = dev->platform_data->gpio0_ctrl(sd, 0); @@ -560,7 +562,7 @@ static int power_down(struct v4l2_subdev *sd) if (ret) dev_err(&client->dev, "vprog failed.\n"); - /*according to DS, 20ms is needed after power down*/ + /* according to DS, 20ms is needed after power down */ msleep(20); return ret; @@ -568,9 +570,9 @@ static int power_down(struct v4l2_subdev *sd) static int mt9m114_s_power(struct v4l2_subdev *sd, int power) { - if (power == 0) + if (power == 0) { return power_down(sd); - else { + } else { if (power_up(sd)) return -EINVAL; @@ -801,7 +803,7 @@ static int mt9m114_get_intg_factor(struct i2c_client *client, } static int mt9m114_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -822,7 +824,7 @@ static int mt9m114_get_fmt(struct v4l2_subdev *sd, } static int mt9m114_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -846,7 +848,7 @@ static int mt9m114_set_fmt(struct v4l2_subdev *sd, mt9m114_try_res(&width, &height); if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; return 0; } res_index = mt9m114_to_res(width, height); @@ -947,7 +949,7 @@ static int mt9m114_g_focal(struct v4l2_subdev *sd, s32 *val) static int mt9m114_g_fnumber(struct v4l2_subdev *sd, s32 *val) { - /*const f number for mt9m114*/ + /* const f number for mt9m114 */ *val = (MT9M114_F_NUMBER_DEFAULT_NUM << 16) | MT9M114_F_NUMBER_DEM; return 0; } @@ -998,44 +1000,48 @@ static long mt9m114_s_exposure(struct v4l2_subdev *sd, struct mt9m114_device *dev = to_mt9m114_sensor(sd); int ret = 0; unsigned int coarse_integration = 0; - unsigned int FLines = 0; - unsigned int FrameLengthLines = 0; /* ExposureTime.FrameLengthLines; */ - unsigned int AnalogGain, DigitalGain; - u32 AnalogGainToWrite = 0; + unsigned int f_lines = 0; + unsigned int frame_len_lines = 0; /* ExposureTime.FrameLengthLines; */ + unsigned int analog_gain, digital_gain; + u32 analog_gain_to_write = 0; dev_dbg(&client->dev, "%s(0x%X 0x%X 0x%X)\n", __func__, exposure->integration_time[0], exposure->gain[0], exposure->gain[1]); coarse_integration = exposure->integration_time[0]; - /* fine_integration = ExposureTime.FineIntegrationTime; */ - /* FrameLengthLines = ExposureTime.FrameLengthLines; */ - FLines = mt9m114_res[dev->res].lines_per_frame; - AnalogGain = exposure->gain[0]; - DigitalGain = exposure->gain[1]; + /* + * fine_integration = ExposureTime.FineIntegrationTime; + * frame_len_lines = ExposureTime.FrameLengthLines; + */ + f_lines = mt9m114_res[dev->res].lines_per_frame; + analog_gain = exposure->gain[0]; + digital_gain = exposure->gain[1]; if (!dev->streamon) { /*Save the first exposure values while stream is off*/ dev->first_exp = coarse_integration; - dev->first_gain = AnalogGain; - dev->first_diggain = DigitalGain; + dev->first_gain = analog_gain; + dev->first_diggain = digital_gain; } - /* DigitalGain = 0x400 * (((u16) DigitalGain) >> 8) + - ((unsigned int)(0x400 * (((u16) DigitalGain) & 0xFF)) >>8); */ + /* digital_gain = 0x400 * (((u16) digital_gain) >> 8) + */ + /* ((unsigned int)(0x400 * (((u16) digital_gain) & 0xFF)) >>8); */ /* set frame length */ - if (FLines < coarse_integration + 6) - FLines = coarse_integration + 6; - if (FLines < FrameLengthLines) - FLines = FrameLengthLines; - ret = mt9m114_write_reg(client, MISENSOR_16BIT, 0x300A, FLines); + if (f_lines < coarse_integration + 6) + f_lines = coarse_integration + 6; + if (f_lines < frame_len_lines) + f_lines = frame_len_lines; + ret = mt9m114_write_reg(client, MISENSOR_16BIT, 0x300A, f_lines); if (ret) { - v4l2_err(client, "%s: fail to set FLines\n", __func__); + v4l2_err(client, "%s: fail to set f_lines\n", __func__); return -EINVAL; } /* set coarse integration */ - /* 3A provide real exposure time. - should not translate to any value here. */ + /* + * 3A provide real exposure time. + * should not translate to any value here. + */ ret = mt9m114_write_reg(client, MISENSOR_16BIT, REG_EXPO_COARSE, (u16)(coarse_integration)); if (ret) { @@ -1044,38 +1050,40 @@ static long mt9m114_s_exposure(struct v4l2_subdev *sd, } /* - // set analog/digital gain - switch(AnalogGain) + * set analog/digital gain + switch(analog_gain) { case 0: - AnalogGainToWrite = 0x0; + analog_gain_to_write = 0x0; break; case 1: - AnalogGainToWrite = 0x20; + analog_gain_to_write = 0x20; break; case 2: - AnalogGainToWrite = 0x60; + analog_gain_to_write = 0x60; break; case 4: - AnalogGainToWrite = 0xA0; + analog_gain_to_write = 0xA0; break; case 8: - AnalogGainToWrite = 0xE0; + analog_gain_to_write = 0xE0; break; default: - AnalogGainToWrite = 0x20; + analog_gain_to_write = 0x20; break; } */ - if (DigitalGain >= 16 || DigitalGain <= 1) - DigitalGain = 1; - /* AnalogGainToWrite = - (u16)((DigitalGain << 12) | AnalogGainToWrite); */ - AnalogGainToWrite = (u16)((DigitalGain << 12) | (u16)AnalogGain); + if (digital_gain >= 16 || digital_gain <= 1) + digital_gain = 1; + /* + * analog_gain_to_write = (u16)((digital_gain << 12) + * | analog_gain_to_write); + */ + analog_gain_to_write = (u16)((digital_gain << 12) | (u16)analog_gain); ret = mt9m114_write_reg(client, MISENSOR_16BIT, - REG_GAIN, AnalogGainToWrite); + REG_GAIN, analog_gain_to_write); if (ret) { - v4l2_err(client, "%s: fail to set AnalogGainToWrite\n", + v4l2_err(client, "%s: fail to set analog_gain_to_write\n", __func__); return -EINVAL; } @@ -1095,8 +1103,10 @@ static long mt9m114_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg) return 0; } -/* This returns the exposure time being used. This should only be used - for filling in EXIF data, not for actual image processing. */ +/* + * This returns the exposure time being used. This should only be used + * for filling in EXIF data, not for actual image processing. + */ static int mt9m114_g_exposure(struct v4l2_subdev *sd, s32 *value) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -1158,7 +1168,7 @@ static int mt9m114_s_exposure_metering(struct v4l2_subdev *sd, s32 val) * This function is for touch exposure feature. */ static int mt9m114_s_exposure_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -1247,7 +1257,8 @@ static int mt9m114_s_ev(struct v4l2_subdev *sd, s32 val) s32 luma = 0x37; int err; - /* EV value only support -2 to 2 + /* + * EV value only support -2 to 2 * 0: 0x37, 1:0x47, 2:0x57, -1:0x27, -2:0x17 */ if (val < -2 || val > 2) @@ -1295,9 +1306,10 @@ static int mt9m114_g_ev(struct v4l2_subdev *sd, s32 *val) return 0; } -/* Fake interface +/* + * Fake interface * mt9m114 now can not support 3a_lock -*/ + */ static int mt9m114_s_3a_lock(struct v4l2_subdev *sd, s32 val) { aaalock = val; @@ -1719,7 +1731,7 @@ static int mt9m114_s_stream(struct v4l2_subdev *sd, int enable) } static int mt9m114_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index) @@ -1730,7 +1742,7 @@ static int mt9m114_enum_mbus_code(struct v4l2_subdev *sd, } static int mt9m114_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { unsigned int index = fse->index; @@ -1843,7 +1855,7 @@ static int mt9m114_probe(struct i2c_client *client) return ret; } - /*TODO add format code here*/ + /* TODO add format code here */ dev->sd.flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; dev->pad.flags = MEDIA_PAD_FL_SOURCE; dev->format.code = MEDIA_BUS_FMT_SGRBG10_1X10; diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c index c90730513438..2111e4a478c1 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c @@ -127,7 +127,7 @@ static int ov2680_g_focal(struct v4l2_subdev *sd, s32 *val) static int ov2680_g_fnumber(struct v4l2_subdev *sd, s32 *val) { - /*const f number for ov2680*/ + /* const f number for ov2680 */ *val = (OV2680_F_NUMBER_DEFAULT_NUM << 16) | OV2680_F_NUMBER_DEM; return 0; @@ -399,7 +399,8 @@ static long ov2680_ioctl(struct v4l2_subdev *sd, unsigned int cmd, void *arg) return 0; } -/* This returns the exposure time being used. This should only be used +/* + * This returns the exposure time being used. This should only be used * for filling in EXIF data, not for actual image processing. */ static int ov2680_q_exposure(struct v4l2_subdev *sd, s32 *value) @@ -461,11 +462,11 @@ static int ov2680_v_flip(struct v4l2_subdev *sd, s32 value) ret = ov2680_read_reg(client, 1, OV2680_FLIP_REG, &val); if (ret) return ret; - if (value) { + if (value) val |= OV2680_FLIP_MIRROR_BIT_ENABLE; - } else { + else val &= ~OV2680_FLIP_MIRROR_BIT_ENABLE; - } + ret = ov2680_write_reg(client, 1, OV2680_FLIP_REG, val); if (ret) @@ -727,11 +728,13 @@ static int gpio_ctrl(struct v4l2_subdev *sd, bool flag) if (!dev || !dev->platform_data) return -ENODEV; - /* The OV2680 documents only one GPIO input (#XSHUTDN), but + /* + * The OV2680 documents only one GPIO input (#XSHUTDN), but * existing integrations often wire two (reset/power_down) * because that is the way other sensors work. There is no * way to tell how it is wired internally, so existing - * firmwares expose both and we drive them symmetrically. */ + * firmwares expose both and we drive them symmetrically. + */ if (flag) { ret = dev->platform_data->gpio0_ctrl(sd, 1); usleep_range(10000, 15000); @@ -911,7 +914,7 @@ static int get_resolution_index(int w, int h) } static int ov2680_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -948,7 +951,7 @@ static int ov2680_set_fmt(struct v4l2_subdev *sd, } fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; mutex_unlock(&dev->input_lock); return 0; } @@ -977,7 +980,8 @@ static int ov2680_set_fmt(struct v4l2_subdev *sd, goto err; } - /*recall flip functions to avoid flip registers + /* + * recall flip functions to avoid flip registers * were overridden by default setting */ if (h_flag) @@ -987,7 +991,8 @@ static int ov2680_set_fmt(struct v4l2_subdev *sd, v4l2_info(client, "\n%s idx %d\n", __func__, dev->fmt_idx); - /*ret = startup(sd); + /* + * ret = startup(sd); * if (ret) * dev_err(&client->dev, "ov2680 startup err\n"); */ @@ -997,7 +1002,7 @@ static int ov2680_set_fmt(struct v4l2_subdev *sd, } static int ov2680_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -1096,7 +1101,8 @@ static int ov2680_s_config(struct v4l2_subdev *sd, (struct camera_sensor_platform_data *)platform_data; mutex_lock(&dev->input_lock); - /* power off the module, then power on it in future + /* + * power off the module, then power on it in future * as first power on by board may not fulfill the * power on sequqence needed by the module */ @@ -1155,7 +1161,7 @@ static int ov2680_g_frame_interval(struct v4l2_subdev *sd, } static int ov2680_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= MAX_FMTS) @@ -1166,7 +1172,7 @@ static int ov2680_enum_mbus_code(struct v4l2_subdev *sd, } static int ov2680_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int index = fse->index; diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c index 1209492c1826..90d0871a78a3 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c @@ -49,8 +49,8 @@ static int ov2722_read_reg(struct i2c_client *client, return -ENODEV; } - if (data_length != OV2722_8BIT && data_length != OV2722_16BIT - && data_length != OV2722_32BIT) { + if (data_length != OV2722_8BIT && data_length != OV2722_16BIT && + data_length != OV2722_32BIT) { dev_err(&client->dev, "%s error, invalid data length\n", __func__); return -EINVAL; @@ -212,8 +212,8 @@ static int __ov2722_buf_reg_array(struct i2c_client *client, } static int __ov2722_write_reg_is_consecutive(struct i2c_client *client, - struct ov2722_write_ctrl *ctrl, - const struct ov2722_reg *next) + struct ov2722_write_ctrl *ctrl, + const struct ov2722_reg *next) { if (ctrl->index == 0) return 1; @@ -774,11 +774,11 @@ static int ov2722_s_power(struct v4l2_subdev *sd, int on) if (on == 0) return power_down(sd); - else { - ret = power_up(sd); - if (!ret) - return ov2722_init(sd); - } + + ret = power_up(sd); + if (!ret) + return ov2722_init(sd); + return ret; } @@ -876,7 +876,7 @@ static int startup(struct v4l2_subdev *sd) } static int ov2722_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -906,7 +906,7 @@ static int ov2722_set_fmt(struct v4l2_subdev *sd, } fmt->code = MEDIA_BUS_FMT_SGRBG10_1X10; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; mutex_unlock(&dev->input_lock); return 0; } @@ -961,7 +961,7 @@ static int ov2722_set_fmt(struct v4l2_subdev *sd, } static int ov2722_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -1104,7 +1104,7 @@ static int ov2722_g_frame_interval(struct v4l2_subdev *sd, } static int ov2722_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= MAX_FMTS) @@ -1115,7 +1115,7 @@ static int ov2722_enum_mbus_code(struct v4l2_subdev *sd, } static int ov2722_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int index = fse->index; diff --git a/drivers/staging/media/atomisp/i2c/mt9m114.h b/drivers/staging/media/atomisp/i2c/mt9m114.h index 787bbf59e895..bcce18b65fa6 100644 --- a/drivers/staging/media/atomisp/i2c/mt9m114.h +++ b/drivers/staging/media/atomisp/i2c/mt9m114.h @@ -764,8 +764,10 @@ static struct misensor_reg const mt9m114_common[] = { {MISENSOR_8BIT, 0xC85C, 0x03}, /* cam_crop_cropmode = 3 */ {MISENSOR_16BIT, 0xC868, 0x0280}, /* cam_output_width = 952 */ {MISENSOR_16BIT, 0xC86A, 0x01E0}, /* cam_output_height = 538 */ - /* LOAD = Step3-Recommended - * Patch,Errata and Sensor optimization Setting */ + /* + * LOAD = Step3-Recommended + * Patch, Errata and Sensor optimization Setting + */ {MISENSOR_16BIT, 0x316A, 0x8270}, /* DAC_TXLO_ROW */ {MISENSOR_16BIT, 0x316C, 0x8270}, /* DAC_TXLO */ {MISENSOR_16BIT, 0x3ED0, 0x2305}, /* DAC_LD_4_5 */ diff --git a/drivers/staging/media/atomisp/i2c/ov2680.h b/drivers/staging/media/atomisp/i2c/ov2680.h index 49920245e064..4d43b45915e5 100644 --- a/drivers/staging/media/atomisp/i2c/ov2680.h +++ b/drivers/staging/media/atomisp/i2c/ov2680.h @@ -459,8 +459,8 @@ static struct ov2680_reg const ov2680_656x496_30fps[] = { }; /* -* 800x600 30fps VBlanking 1lane 10Bit (binning) -*/ + * 800x600 30fps VBlanking 1lane 10Bit (binning) + */ static struct ov2680_reg const ov2680_720x592_30fps[] = { {0x3086, 0x01}, {0x3501, 0x26}, @@ -504,8 +504,8 @@ static struct ov2680_reg const ov2680_720x592_30fps[] = { }; /* -* 800x600 30fps VBlanking 1lane 10Bit (binning) -*/ + * 800x600 30fps VBlanking 1lane 10Bit (binning) + */ static struct ov2680_reg const ov2680_800x600_30fps[] = { {0x3086, 0x01}, {0x3501, 0x26}, @@ -634,7 +634,7 @@ static struct ov2680_reg const ov2680_1296x976_30fps[] = { /* * 1456*1096 30fps VBlanking 1lane 10bit(no-scaling) -*/ + */ static struct ov2680_reg const ov2680_1456x1096_30fps[] = { {0x3086, 0x00}, {0x3501, 0x48}, diff --git a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c index e698b63d6cb7..0828ca9ab6f2 100644 --- a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c +++ b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c @@ -1577,7 +1577,7 @@ static int startup(struct v4l2_subdev *sd) } static int ov5693_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -1608,7 +1608,7 @@ static int ov5693_set_fmt(struct v4l2_subdev *sd, fmt->code = MEDIA_BUS_FMT_SBGGR10_1X10; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; mutex_unlock(&dev->input_lock); return 0; } @@ -1676,7 +1676,7 @@ static int ov5693_set_fmt(struct v4l2_subdev *sd, } static int ov5693_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -1825,7 +1825,7 @@ static int ov5693_g_frame_interval(struct v4l2_subdev *sd, } static int ov5693_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= MAX_FMTS) @@ -1836,7 +1836,7 @@ static int ov5693_enum_mbus_code(struct v4l2_subdev *sd, } static int ov5693_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { int index = fse->index; diff --git a/drivers/staging/media/atomisp/pci/atomisp_acc.c b/drivers/staging/media/atomisp/pci/atomisp_acc.c index f638d0bd09fe..9a1751895ab0 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_acc.c +++ b/drivers/staging/media/atomisp/pci/atomisp_acc.c @@ -77,8 +77,8 @@ acc_get_fw(struct atomisp_sub_device *asd, unsigned int handle) struct atomisp_acc_fw *acc_fw; list_for_each_entry(acc_fw, &asd->acc.fw, list) - if (acc_fw->handle == handle) - return acc_fw; + if (acc_fw->handle == handle) + return acc_fw; return NULL; } @@ -464,9 +464,11 @@ int atomisp_acc_load_extensions(struct atomisp_sub_device *asd) continue; for (i = 0; i < ARRAY_SIZE(acc_flag_to_pipe); i++) { - /* QoS (ACC pipe) acceleration stages are currently - * allowed only in continuous mode. Skip them for - * all other modes. */ + /* + * QoS (ACC pipe) acceleration stages are + * currently allowed only in continuous mode. + * Skip them for all other modes. + */ if (!continuous && acc_flag_to_pipe[i].flag == ATOMISP_ACC_FW_LOAD_FL_ACC) diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c index 14abc1ca00e8..366161cff560 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c +++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c @@ -1138,9 +1138,10 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, asd->frame_status[vb->i] = ATOMISP_FRAME_STATUS_OK; } - } else + } else { asd->frame_status[vb->i] = ATOMISP_FRAME_STATUS_OK; + } } else { asd->frame_status[vb->i] = ATOMISP_FRAME_STATUS_OK; } @@ -4841,6 +4842,9 @@ int atomisp_try_fmt(struct video_device *vdev, struct v4l2_pix_format *f, struct atomisp_device *isp = video_get_drvdata(vdev); struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format format = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -4876,7 +4880,7 @@ int atomisp_try_fmt(struct video_device *vdev, struct v4l2_pix_format *f, snr_mbus_fmt->width, snr_mbus_fmt->height); ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera, - pad, set_fmt, &pad_cfg, &format); + pad, set_fmt, &pad_state, &format); if (ret) return ret; @@ -4941,9 +4945,9 @@ atomisp_try_fmt_file(struct atomisp_device *isp, struct v4l2_format *f) depth = get_pixel_depth(pixelformat); - if (field == V4L2_FIELD_ANY) + if (field == V4L2_FIELD_ANY) { field = V4L2_FIELD_NONE; - else if (field != V4L2_FIELD_NONE) { + } else if (field != V4L2_FIELD_NONE) { dev_err(isp->dev, "Wrong output field\n"); return -EINVAL; } @@ -5251,11 +5255,11 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, atomisp_output_fmts[] in atomisp_v4l2.c */ vf_ffmt.code = V4L2_MBUS_FMT_CUSTOM_YUV420; - atomisp_subdev_set_selection(&asd->subdev, fh.pad, + atomisp_subdev_set_selection(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SOURCE_VF, V4L2_SEL_TGT_COMPOSE, 0, &vf_size); - atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, + atomisp_subdev_set_ffmt(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SOURCE_VF, &vf_ffmt); asd->video_out_vf.sh_fmt = IA_CSS_FRAME_FORMAT_NV12; @@ -5492,6 +5496,9 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev, struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd; const struct atomisp_format_bridge *format; struct v4l2_subdev_pad_config pad_cfg; + struct v4l2_subdev_state pad_state = { + .pads = &pad_cfg + }; struct v4l2_subdev_format vformat = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -5530,7 +5537,7 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev, source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) { vformat.which = V4L2_SUBDEV_FORMAT_TRY; ret = v4l2_subdev_call(isp->inputs[asd->input_curr].camera, - pad, set_fmt, &pad_cfg, &vformat); + pad, set_fmt, &pad_state, &vformat); if (ret) return ret; if (ffmt->width < req_ffmt->width || @@ -5568,7 +5575,7 @@ static int atomisp_set_fmt_to_snr(struct video_device *vdev, asd->params.video_dis_en = false; } - atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, + atomisp_subdev_set_ffmt(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK, ffmt); @@ -5647,7 +5654,7 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) } atomisp_subdev_set_selection( - &asd->subdev, fh.pad, + &asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, source_pad, V4L2_SEL_TGT_COMPOSE, 0, &r); @@ -5777,7 +5784,7 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) ATOMISP_SUBDEV_PAD_SINK); isp_source_fmt.code = format_bridge->mbus_code; - atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, + atomisp_subdev_set_ffmt(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, source_pad, &isp_source_fmt); @@ -5896,13 +5903,13 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) isp_sink_crop.height = f->fmt.pix.height; } - atomisp_subdev_set_selection(&asd->subdev, fh.pad, + atomisp_subdev_set_selection(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK, V4L2_SEL_TGT_CROP, V4L2_SEL_FLAG_KEEP_CONFIG, &isp_sink_crop); - atomisp_subdev_set_selection(&asd->subdev, fh.pad, + atomisp_subdev_set_selection(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, source_pad, V4L2_SEL_TGT_COMPOSE, 0, &isp_sink_crop); @@ -5921,7 +5928,7 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) f->fmt.pix.height); } - atomisp_subdev_set_selection(&asd->subdev, fh.pad, + atomisp_subdev_set_selection(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, source_pad, V4L2_SEL_TGT_COMPOSE, 0, @@ -5955,14 +5962,14 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) f->fmt.pix.width, ATOM_ISP_STEP_HEIGHT); } - atomisp_subdev_set_selection(&asd->subdev, fh.pad, + atomisp_subdev_set_selection(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK, V4L2_SEL_TGT_CROP, V4L2_SEL_FLAG_KEEP_CONFIG, &sink_crop); } - atomisp_subdev_set_selection(&asd->subdev, fh.pad, + atomisp_subdev_set_selection(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, source_pad, V4L2_SEL_TGT_COMPOSE, 0, @@ -6053,7 +6060,8 @@ int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f) ffmt.height = f->fmt.pix.height; ffmt.code = format_bridge->mbus_code; - atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, V4L2_SUBDEV_FORMAT_ACTIVE, + atomisp_subdev_set_ffmt(&asd->subdev, fh.state, + V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK, &ffmt); return 0; @@ -6564,17 +6572,17 @@ static int atomisp_get_pipe_id(struct atomisp_video_pipe *pipe) { struct atomisp_sub_device *asd = pipe->asd; - if (ATOMISP_USE_YUVPP(asd)) + if (ATOMISP_USE_YUVPP(asd)) { return IA_CSS_PIPE_ID_YUVPP; - else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) + } else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) { return IA_CSS_PIPE_ID_VIDEO; - else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) + } else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) { return IA_CSS_PIPE_ID_CAPTURE; - else if (pipe == &asd->video_out_video_capture) + } else if (pipe == &asd->video_out_video_capture) { return IA_CSS_PIPE_ID_VIDEO; - else if (pipe == &asd->video_out_vf) + } else if (pipe == &asd->video_out_vf) { return IA_CSS_PIPE_ID_CAPTURE; - else if (pipe == &asd->video_out_preview) { + } else if (pipe == &asd->video_out_preview) { if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) return IA_CSS_PIPE_ID_VIDEO; else diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.h b/drivers/staging/media/atomisp/pci/atomisp_cmd.h index 412baeb91944..e8bdd264d31b 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_cmd.h +++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.h @@ -49,9 +49,7 @@ struct ia_css_frame; /* FIXME: check if can go */ extern int atomisp_punit_hpll_freq; -/* - * Helper function - */ +/* Helper function */ void dump_sp_dmem(struct atomisp_device *isp, unsigned int addr, unsigned int size); struct camera_mipi_info *atomisp_to_sensor_mipi_info(struct v4l2_subdev *sd); @@ -65,9 +63,7 @@ bool atomisp_buffers_queued(struct atomisp_sub_device *asd); /* ISP2401 */ bool atomisp_buffers_queued_pipe(struct atomisp_video_pipe *pipe); -/* - * Interrupt functions - */ +/* Interrupt functions */ void atomisp_msi_irq_init(struct atomisp_device *isp); void atomisp_msi_irq_uninit(struct atomisp_device *isp); void atomisp_wdt_work(struct work_struct *work); @@ -82,15 +78,10 @@ int atomisp_get_frame_pgnr(struct atomisp_device *isp, const struct ia_css_frame *frame, u32 *p_pgnr); void atomisp_delayed_init_work(struct work_struct *work); -/* - * Get internal fmt according to V4L2 fmt - */ - +/* Get internal fmt according to V4L2 fmt */ bool atomisp_is_viewfinder_support(struct atomisp_device *isp); -/* - * ISP features control function - */ +/* ISP features control function */ /* * Function to set sensor runmode by user when @@ -105,9 +96,7 @@ int atomisp_set_sensor_runmode(struct atomisp_sub_device *asd, int atomisp_gdc_cac(struct atomisp_sub_device *asd, int flag, __s32 *value); -/* - * Function to enable/disable low light mode (including ANR) - */ +/* Function to enable/disable low light mode (including ANR) */ int atomisp_low_light(struct atomisp_sub_device *asd, int flag, __s32 *value); @@ -120,91 +109,63 @@ int atomisp_xnr(struct atomisp_sub_device *asd, int flag, int *arg); int atomisp_formats(struct atomisp_sub_device *asd, int flag, struct atomisp_formats_config *config); -/* - * Function to configure noise reduction - */ +/* Function to configure noise reduction */ int atomisp_nr(struct atomisp_sub_device *asd, int flag, struct atomisp_nr_config *config); -/* - * Function to configure temporal noise reduction (TNR) - */ +/* Function to configure temporal noise reduction (TNR) */ int atomisp_tnr(struct atomisp_sub_device *asd, int flag, struct atomisp_tnr_config *config); -/* - * Function to configure black level compensation - */ +/* Function to configure black level compensation */ int atomisp_black_level(struct atomisp_sub_device *asd, int flag, struct atomisp_ob_config *config); -/* - * Function to configure edge enhancement - */ +/* Function to configure edge enhancement */ int atomisp_ee(struct atomisp_sub_device *asd, int flag, struct atomisp_ee_config *config); -/* - * Function to update Gamma table for gamma, brightness and contrast config - */ +/* Function to update Gamma table for gamma, brightness and contrast config */ int atomisp_gamma(struct atomisp_sub_device *asd, int flag, struct atomisp_gamma_table *config); -/* - * Function to update Ctc table for Chroma Enhancement - */ + +/* Function to update Ctc table for Chroma Enhancement */ int atomisp_ctc(struct atomisp_sub_device *asd, int flag, struct atomisp_ctc_table *config); -/* - * Function to update gamma correction parameters - */ +/* Function to update gamma correction parameters */ int atomisp_gamma_correction(struct atomisp_sub_device *asd, int flag, struct atomisp_gc_config *config); -/* - * Function to update Gdc table for gdc - */ +/* Function to update Gdc table for gdc */ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag, struct atomisp_morph_table *config); -/* - * Function to update table for macc - */ +/* Function to update table for macc */ int atomisp_macc_table(struct atomisp_sub_device *asd, int flag, struct atomisp_macc_config *config); -/* - * Function to get DIS statistics. - */ + +/* Function to get DIS statistics. */ int atomisp_get_dis_stat(struct atomisp_sub_device *asd, struct atomisp_dis_statistics *stats); -/* - * Function to get DVS2 BQ resolution settings - */ +/* Function to get DVS2 BQ resolution settings */ int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd, struct atomisp_dvs2_bq_resolutions *bq_res); -/* - * Function to set the DIS coefficients. - */ +/* Function to set the DIS coefficients. */ int atomisp_set_dis_coefs(struct atomisp_sub_device *asd, struct atomisp_dis_coefficients *coefs); -/* - * Function to set the DIS motion vector. - */ +/* Function to set the DIS motion vector. */ int atomisp_set_dis_vector(struct atomisp_sub_device *asd, struct atomisp_dis_vector *vector); -/* - * Function to set/get 3A stat from isp - */ +/* Function to set/get 3A stat from isp */ int atomisp_3a_stat(struct atomisp_sub_device *asd, int flag, struct atomisp_3a_statistics *config); -/* - * Function to get metadata from isp - */ +/* Function to get metadata from isp */ int atomisp_get_metadata(struct atomisp_sub_device *asd, int flag, struct atomisp_metadata *config); @@ -213,84 +174,59 @@ int atomisp_get_metadata_by_type(struct atomisp_sub_device *asd, int flag, int atomisp_set_parameters(struct video_device *vdev, struct atomisp_parameters *arg); -/* - * Function to set/get isp parameters to isp - */ + +/* Function to set/get isp parameters to isp */ int atomisp_param(struct atomisp_sub_device *asd, int flag, struct atomisp_parm *config); -/* - * Function to configure color effect of the image - */ +/* Function to configure color effect of the image */ int atomisp_color_effect(struct atomisp_sub_device *asd, int flag, __s32 *effect); -/* - * Function to configure bad pixel correction - */ +/* Function to configure bad pixel correction */ int atomisp_bad_pixel(struct atomisp_sub_device *asd, int flag, __s32 *value); -/* - * Function to configure bad pixel correction params - */ +/* Function to configure bad pixel correction params */ int atomisp_bad_pixel_param(struct atomisp_sub_device *asd, int flag, struct atomisp_dp_config *config); -/* - * Function to enable/disable video image stablization - */ +/* Function to enable/disable video image stablization */ int atomisp_video_stable(struct atomisp_sub_device *asd, int flag, __s32 *value); -/* - * Function to configure fixed pattern noise - */ +/* Function to configure fixed pattern noise */ int atomisp_fixed_pattern(struct atomisp_sub_device *asd, int flag, __s32 *value); -/* - * Function to configure fixed pattern noise table - */ +/* Function to configure fixed pattern noise table */ int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd, struct v4l2_framebuffer *config); -/* - * Function to configure false color correction - */ +/* Function to configure false color correction */ int atomisp_false_color(struct atomisp_sub_device *asd, int flag, __s32 *value); -/* - * Function to configure false color correction params - */ +/* Function to configure false color correction params */ int atomisp_false_color_param(struct atomisp_sub_device *asd, int flag, struct atomisp_de_config *config); -/* - * Function to configure white balance params - */ +/* Function to configure white balance params */ int atomisp_white_balance_param(struct atomisp_sub_device *asd, int flag, struct atomisp_wb_config *config); int atomisp_3a_config_param(struct atomisp_sub_device *asd, int flag, struct atomisp_3a_config *config); -/* - * Function to setup digital zoom - */ +/* Function to setup digital zoom */ int atomisp_digital_zoom(struct atomisp_sub_device *asd, int flag, __s32 *value); -/* - * Function set camera_prefiles.xml current sensor pixel array size - */ +/* Function set camera_prefiles.xml current sensor pixel array size */ int atomisp_set_array_res(struct atomisp_sub_device *asd, struct atomisp_resolution *config); -/* - * Function to calculate real zoom region for every pipe - */ +/* Function to calculate real zoom region for every pipe */ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd, struct ia_css_dz_config *dz_config, enum ia_css_pipe_id css_pipe_id); @@ -371,9 +307,7 @@ void atomisp_css_flush(struct atomisp_device *isp); int atomisp_source_pad_to_stream_id(struct atomisp_sub_device *asd, uint16_t source_pad); -/* - * Events. Only one event has to be exported for now. - */ +/* Events. Only one event has to be exported for now. */ void atomisp_eof_event(struct atomisp_sub_device *asd, uint8_t exp_id); enum mipi_port_id __get_mipi_port(struct atomisp_device *isp, @@ -389,34 +323,25 @@ void atomisp_free_css_parameters(struct atomisp_css_params *css_param); void atomisp_handle_parameter_and_buffer(struct atomisp_video_pipe *pipe); void atomisp_flush_params_queue(struct atomisp_video_pipe *asd); -/* - * Function to do Raw Buffer related operation, after enable Lock Unlock Raw Buffer - */ + +/* Function to do Raw Buffer related operation, after enable Lock Unlock Raw Buffer */ int atomisp_exp_id_unlock(struct atomisp_sub_device *asd, int *exp_id); int atomisp_exp_id_capture(struct atomisp_sub_device *asd, int *exp_id); -/* - * Function to update Raw Buffer bitmap - */ +/* Function to update Raw Buffer bitmap */ int atomisp_set_raw_buffer_bitmap(struct atomisp_sub_device *asd, int exp_id); void atomisp_init_raw_buffer_bitmap(struct atomisp_sub_device *asd); -/* - * Function to enable/disable zoom for capture pipe - */ +/* Function to enable/disable zoom for capture pipe */ int atomisp_enable_dz_capt_pipe(struct atomisp_sub_device *asd, unsigned int *enable); -/* - * Function to get metadata type bu pipe id - */ +/* Function to get metadata type bu pipe id */ enum atomisp_metadata_type atomisp_get_metadata_type(struct atomisp_sub_device *asd, enum ia_css_pipe_id pipe_id); -/* - * Function for HAL to inject a fake event to wake up poll thread - */ +/* Function for HAL to inject a fake event to wake up poll thread */ int atomisp_inject_a_fake_event(struct atomisp_sub_device *asd, int *event); /* diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c index ce3165291eec..f60198bb8a1a 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c +++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c @@ -2782,9 +2782,9 @@ int atomisp_get_css_frame_info(struct atomisp_sub_device *asd, int stream_index; struct atomisp_device *isp = asd->isp; - if (ATOMISP_SOC_CAMERA(asd)) + if (ATOMISP_SOC_CAMERA(asd)) { stream_index = atomisp_source_pad_to_stream_id(asd, source_pad); - else { + } else { stream_index = (pipe_index == IA_CSS_PIPE_ID_YUVPP) ? ATOMISP_INPUT_STREAM_VIDEO : atomisp_source_pad_to_stream_id(asd, source_pad); diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c deleted file mode 100644 index e5553df5bad4..000000000000 --- a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c +++ /dev/null @@ -1,1202 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Support for Intel Camera Imaging ISP subsystem. - * - * Copyright (c) 2013 Intel Corporation. All Rights Reserved. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * - */ -#ifdef CONFIG_COMPAT -#include - -#include - -#include "atomisp_internal.h" -#include "atomisp_compat.h" -#include "atomisp_ioctl.h" -#include "atomisp_compat_ioctl32.h" - -/* Macros borrowed from v4l2-compat-ioctl32.c */ - -#define get_user_cast(__x, __ptr) \ -({ \ - get_user(__x, (typeof(*__ptr) __user *)(__ptr)); \ -}) - -#define put_user_force(__x, __ptr) \ -({ \ - put_user((typeof(*__x) __force *)(__x), __ptr); \ -}) - -/* Use the same argument order as copy_in_user */ -#define assign_in_user(to, from) \ -({ \ - typeof(*from) __assign_tmp; \ - \ - get_user_cast(__assign_tmp, from) || put_user(__assign_tmp, to);\ -}) - -static int get_atomisp_histogram32(struct atomisp_histogram __user *kp, - struct atomisp_histogram32 __user *up) -{ - compat_uptr_t tmp; - - if (!access_ok(up, sizeof(struct atomisp_histogram32)) || - assign_in_user(&kp->num_elements, &up->num_elements) || - get_user(tmp, &up->data) || - put_user(compat_ptr(tmp), &kp->data)) - return -EFAULT; - - return 0; -} - -static int put_atomisp_histogram32(struct atomisp_histogram __user *kp, - struct atomisp_histogram32 __user *up) -{ - void __user *tmp; - - if (!access_ok(up, sizeof(struct atomisp_histogram32)) || - assign_in_user(&up->num_elements, &kp->num_elements) || - get_user(tmp, &kp->data) || - put_user(ptr_to_compat(tmp), &up->data)) - return -EFAULT; - - return 0; -} - -static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp, - struct v4l2_framebuffer32 __user *up) -{ - compat_uptr_t tmp; - - if (!access_ok(up, sizeof(struct v4l2_framebuffer32)) || - get_user(tmp, &up->base) || - put_user_force(compat_ptr(tmp), &kp->base) || - assign_in_user(&kp->capability, &up->capability) || - assign_in_user(&kp->flags, &up->flags) || - copy_in_user(&kp->fmt, &up->fmt, sizeof(kp->fmt))) - return -EFAULT; - - return 0; -} - -static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics __user *kp, - struct atomisp_dis_statistics32 __user *up) -{ - compat_uptr_t hor_prod_odd_real; - compat_uptr_t hor_prod_odd_imag; - compat_uptr_t hor_prod_even_real; - compat_uptr_t hor_prod_even_imag; - compat_uptr_t ver_prod_odd_real; - compat_uptr_t ver_prod_odd_imag; - compat_uptr_t ver_prod_even_real; - compat_uptr_t ver_prod_even_imag; - - if (!access_ok(up, sizeof(struct atomisp_dis_statistics32)) || - copy_in_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) || - get_user(hor_prod_odd_real, - &up->dvs2_stat.hor_prod.odd_real) || - get_user(hor_prod_odd_imag, - &up->dvs2_stat.hor_prod.odd_imag) || - get_user(hor_prod_even_real, - &up->dvs2_stat.hor_prod.even_real) || - get_user(hor_prod_even_imag, - &up->dvs2_stat.hor_prod.even_imag) || - get_user(ver_prod_odd_real, - &up->dvs2_stat.ver_prod.odd_real) || - get_user(ver_prod_odd_imag, - &up->dvs2_stat.ver_prod.odd_imag) || - get_user(ver_prod_even_real, - &up->dvs2_stat.ver_prod.even_real) || - get_user(ver_prod_even_imag, - &up->dvs2_stat.ver_prod.even_imag) || - assign_in_user(&kp->exp_id, &up->exp_id) || - put_user(compat_ptr(hor_prod_odd_real), - &kp->dvs2_stat.hor_prod.odd_real) || - put_user(compat_ptr(hor_prod_odd_imag), - &kp->dvs2_stat.hor_prod.odd_imag) || - put_user(compat_ptr(hor_prod_even_real), - &kp->dvs2_stat.hor_prod.even_real) || - put_user(compat_ptr(hor_prod_even_imag), - &kp->dvs2_stat.hor_prod.even_imag) || - put_user(compat_ptr(ver_prod_odd_real), - &kp->dvs2_stat.ver_prod.odd_real) || - put_user(compat_ptr(ver_prod_odd_imag), - &kp->dvs2_stat.ver_prod.odd_imag) || - put_user(compat_ptr(ver_prod_even_real), - &kp->dvs2_stat.ver_prod.even_real) || - put_user(compat_ptr(ver_prod_even_imag), - &kp->dvs2_stat.ver_prod.even_imag)) - return -EFAULT; - - return 0; -} - -static int put_atomisp_dis_statistics32(struct atomisp_dis_statistics __user *kp, - struct atomisp_dis_statistics32 __user *up) -{ - void __user *hor_prod_odd_real; - void __user *hor_prod_odd_imag; - void __user *hor_prod_even_real; - void __user *hor_prod_even_imag; - void __user *ver_prod_odd_real; - void __user *ver_prod_odd_imag; - void __user *ver_prod_even_real; - void __user *ver_prod_even_imag; - - if (!!access_ok(up, sizeof(struct atomisp_dis_statistics32)) || - copy_in_user(up, kp, sizeof(struct atomisp_dvs_grid_info)) || - get_user(hor_prod_odd_real, - &kp->dvs2_stat.hor_prod.odd_real) || - get_user(hor_prod_odd_imag, - &kp->dvs2_stat.hor_prod.odd_imag) || - get_user(hor_prod_even_real, - &kp->dvs2_stat.hor_prod.even_real) || - get_user(hor_prod_even_imag, - &kp->dvs2_stat.hor_prod.even_imag) || - get_user(ver_prod_odd_real, - &kp->dvs2_stat.ver_prod.odd_real) || - get_user(ver_prod_odd_imag, - &kp->dvs2_stat.ver_prod.odd_imag) || - get_user(ver_prod_even_real, - &kp->dvs2_stat.ver_prod.even_real) || - get_user(ver_prod_even_imag, - &kp->dvs2_stat.ver_prod.even_imag) || - put_user(ptr_to_compat(hor_prod_odd_real), - &up->dvs2_stat.hor_prod.odd_real) || - put_user(ptr_to_compat(hor_prod_odd_imag), - &up->dvs2_stat.hor_prod.odd_imag) || - put_user(ptr_to_compat(hor_prod_even_real), - &up->dvs2_stat.hor_prod.even_real) || - put_user(ptr_to_compat(hor_prod_even_imag), - &up->dvs2_stat.hor_prod.even_imag) || - put_user(ptr_to_compat(ver_prod_odd_real), - &up->dvs2_stat.ver_prod.odd_real) || - put_user(ptr_to_compat(ver_prod_odd_imag), - &up->dvs2_stat.ver_prod.odd_imag) || - put_user(ptr_to_compat(ver_prod_even_real), - &up->dvs2_stat.ver_prod.even_real) || - put_user(ptr_to_compat(ver_prod_even_imag), - &up->dvs2_stat.ver_prod.even_imag) || - assign_in_user(&up->exp_id, &kp->exp_id)) - return -EFAULT; - - return 0; -} - -static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients __user *kp, - struct atomisp_dis_coefficients32 __user *up) -{ - compat_uptr_t hor_coefs_odd_real; - compat_uptr_t hor_coefs_odd_imag; - compat_uptr_t hor_coefs_even_real; - compat_uptr_t hor_coefs_even_imag; - compat_uptr_t ver_coefs_odd_real; - compat_uptr_t ver_coefs_odd_imag; - compat_uptr_t ver_coefs_even_real; - compat_uptr_t ver_coefs_even_imag; - - if (!access_ok(up, sizeof(struct atomisp_dis_coefficients32)) || - copy_in_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) || - get_user(hor_coefs_odd_real, &up->hor_coefs.odd_real) || - get_user(hor_coefs_odd_imag, &up->hor_coefs.odd_imag) || - get_user(hor_coefs_even_real, &up->hor_coefs.even_real) || - get_user(hor_coefs_even_imag, &up->hor_coefs.even_imag) || - get_user(ver_coefs_odd_real, &up->ver_coefs.odd_real) || - get_user(ver_coefs_odd_imag, &up->ver_coefs.odd_imag) || - get_user(ver_coefs_even_real, &up->ver_coefs.even_real) || - get_user(ver_coefs_even_imag, &up->ver_coefs.even_imag) || - put_user(compat_ptr(hor_coefs_odd_real), - &kp->hor_coefs.odd_real) || - put_user(compat_ptr(hor_coefs_odd_imag), - &kp->hor_coefs.odd_imag) || - put_user(compat_ptr(hor_coefs_even_real), - &kp->hor_coefs.even_real) || - put_user(compat_ptr(hor_coefs_even_imag), - &kp->hor_coefs.even_imag) || - put_user(compat_ptr(ver_coefs_odd_real), - &kp->ver_coefs.odd_real) || - put_user(compat_ptr(ver_coefs_odd_imag), - &kp->ver_coefs.odd_imag) || - put_user(compat_ptr(ver_coefs_even_real), - &kp->ver_coefs.even_real) || - put_user(compat_ptr(ver_coefs_even_imag), - &kp->ver_coefs.even_imag)) - return -EFAULT; - - return 0; -} - -static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config __user *kp, - struct atomisp_dvs_6axis_config32 __user *up) -{ - compat_uptr_t xcoords_y; - compat_uptr_t ycoords_y; - compat_uptr_t xcoords_uv; - compat_uptr_t ycoords_uv; - - if (!access_ok(up, sizeof(struct atomisp_dvs_6axis_config32)) || - assign_in_user(&kp->exp_id, &up->exp_id) || - assign_in_user(&kp->width_y, &up->width_y) || - assign_in_user(&kp->height_y, &up->height_y) || - assign_in_user(&kp->width_uv, &up->width_uv) || - assign_in_user(&kp->height_uv, &up->height_uv) || - get_user(xcoords_y, &up->xcoords_y) || - get_user(ycoords_y, &up->ycoords_y) || - get_user(xcoords_uv, &up->xcoords_uv) || - get_user(ycoords_uv, &up->ycoords_uv) || - put_user_force(compat_ptr(xcoords_y), &kp->xcoords_y) || - put_user_force(compat_ptr(ycoords_y), &kp->ycoords_y) || - put_user_force(compat_ptr(xcoords_uv), &kp->xcoords_uv) || - put_user_force(compat_ptr(ycoords_uv), &kp->ycoords_uv)) - return -EFAULT; - - return 0; -} - -static int get_atomisp_3a_statistics32(struct atomisp_3a_statistics __user *kp, - struct atomisp_3a_statistics32 __user *up) -{ - compat_uptr_t data; - compat_uptr_t rgby_data; - - if (!access_ok(up, sizeof(struct atomisp_3a_statistics32)) || - copy_in_user(kp, up, sizeof(struct atomisp_grid_info)) || - get_user(rgby_data, &up->rgby_data) || - put_user(compat_ptr(rgby_data), &kp->rgby_data) || - get_user(data, &up->data) || - put_user(compat_ptr(data), &kp->data) || - assign_in_user(&kp->exp_id, &up->exp_id) || - assign_in_user(&kp->isp_config_id, &up->isp_config_id)) - return -EFAULT; - - return 0; -} - -static int put_atomisp_3a_statistics32(struct atomisp_3a_statistics __user *kp, - struct atomisp_3a_statistics32 __user *up) -{ - void __user *data; - void __user *rgby_data; - - if (!access_ok(up, sizeof(struct atomisp_3a_statistics32)) || - copy_in_user(up, kp, sizeof(struct atomisp_grid_info)) || - get_user(rgby_data, &kp->rgby_data) || - put_user(ptr_to_compat(rgby_data), &up->rgby_data) || - get_user(data, &kp->data) || - put_user(ptr_to_compat(data), &up->data) || - assign_in_user(&up->exp_id, &kp->exp_id) || - assign_in_user(&up->isp_config_id, &kp->isp_config_id)) - return -EFAULT; - - return 0; -} - -static int get_atomisp_metadata_stat32(struct atomisp_metadata __user *kp, - struct atomisp_metadata32 __user *up) -{ - compat_uptr_t data; - compat_uptr_t effective_width; - - if (!access_ok(up, sizeof(struct atomisp_metadata32)) || - get_user(data, &up->data) || - put_user(compat_ptr(data), &kp->data) || - assign_in_user(&kp->width, &up->width) || - assign_in_user(&kp->height, &up->height) || - assign_in_user(&kp->stride, &up->stride) || - assign_in_user(&kp->exp_id, &up->exp_id) || - get_user(effective_width, &up->effective_width) || - put_user_force(compat_ptr(effective_width), &kp->effective_width)) - return -EFAULT; - - return 0; -} - -static int put_atomisp_metadata_stat32(struct atomisp_metadata __user *kp, - struct atomisp_metadata32 __user *up) -{ - void __user *data; - void *effective_width; - - if (!access_ok(up, sizeof(struct atomisp_metadata32)) || - get_user(data, &kp->data) || - put_user(ptr_to_compat(data), &up->data) || - assign_in_user(&up->width, &kp->width) || - assign_in_user(&up->height, &kp->height) || - assign_in_user(&up->stride, &kp->stride) || - assign_in_user(&up->exp_id, &kp->exp_id) || - get_user(effective_width, &kp->effective_width) || - put_user(ptr_to_compat((void __user *)effective_width), - &up->effective_width)) - return -EFAULT; - - return 0; -} - -static int -put_atomisp_metadata_by_type_stat32(struct atomisp_metadata_with_type __user *kp, - struct atomisp_metadata_with_type32 __user *up) -{ - void __user *data; - u32 *effective_width; - - if (!access_ok(up, sizeof(struct atomisp_metadata_with_type32)) || - get_user(data, &kp->data) || - put_user(ptr_to_compat(data), &up->data) || - assign_in_user(&up->width, &kp->width) || - assign_in_user(&up->height, &kp->height) || - assign_in_user(&up->stride, &kp->stride) || - assign_in_user(&up->exp_id, &kp->exp_id) || - get_user(effective_width, &kp->effective_width) || - put_user(ptr_to_compat((void __user *)effective_width), - &up->effective_width) || - assign_in_user(&up->type, &kp->type)) - return -EFAULT; - - return 0; -} - -static int -get_atomisp_metadata_by_type_stat32(struct atomisp_metadata_with_type __user *kp, - struct atomisp_metadata_with_type32 __user *up) -{ - compat_uptr_t data; - compat_uptr_t effective_width; - - if (!access_ok(up, sizeof(struct atomisp_metadata_with_type32)) || - get_user(data, &up->data) || - put_user(compat_ptr(data), &kp->data) || - assign_in_user(&kp->width, &up->width) || - assign_in_user(&kp->height, &up->height) || - assign_in_user(&kp->stride, &up->stride) || - assign_in_user(&kp->exp_id, &up->exp_id) || - get_user(effective_width, &up->effective_width) || - put_user_force(compat_ptr(effective_width), &kp->effective_width) || - assign_in_user(&kp->type, &up->type)) - return -EFAULT; - - return 0; -} - -static int -get_atomisp_morph_table32(struct atomisp_morph_table __user *kp, - struct atomisp_morph_table32 __user *up) -{ - unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES; - - if (!access_ok(up, sizeof(struct atomisp_morph_table32)) || - assign_in_user(&kp->enabled, &up->enabled) || - assign_in_user(&kp->width, &up->width) || - assign_in_user(&kp->height, &up->height)) - return -EFAULT; - - while (n-- > 0) { - compat_uptr_t coord_kp; - - if (get_user(coord_kp, &up->coordinates_x[n]) || - put_user(compat_ptr(coord_kp), &kp->coordinates_x[n]) || - get_user(coord_kp, &up->coordinates_y[n]) || - put_user(compat_ptr(coord_kp), &kp->coordinates_y[n])) - return -EFAULT; - } - return 0; -} - -static int put_atomisp_morph_table32(struct atomisp_morph_table __user *kp, - struct atomisp_morph_table32 __user *up) -{ - unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES; - - if (!access_ok(up, sizeof(struct atomisp_morph_table32)) || - assign_in_user(&up->enabled, &kp->enabled) || - assign_in_user(&up->width, &kp->width) || - assign_in_user(&up->height, &kp->height)) - return -EFAULT; - - while (n-- > 0) { - void __user *coord_kp; - - if (get_user(coord_kp, &kp->coordinates_x[n]) || - put_user(ptr_to_compat(coord_kp), &up->coordinates_x[n]) || - get_user(coord_kp, &kp->coordinates_y[n]) || - put_user(ptr_to_compat(coord_kp), &up->coordinates_y[n])) - return -EFAULT; - } - return 0; -} - -static int get_atomisp_overlay32(struct atomisp_overlay __user *kp, - struct atomisp_overlay32 __user *up) -{ - compat_uptr_t frame; - - if (!access_ok(up, sizeof(struct atomisp_overlay32)) || - get_user(frame, &up->frame) || - put_user_force(compat_ptr(frame), &kp->frame) || - assign_in_user(&kp->bg_y, &up->bg_y) || - assign_in_user(&kp->bg_u, &up->bg_u) || - assign_in_user(&kp->bg_v, &up->bg_v) || - assign_in_user(&kp->blend_input_perc_y, - &up->blend_input_perc_y) || - assign_in_user(&kp->blend_input_perc_u, - &up->blend_input_perc_u) || - assign_in_user(&kp->blend_input_perc_v, - &up->blend_input_perc_v) || - assign_in_user(&kp->blend_overlay_perc_y, - &up->blend_overlay_perc_y) || - assign_in_user(&kp->blend_overlay_perc_u, - &up->blend_overlay_perc_u) || - assign_in_user(&kp->blend_overlay_perc_v, - &up->blend_overlay_perc_v) || - assign_in_user(&kp->overlay_start_x, &up->overlay_start_x) || - assign_in_user(&kp->overlay_start_y, &up->overlay_start_y)) - return -EFAULT; - - return 0; -} - -static int put_atomisp_overlay32(struct atomisp_overlay __user *kp, - struct atomisp_overlay32 __user *up) -{ - void *frame; - - if (!access_ok(up, sizeof(struct atomisp_overlay32)) || - get_user(frame, &kp->frame) || - put_user(ptr_to_compat((void __user *)frame), &up->frame) || - assign_in_user(&up->bg_y, &kp->bg_y) || - assign_in_user(&up->bg_u, &kp->bg_u) || - assign_in_user(&up->bg_v, &kp->bg_v) || - assign_in_user(&up->blend_input_perc_y, - &kp->blend_input_perc_y) || - assign_in_user(&up->blend_input_perc_u, - &kp->blend_input_perc_u) || - assign_in_user(&up->blend_input_perc_v, - &kp->blend_input_perc_v) || - assign_in_user(&up->blend_overlay_perc_y, - &kp->blend_overlay_perc_y) || - assign_in_user(&up->blend_overlay_perc_u, - &kp->blend_overlay_perc_u) || - assign_in_user(&up->blend_overlay_perc_v, - &kp->blend_overlay_perc_v) || - assign_in_user(&up->overlay_start_x, &kp->overlay_start_x) || - assign_in_user(&up->overlay_start_y, &kp->overlay_start_y)) - return -EFAULT; - - return 0; -} - -static int -get_atomisp_calibration_group32(struct atomisp_calibration_group __user *kp, - struct atomisp_calibration_group32 __user *up) -{ - compat_uptr_t calb_grp_values; - - if (!access_ok(up, sizeof(struct atomisp_calibration_group32)) || - assign_in_user(&kp->size, &up->size) || - assign_in_user(&kp->type, &up->type) || - get_user(calb_grp_values, &up->calb_grp_values) || - put_user_force(compat_ptr(calb_grp_values), &kp->calb_grp_values)) - return -EFAULT; - - return 0; -} - -static int -put_atomisp_calibration_group32(struct atomisp_calibration_group __user *kp, - struct atomisp_calibration_group32 __user *up) -{ - void *calb_grp_values; - - if (!access_ok(up, sizeof(struct atomisp_calibration_group32)) || - assign_in_user(&up->size, &kp->size) || - assign_in_user(&up->type, &kp->type) || - get_user(calb_grp_values, &kp->calb_grp_values) || - put_user(ptr_to_compat((void __user *)calb_grp_values), - &up->calb_grp_values)) - return -EFAULT; - - return 0; -} - -static int get_atomisp_acc_fw_load32(struct atomisp_acc_fw_load __user *kp, - struct atomisp_acc_fw_load32 __user *up) -{ - compat_uptr_t data; - - if (!access_ok(up, sizeof(struct atomisp_acc_fw_load32)) || - assign_in_user(&kp->size, &up->size) || - assign_in_user(&kp->fw_handle, &up->fw_handle) || - get_user_cast(data, &up->data) || - put_user(compat_ptr(data), &kp->data)) - return -EFAULT; - - return 0; -} - -static int put_atomisp_acc_fw_load32(struct atomisp_acc_fw_load __user *kp, - struct atomisp_acc_fw_load32 __user *up) -{ - void __user *data; - - if (!access_ok(up, sizeof(struct atomisp_acc_fw_load32)) || - assign_in_user(&up->size, &kp->size) || - assign_in_user(&up->fw_handle, &kp->fw_handle) || - get_user(data, &kp->data) || - put_user(ptr_to_compat(data), &up->data)) - return -EFAULT; - - return 0; -} - -static int get_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg __user *kp, - struct atomisp_acc_fw_arg32 __user *up) -{ - compat_uptr_t value; - - if (!access_ok(up, sizeof(struct atomisp_acc_fw_arg32)) || - assign_in_user(&kp->fw_handle, &up->fw_handle) || - assign_in_user(&kp->index, &up->index) || - get_user(value, &up->value) || - put_user(compat_ptr(value), &kp->value) || - assign_in_user(&kp->size, &up->size)) - return -EFAULT; - - return 0; -} - -static int put_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg __user *kp, - struct atomisp_acc_fw_arg32 __user *up) -{ - void __user *value; - - if (!access_ok(up, sizeof(struct atomisp_acc_fw_arg32)) || - assign_in_user(&up->fw_handle, &kp->fw_handle) || - assign_in_user(&up->index, &kp->index) || - get_user(value, &kp->value) || - put_user(ptr_to_compat(value), &up->value) || - assign_in_user(&up->size, &kp->size)) - return -EFAULT; - - return 0; -} - -static int get_v4l2_private_int_data32(struct v4l2_private_int_data __user *kp, - struct v4l2_private_int_data32 __user *up) -{ - compat_uptr_t data; - - if (!access_ok(up, sizeof(struct v4l2_private_int_data32)) || - assign_in_user(&kp->size, &up->size) || - get_user(data, &up->data) || - put_user(compat_ptr(data), &kp->data) || - assign_in_user(&kp->reserved[0], &up->reserved[0]) || - assign_in_user(&kp->reserved[1], &up->reserved[1])) - return -EFAULT; - - return 0; -} - -static int put_v4l2_private_int_data32(struct v4l2_private_int_data __user *kp, - struct v4l2_private_int_data32 __user *up) -{ - void __user *data; - - if (!access_ok(up, sizeof(struct v4l2_private_int_data32)) || - assign_in_user(&up->size, &kp->size) || - get_user(data, &kp->data) || - put_user(ptr_to_compat(data), &up->data) || - assign_in_user(&up->reserved[0], &kp->reserved[0]) || - assign_in_user(&up->reserved[1], &kp->reserved[1])) - return -EFAULT; - - return 0; -} - -static int get_atomisp_shading_table32(struct atomisp_shading_table __user *kp, - struct atomisp_shading_table32 __user *up) -{ - unsigned int n = ATOMISP_NUM_SC_COLORS; - - if (!access_ok(up, sizeof(struct atomisp_shading_table32)) || - assign_in_user(&kp->enable, &up->enable) || - assign_in_user(&kp->sensor_width, &up->sensor_width) || - assign_in_user(&kp->sensor_height, &up->sensor_height) || - assign_in_user(&kp->width, &up->width) || - assign_in_user(&kp->height, &up->height) || - assign_in_user(&kp->fraction_bits, &up->fraction_bits)) - return -EFAULT; - - while (n-- > 0) { - compat_uptr_t tmp; - - if (get_user(tmp, &up->data[n]) || - put_user_force(compat_ptr(tmp), &kp->data[n])) - return -EFAULT; - } - return 0; -} - -static int get_atomisp_acc_map32(struct atomisp_acc_map __user *kp, - struct atomisp_acc_map32 __user *up) -{ - compat_uptr_t user_ptr; - - if (!access_ok(up, sizeof(struct atomisp_acc_map32)) || - assign_in_user(&kp->flags, &up->flags) || - assign_in_user(&kp->length, &up->length) || - get_user(user_ptr, &up->user_ptr) || - put_user(compat_ptr(user_ptr), &kp->user_ptr) || - assign_in_user(&kp->css_ptr, &up->css_ptr) || - assign_in_user(&kp->reserved[0], &up->reserved[0]) || - assign_in_user(&kp->reserved[1], &up->reserved[1]) || - assign_in_user(&kp->reserved[2], &up->reserved[2]) || - assign_in_user(&kp->reserved[3], &up->reserved[3])) - return -EFAULT; - - return 0; -} - -static int put_atomisp_acc_map32(struct atomisp_acc_map __user *kp, - struct atomisp_acc_map32 __user *up) -{ - void __user *user_ptr; - - if (!access_ok(up, sizeof(struct atomisp_acc_map32)) || - assign_in_user(&up->flags, &kp->flags) || - assign_in_user(&up->length, &kp->length) || - get_user(user_ptr, &kp->user_ptr) || - put_user(ptr_to_compat(user_ptr), &up->user_ptr) || - assign_in_user(&up->css_ptr, &kp->css_ptr) || - assign_in_user(&up->reserved[0], &kp->reserved[0]) || - assign_in_user(&up->reserved[1], &kp->reserved[1]) || - assign_in_user(&up->reserved[2], &kp->reserved[2]) || - assign_in_user(&up->reserved[3], &kp->reserved[3])) - return -EFAULT; - - return 0; -} - -static int -get_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg __user *kp, - struct atomisp_acc_s_mapped_arg32 __user *up) -{ - if (!access_ok(up, sizeof(struct atomisp_acc_s_mapped_arg32)) || - assign_in_user(&kp->fw_handle, &up->fw_handle) || - assign_in_user(&kp->memory, &up->memory) || - assign_in_user(&kp->length, &up->length) || - assign_in_user(&kp->css_ptr, &up->css_ptr)) - return -EFAULT; - - return 0; -} - -static int -put_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg __user *kp, - struct atomisp_acc_s_mapped_arg32 __user *up) -{ - if (!access_ok(up, sizeof(struct atomisp_acc_s_mapped_arg32)) || - assign_in_user(&up->fw_handle, &kp->fw_handle) || - assign_in_user(&up->memory, &kp->memory) || - assign_in_user(&up->length, &kp->length) || - assign_in_user(&up->css_ptr, &kp->css_ptr)) - return -EFAULT; - - return 0; -} - -static int get_atomisp_parameters32(struct atomisp_parameters __user *kp, - struct atomisp_parameters32 __user *up) -{ - int n = offsetof(struct atomisp_parameters32, output_frame) / - sizeof(compat_uptr_t); - compat_uptr_t stp, mtp, dcp, dscp; - struct { - struct atomisp_shading_table shading_table; - struct atomisp_morph_table morph_table; - struct atomisp_dis_coefficients dvs2_coefs; - struct atomisp_dvs_6axis_config dvs_6axis_config; - } __user *karg = (void __user *)(kp + 1); - - if (!access_ok(up, sizeof(struct atomisp_parameters32))) - return -EFAULT; - - while (n >= 0) { - compat_uptr_t __user *src = (compat_uptr_t __user *)up + n; - void * __user *dst = (void * __user *)kp + n; - compat_uptr_t tmp; - - if (get_user_cast(tmp, src) || put_user_force(compat_ptr(tmp), dst)) - return -EFAULT; - n--; - } - - if (assign_in_user(&kp->isp_config_id, &up->isp_config_id) || - assign_in_user(&kp->per_frame_setting, &up->per_frame_setting) || - get_user(stp, &up->shading_table) || - get_user(mtp, &up->morph_table) || - get_user(dcp, &up->dvs2_coefs) || - get_user(dscp, &up->dvs_6axis_config)) - return -EFAULT; - - /* handle shading table */ - if (stp && (get_atomisp_shading_table32(&karg->shading_table, - compat_ptr(stp)) || - put_user_force(&karg->shading_table, &kp->shading_table))) - return -EFAULT; - - /* handle morph table */ - if (mtp && (get_atomisp_morph_table32(&karg->morph_table, - compat_ptr(mtp)) || - put_user_force(&karg->morph_table, &kp->morph_table))) - return -EFAULT; - - /* handle dvs2 coefficients */ - if (dcp && (get_atomisp_dis_coefficients32(&karg->dvs2_coefs, - compat_ptr(dcp)) || - put_user_force(&karg->dvs2_coefs, &kp->dvs2_coefs))) - return -EFAULT; - - /* handle dvs 6axis configuration */ - if (dscp && - (get_atomisp_dvs_6axis_config32(&karg->dvs_6axis_config, - compat_ptr(dscp)) || - put_user_force(&karg->dvs_6axis_config, &kp->dvs_6axis_config))) - return -EFAULT; - - return 0; -} - -static int -get_atomisp_acc_fw_load_to_pipe32(struct atomisp_acc_fw_load_to_pipe __user *kp, - struct atomisp_acc_fw_load_to_pipe32 __user *up) -{ - compat_uptr_t data; - - if (!access_ok(up, sizeof(struct atomisp_acc_fw_load_to_pipe32)) || - assign_in_user(&kp->flags, &up->flags) || - assign_in_user(&kp->fw_handle, &up->fw_handle) || - assign_in_user(&kp->size, &up->size) || - assign_in_user(&kp->type, &up->type) || - assign_in_user(&kp->reserved[0], &up->reserved[0]) || - assign_in_user(&kp->reserved[1], &up->reserved[1]) || - assign_in_user(&kp->reserved[2], &up->reserved[2]) || - get_user(data, &up->data) || - put_user(compat_ptr(data), &kp->data)) - return -EFAULT; - - return 0; -} - -static int -put_atomisp_acc_fw_load_to_pipe32(struct atomisp_acc_fw_load_to_pipe __user *kp, - struct atomisp_acc_fw_load_to_pipe32 __user *up) -{ - void __user *data; - - if (!access_ok(up, sizeof(struct atomisp_acc_fw_load_to_pipe32)) || - assign_in_user(&up->flags, &kp->flags) || - assign_in_user(&up->fw_handle, &kp->fw_handle) || - assign_in_user(&up->size, &kp->size) || - assign_in_user(&up->type, &kp->type) || - assign_in_user(&up->reserved[0], &kp->reserved[0]) || - assign_in_user(&up->reserved[1], &kp->reserved[1]) || - assign_in_user(&up->reserved[2], &kp->reserved[2]) || - get_user(data, &kp->data) || - put_user(ptr_to_compat(data), &up->data)) - return -EFAULT; - - return 0; -} - -static int -get_atomisp_sensor_ae_bracketing_lut(struct atomisp_sensor_ae_bracketing_lut __user *kp, - struct atomisp_sensor_ae_bracketing_lut32 __user *up) -{ - compat_uptr_t lut; - - if (!access_ok(up, sizeof(struct atomisp_sensor_ae_bracketing_lut32)) || - assign_in_user(&kp->lut_size, &up->lut_size) || - get_user(lut, &up->lut) || - put_user_force(compat_ptr(lut), &kp->lut)) - return -EFAULT; - - return 0; -} - -static long native_ioctl(struct file *file, unsigned int cmd, unsigned long arg) -{ - long ret = -ENOIOCTLCMD; - - if (file->f_op->unlocked_ioctl) - ret = file->f_op->unlocked_ioctl(file, cmd, arg); - - return ret; -} - -static long atomisp_do_compat_ioctl(struct file *file, - unsigned int cmd, unsigned long arg) -{ - union { - struct atomisp_histogram his; - struct atomisp_dis_statistics dis_s; - struct atomisp_dis_coefficients dis_c; - struct atomisp_dvs_6axis_config dvs_c; - struct atomisp_3a_statistics s3a_s; - struct atomisp_morph_table mor_t; - struct v4l2_framebuffer v4l2_buf; - struct atomisp_overlay overlay; - struct atomisp_calibration_group cal_grp; - struct atomisp_acc_fw_load acc_fw_load; - struct atomisp_acc_fw_arg acc_fw_arg; - struct v4l2_private_int_data v4l2_pri_data; - struct atomisp_shading_table shd_tbl; - struct atomisp_acc_map acc_map; - struct atomisp_acc_s_mapped_arg acc_map_arg; - struct atomisp_parameters param; - struct atomisp_acc_fw_load_to_pipe acc_fw_to_pipe; - struct atomisp_metadata md; - struct atomisp_metadata_with_type md_with_type; - struct atomisp_sensor_ae_bracketing_lut lut; - } __user *karg; - void __user *up = compat_ptr(arg); - long err = -ENOIOCTLCMD; - - karg = compat_alloc_user_space( - sizeof(*karg) + (cmd == ATOMISP_IOC_S_PARAMETERS32 ? - sizeof(struct atomisp_shading_table) + - sizeof(struct atomisp_morph_table) + - sizeof(struct atomisp_dis_coefficients) + - sizeof(struct atomisp_dvs_6axis_config) : 0)); - if (!karg) - return -ENOMEM; - - /* First, convert the command. */ - switch (cmd) { - case ATOMISP_IOC_G_HISTOGRAM32: - cmd = ATOMISP_IOC_G_HISTOGRAM; - break; - case ATOMISP_IOC_S_HISTOGRAM32: - cmd = ATOMISP_IOC_S_HISTOGRAM; - break; - case ATOMISP_IOC_G_DIS_STAT32: - cmd = ATOMISP_IOC_G_DIS_STAT; - break; - case ATOMISP_IOC_S_DIS_COEFS32: - cmd = ATOMISP_IOC_S_DIS_COEFS; - break; - case ATOMISP_IOC_S_DIS_VECTOR32: - cmd = ATOMISP_IOC_S_DIS_VECTOR; - break; - case ATOMISP_IOC_G_3A_STAT32: - cmd = ATOMISP_IOC_G_3A_STAT; - break; - case ATOMISP_IOC_G_ISP_GDC_TAB32: - cmd = ATOMISP_IOC_G_ISP_GDC_TAB; - break; - case ATOMISP_IOC_S_ISP_GDC_TAB32: - cmd = ATOMISP_IOC_S_ISP_GDC_TAB; - break; - case ATOMISP_IOC_S_ISP_FPN_TABLE32: - cmd = ATOMISP_IOC_S_ISP_FPN_TABLE; - break; - case ATOMISP_IOC_G_ISP_OVERLAY32: - cmd = ATOMISP_IOC_G_ISP_OVERLAY; - break; - case ATOMISP_IOC_S_ISP_OVERLAY32: - cmd = ATOMISP_IOC_S_ISP_OVERLAY; - break; - case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP32: - cmd = ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP; - break; - case ATOMISP_IOC_ACC_LOAD32: - cmd = ATOMISP_IOC_ACC_LOAD; - break; - case ATOMISP_IOC_ACC_S_ARG32: - cmd = ATOMISP_IOC_ACC_S_ARG; - break; - case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA32: - cmd = ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA; - break; - case ATOMISP_IOC_S_ISP_SHD_TAB32: - cmd = ATOMISP_IOC_S_ISP_SHD_TAB; - break; - case ATOMISP_IOC_ACC_DESTAB32: - cmd = ATOMISP_IOC_ACC_DESTAB; - break; - case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA32: - cmd = ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA; - break; - case ATOMISP_IOC_ACC_MAP32: - cmd = ATOMISP_IOC_ACC_MAP; - break; - case ATOMISP_IOC_ACC_UNMAP32: - cmd = ATOMISP_IOC_ACC_UNMAP; - break; - case ATOMISP_IOC_ACC_S_MAPPED_ARG32: - cmd = ATOMISP_IOC_ACC_S_MAPPED_ARG; - break; - case ATOMISP_IOC_S_PARAMETERS32: - cmd = ATOMISP_IOC_S_PARAMETERS; - break; - case ATOMISP_IOC_ACC_LOAD_TO_PIPE32: - cmd = ATOMISP_IOC_ACC_LOAD_TO_PIPE; - break; - case ATOMISP_IOC_G_METADATA32: - cmd = ATOMISP_IOC_G_METADATA; - break; - case ATOMISP_IOC_G_METADATA_BY_TYPE32: - cmd = ATOMISP_IOC_G_METADATA_BY_TYPE; - break; - case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT32: - cmd = ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT; - break; - } - - switch (cmd) { - case ATOMISP_IOC_G_HISTOGRAM: - case ATOMISP_IOC_S_HISTOGRAM: - err = get_atomisp_histogram32(&karg->his, up); - break; - case ATOMISP_IOC_G_DIS_STAT: - err = get_atomisp_dis_statistics32(&karg->dis_s, up); - break; - case ATOMISP_IOC_S_DIS_COEFS: - err = get_atomisp_dis_coefficients32(&karg->dis_c, up); - break; - case ATOMISP_IOC_S_DIS_VECTOR: - err = get_atomisp_dvs_6axis_config32(&karg->dvs_c, up); - break; - case ATOMISP_IOC_G_3A_STAT: - err = get_atomisp_3a_statistics32(&karg->s3a_s, up); - break; - case ATOMISP_IOC_G_ISP_GDC_TAB: - case ATOMISP_IOC_S_ISP_GDC_TAB: - err = get_atomisp_morph_table32(&karg->mor_t, up); - break; - case ATOMISP_IOC_S_ISP_FPN_TABLE: - err = get_v4l2_framebuffer32(&karg->v4l2_buf, up); - break; - case ATOMISP_IOC_G_ISP_OVERLAY: - case ATOMISP_IOC_S_ISP_OVERLAY: - err = get_atomisp_overlay32(&karg->overlay, up); - break; - case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP: - err = get_atomisp_calibration_group32(&karg->cal_grp, up); - break; - case ATOMISP_IOC_ACC_LOAD: - err = get_atomisp_acc_fw_load32(&karg->acc_fw_load, up); - break; - case ATOMISP_IOC_ACC_S_ARG: - case ATOMISP_IOC_ACC_DESTAB: - err = get_atomisp_acc_fw_arg32(&karg->acc_fw_arg, up); - break; - case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA: - case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA: - err = get_v4l2_private_int_data32(&karg->v4l2_pri_data, up); - break; - case ATOMISP_IOC_S_ISP_SHD_TAB: - err = get_atomisp_shading_table32(&karg->shd_tbl, up); - break; - case ATOMISP_IOC_ACC_MAP: - case ATOMISP_IOC_ACC_UNMAP: - err = get_atomisp_acc_map32(&karg->acc_map, up); - break; - case ATOMISP_IOC_ACC_S_MAPPED_ARG: - err = get_atomisp_acc_s_mapped_arg32(&karg->acc_map_arg, up); - break; - case ATOMISP_IOC_S_PARAMETERS: - err = get_atomisp_parameters32(&karg->param, up); - break; - case ATOMISP_IOC_ACC_LOAD_TO_PIPE: - err = get_atomisp_acc_fw_load_to_pipe32(&karg->acc_fw_to_pipe, - up); - break; - case ATOMISP_IOC_G_METADATA: - err = get_atomisp_metadata_stat32(&karg->md, up); - break; - case ATOMISP_IOC_G_METADATA_BY_TYPE: - err = get_atomisp_metadata_by_type_stat32(&karg->md_with_type, - up); - break; - case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT: - err = get_atomisp_sensor_ae_bracketing_lut(&karg->lut, up); - break; - } - if (err) - return err; - - err = native_ioctl(file, cmd, (unsigned long)karg); - if (err) - return err; - - switch (cmd) { - case ATOMISP_IOC_G_HISTOGRAM: - err = put_atomisp_histogram32(&karg->his, up); - break; - case ATOMISP_IOC_G_DIS_STAT: - err = put_atomisp_dis_statistics32(&karg->dis_s, up); - break; - case ATOMISP_IOC_G_3A_STAT: - err = put_atomisp_3a_statistics32(&karg->s3a_s, up); - break; - case ATOMISP_IOC_G_ISP_GDC_TAB: - err = put_atomisp_morph_table32(&karg->mor_t, up); - break; - case ATOMISP_IOC_G_ISP_OVERLAY: - err = put_atomisp_overlay32(&karg->overlay, up); - break; - case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP: - err = put_atomisp_calibration_group32(&karg->cal_grp, up); - break; - case ATOMISP_IOC_ACC_LOAD: - err = put_atomisp_acc_fw_load32(&karg->acc_fw_load, up); - break; - case ATOMISP_IOC_ACC_S_ARG: - case ATOMISP_IOC_ACC_DESTAB: - err = put_atomisp_acc_fw_arg32(&karg->acc_fw_arg, up); - break; - case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA: - case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA: - err = put_v4l2_private_int_data32(&karg->v4l2_pri_data, up); - break; - case ATOMISP_IOC_ACC_MAP: - case ATOMISP_IOC_ACC_UNMAP: - err = put_atomisp_acc_map32(&karg->acc_map, up); - break; - case ATOMISP_IOC_ACC_S_MAPPED_ARG: - err = put_atomisp_acc_s_mapped_arg32(&karg->acc_map_arg, up); - break; - case ATOMISP_IOC_ACC_LOAD_TO_PIPE: - err = put_atomisp_acc_fw_load_to_pipe32(&karg->acc_fw_to_pipe, - up); - break; - case ATOMISP_IOC_G_METADATA: - err = put_atomisp_metadata_stat32(&karg->md, up); - break; - case ATOMISP_IOC_G_METADATA_BY_TYPE: - err = put_atomisp_metadata_by_type_stat32(&karg->md_with_type, - up); - break; - } - - return err; -} - -long atomisp_compat_ioctl32(struct file *file, - unsigned int cmd, unsigned long arg) -{ - struct video_device *vdev = video_devdata(file); - struct atomisp_device *isp = video_get_drvdata(vdev); - long ret = -ENOIOCTLCMD; - - if (!file->f_op->unlocked_ioctl) - return ret; - - switch (cmd) { - case ATOMISP_IOC_G_XNR: - case ATOMISP_IOC_S_XNR: - case ATOMISP_IOC_G_NR: - case ATOMISP_IOC_S_NR: - case ATOMISP_IOC_G_TNR: - case ATOMISP_IOC_S_TNR: - case ATOMISP_IOC_G_BLACK_LEVEL_COMP: - case ATOMISP_IOC_S_BLACK_LEVEL_COMP: - case ATOMISP_IOC_G_EE: - case ATOMISP_IOC_S_EE: - case ATOMISP_IOC_S_DIS_VECTOR: - case ATOMISP_IOC_G_ISP_PARM: - case ATOMISP_IOC_S_ISP_PARM: - case ATOMISP_IOC_G_ISP_GAMMA: - case ATOMISP_IOC_S_ISP_GAMMA: - case ATOMISP_IOC_ISP_MAKERNOTE: - case ATOMISP_IOC_G_ISP_MACC: - case ATOMISP_IOC_S_ISP_MACC: - case ATOMISP_IOC_G_ISP_BAD_PIXEL_DETECTION: - case ATOMISP_IOC_S_ISP_BAD_PIXEL_DETECTION: - case ATOMISP_IOC_G_ISP_FALSE_COLOR_CORRECTION: - case ATOMISP_IOC_S_ISP_FALSE_COLOR_CORRECTION: - case ATOMISP_IOC_G_ISP_CTC: - case ATOMISP_IOC_S_ISP_CTC: - case ATOMISP_IOC_G_ISP_WHITE_BALANCE: - case ATOMISP_IOC_S_ISP_WHITE_BALANCE: - case ATOMISP_IOC_CAMERA_BRIDGE: - case ATOMISP_IOC_G_SENSOR_MODE_DATA: - case ATOMISP_IOC_S_EXPOSURE: - case ATOMISP_IOC_G_3A_CONFIG: - case ATOMISP_IOC_S_3A_CONFIG: - case ATOMISP_IOC_ACC_UNLOAD: - case ATOMISP_IOC_ACC_START: - case ATOMISP_IOC_ACC_WAIT: - case ATOMISP_IOC_ACC_ABORT: - case ATOMISP_IOC_G_ISP_GAMMA_CORRECTION: - case ATOMISP_IOC_S_ISP_GAMMA_CORRECTION: - case ATOMISP_IOC_S_CONT_CAPTURE_CONFIG: - case ATOMISP_IOC_G_DVS2_BQ_RESOLUTIONS: - case ATOMISP_IOC_EXT_ISP_CTRL: - case ATOMISP_IOC_EXP_ID_UNLOCK: - case ATOMISP_IOC_EXP_ID_CAPTURE: - case ATOMISP_IOC_S_ENABLE_DZ_CAPT_PIPE: - case ATOMISP_IOC_G_FORMATS_CONFIG: - case ATOMISP_IOC_S_FORMATS_CONFIG: - case ATOMISP_IOC_S_EXPOSURE_WINDOW: - case ATOMISP_IOC_S_ACC_STATE: - case ATOMISP_IOC_G_ACC_STATE: - case ATOMISP_IOC_INJECT_A_FAKE_EVENT: - case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_INFO: - case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_MODE: - case ATOMISP_IOC_G_SENSOR_AE_BRACKETING_MODE: - case ATOMISP_IOC_G_INVALID_FRAME_NUM: - case ATOMISP_IOC_S_ARRAY_RESOLUTION: - case ATOMISP_IOC_S_SENSOR_RUNMODE: - case ATOMISP_IOC_G_UPDATE_EXPOSURE: - ret = native_ioctl(file, cmd, arg); - break; - - case ATOMISP_IOC_G_HISTOGRAM32: - case ATOMISP_IOC_S_HISTOGRAM32: - case ATOMISP_IOC_G_DIS_STAT32: - case ATOMISP_IOC_S_DIS_COEFS32: - case ATOMISP_IOC_S_DIS_VECTOR32: - case ATOMISP_IOC_G_3A_STAT32: - case ATOMISP_IOC_G_ISP_GDC_TAB32: - case ATOMISP_IOC_S_ISP_GDC_TAB32: - case ATOMISP_IOC_S_ISP_FPN_TABLE32: - case ATOMISP_IOC_G_ISP_OVERLAY32: - case ATOMISP_IOC_S_ISP_OVERLAY32: - case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP32: - case ATOMISP_IOC_ACC_LOAD32: - case ATOMISP_IOC_ACC_S_ARG32: - case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA32: - case ATOMISP_IOC_S_ISP_SHD_TAB32: - case ATOMISP_IOC_ACC_DESTAB32: - case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA32: - case ATOMISP_IOC_ACC_MAP32: - case ATOMISP_IOC_ACC_UNMAP32: - case ATOMISP_IOC_ACC_S_MAPPED_ARG32: - case ATOMISP_IOC_S_PARAMETERS32: - case ATOMISP_IOC_ACC_LOAD_TO_PIPE32: - case ATOMISP_IOC_G_METADATA32: - case ATOMISP_IOC_G_METADATA_BY_TYPE32: - case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT32: - ret = atomisp_do_compat_ioctl(file, cmd, arg); - break; - - default: - dev_warn(isp->dev, - "%s: unknown ioctl '%c', dir=%d, #%d (0x%08x)\n", - __func__, _IOC_TYPE(cmd), _IOC_DIR(cmd), _IOC_NR(cmd), - cmd); - break; - } - return ret; -} -#endif /* CONFIG_COMPAT */ diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2.c b/drivers/staging/media/atomisp/pci/atomisp_csi2.c index 060b8765ae96..56456e59bf89 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_csi2.c +++ b/drivers/staging/media/atomisp/pci/atomisp_csi2.c @@ -25,13 +25,13 @@ static struct v4l2_mbus_framefmt *__csi2_get_format(struct atomisp_mipi_csi2_device * csi2, - struct - v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which, unsigned int pad) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&csi2->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&csi2->subdev, sd_state, + pad); else return &csi2->formats[pad]; } @@ -44,7 +44,7 @@ static struct v4l2_mbus_framefmt *__csi2_get_format(struct * return -EINVAL or zero on success */ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { const struct atomisp_in_fmt_conv *ic = atomisp_in_fmt_conv; @@ -70,13 +70,13 @@ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int csi2_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct atomisp_mipi_csi2_device *csi2 = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csi2_get_format(csi2, cfg, fmt->which, fmt->pad); + format = __csi2_get_format(csi2, sd_state, fmt->which, fmt->pad); fmt->format = *format; @@ -84,12 +84,14 @@ static int csi2_get_format(struct v4l2_subdev *sd, } int atomisp_csi2_set_ffmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int which, uint16_t pad, struct v4l2_mbus_framefmt *ffmt) { struct atomisp_mipi_csi2_device *csi2 = v4l2_get_subdevdata(sd); - struct v4l2_mbus_framefmt *actual_ffmt = __csi2_get_format(csi2, cfg, which, pad); + struct v4l2_mbus_framefmt *actual_ffmt = __csi2_get_format(csi2, + sd_state, + which, pad); if (pad == CSI2_PAD_SINK) { const struct atomisp_in_fmt_conv *ic; @@ -110,12 +112,14 @@ int atomisp_csi2_set_ffmt(struct v4l2_subdev *sd, tmp_ffmt = *ffmt = *actual_ffmt; - return atomisp_csi2_set_ffmt(sd, cfg, which, CSI2_PAD_SOURCE, + return atomisp_csi2_set_ffmt(sd, sd_state, which, + CSI2_PAD_SOURCE, &tmp_ffmt); } /* FIXME: DPCM decompression */ - *actual_ffmt = *ffmt = *__csi2_get_format(csi2, cfg, which, CSI2_PAD_SINK); + *actual_ffmt = *ffmt = *__csi2_get_format(csi2, sd_state, which, + CSI2_PAD_SINK); return 0; } @@ -129,10 +133,10 @@ int atomisp_csi2_set_ffmt(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int csi2_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - return atomisp_csi2_set_ffmt(sd, cfg, fmt->which, fmt->pad, + return atomisp_csi2_set_ffmt(sd, sd_state, fmt->which, fmt->pad, &fmt->format); } diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2.h b/drivers/staging/media/atomisp/pci/atomisp_csi2.h index 59261e8f1a1a..e35711be8a37 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_csi2.h +++ b/drivers/staging/media/atomisp/pci/atomisp_csi2.h @@ -44,7 +44,7 @@ struct atomisp_mipi_csi2_device { }; int atomisp_csi2_set_ffmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int which, uint16_t pad, struct v4l2_mbus_framefmt *ffmt); int atomisp_mipi_csi2_init(struct atomisp_device *isp); diff --git a/drivers/staging/media/atomisp/pci/atomisp_file.c b/drivers/staging/media/atomisp/pci/atomisp_file.c index e568ca99c45a..4570a9ab100b 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_file.c +++ b/drivers/staging/media/atomisp/pci/atomisp_file.c @@ -80,7 +80,7 @@ static int file_input_s_stream(struct v4l2_subdev *sd, int enable) } static int file_input_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -104,16 +104,16 @@ static int file_input_get_fmt(struct v4l2_subdev *sd, } static int file_input_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; if (format->pad) return -EINVAL; - file_input_get_fmt(sd, cfg, format); + file_input_get_fmt(sd, sd_state, format); if (format->which == V4L2_SUBDEV_FORMAT_TRY) - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; return 0; } @@ -130,7 +130,7 @@ static int file_input_s_power(struct v4l2_subdev *sd, int on) } static int file_input_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { /*to fake*/ @@ -138,7 +138,7 @@ static int file_input_enum_mbus_code(struct v4l2_subdev *sd, } static int file_input_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { /*to fake*/ @@ -146,7 +146,7 @@ static int file_input_enum_frame_size(struct v4l2_subdev *sd, } static int file_input_enum_frame_ival(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp_fops.c index f1e6b2597853..f82bf082aa79 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_fops.c +++ b/drivers/staging/media/atomisp/pci/atomisp_fops.c @@ -837,7 +837,7 @@ static int atomisp_open(struct file *file) } /* runtime power management, turn on ISP */ - ret = pm_runtime_get_sync(vdev->v4l2_dev->dev); + ret = pm_runtime_resume_and_get(vdev->v4l2_dev->dev); if (ret < 0) { dev_err(isp->dev, "Failed to power on device\n"); goto error; @@ -881,9 +881,9 @@ static int atomisp_open(struct file *file) css_error: atomisp_css_uninit(isp); + pm_runtime_put(vdev->v4l2_dev->dev); error: hmm_pool_unregister(HMM_POOL_TYPE_DYNAMIC); - pm_runtime_put(vdev->v4l2_dev->dev); rt_mutex_unlock(&isp->mutex); return ret; } @@ -963,7 +963,7 @@ static int atomisp_release(struct file *file) if (!isp->sw_contex.file_input && asd->fmt_auto->val) { struct v4l2_mbus_framefmt isp_sink_fmt = { 0 }; - atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, + atomisp_subdev_set_ffmt(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK, &isp_sink_fmt); } @@ -975,7 +975,7 @@ static int atomisp_release(struct file *file) if (isp->sw_contex.file_input && asd->fmt_auto->val) { struct v4l2_mbus_framefmt isp_sink_fmt = { 0 }; - atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, + atomisp_subdev_set_ffmt(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK, &isp_sink_fmt); } @@ -1016,7 +1016,7 @@ static int atomisp_release(struct file *file) done: if (!acc_node) { - atomisp_subdev_set_selection(&asd->subdev, fh.pad, + atomisp_subdev_set_selection(&asd->subdev, fh.state, V4L2_SUBDEV_FORMAT_ACTIVE, atomisp_subdev_source_pad(vdev), V4L2_SEL_TGT_COMPOSE, 0, @@ -1283,7 +1283,8 @@ const struct v4l2_file_operations atomisp_fops = { .unlocked_ioctl = video_ioctl2, #ifdef CONFIG_COMPAT /* - * There are problems with this code. Disable this for now. + * this was removed because of bugs, the interface + * needs to be made safe for compat tasks instead. .compat_ioctl32 = atomisp_compat_ioctl32, */ #endif @@ -1297,10 +1298,7 @@ const struct v4l2_file_operations atomisp_file_fops = { .mmap = atomisp_file_mmap, .unlocked_ioctl = video_ioctl2, #ifdef CONFIG_COMPAT - /* - * There are problems with this code. Disable this for now. - .compat_ioctl32 = atomisp_compat_ioctl32, - */ + /* .compat_ioctl32 = atomisp_compat_ioctl32, */ #endif .poll = atomisp_poll, }; diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp_subdev.c index 2ef5f44e4b6b..12f22ad007c7 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_subdev.c +++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.c @@ -213,7 +213,7 @@ static int isp_subdev_unsubscribe_event(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int isp_subdev_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->index >= ARRAY_SIZE(atomisp_in_fmt_conv) - 1) @@ -246,7 +246,7 @@ static int isp_subdev_validate_rect(struct v4l2_subdev *sd, uint32_t pad, } struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, u32 which, uint32_t pad, uint32_t target) { @@ -255,9 +255,9 @@ struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd, if (which == V4L2_SUBDEV_FORMAT_TRY) { switch (target) { case V4L2_SEL_TGT_CROP: - return v4l2_subdev_get_try_crop(sd, cfg, pad); + return v4l2_subdev_get_try_crop(sd, sd_state, pad); case V4L2_SEL_TGT_COMPOSE: - return v4l2_subdev_get_try_compose(sd, cfg, pad); + return v4l2_subdev_get_try_compose(sd, sd_state, pad); } } @@ -273,19 +273,20 @@ struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd, struct v4l2_mbus_framefmt *atomisp_subdev_get_ffmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, uint32_t which, + struct v4l2_subdev_state *sd_state, uint32_t which, uint32_t pad) { struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd); if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(sd, cfg, pad); + return v4l2_subdev_get_try_format(sd, sd_state, pad); return &isp_sd->fmt[pad].fmt; } static void isp_get_fmt_rect(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, uint32_t which, + struct v4l2_subdev_state *sd_state, + uint32_t which, struct v4l2_mbus_framefmt **ffmt, struct v4l2_rect *crop[ATOMISP_SUBDEV_PADS_NUM], struct v4l2_rect *comp[ATOMISP_SUBDEV_PADS_NUM]) @@ -293,16 +294,16 @@ static void isp_get_fmt_rect(struct v4l2_subdev *sd, unsigned int i; for (i = 0; i < ATOMISP_SUBDEV_PADS_NUM; i++) { - ffmt[i] = atomisp_subdev_get_ffmt(sd, cfg, which, i); - crop[i] = atomisp_subdev_get_rect(sd, cfg, which, i, + ffmt[i] = atomisp_subdev_get_ffmt(sd, sd_state, which, i); + crop[i] = atomisp_subdev_get_rect(sd, sd_state, which, i, V4L2_SEL_TGT_CROP); - comp[i] = atomisp_subdev_get_rect(sd, cfg, which, i, + comp[i] = atomisp_subdev_get_rect(sd, sd_state, which, i, V4L2_SEL_TGT_COMPOSE); } } static void isp_subdev_propagate(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, u32 which, uint32_t pad, uint32_t target, uint32_t flags) { @@ -313,7 +314,7 @@ static void isp_subdev_propagate(struct v4l2_subdev *sd, if (flags & V4L2_SEL_FLAG_KEEP_CONFIG) return; - isp_get_fmt_rect(sd, cfg, which, ffmt, crop, comp); + isp_get_fmt_rect(sd, sd_state, which, ffmt, crop, comp); switch (pad) { case ATOMISP_SUBDEV_PAD_SINK: { @@ -323,7 +324,7 @@ static void isp_subdev_propagate(struct v4l2_subdev *sd, r.width = ffmt[pad]->width; r.height = ffmt[pad]->height; - atomisp_subdev_set_selection(sd, cfg, which, pad, + atomisp_subdev_set_selection(sd, sd_state, which, pad, target, flags, &r); break; } @@ -331,7 +332,7 @@ static void isp_subdev_propagate(struct v4l2_subdev *sd, } static int isp_subdev_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct v4l2_rect *rec; @@ -340,7 +341,7 @@ static int isp_subdev_get_selection(struct v4l2_subdev *sd, if (rval) return rval; - rec = atomisp_subdev_get_rect(sd, cfg, sel->which, sel->pad, + rec = atomisp_subdev_get_rect(sd, sd_state, sel->which, sel->pad, sel->target); if (!rec) return -EINVAL; @@ -365,7 +366,7 @@ static const char *atomisp_pad_str(unsigned int pad) } int atomisp_subdev_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, u32 which, uint32_t pad, uint32_t target, u32 flags, struct v4l2_rect *r) { @@ -382,7 +383,7 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd, stream_id = atomisp_source_pad_to_stream_id(isp_sd, vdev_pad); - isp_get_fmt_rect(sd, cfg, which, ffmt, crop, comp); + isp_get_fmt_rect(sd, sd_state, which, ffmt, crop, comp); dev_dbg(isp->dev, "sel: pad %s tgt %s l %d t %d w %d h %d which %s f 0x%8.8x\n", @@ -450,7 +451,8 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd, struct v4l2_rect tmp = *crop[pad]; atomisp_subdev_set_selection( - sd, cfg, which, i, V4L2_SEL_TGT_COMPOSE, + sd, sd_state, which, i, + V4L2_SEL_TGT_COMPOSE, flags, &tmp); } } @@ -472,9 +474,9 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd, * when dvs is disabled. */ dvs_w = dvs_h = 12; - } else + } else { dvs_w = dvs_h = 0; - + } atomisp_css_video_set_dis_envelope(isp_sd, dvs_w, dvs_h); atomisp_css_input_set_effective_resolution(isp_sd, stream_id, crop[pad]->width, crop[pad]->height); @@ -551,9 +553,9 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd, ffmt[pad]->height = comp[pad]->height; } - if (!atomisp_subdev_get_rect(sd, cfg, which, pad, target)) + if (!atomisp_subdev_get_rect(sd, sd_state, which, pad, target)) return -EINVAL; - *r = *atomisp_subdev_get_rect(sd, cfg, which, pad, target); + *r = *atomisp_subdev_get_rect(sd, sd_state, which, pad, target); dev_dbg(isp->dev, "sel actual: l %d t %d w %d h %d\n", r->left, r->top, r->width, r->height); @@ -562,7 +564,7 @@ int atomisp_subdev_set_selection(struct v4l2_subdev *sd, } static int isp_subdev_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { int rval = isp_subdev_validate_rect(sd, sel->pad, sel->target); @@ -570,7 +572,8 @@ static int isp_subdev_set_selection(struct v4l2_subdev *sd, if (rval) return rval; - return atomisp_subdev_set_selection(sd, cfg, sel->which, sel->pad, + return atomisp_subdev_set_selection(sd, sd_state, sel->which, + sel->pad, sel->target, sel->flags, &sel->r); } @@ -609,13 +612,14 @@ static int atomisp_get_sensor_bin_factor(struct atomisp_sub_device *asd) } void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, uint32_t which, + struct v4l2_subdev_state *sd_state, + uint32_t which, u32 pad, struct v4l2_mbus_framefmt *ffmt) { struct atomisp_sub_device *isp_sd = v4l2_get_subdevdata(sd); struct atomisp_device *isp = isp_sd->isp; struct v4l2_mbus_framefmt *__ffmt = - atomisp_subdev_get_ffmt(sd, cfg, which, pad); + atomisp_subdev_get_ffmt(sd, sd_state, which, pad); u16 vdev_pad = atomisp_subdev_source_pad(sd->devnode); enum atomisp_input_stream_id stream_id; @@ -640,7 +644,7 @@ void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd, *__ffmt = *ffmt; - isp_subdev_propagate(sd, cfg, which, pad, + isp_subdev_propagate(sd, sd_state, which, pad, V4L2_SEL_TGT_CROP, 0); if (which == V4L2_SUBDEV_FORMAT_ACTIVE) { @@ -679,10 +683,11 @@ void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd, * to the format type. */ static int isp_subdev_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - fmt->format = *atomisp_subdev_get_ffmt(sd, cfg, fmt->which, fmt->pad); + fmt->format = *atomisp_subdev_get_ffmt(sd, sd_state, fmt->which, + fmt->pad); return 0; } @@ -698,10 +703,11 @@ static int isp_subdev_get_format(struct v4l2_subdev *sd, * to the format type. */ static int isp_subdev_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { - atomisp_subdev_set_ffmt(sd, cfg, fmt->which, fmt->pad, &fmt->format); + atomisp_subdev_set_ffmt(sd, sd_state, fmt->which, fmt->pad, + &fmt->format); return 0; } diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.h b/drivers/staging/media/atomisp/pci/atomisp_subdev.h index 330a77eed8aa..d6fcfab6352d 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_subdev.h +++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.h @@ -437,19 +437,20 @@ uint16_t atomisp_subdev_source_pad(struct video_device *vdev); /* Get pointer to appropriate format */ struct v4l2_mbus_framefmt *atomisp_subdev_get_ffmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, uint32_t which, + struct v4l2_subdev_state *sd_state, uint32_t which, uint32_t pad); struct v4l2_rect *atomisp_subdev_get_rect(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, u32 which, uint32_t pad, uint32_t target); int atomisp_subdev_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, u32 which, uint32_t pad, uint32_t target, u32 flags, struct v4l2_rect *r); /* Actually set the format */ void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, uint32_t which, + struct v4l2_subdev_state *sd_state, + uint32_t which, u32 pad, struct v4l2_mbus_framefmt *ffmt); int atomisp_update_run_mode(struct atomisp_sub_device *asd); diff --git a/drivers/staging/media/atomisp/pci/atomisp_tpg.c b/drivers/staging/media/atomisp/pci/atomisp_tpg.c index 1def80bab180..e29a96da5f98 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_tpg.c +++ b/drivers/staging/media/atomisp/pci/atomisp_tpg.c @@ -29,7 +29,7 @@ static int tpg_s_stream(struct v4l2_subdev *sd, int enable) } static int tpg_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { /*to fake*/ @@ -37,7 +37,7 @@ static int tpg_get_fmt(struct v4l2_subdev *sd, } static int tpg_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *format) { struct v4l2_mbus_framefmt *fmt = &format->format; @@ -47,7 +47,7 @@ static int tpg_set_fmt(struct v4l2_subdev *sd, /* only raw8 grbg is supported by TPG */ fmt->code = MEDIA_BUS_FMT_SGRBG8_1X8; if (format->which == V4L2_SUBDEV_FORMAT_TRY) { - cfg->try_fmt = *fmt; + sd_state->pads->try_fmt = *fmt; return 0; } return 0; @@ -65,7 +65,7 @@ static int tpg_s_power(struct v4l2_subdev *sd, int on) } static int tpg_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { /*to fake*/ @@ -73,7 +73,7 @@ static int tpg_enum_mbus_code(struct v4l2_subdev *sd, } static int tpg_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { /*to fake*/ @@ -81,7 +81,7 @@ static int tpg_enum_frame_size(struct v4l2_subdev *sd, } static int tpg_enum_frame_ival(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { /*to fake*/ diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c index 0295e2e32d79..948769ca6539 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c +++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c @@ -1178,7 +1178,7 @@ static void atomisp_unregister_entities(struct atomisp_device *isp) atomisp_mipi_csi2_unregister_entities(&isp->csi2_port[i]); list_for_each_entry_safe(sd, next, &isp->v4l2_dev.subdevs, list) - v4l2_device_unregister_subdev(sd); + v4l2_device_unregister_subdev(sd); v4l2_device_unregister(&isp->v4l2_dev); media_device_unregister(&isp->media_dev); @@ -1500,9 +1500,9 @@ static int init_atomisp_wdts(struct atomisp_device *isp) for (i = 0; i < isp->num_of_streams; i++) { struct atomisp_sub_device *asd = &isp->asd[i]; - if (!IS_ISP2401) + if (!IS_ISP2401) { timer_setup(&asd->wdt, atomisp_wdt, 0); - else { + } else { timer_setup(&asd->video_out_capture.wdt, atomisp_wdt, 0); timer_setup(&asd->video_out_preview.wdt, diff --git a/drivers/staging/media/atomisp/pci/sh_css.c b/drivers/staging/media/atomisp/pci/sh_css.c index 27dd8ce8ba0a..d26b1301eeb7 100644 --- a/drivers/staging/media/atomisp/pci/sh_css.c +++ b/drivers/staging/media/atomisp/pci/sh_css.c @@ -239,8 +239,8 @@ ia_css_reset_defaults(struct sh_css *css); static void sh_css_init_host_sp_control_vars(void); -static int set_num_primary_stages(unsigned int *num, - enum ia_css_pipe_version version); +static int +set_num_primary_stages(unsigned int *num, enum ia_css_pipe_version version); static bool need_capture_pp(const struct ia_css_pipe *pipe); @@ -308,8 +308,7 @@ sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, const void *acc_fw); static int -alloc_continuous_frames( - struct ia_css_pipe *pipe, bool init_time); +alloc_continuous_frames(struct ia_css_pipe *pipe, bool init_time); static void pipe_global_init(void); @@ -413,7 +412,6 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, static void sh_css_pipe_free_shading_table(struct ia_css_pipe *pipe) { - assert(pipe); if (!pipe) { IA_CSS_ERROR("NULL input parameter"); return; @@ -453,15 +451,15 @@ static enum ia_css_frame_format yuv422_copy_formats[] = { * by the copy binary given the stream format. * */ static int -verify_copy_out_frame_format(struct ia_css_pipe *pipe) { +verify_copy_out_frame_format(struct ia_css_pipe *pipe) +{ enum ia_css_frame_format out_fmt = pipe->output_info[0].format; unsigned int i, found = 0; assert(pipe); assert(pipe->stream); - switch (pipe->stream->config.input_config.format) - { + switch (pipe->stream->config.input_config.format) { case ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY: case ATOMISP_INPUT_FORMAT_YUV420_8: for (i = 0; i < ARRAY_SIZE(yuv420_copy_formats) && !found; i++) @@ -528,7 +526,8 @@ ia_css_stream_input_format_bits_per_pixel(struct ia_css_stream *stream) #if !defined(ISP2401) static int -sh_css_config_input_network(struct ia_css_stream *stream) { +sh_css_config_input_network(struct ia_css_stream *stream) +{ unsigned int fmt_type; struct ia_css_pipe *pipe = stream->last_pipe; struct ia_css_binary *binary = NULL; @@ -554,8 +553,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { stream->config.mode); if ((binary && (binary->online || stream->config.continuous)) || - pipe->config.mode == IA_CSS_PIPE_MODE_COPY) - { + pipe->config.mode == IA_CSS_PIPE_MODE_COPY) { err = ia_css_ifmtr_configure(&stream->config, binary); if (err) @@ -563,8 +561,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { } if (stream->config.mode == IA_CSS_INPUT_MODE_TPG || - stream->config.mode == IA_CSS_INPUT_MODE_PRBS) - { + stream->config.mode == IA_CSS_INPUT_MODE_PRBS) { unsigned int hblank_cycles = 100, vblank_lines = 6, width, @@ -723,35 +720,32 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_id( switch (stream_cfg->mode) { case IA_CSS_INPUT_MODE_TPG: - if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID0) { + if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID0) isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT0_ID; - } else if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID1) { + else if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID1) isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT1_ID; - } else if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID2) { + else if (stream_cfg->source.tpg.id == IA_CSS_TPG_ID2) isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT2_ID; - } break; case IA_CSS_INPUT_MODE_PRBS: - if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID0) { + if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID0) isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT0_ID; - } else if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID1) { + else if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID1) isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT1_ID; - } else if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID2) { + else if (stream_cfg->source.prbs.id == IA_CSS_PRBS_ID2) isys_stream_descr->input_port_id = INPUT_SYSTEM_PIXELGEN_PORT2_ID; - } break; case IA_CSS_INPUT_MODE_BUFFERED_SENSOR: - if (stream_cfg->source.port.port == MIPI_PORT0_ID) { + if (stream_cfg->source.port.port == MIPI_PORT0_ID) isys_stream_descr->input_port_id = INPUT_SYSTEM_CSI_PORT0_ID; - } else if (stream_cfg->source.port.port == MIPI_PORT1_ID) { + else if (stream_cfg->source.port.port == MIPI_PORT1_ID) isys_stream_descr->input_port_id = INPUT_SYSTEM_CSI_PORT1_ID; - } else if (stream_cfg->source.port.port == MIPI_PORT2_ID) { + else if (stream_cfg->source.port.port == MIPI_PORT2_ID) isys_stream_descr->input_port_id = INPUT_SYSTEM_CSI_PORT2_ID; - } break; default: @@ -804,15 +798,14 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr( rc = true; switch (stream_cfg->mode) { case IA_CSS_INPUT_MODE_TPG: - if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_RAMP) { + if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_RAMP) isys_stream_descr->tpg_port_attr.mode = PIXELGEN_TPG_MODE_RAMP; - } else if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_CHECKERBOARD) { + else if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_CHECKERBOARD) isys_stream_descr->tpg_port_attr.mode = PIXELGEN_TPG_MODE_CHBO; - } else if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_MONO) { + else if (stream_cfg->source.tpg.mode == IA_CSS_TPG_MODE_MONO) isys_stream_descr->tpg_port_attr.mode = PIXELGEN_TPG_MODE_MONO; - } else { + else rc = false; - } /* * TODO @@ -951,12 +944,12 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_resolution( stream_cfg->mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) && stream_cfg->source.port.compression.type != IA_CSS_CSI2_COMPRESSION_TYPE_NONE) { if (stream_cfg->source.port.compression.uncompressed_bits_per_pixel == - UNCOMPRESSED_BITS_PER_PIXEL_10) { + UNCOMPRESSED_BITS_PER_PIXEL_10) fmt_type = ATOMISP_INPUT_FORMAT_RAW_10; - } else if (stream_cfg->source.port.compression.uncompressed_bits_per_pixel == - UNCOMPRESSED_BITS_PER_PIXEL_12) { + else if (stream_cfg->source.port.compression.uncompressed_bits_per_pixel == + UNCOMPRESSED_BITS_PER_PIXEL_12) fmt_type = ATOMISP_INPUT_FORMAT_RAW_12; - } else + else return false; } @@ -1045,7 +1038,8 @@ static bool sh_css_translate_binary_info_to_input_system_output_port_attr( } static int -sh_css_config_input_network(struct ia_css_stream *stream) { +sh_css_config_input_network(struct ia_css_stream *stream) +{ bool rc; ia_css_isys_descr_t isys_stream_descr; unsigned int sp_thread_id; @@ -1060,19 +1054,16 @@ sh_css_config_input_network(struct ia_css_stream *stream) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_config_input_network() enter 0x%p:\n", stream); - if (stream->config.continuous) - { - if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_CAPTURE) { + if (stream->config.continuous) { + if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_CAPTURE) pipe = stream->last_pipe; - } else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_YUVPP) { + else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_YUVPP) pipe = stream->last_pipe; - } else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_PREVIEW) { + else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_PREVIEW) pipe = stream->last_pipe->pipe_settings.preview.copy_pipe; - } else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_VIDEO) { + else if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_VIDEO) pipe = stream->last_pipe->pipe_settings.video.copy_pipe; - } - } else - { + } else { pipe = stream->last_pipe; if (stream->last_pipe->config.mode == IA_CSS_PIPE_MODE_CAPTURE) { /* @@ -1087,7 +1078,6 @@ sh_css_config_input_network(struct ia_css_stream *stream) { } } - assert(pipe); if (!pipe) return -EINVAL; @@ -1095,8 +1085,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { if (pipe->pipeline.stages->binary) binary = pipe->pipeline.stages->binary; - if (binary) - { + if (binary) { /* this was being done in ifmtr in 2400. * online and cont bypass the init_in_frameinfo_memory_defaults * so need to do it here @@ -1111,8 +1100,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { /* get the target input terminal */ sp_pipeline_input_terminal = &sh_css_sp_group.pipe_io[sp_thread_id].input; - for (i = 0; i < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; i++) - { + for (i = 0; i < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH; i++) { /* initialization */ memset((void *)(&isys_stream_descr), 0, sizeof(ia_css_isys_descr_t)); sp_pipeline_input_terminal->context.virtual_input_system_stream[i].valid = 0; @@ -1210,11 +1198,10 @@ static inline struct ia_css_pipe *stream_get_target_pipe( struct ia_css_pipe *target_pipe; /* get the pipe that consumes the stream */ - if (stream->config.continuous) { + if (stream->config.continuous) target_pipe = stream_get_copy_pipe(stream); - } else { + else target_pipe = stream_get_last_pipe(stream); - } return target_pipe; } @@ -1388,10 +1375,9 @@ start_binary(struct ia_css_pipe *pipe, /* start the copy function on the SP */ static int start_copy_on_sp(struct ia_css_pipe *pipe, - struct ia_css_frame *out_frame) { + struct ia_css_frame *out_frame) +{ (void)out_frame; - assert(pipe); - assert(pipe->stream); if ((!pipe) || (!pipe->stream)) return -EINVAL; @@ -1406,8 +1392,7 @@ start_copy_on_sp(struct ia_css_pipe *pipe, sh_css_sp_start_binary_copy(ia_css_pipe_get_pipe_num(pipe), out_frame, pipe->stream->config.pixels_per_clock == 2); #if !defined(ISP2401) - if (pipe->stream->reconfigure_css_rx) - { + if (pipe->stream->reconfigure_css_rx) { ia_css_isys_rx_configure(&pipe->stream->csi_rx_config, pipe->stream->config.mode); pipe->stream->reconfigure_css_rx = false; @@ -1596,7 +1581,8 @@ ia_css_reset_defaults(struct sh_css *css) int ia_css_load_firmware(struct device *dev, const struct ia_css_env *env, - const struct ia_css_fw *fw) { + const struct ia_css_fw *fw) +{ int err; if (!env) @@ -1607,16 +1593,14 @@ ia_css_load_firmware(struct device *dev, const struct ia_css_env *env, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_load_firmware() enter\n"); /* make sure we initialize my_css */ - if (my_css.flush != env->cpu_mem_env.flush) - { + if (my_css.flush != env->cpu_mem_env.flush) { ia_css_reset_defaults(&my_css); my_css.flush = env->cpu_mem_env.flush; } ia_css_unload_firmware(); /* in case we are called twice */ err = sh_css_load_firmware(dev, fw->data, fw->bytes); - if (!err) - { + if (!err) { err = ia_css_binary_init_infos(); if (!err) fw_explicitly_loaded = true; @@ -1630,7 +1614,8 @@ int ia_css_init(struct device *dev, const struct ia_css_env *env, const struct ia_css_fw *fw, u32 mmu_l1_base, - enum ia_css_irq_type irq_type) { + enum ia_css_irq_type irq_type) +{ int err; ia_css_spctrl_cfg spctrl_cfg; @@ -1704,16 +1689,14 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, my_css.flush = flush_func; err = ia_css_rmgr_init(); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } IA_CSS_LOG("init: %d", my_css_save_initialized); - if (!my_css_save_initialized) - { + if (!my_css_save_initialized) { my_css_save_initialized = true; my_css_save.mode = sh_css_mode_working; memset(my_css_save.stream_seeds, 0, @@ -1741,19 +1724,16 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, gpio_reg_store(GPIO0_ID, _gpio_block_reg_do_0, 0); err = ia_css_refcount_init(REFCOUNT_SIZE); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } err = sh_css_params_init(); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } - if (fw) - { + if (fw) { ia_css_unload_firmware(); /* in case we already had firmware loaded */ err = sh_css_load_firmware(dev, fw->data, fw->bytes); if (err) { @@ -1774,23 +1754,20 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, return -EINVAL; err = ia_css_spctrl_load_fw(SP0_ID, &spctrl_cfg); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } #if WITH_PC_MONITORING - if (!thread_alive) - { + if (!thread_alive) { thread_alive++; sh_css_print("PC_MONITORING: %s() -- create thread DISABLED\n", __func__); spying_thread_create(); } #endif - if (!sh_css_hrt_system_is_idle()) - { + if (!sh_css_hrt_system_is_idle()) { IA_CSS_LEAVE_ERR(-EBUSY); return -EBUSY; } @@ -1823,7 +1800,8 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, } int -ia_css_enable_isys_event_queue(bool enable) { +ia_css_enable_isys_event_queue(bool enable) +{ if (sh_css_sp_is_running()) return -EBUSY; sh_css_sp_enable_isys_event_queue(enable); @@ -1844,7 +1822,8 @@ sh_css_flush(struct ia_css_acc_fw *fw) * doing it from stream_create since we could run out of sp threads due to * allocation on inactive pipelines. */ static int -map_sp_threads(struct ia_css_stream *stream, bool map) { +map_sp_threads(struct ia_css_stream *stream, bool map) +{ struct ia_css_pipe *main_pipe = NULL; struct ia_css_pipe *copy_pipe = NULL; struct ia_css_pipe *capture_pipe = NULL; @@ -1852,12 +1831,10 @@ map_sp_threads(struct ia_css_stream *stream, bool map) { int err = 0; enum ia_css_pipe_id pipe_id; - assert(stream); IA_CSS_ENTER_PRIVATE("stream = %p, map = %s", stream, map ? "true" : "false"); - if (!stream) - { + if (!stream) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -1867,8 +1844,7 @@ map_sp_threads(struct ia_css_stream *stream, bool map) { ia_css_pipeline_map(main_pipe->pipe_num, map); - switch (pipe_id) - { + switch (pipe_id) { case IA_CSS_PIPE_ID_PREVIEW: copy_pipe = main_pipe->pipe_settings.preview.copy_pipe; capture_pipe = main_pipe->pipe_settings.preview.capture_pipe; @@ -1887,23 +1863,17 @@ map_sp_threads(struct ia_css_stream *stream, bool map) { } if (acc_pipe) - { ia_css_pipeline_map(acc_pipe->pipe_num, map); - } if (capture_pipe) - { ia_css_pipeline_map(capture_pipe->pipe_num, map); - } /* Firmware expects copy pipe to be the last pipe mapped. (if needed) */ if (copy_pipe) - { ia_css_pipeline_map(copy_pipe->pipe_num, map); - } + /* DH regular multi pipe - not continuous mode: map the next pipes too */ - if (!stream->config.continuous) - { + if (!stream->config.continuous) { int i; for (i = 1; i < stream->num_pipes; i++) @@ -1917,7 +1887,8 @@ map_sp_threads(struct ia_css_stream *stream, bool map) { /* creates a host pipeline skeleton for all pipes in a stream. Called during * stream_create. */ static int -create_host_pipeline_structure(struct ia_css_stream *stream) { +create_host_pipeline_structure(struct ia_css_stream *stream) +{ struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL; struct ia_css_pipe *acc_pipe = NULL; enum ia_css_pipe_id pipe_id; @@ -1926,27 +1897,22 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { unsigned int copy_pipe_delay = 0, capture_pipe_delay = 0; - assert(stream); IA_CSS_ENTER_PRIVATE("stream = %p", stream); - if (!stream) - { + if (!stream) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } main_pipe = stream->last_pipe; - assert(main_pipe); - if (!main_pipe) - { + if (!main_pipe) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } pipe_id = main_pipe->mode; - switch (pipe_id) - { + switch (pipe_id) { case IA_CSS_PIPE_ID_PREVIEW: copy_pipe = main_pipe->pipe_settings.preview.copy_pipe; copy_pipe_delay = main_pipe->dvs_frame_delay; @@ -1986,30 +1952,23 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { } if (!(err) && copy_pipe) - { err = ia_css_pipeline_create(©_pipe->pipeline, copy_pipe->mode, copy_pipe->pipe_num, copy_pipe_delay); - } if (!(err) && capture_pipe) - { err = ia_css_pipeline_create(&capture_pipe->pipeline, capture_pipe->mode, capture_pipe->pipe_num, capture_pipe_delay); - } if (!(err) && acc_pipe) - { err = ia_css_pipeline_create(&acc_pipe->pipeline, acc_pipe->mode, acc_pipe->pipe_num, main_pipe->dvs_frame_delay); - } /* DH regular multi pipe - not continuous mode: create the next pipelines too */ - if (!stream->config.continuous) - { + if (!stream->config.continuous) { int i; for (i = 1; i < stream->num_pipes && 0 == err; i++) { @@ -2028,7 +1987,8 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { /* creates a host pipeline for all pipes in a stream. Called during * stream_start. */ static int -create_host_pipeline(struct ia_css_stream *stream) { +create_host_pipeline(struct ia_css_stream *stream) +{ struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL; struct ia_css_pipe *acc_pipe = NULL; enum ia_css_pipe_id pipe_id; @@ -2037,8 +1997,7 @@ create_host_pipeline(struct ia_css_stream *stream) { unsigned int max_input_width = 0; IA_CSS_ENTER_PRIVATE("stream = %p", stream); - if (!stream) - { + if (!stream) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -2049,8 +2008,7 @@ create_host_pipeline(struct ia_css_stream *stream) { /* No continuous frame allocation for capture pipe. It uses the * "main" pipe's frames. */ if ((pipe_id == IA_CSS_PIPE_ID_PREVIEW) || - (pipe_id == IA_CSS_PIPE_ID_VIDEO)) - { + (pipe_id == IA_CSS_PIPE_ID_VIDEO)) { /* About pipe_id == IA_CSS_PIPE_ID_PREVIEW && stream->config.mode != IA_CSS_INPUT_MODE_MEMORY: * The original condition pipe_id == IA_CSS_PIPE_ID_PREVIEW is too strong. E.g. in SkyCam (with memory * based input frames) there is no continuous mode and thus no need for allocated continuous frames @@ -2068,24 +2026,21 @@ create_host_pipeline(struct ia_css_stream *stream) { #if !defined(ISP2401) /* old isys: need to allocate_mipi_frames() even in IA_CSS_PIPE_MODE_COPY */ - if (pipe_id != IA_CSS_PIPE_ID_ACC) - { + if (pipe_id != IA_CSS_PIPE_ID_ACC) { err = allocate_mipi_frames(main_pipe, &stream->info); if (err) goto ERR; } #elif defined(ISP2401) if ((pipe_id != IA_CSS_PIPE_ID_ACC) && - (main_pipe->config.mode != IA_CSS_PIPE_MODE_COPY)) - { + (main_pipe->config.mode != IA_CSS_PIPE_MODE_COPY)) { err = allocate_mipi_frames(main_pipe, &stream->info); if (err) goto ERR; } #endif - switch (pipe_id) - { + switch (pipe_id) { case IA_CSS_PIPE_ID_PREVIEW: copy_pipe = main_pipe->pipe_settings.preview.copy_pipe; capture_pipe = main_pipe->pipe_settings.preview.capture_pipe; @@ -2135,31 +2090,27 @@ create_host_pipeline(struct ia_css_stream *stream) { if (err) goto ERR; - if (copy_pipe) - { + if (copy_pipe) { err = create_host_copy_pipeline(copy_pipe, max_input_width, main_pipe->continuous_frames[0]); if (err) goto ERR; } - if (capture_pipe) - { + if (capture_pipe) { err = create_host_capture_pipeline(capture_pipe); if (err) goto ERR; } - if (acc_pipe) - { + if (acc_pipe) { err = create_host_acc_pipeline(acc_pipe); if (err) goto ERR; } /* DH regular multi pipe - not continuous mode: create the next pipelines too */ - if (!stream->config.continuous) - { + if (!stream->config.continuous) { int i; for (i = 1; i < stream->num_pipes && 0 == err; i++) { @@ -2201,10 +2152,9 @@ static const struct ia_css_yuvpp_settings yuvpp = IA_CSS_DEFAULT_YUVPP_SETTINGS; static int init_pipe_defaults(enum ia_css_pipe_mode mode, struct ia_css_pipe *pipe, - bool copy_pipe) { - - if (!pipe) - { + bool copy_pipe) +{ + if (!pipe) { IA_CSS_ERROR("NULL pipe parameter"); return -EINVAL; } @@ -2213,18 +2163,17 @@ init_pipe_defaults(enum ia_css_pipe_mode mode, memcpy(pipe, &default_pipe, sizeof(default_pipe)); /* TODO: JB should not be needed, but temporary backward reference */ - switch (mode) - { + switch (mode) { case IA_CSS_PIPE_MODE_PREVIEW: pipe->mode = IA_CSS_PIPE_ID_PREVIEW; memcpy(&pipe->pipe_settings.preview, &preview, sizeof(preview)); break; case IA_CSS_PIPE_MODE_CAPTURE: - if (copy_pipe) { + if (copy_pipe) pipe->mode = IA_CSS_PIPE_ID_COPY; - } else { + else pipe->mode = IA_CSS_PIPE_ID_CAPTURE; - } + memcpy(&pipe->pipe_settings.capture, &capture, sizeof(capture)); break; case IA_CSS_PIPE_MODE_VIDEO: @@ -2254,27 +2203,25 @@ pipe_global_init(void) u8 i; my_css.pipe_counter = 0; - for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++) { + for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++) my_css.all_pipes[i] = NULL; - } } static int pipe_generate_pipe_num(const struct ia_css_pipe *pipe, - unsigned int *pipe_number) { + unsigned int *pipe_number) +{ const u8 INVALID_PIPE_NUM = (uint8_t)~(0); u8 pipe_num = INVALID_PIPE_NUM; u8 i; - if (!pipe) - { + if (!pipe) { IA_CSS_ERROR("NULL pipe parameter"); return -EINVAL; } /* Assign a new pipe_num .... search for empty place */ - for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++) - { + for (i = 0; i < IA_CSS_PIPELINE_NUM_MAX; i++) { if (!my_css.all_pipes[i]) { /*position is reserved */ my_css.all_pipes[i] = (struct ia_css_pipe *)pipe; @@ -2282,8 +2229,7 @@ pipe_generate_pipe_num(const struct ia_css_pipe *pipe, break; } } - if (pipe_num == INVALID_PIPE_NUM) - { + if (pipe_num == INVALID_PIPE_NUM) { /* Max number of pipes already allocated */ IA_CSS_ERROR("Max number of pipes already created"); return -ENOSPC; @@ -2309,12 +2255,12 @@ pipe_release_pipe_num(unsigned int pipe_num) static int create_pipe(enum ia_css_pipe_mode mode, struct ia_css_pipe **pipe, - bool copy_pipe) { + bool copy_pipe) +{ int err = 0; struct ia_css_pipe *me; - if (!pipe) - { + if (!pipe) { IA_CSS_ERROR("NULL pipe parameter"); return -EINVAL; } @@ -2324,15 +2270,13 @@ create_pipe(enum ia_css_pipe_mode mode, return -ENOMEM; err = init_pipe_defaults(mode, me, copy_pipe); - if (err) - { + if (err) { kfree(me); return err; } err = pipe_generate_pipe_num(me, &me->pipe_num); - if (err) - { + if (err) { kfree(me); return err; } @@ -2361,7 +2305,6 @@ static void sh_css_pipe_free_acc_binaries( struct ia_css_pipeline *pipeline; struct ia_css_pipeline_stage *stage; - assert(pipe); if (!pipe) { IA_CSS_ERROR("NULL input pointer"); return; @@ -2378,26 +2321,24 @@ static void sh_css_pipe_free_acc_binaries( } int -ia_css_pipe_destroy(struct ia_css_pipe *pipe) { +ia_css_pipe_destroy(struct ia_css_pipe *pipe) +{ int err = 0; IA_CSS_ENTER("pipe = %p", pipe); - if (!pipe) - { + if (!pipe) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } - if (pipe->stream) - { + if (pipe->stream) { IA_CSS_LOG("ia_css_stream_destroy not called!"); IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } - switch (pipe->config.mode) - { + switch (pipe->config.mode) { case IA_CSS_PIPE_MODE_PREVIEW: /* need to take into account that this function is also called on the internal copy pipe */ @@ -2461,9 +2402,8 @@ ia_css_pipe_destroy(struct ia_css_pipe *pipe) { /* Temporarily, not every sh_css_pipe has an acc_extension. */ if (pipe->config.acc_extension) - { ia_css_pipe_unload_extension(pipe, pipe->config.acc_extension); - } + kfree(pipe); IA_CSS_LEAVE("err = %d", err); return err; @@ -2493,9 +2433,9 @@ ia_css_uninit(void) ifmtr_set_if_blocking_mode_reset = true; #endif - if (!fw_explicitly_loaded) { + if (!fw_explicitly_loaded) ia_css_unload_firmware(); - } + ia_css_spctrl_unload_fw(SP0_ID); sh_css_sp_set_sp_running(false); /* check and free any remaining mipi frames */ @@ -2681,8 +2621,8 @@ static int load_copy_binary( } static int -alloc_continuous_frames( - struct ia_css_pipe *pipe, bool init_time) { +alloc_continuous_frames(struct ia_css_pipe *pipe, bool init_time) +{ int err = 0; struct ia_css_frame_info ref_info; enum ia_css_pipe_id pipe_id; @@ -2692,8 +2632,7 @@ alloc_continuous_frames( IA_CSS_ENTER_PRIVATE("pipe = %p, init_time = %d", pipe, init_time); - if ((!pipe) || (!pipe->stream)) - { + if ((!pipe) || (!pipe->stream)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -2701,26 +2640,22 @@ alloc_continuous_frames( pipe_id = pipe->mode; continuous = pipe->stream->config.continuous; - if (continuous) - { + if (continuous) { if (init_time) { num_frames = pipe->stream->config.init_num_cont_raw_buf; pipe->stream->continuous_pipe = pipe; - } else + } else { num_frames = pipe->stream->config.target_num_cont_raw_buf; - } else - { + } + } else { num_frames = NUM_ONLINE_INIT_CONTINUOUS_FRAMES; } - if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) - { + if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) { ref_info = pipe->pipe_settings.preview.preview_binary.in_frame_info; - } else if (pipe_id == IA_CSS_PIPE_ID_VIDEO) - { + } else if (pipe_id == IA_CSS_PIPE_ID_VIDEO) { ref_info = pipe->pipe_settings.video.video_binary.in_frame_info; - } else - { + } else { /* should not happen */ IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; @@ -2736,8 +2671,7 @@ alloc_continuous_frames( #endif #if !defined(HAS_NO_PACKED_RAW_PIXELS) - if (pipe->stream->config.pack_raw_pixels) - { + if (pipe->stream->config.pack_raw_pixels) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "alloc_continuous_frames() IA_CSS_FRAME_FORMAT_RAW_PACKED\n"); ref_info.format = IA_CSS_FRAME_FORMAT_RAW_PACKED; @@ -2766,8 +2700,7 @@ alloc_continuous_frames( else idx = pipe->stream->config.init_num_cont_raw_buf; - for (i = idx; i < NUM_CONTINUOUS_FRAMES; i++) - { + for (i = idx; i < NUM_CONTINUOUS_FRAMES; i++) { /* free previous frame */ if (pipe->continuous_frames[i]) { ia_css_frame_free(pipe->continuous_frames[i]); @@ -2797,14 +2730,16 @@ alloc_continuous_frames( } int -ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream) { +ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream) +{ if (!stream) return -EINVAL; return alloc_continuous_frames(stream->continuous_pipe, false); } static int -load_preview_binaries(struct ia_css_pipe *pipe) { +load_preview_binaries(struct ia_css_pipe *pipe) +{ struct ia_css_frame_info prev_in_info, prev_bds_out_info, prev_out_info, @@ -2912,8 +2847,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { * then the preview binary selection is done again. */ if (need_vf_pp && - (mycs->preview_binary.out_frame_info[0].format != IA_CSS_FRAME_FORMAT_YUV_LINE)) - { + (mycs->preview_binary.out_frame_info[0].format != IA_CSS_FRAME_FORMAT_YUV_LINE)) { /* Preview step 2 */ if (pipe->vf_yuv_ds_input_info.res.width) prev_vf_info = pipe->vf_yuv_ds_input_info; @@ -2938,8 +2872,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { return err; } - if (need_vf_pp) - { + if (need_vf_pp) { struct ia_css_binary_descr vf_pp_descr; /* Viewfinder post-processing */ @@ -2970,8 +2903,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { #endif /* Copy */ - if (need_isp_copy_binary) - { + if (need_isp_copy_binary) { err = load_copy_binary(pipe, &mycs->copy_binary, &mycs->preview_binary); @@ -2979,8 +2911,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { return err; } - if (pipe->shading_table) - { + if (pipe->shading_table) { ia_css_shading_table_free(pipe->shading_table); pipe->shading_table = NULL; } @@ -2995,11 +2926,11 @@ ia_css_binary_unload(struct ia_css_binary *binary) } static int -unload_preview_binaries(struct ia_css_pipe *pipe) { +unload_preview_binaries(struct ia_css_pipe *pipe) +{ IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) - { + if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -3052,20 +2983,19 @@ static int add_firmwares( struct ia_css_frame *in = NULL; struct ia_css_frame *vf = NULL; - if ((fw == last_fw) && (fw->info.isp.sp.enable.out_frame != 0)) { + if ((fw == last_fw) && (fw->info.isp.sp.enable.out_frame != 0)) out[0] = out_frame; - } - if (fw->info.isp.sp.enable.in_frame != 0) { + + if (fw->info.isp.sp.enable.in_frame != 0) in = in_frame; - } - if (fw->info.isp.sp.enable.out_frame != 0) { + + if (fw->info.isp.sp.enable.out_frame != 0) vf = vf_frame; - } + ia_css_pipe_get_firmwares_stage_desc(&stage_desc, binary, out, in, vf, fw, binary_mode); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - &extra_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + &extra_stage); if (err) return err; if (fw->info.isp.sp.enable.output != 0) @@ -3173,9 +3103,8 @@ static int add_yuv_scaler_stage( ia_css_pipe_get_generic_stage_desc(&stage_desc, yuv_scaler_binary, out_frames, in_frame, vf_frame); } - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - pre_vf_pp_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + pre_vf_pp_stage); if (err) return err; in_frame = (*pre_vf_pp_stage)->args.out_frame[0]; @@ -3233,9 +3162,8 @@ static int add_capture_pp_stage( ia_css_pipe_get_generic_stage_desc(&stage_desc, capture_pp_binary, out_frames, NULL, vf_frame); } - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - capture_pp_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + capture_pp_stage); if (err) return err; err = add_firmwares(me, capture_pp_binary, pipe->output_stage, last_fw, @@ -3274,7 +3202,8 @@ static void sh_css_setup_queues(void) static int init_vf_frameinfo_defaults(struct ia_css_pipe *pipe, - struct ia_css_frame *vf_frame, unsigned int idx) { + struct ia_css_frame *vf_frame, unsigned int idx) +{ int err = 0; unsigned int thread_id; enum sh_css_queue_id queue_id; @@ -3439,7 +3368,8 @@ ia_css_get_crop_offsets( static int init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe, - struct ia_css_frame *frame, enum ia_css_frame_format format) { + struct ia_css_frame *frame, enum ia_css_frame_format format) +{ struct ia_css_frame *in_frame; int err = 0; unsigned int thread_id; @@ -3480,7 +3410,8 @@ init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe, static int init_out_frameinfo_defaults(struct ia_css_pipe *pipe, - struct ia_css_frame *out_frame, unsigned int idx) { + struct ia_css_frame *out_frame, unsigned int idx) +{ int err = 0; unsigned int thread_id; enum sh_css_queue_id queue_id; @@ -3587,9 +3518,8 @@ static int create_host_video_pipeline(struct ia_css_pipe *pipe) ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, out_frames, NULL, NULL); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - ©_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + ©_stage); if (err) goto ERR; in_frame = me->stages->args.out_frame[0]; @@ -3616,9 +3546,8 @@ static int create_host_video_pipeline(struct ia_css_pipe *pipe) ia_css_pipe_get_generic_stage_desc(&stage_desc, video_binary, out_frames, in_frame, vf_frame); } - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - &video_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + &video_stage); if (err) goto ERR; @@ -3681,13 +3610,10 @@ static int create_host_video_pipeline(struct ia_css_pipe *pipe) struct ia_css_frame *tmp_out_frame = NULL; for (i = 0; i < num_yuv_scaler; i++) { - if (is_output_stage[i]) { - tmp_out_frame = out_frame; - } else { - tmp_out_frame = NULL; - } - err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, tmp_out_frame, - NULL, + tmp_out_frame = is_output_stage[i] ? out_frame : NULL; + + err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, + tmp_out_frame, NULL, &yuv_scaler_binary[i], &yuv_scaler_stage); @@ -3711,14 +3637,14 @@ static int create_host_video_pipeline(struct ia_css_pipe *pipe) } static int -create_host_acc_pipeline(struct ia_css_pipe *pipe) { +create_host_acc_pipeline(struct ia_css_pipe *pipe) +{ int err = 0; const struct ia_css_fw_info *fw; unsigned int i; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || (!pipe->stream)) - { + if ((!pipe) || (!pipe->stream)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -3729,15 +3655,13 @@ create_host_acc_pipeline(struct ia_css_pipe *pipe) { pipe->pipeline.pipe_qos_config = 0; fw = pipe->vf_stage; - for (i = 0; fw; fw = fw->next) - { + for (i = 0; fw; fw = fw->next) { err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw); if (err) goto ERR; } - for (i = 0; i < pipe->config.num_acc_stages; i++) - { + for (i = 0; i < pipe->config.num_acc_stages; i++) { struct ia_css_fw_info *fw = pipe->config.acc_stages[i]; err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw); @@ -3754,7 +3678,8 @@ create_host_acc_pipeline(struct ia_css_pipe *pipe) { /* Create stages for preview */ static int -create_host_preview_pipeline(struct ia_css_pipe *pipe) { +create_host_preview_pipeline(struct ia_css_pipe *pipe) +{ struct ia_css_pipeline_stage *copy_stage = NULL; struct ia_css_pipeline_stage *preview_stage = NULL; struct ia_css_pipeline_stage *vf_pp_stage = NULL; @@ -3774,8 +3699,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { #endif IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) - { + if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -3803,16 +3727,14 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { /* Construct in_frame info (only in case we have dynamic input */ need_in_frameinfo_memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY; #endif - if (need_in_frameinfo_memory) - { + if (need_in_frameinfo_memory) { err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame, IA_CSS_FRAME_FORMAT_RAW); if (err) goto ERR; in_frame = &me->in_frame; - } else - { + } else { in_frame = NULL; } @@ -3826,14 +3748,12 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { if (pipe->pipe_settings.preview.vf_pp_binary.info) vf_pp_binary = &pipe->pipe_settings.preview.vf_pp_binary; - if (pipe->pipe_settings.preview.copy_binary.info) - { + if (pipe->pipe_settings.preview.copy_binary.info) { ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, out_frames, NULL, NULL); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - ©_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + ©_stage); if (err) goto ERR; in_frame = me->stages->args.out_frame[0]; @@ -3842,42 +3762,37 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { /* When continuous is enabled, configure in_frame with the * last pipe, which is the copy pipe. */ - if (continuous || !online) { + if (continuous || !online) in_frame = pipe->stream->last_pipe->continuous_frames[0]; - } + #else in_frame = pipe->continuous_frames[0]; #endif } - if (vf_pp_binary) - { + if (vf_pp_binary) { ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); ia_css_pipe_get_generic_stage_desc(&stage_desc, preview_binary, out_frames, in_frame, NULL); - } else - { + } else { ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame); ia_css_pipe_get_generic_stage_desc(&stage_desc, preview_binary, out_frames, in_frame, NULL); } - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - &preview_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + &preview_stage); if (err) goto ERR; /* If we use copy iso preview, the input must be yuv iso raw */ preview_stage->args.copy_vf = preview_binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_COPY; preview_stage->args.copy_output = !preview_stage->args.copy_vf; - if (preview_stage->args.copy_vf && !preview_stage->args.out_vf_frame) - { + if (preview_stage->args.copy_vf && !preview_stage->args.out_vf_frame) { /* in case of copy, use the vf frame as output frame */ preview_stage->args.out_vf_frame = preview_stage->args.out_frame[0]; } - if (vf_pp_binary) - { + if (vf_pp_binary) { if (preview_binary->info->sp.pipeline.mode == IA_CSS_BINARY_MODE_COPY) in_frame = preview_stage->args.out_vf_frame; else @@ -3917,7 +3832,8 @@ static void send_raw_frames(struct ia_css_pipe *pipe) } static int -preview_start(struct ia_css_pipe *pipe) { +preview_start(struct ia_css_pipe *pipe) +{ int err = 0; struct ia_css_pipe *copy_pipe, *capture_pipe; struct ia_css_pipe *acc_pipe; @@ -3927,8 +3843,7 @@ preview_start(struct ia_css_pipe *pipe) { const struct ia_css_isp_parameters *params = NULL; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) - { + if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -3955,8 +3870,7 @@ preview_start(struct ia_css_pipe *pipe) { ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); copy_ovrd = 1 << thread_id; - if (pipe->stream->cont_capt) - { + if (pipe->stream->cont_capt) { ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(capture_pipe), &thread_id); copy_ovrd |= 1 << thread_id; @@ -3969,8 +3883,7 @@ preview_start(struct ia_css_pipe *pipe) { } /* Construct and load the copy pipe */ - if (pipe->stream->config.continuous) - { + if (pipe->stream->config.continuous) { sh_css_sp_init_pipeline(©_pipe->pipeline, IA_CSS_PIPE_ID_COPY, (uint8_t)ia_css_pipe_get_pipe_num(copy_pipe), @@ -3991,8 +3904,7 @@ preview_start(struct ia_css_pipe *pipe) { } /* Construct and load the capture pipe */ - if (pipe->stream->cont_capt) - { + if (pipe->stream->cont_capt) { sh_css_sp_init_pipeline(&capture_pipe->pipeline, IA_CSS_PIPE_ID_CAPTURE, (uint8_t)ia_css_pipe_get_pipe_num(capture_pipe), @@ -4010,8 +3922,7 @@ preview_start(struct ia_css_pipe *pipe) { params); } - if (acc_pipe) - { + if (acc_pipe) { sh_css_sp_init_pipeline(&acc_pipe->pipeline, IA_CSS_PIPE_ID_ACC, (uint8_t)ia_css_pipe_get_pipe_num(acc_pipe), @@ -4037,7 +3948,8 @@ preview_start(struct ia_css_pipe *pipe) { int ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, - const struct ia_css_buffer *buffer) { + const struct ia_css_buffer *buffer) +{ int return_err = 0; unsigned int thread_id; enum sh_css_queue_id queue_id; @@ -4052,8 +3964,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, IA_CSS_ENTER("pipe=%p, buffer=%p", pipe, buffer); - if ((!pipe) || (!buffer)) - { + if ((!pipe) || (!buffer)) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } @@ -4062,8 +3973,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, /* following code will be enabled when IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME is removed */ #if 0 - if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) - { + if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) { bool found_pipe = false; for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) { @@ -4077,8 +3987,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, if (!found_pipe) return -EINVAL; } - if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) - { + if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) { bool found_pipe = false; for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) { @@ -4099,36 +4008,31 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, assert(pipe_id < IA_CSS_PIPE_ID_NUM); assert(buf_type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE); - if ((buf_type == IA_CSS_BUFFER_TYPE_INVALID) || - (buf_type >= IA_CSS_NUM_DYNAMIC_BUFFER_TYPE) || - (pipe_id >= IA_CSS_PIPE_ID_NUM)) - { + if (buf_type == IA_CSS_BUFFER_TYPE_INVALID || + buf_type >= IA_CSS_NUM_DYNAMIC_BUFFER_TYPE || + pipe_id >= IA_CSS_PIPE_ID_NUM) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); - if (!ret_err) - { + if (!ret_err) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id); - if (!ret_err) - { + if (!ret_err) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } - if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) - { + if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } - if (!sh_css_sp_is_running()) - { + if (!sh_css_sp_is_running()) { IA_CSS_LOG("SP is not running!"); IA_CSS_LEAVE_ERR(-EBUSY); /* SP is not running. The queues are not valid */ @@ -4146,36 +4050,32 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, ddr_buffer.cookie_ptr = buffer->driver_cookie; ddr_buffer.timing_data = buffer->timing_data; - if (buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS) - { + if (buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS) { if (!buffer->data.stats_3a) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_3a); ddr_buffer.payload.s3a = *buffer->data.stats_3a; - } else if (buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS) - { + } else if (buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS) { if (!buffer->data.stats_dvs) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_dvs); ddr_buffer.payload.dis = *buffer->data.stats_dvs; - } else if (buf_type == IA_CSS_BUFFER_TYPE_METADATA) - { + } else if (buf_type == IA_CSS_BUFFER_TYPE_METADATA) { if (!buffer->data.metadata) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.metadata); ddr_buffer.payload.metadata = *buffer->data.metadata; - } else if ((buf_type == IA_CSS_BUFFER_TYPE_INPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME)) - { + } else if (buf_type == IA_CSS_BUFFER_TYPE_INPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME) { if (!buffer->data.frame) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; @@ -4207,22 +4107,17 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, /* TODO: change next to correct pool for optimization */ ia_css_rmgr_acq_vbuf(hmm_buffer_pool, &h_vbuf); - assert(h_vbuf); - assert(h_vbuf->vptr != 0x0); - - if ((!h_vbuf) || (h_vbuf->vptr == 0x0)) - { + if ((!h_vbuf) || (h_vbuf->vptr == 0x0)) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } hmm_store(h_vbuf->vptr, - (void *)(&ddr_buffer), - sizeof(struct sh_css_hmm_buffer)); - if ((buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS) - || (buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS) - || (buf_type == IA_CSS_BUFFER_TYPE_LACE_STATISTICS)) - { + (void *)(&ddr_buffer), + sizeof(struct sh_css_hmm_buffer)); + if (buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS || + buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS || + buf_type == IA_CSS_BUFFER_TYPE_LACE_STATISTICS) { if (!pipeline) { ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &h_vbuf); IA_CSS_LOG("pipeline is empty!"); @@ -4240,19 +4135,18 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, (uint32_t)h_vbuf->vptr); } } - } else if ((buf_type == IA_CSS_BUFFER_TYPE_INPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME) - || (buf_type == IA_CSS_BUFFER_TYPE_METADATA)) - { + } else if (buf_type == IA_CSS_BUFFER_TYPE_INPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_METADATA) { return_err = ia_css_bufq_enqueue_buffer(thread_id, queue_id, (uint32_t)h_vbuf->vptr); #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS) - if (!(return_err) && - (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)) { + if (!return_err && + buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) { IA_CSS_LOG("pfp: enqueued OF %d to q %d thread %d", ddr_buffer.payload.frame.frame_data, queue_id, thread_id); @@ -4260,8 +4154,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, #endif } - if (!return_err) - { + if (!return_err) { if (sh_css_hmm_buffer_record_acquire( h_vbuf, buf_type, HOST_ADDRESS(ddr_buffer.kernel_ptr))) { @@ -4276,8 +4169,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, * Tell the SP which queues are not empty, * by sending the software event. */ - if (!return_err) - { + if (!return_err) { if (!sh_css_sp_is_running()) { /* SP is not running. The queues are not valid */ IA_CSS_LOG("SP is not running!"); @@ -4289,8 +4181,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, (uint8_t)thread_id, queue_id, 0); - } else - { + } else { ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &h_vbuf); IA_CSS_ERROR("buffer not enqueued"); } @@ -4305,7 +4196,8 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, */ int ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, - struct ia_css_buffer *buffer) { + struct ia_css_buffer *buffer) +{ int return_err; enum sh_css_queue_id queue_id; ia_css_ptr ddr_buffer_addr = (ia_css_ptr)0; @@ -4318,8 +4210,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, IA_CSS_ENTER("pipe=%p, buffer=%p", pipe, buffer); - if ((!pipe) || (!buffer)) - { + if ((!pipe) || (!buffer)) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } @@ -4333,27 +4224,23 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, ddr_buffer.kernel_ptr = 0; ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); - if (!ret_err) - { + if (!ret_err) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id); - if (!ret_err) - { + if (!ret_err) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } - if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) - { + if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } - if (!sh_css_sp_is_running()) - { + if (!sh_css_sp_is_running()) { IA_CSS_LOG("SP is not running!"); IA_CSS_LEAVE_ERR(-EBUSY); /* SP is not running. The queues are not valid */ @@ -4363,8 +4250,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, return_err = ia_css_bufq_dequeue_buffer(queue_id, (uint32_t *)&ddr_buffer_addr); - if (!return_err) - { + if (!return_err) { struct ia_css_frame *frame; struct sh_css_hmm_buffer_record *hmm_buffer_record = NULL; @@ -4389,8 +4275,8 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, } hmm_load(ddr_buffer_addr, - &ddr_buffer, - sizeof(struct sh_css_hmm_buffer)); + &ddr_buffer, + sizeof(struct sh_css_hmm_buffer)); /* if the kernel_ptr is 0 or an invalid, return an error. * do not access the buffer via the kernal_ptr. @@ -4412,8 +4298,8 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, buffer->driver_cookie = ddr_buffer.cookie_ptr; buffer->timing_data = ddr_buffer.timing_data; - if ((buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) || - (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME)) { + if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME || + buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) { buffer->isys_eof_clock_tick.ticks = ddr_buffer.isys_eof_clock_tick; } @@ -4506,8 +4392,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, * Tell the SP which queues are not full, * by sending the software event. */ - if (!return_err) - { + if (!return_err) { if (!sh_css_sp_is_running()) { IA_CSS_LOG("SP is not running!"); IA_CSS_LEAVE_ERR(-EBUSY); @@ -4556,12 +4441,14 @@ static enum ia_css_event_type convert_event_sp_to_host_domain[] = { }; int -ia_css_dequeue_event(struct ia_css_event *event) { +ia_css_dequeue_event(struct ia_css_event *event) +{ return ia_css_dequeue_psys_event(event); } int -ia_css_dequeue_psys_event(struct ia_css_event *event) { +ia_css_dequeue_psys_event(struct ia_css_event *event) +{ enum ia_css_pipe_id pipe_id = 0; u8 payload[4] = {0, 0, 0, 0}; int ret_err; @@ -4576,11 +4463,9 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { if (!event) return -EINVAL; + /* SP is not running. The queues are not valid */ if (!sh_css_sp_is_running()) - { - /* SP is not running. The queues are not valid */ return -EBUSY; - } /* dequeue the event (if any) from the psys event queue */ ret_err = ia_css_bufq_dequeue_psys_event(payload); @@ -4607,8 +4492,7 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { event->timer_code = 0; event->timer_subcode = 0; - if (event->type == IA_CSS_EVENT_TYPE_TIMER) - { + if (event->type == IA_CSS_EVENT_TYPE_TIMER) { /* timer event ??? get the 2nd event and decode the data into the event struct */ u32 tmp_data; /* 1st event: LSB 16-bit timer data and code */ @@ -4632,37 +4516,32 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { tmp_data = ((payload[1] & 0xFF) | ((payload[3] & 0xFF) << 8)); event->timer_data |= (tmp_data << 16); event->timer_subcode = payload[2]; - } + } else { /* It's a non timer event. So clear first half of the timer event data. * If the second part of the TIMER event is not received, we discard * the first half of the timer data and process the non timer event without * affecting the flow. So the non timer event falls through * the code. */ - else { event->timer_data = 0; event->timer_code = 0; event->timer_subcode = 0; IA_CSS_ERROR("Missing 2nd timer event. Timer event discarded"); } } - if (event->type == IA_CSS_EVENT_TYPE_PORT_EOF) - { + if (event->type == IA_CSS_EVENT_TYPE_PORT_EOF) { event->port = (enum mipi_port_id)payload[1]; event->exp_id = payload[3]; - } else if (event->type == IA_CSS_EVENT_TYPE_FW_WARNING) - { + } else if (event->type == IA_CSS_EVENT_TYPE_FW_WARNING) { event->fw_warning = (enum ia_css_fw_warning)payload[1]; /* exp_id is only available in these warning types */ if (event->fw_warning == IA_CSS_FW_WARNING_EXP_ID_LOCKED || event->fw_warning == IA_CSS_FW_WARNING_TAG_EXP_ID_FAILED) event->exp_id = payload[3]; - } else if (event->type == IA_CSS_EVENT_TYPE_FW_ASSERT) - { + } else if (event->type == IA_CSS_EVENT_TYPE_FW_ASSERT) { event->fw_assert_module_id = payload[1]; /* module */ event->fw_assert_line_no = (payload[2] << 8) + payload[3]; /* payload[2] is line_no>>8, payload[3] is line_no&0xff */ - } else if (event->type != IA_CSS_EVENT_TYPE_TIMER) - { + } else if (event->type != IA_CSS_EVENT_TYPE_TIMER) { /* pipe related events. * payload[1] contains the pipe_num, * payload[2] contains the pipe_id. These are different. */ @@ -4712,7 +4591,8 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { } int -ia_css_dequeue_isys_event(struct ia_css_event *event) { +ia_css_dequeue_isys_event(struct ia_css_event *event) +{ u8 payload[4] = {0, 0, 0, 0}; int err = 0; @@ -4722,11 +4602,9 @@ ia_css_dequeue_isys_event(struct ia_css_event *event) { if (!event) return -EINVAL; + /* SP is not running. The queues are not valid */ if (!sh_css_sp_is_running()) - { - /* SP is not running. The queues are not valid */ return -EBUSY; - } err = ia_css_bufq_dequeue_isys_event(payload); if (err) @@ -4759,7 +4637,8 @@ acc_start(struct ia_css_pipe *pipe) } static int -sh_css_pipe_start(struct ia_css_stream *stream) { +sh_css_pipe_start(struct ia_css_stream *stream) +{ int err = 0; struct ia_css_pipe *pipe; @@ -4768,22 +4647,19 @@ sh_css_pipe_start(struct ia_css_stream *stream) { IA_CSS_ENTER_PRIVATE("stream = %p", stream); - if (!stream) - { + if (!stream) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } pipe = stream->last_pipe; - if (!pipe) - { + if (!pipe) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } pipe_id = pipe->mode; - if (stream->started) - { + if (stream->started) { IA_CSS_WARNING("Cannot start stream that is already started"); IA_CSS_LEAVE_ERR(err); return err; @@ -4791,8 +4667,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { pipe->stop_requested = false; - switch (pipe_id) - { + switch (pipe_id) { case IA_CSS_PIPE_ID_PREVIEW: err = preview_start(pipe); break; @@ -4812,8 +4687,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { err = -EINVAL; } /* DH regular multi pipe - not continuous mode: start the next pipes too */ - if (!stream->config.continuous) - { + if (!stream->config.continuous) { int i; for (i = 1; i < stream->num_pipes && 0 == err ; i++) { @@ -4843,8 +4717,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { } } } - if (err) - { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4854,8 +4727,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { * don't use ISP parameters anyway. So this should be okay. * The SP binary (jpeg) copy does not use any parameters. */ - if (!copy_on_sp(pipe)) - { + if (!copy_on_sp(pipe)) { sh_css_invalidate_params(stream); err = sh_css_param_update_isp_params(pipe, stream->isp_params_configs, true, NULL); @@ -4869,8 +4741,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); - if (!sh_css_sp_is_running()) - { + if (!sh_css_sp_is_running()) { IA_CSS_LEAVE_ERR_PRIVATE(-EBUSY); /* SP is not running. The queues are not valid */ return -EBUSY; @@ -4879,8 +4750,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { (uint8_t)thread_id, 0, 0); /* DH regular multi pipe - not continuous mode: enqueue event to the next pipes too */ - if (!stream->config.continuous) - { + if (!stream->config.continuous) { int i; for (i = 1; i < stream->num_pipes; i++) { @@ -4894,8 +4764,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { } /* in case of continuous capture mode, we also start capture thread and copy thread*/ - if (pipe->stream->config.continuous) - { + if (pipe->stream->config.continuous) { struct ia_css_pipe *copy_pipe = NULL; if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) @@ -4914,8 +4783,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { IA_CSS_PSYS_SW_EVENT_START_STREAM, (uint8_t)thread_id, 0, 0); } - if (pipe->stream->cont_capt) - { + if (pipe->stream->cont_capt) { struct ia_css_pipe *capture_pipe = NULL; if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) @@ -4936,8 +4804,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { } /* in case of PREVIEW mode, check whether QOS acc_pipe is available, then start the qos pipe */ - if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) - { + if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) { struct ia_css_pipe *acc_pipe = NULL; acc_pipe = pipe->pipe_settings.preview.acc_pipe; @@ -4988,7 +4855,8 @@ sh_css_continuous_is_enabled(uint8_t pipe_num) /* ISP2400 */ int ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, - int *buffer_depth) { + int *buffer_depth) +{ if (!buffer_depth) return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_max_buffer_depth() enter: void\n"); @@ -4998,7 +4866,8 @@ ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, } int -ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth) { +ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth) +{ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_set_buffer_depth() enter: num_frames=%d\n", buffer_depth); (void)stream; if (buffer_depth > NUM_CONTINUOUS_FRAMES || buffer_depth < 1) @@ -5012,7 +4881,8 @@ ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth) { /* ISP2401 */ int ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, - int *buffer_depth) { + int *buffer_depth) +{ if (!buffer_depth) return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_buffer_depth() enter: void\n"); @@ -5036,18 +4906,14 @@ sh_css_pipes_stop(struct ia_css_stream *stream) enum ia_css_pipe_id main_pipe_id; int i; - assert(stream); - if (!stream) - { + if (!stream) { IA_CSS_LOG("stream does NOT exist!"); err = -EINVAL; goto ERR; } main_pipe = stream->last_pipe; - assert(main_pipe); - if (!main_pipe) - { + if (!main_pipe) { IA_CSS_LOG("main_pipe does NOT exist!"); err = -EINVAL; goto ERR; @@ -5060,11 +4926,10 @@ sh_css_pipes_stop(struct ia_css_stream *stream) * Stop all "ia_css_pipe" instances in this target * "ia_css_stream" instance. */ - for (i = 0; i < stream->num_pipes; i++) - { + for (i = 0; i < stream->num_pipes; i++) { /* send the "stop" request to the "ia_css_pipe" instance */ IA_CSS_LOG("Send the stop-request to the pipe: pipe_id=%d", - stream->pipes[i]->pipeline.pipe_id); + stream->pipes[i]->pipeline.pipe_id); err = ia_css_pipeline_request_stop(&stream->pipes[i]->pipeline); /* @@ -5095,8 +4960,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream) * * We need to stop this "Copy Pipe", as well. */ - if (main_pipe->stream->config.continuous) - { + if (main_pipe->stream->config.continuous) { struct ia_css_pipe *copy_pipe = NULL; /* get the reference to "Copy Pipe" */ @@ -5106,7 +4970,6 @@ sh_css_pipes_stop(struct ia_css_stream *stream) copy_pipe = main_pipe->pipe_settings.video.copy_pipe; /* return the error code if "Copy Pipe" does NOT exist */ - assert(copy_pipe); if (!copy_pipe) { IA_CSS_LOG("Copy Pipe does NOT exist!"); err = -EINVAL; @@ -5115,7 +4978,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream) /* send the "stop" request to "Copy Pipe" */ IA_CSS_LOG("Send the stop-request to the pipe: pipe_id=%d", - copy_pipe->pipeline.pipe_id); + copy_pipe->pipeline.pipe_id); err = ia_css_pipeline_request_stop(©_pipe->pipeline); } @@ -5141,7 +5004,6 @@ sh_css_pipes_have_stopped(struct ia_css_stream *stream) int i; - assert(stream); if (!stream) { IA_CSS_LOG("stream does NOT exist!"); rval = false; @@ -5149,7 +5011,6 @@ sh_css_pipes_have_stopped(struct ia_css_stream *stream) } main_pipe = stream->last_pipe; - assert(main_pipe); if (!main_pipe) { IA_CSS_LOG("main_pipe does NOT exist!"); @@ -5190,7 +5051,6 @@ sh_css_pipes_have_stopped(struct ia_css_stream *stream) copy_pipe = main_pipe->pipe_settings.video.copy_pipe; /* return if "Copy Pipe" does NOT exist */ - assert(copy_pipe); if (!copy_pipe) { IA_CSS_LOG("Copy Pipe does NOT exist!"); @@ -5272,8 +5132,7 @@ sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe, binary = ia_css_pipe_get_shading_correction_binary(pipe); - if (binary) - { + if (binary) { err = ia_css_binary_get_shading_info(binary, IA_CSS_SHADING_CORRECTION_TYPE_1, pipe->required_bds_factor, @@ -5283,8 +5142,7 @@ sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe, /* Other function calls can be added here when other shading correction types will be added * in the future. */ - } else - { + } else { /* When the pipe does not have a binary which has the shading * correction, this function does not need to fill the shading * information. It is not a error case, and then @@ -5297,7 +5155,8 @@ sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe, static int sh_css_pipe_get_grid_info(struct ia_css_pipe *pipe, - struct ia_css_grid_info *info) { + struct ia_css_grid_info *info) +{ int err = 0; struct ia_css_binary *binary = NULL; @@ -5308,30 +5167,27 @@ sh_css_pipe_get_grid_info(struct ia_css_pipe *pipe, binary = ia_css_pipe_get_s3a_binary(pipe); - if (binary) - { + if (binary) { err = ia_css_binary_3a_grid_info(binary, info, pipe); if (err) goto ERR; - } else + } else { memset(&info->s3a_grid, 0, sizeof(info->s3a_grid)); + } binary = ia_css_pipe_get_sdis_binary(pipe); - if (binary) - { + if (binary) { ia_css_binary_dvs_grid_info(binary, info, pipe); ia_css_binary_dvs_stat_grid_info(binary, info, pipe); - } else - { + } else { memset(&info->dvs_grid.dvs_grid_info, 0, sizeof(info->dvs_grid.dvs_grid_info)); memset(&info->dvs_grid.dvs_stat_grid_info, 0, sizeof(info->dvs_grid.dvs_stat_grid_info)); } - if (binary) - { + if (binary) { /* copy pipe does not have ISP binary*/ info->isp_in_width = binary->internal_frame_info.res.width; info->isp_in_height = binary->internal_frame_info.res.height; @@ -5351,7 +5207,8 @@ ERR : */ static int ia_css_pipe_check_format(struct ia_css_pipe *pipe, - enum ia_css_frame_format format) { + enum ia_css_frame_format format) +{ const enum ia_css_frame_format *supported_formats; int number_of_formats; int found = 0; @@ -5359,8 +5216,7 @@ ia_css_pipe_check_format(struct ia_css_pipe *pipe, IA_CSS_ENTER_PRIVATE(""); - if (NULL == pipe || NULL == pipe->pipe_settings.video.video_binary.info) - { + if (NULL == pipe || NULL == pipe->pipe_settings.video.video_binary.info) { IA_CSS_ERROR("Pipe or binary info is not set"); IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; @@ -5369,23 +5225,19 @@ ia_css_pipe_check_format(struct ia_css_pipe *pipe, supported_formats = pipe->pipe_settings.video.video_binary.info->output_formats; number_of_formats = sizeof(pipe->pipe_settings.video.video_binary.info->output_formats) / sizeof(enum ia_css_frame_format); - for (i = 0; i < number_of_formats && !found; i++) - { + for (i = 0; i < number_of_formats && !found; i++) { if (supported_formats[i] == format) { found = 1; break; } } - if (!found) - { + if (!found) { IA_CSS_ERROR("Requested format is not supported by binary"); IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; - } else - { - IA_CSS_LEAVE_ERR_PRIVATE(0); - return 0; } + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } static int load_video_binaries(struct ia_css_pipe *pipe) @@ -5528,10 +5380,10 @@ static int load_video_binaries(struct ia_css_pipe *pipe) &mycs->video_binary); if (err) { - if (video_vf_info) { - /* This will do another video binary lookup later for YUV_LINE format*/ + /* This will do another video binary lookup later for YUV_LINE format*/ + if (video_vf_info) need_vf_pp = true; - } else + else return err; } else if (video_vf_info) { /* The first video binary lookup is successful, but we may @@ -5694,13 +5546,13 @@ static int load_video_binaries(struct ia_css_pipe *pipe) } static int -unload_video_binaries(struct ia_css_pipe *pipe) { +unload_video_binaries(struct ia_css_pipe *pipe) +{ unsigned int i; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) - { + if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -5850,31 +5702,29 @@ static int sh_css_pipe_configure_viewfinder(struct ia_css_pipe *pipe, unsigned int width, unsigned int height, unsigned int min_width, enum ia_css_frame_format format, - unsigned int idx) { + unsigned int idx) +{ int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, min_width = %d, format = %d, idx = %d\n", pipe, width, height, min_width, format, idx); - if (!pipe) - { + if (!pipe) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } err = ia_css_util_check_res(width, height); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } if (pipe->vf_output_info[idx].res.width != width || pipe->vf_output_info[idx].res.height != height || pipe->vf_output_info[idx].format != format) - { ia_css_frame_info_init(&pipe->vf_output_info[idx], width, height, format, min_width); - } + IA_CSS_LEAVE_ERR_PRIVATE(0); return 0; } @@ -5955,7 +5805,7 @@ static bool need_capt_ldc( } static int set_num_primary_stages(unsigned int *num, - enum ia_css_pipe_version version) + enum ia_css_pipe_version version) { int err = 0; @@ -6155,10 +6005,13 @@ static int load_primary_binaries( capt_pp_in_info = &prim_out_info; ia_css_pipe_get_capturepp_binarydesc(pipe, - &capture_pp_descr, capt_pp_in_info, - &capt_pp_out_info, &vf_info); + &capture_pp_descr, + capt_pp_in_info, + &capt_pp_out_info, + &vf_info); + err = ia_css_binary_find(&capture_pp_descr, - &mycs->capture_pp_binary); + &mycs->capture_pp_binary); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -6168,11 +6021,12 @@ static int load_primary_binaries( struct ia_css_binary_descr capt_ldc_descr; ia_css_pipe_get_ldc_binarydesc(pipe, - &capt_ldc_descr, &prim_out_info, - &capt_ldc_out_info); + &capt_ldc_descr, + &prim_out_info, + &capt_ldc_out_info); err = ia_css_binary_find(&capt_ldc_descr, - &mycs->capture_ldc_binary); + &mycs->capture_ldc_binary); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -6189,8 +6043,9 @@ static int load_primary_binaries( if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] && (i == mycs->num_primary_stage - 1)) local_vf_info = &vf_info; - ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], &prim_in_info, - &prim_out_info, local_vf_info, i); + ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], + &prim_in_info, &prim_out_info, + local_vf_info, i); err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -6242,8 +6097,8 @@ static int load_primary_binaries( /* ISP Copy */ if (need_isp_copy_binary) { err = load_copy_binary(pipe, - &mycs->copy_binary, - &mycs->primary_binary[0]); + &mycs->copy_binary, + &mycs->primary_binary[0]); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -6254,7 +6109,8 @@ static int load_primary_binaries( } static int -allocate_delay_frames(struct ia_css_pipe *pipe) { +allocate_delay_frames(struct ia_css_pipe *pipe) +{ unsigned int num_delay_frames = 0, i = 0; unsigned int dvs_frame_delay = 0; struct ia_css_frame_info ref_info; @@ -6264,8 +6120,7 @@ allocate_delay_frames(struct ia_css_pipe *pipe) { IA_CSS_ENTER_PRIVATE(""); - if (!pipe) - { + if (!pipe) { IA_CSS_ERROR("Invalid args - pipe %p", pipe); return -EINVAL; } @@ -6276,8 +6131,7 @@ allocate_delay_frames(struct ia_css_pipe *pipe) { if (dvs_frame_delay > 0) num_delay_frames = dvs_frame_delay + 1; - switch (mode) - { + switch (mode) { case IA_CSS_PIPE_ID_CAPTURE: { struct ia_css_capture_settings *mycs_capture = &pipe->pipe_settings.capture; (void)mycs_capture; @@ -6329,8 +6183,7 @@ allocate_delay_frames(struct ia_css_pipe *pipe) { ref_info.raw_bit_depth = SH_CSS_REF_BIT_DEPTH; assert(num_delay_frames <= MAX_NUM_VIDEO_DELAY_FRAMES); - for (i = 0; i < num_delay_frames; i++) - { + for (i = 0; i < num_delay_frames; i++) { err = ia_css_frame_allocate_from_info(&delay_frames[i], &ref_info); if (err) return err; @@ -6339,8 +6192,8 @@ allocate_delay_frames(struct ia_css_pipe *pipe) { return 0; } -static int load_advanced_binaries( - struct ia_css_pipe *pipe) { +static int load_advanced_binaries(struct ia_css_pipe *pipe) +{ struct ia_css_frame_info pre_in_info, gdc_in_info, post_in_info, post_out_info, vf_info, *vf_pp_in_info, *pipe_out_info, @@ -6353,7 +6206,7 @@ static int load_advanced_binaries( assert(pipe); assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || - pipe->mode == IA_CSS_PIPE_ID_COPY); + pipe->mode == IA_CSS_PIPE_ID_COPY); if (pipe->pipe_settings.capture.pre_isp_binary.info) return 0; pipe_out_info = &pipe->output_info[0]; @@ -6366,17 +6219,18 @@ static int load_advanced_binaries( need_pp = need_capture_pp(pipe); ia_css_frame_info_set_format(&vf_info, - IA_CSS_FRAME_FORMAT_YUV_LINE); + IA_CSS_FRAME_FORMAT_YUV_LINE); /* we build up the pipeline starting at the end */ /* Capture post-processing */ if (need_pp) { struct ia_css_binary_descr capture_pp_descr; - ia_css_pipe_get_capturepp_binarydesc(pipe, - &capture_pp_descr, &post_out_info, pipe_out_info, &vf_info); + ia_css_pipe_get_capturepp_binarydesc(pipe, &capture_pp_descr, + &post_out_info, + pipe_out_info, &vf_info); err = ia_css_binary_find(&capture_pp_descr, - &pipe->pipe_settings.capture.capture_pp_binary); + &pipe->pipe_settings.capture.capture_pp_binary); if (err) return err; } else { @@ -6387,10 +6241,11 @@ static int load_advanced_binaries( { struct ia_css_binary_descr post_gdc_descr; - ia_css_pipe_get_post_gdc_binarydesc(pipe, - &post_gdc_descr, &post_in_info, &post_out_info, &vf_info); + ia_css_pipe_get_post_gdc_binarydesc(pipe, &post_gdc_descr, + &post_in_info, + &post_out_info, &vf_info); err = ia_css_binary_find(&post_gdc_descr, - &pipe->pipe_settings.capture.post_isp_binary); + &pipe->pipe_settings.capture.post_isp_binary); if (err) return err; } @@ -6400,9 +6255,9 @@ static int load_advanced_binaries( struct ia_css_binary_descr gdc_descr; ia_css_pipe_get_gdc_binarydesc(pipe, &gdc_descr, &gdc_in_info, - &pipe->pipe_settings.capture.post_isp_binary.in_frame_info); + &pipe->pipe_settings.capture.post_isp_binary.in_frame_info); err = ia_css_binary_find(&gdc_descr, - &pipe->pipe_settings.capture.anr_gdc_binary); + &pipe->pipe_settings.capture.anr_gdc_binary); if (err) return err; } @@ -6414,9 +6269,9 @@ static int load_advanced_binaries( struct ia_css_binary_descr pre_gdc_descr; ia_css_pipe_get_pre_gdc_binarydesc(pipe, &pre_gdc_descr, &pre_in_info, - &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info); + &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info); err = ia_css_binary_find(&pre_gdc_descr, - &pipe->pipe_settings.capture.pre_isp_binary); + &pipe->pipe_settings.capture.pre_isp_binary); if (err) return err; } @@ -6438,7 +6293,7 @@ static int load_advanced_binaries( ia_css_pipe_get_vfpp_binarydesc(pipe, &vf_pp_descr, vf_pp_in_info, pipe_vf_out_info); err = ia_css_binary_find(&vf_pp_descr, - &pipe->pipe_settings.capture.vf_pp_binary); + &pipe->pipe_settings.capture.vf_pp_binary); if (err) return err; } @@ -6450,14 +6305,14 @@ static int load_advanced_binaries( #endif if (need_isp_copy) load_copy_binary(pipe, - &pipe->pipe_settings.capture.copy_binary, - &pipe->pipe_settings.capture.pre_isp_binary); + &pipe->pipe_settings.capture.copy_binary, + &pipe->pipe_settings.capture.pre_isp_binary); return err; } -static int load_bayer_isp_binaries( - struct ia_css_pipe *pipe) { +static int load_bayer_isp_binaries(struct ia_css_pipe *pipe) +{ struct ia_css_frame_info pre_isp_in_info, *pipe_out_info; int err = 0; struct ia_css_binary_descr pre_de_descr; @@ -6465,7 +6320,7 @@ static int load_bayer_isp_binaries( IA_CSS_ENTER_PRIVATE(""); assert(pipe); assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || - pipe->mode == IA_CSS_PIPE_ID_COPY); + pipe->mode == IA_CSS_PIPE_ID_COPY); pipe_out_info = &pipe->output_info[0]; if (pipe->pipe_settings.capture.pre_isp_binary.info) @@ -6476,17 +6331,17 @@ static int load_bayer_isp_binaries( return err; ia_css_pipe_get_pre_de_binarydesc(pipe, &pre_de_descr, - &pre_isp_in_info, - pipe_out_info); + &pre_isp_in_info, + pipe_out_info); err = ia_css_binary_find(&pre_de_descr, - &pipe->pipe_settings.capture.pre_isp_binary); + &pipe->pipe_settings.capture.pre_isp_binary); return err; } -static int load_low_light_binaries( - struct ia_css_pipe *pipe) { +static int load_low_light_binaries(struct ia_css_pipe *pipe) +{ struct ia_css_frame_info pre_in_info, anr_in_info, post_in_info, post_out_info, vf_info, *pipe_vf_out_info, *pipe_out_info, @@ -6498,7 +6353,7 @@ static int load_low_light_binaries( IA_CSS_ENTER_PRIVATE(""); assert(pipe); assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || - pipe->mode == IA_CSS_PIPE_ID_COPY); + pipe->mode == IA_CSS_PIPE_ID_COPY); if (pipe->pipe_settings.capture.pre_isp_binary.info) return 0; @@ -6513,17 +6368,18 @@ static int load_low_light_binaries( need_pp = need_capture_pp(pipe); ia_css_frame_info_set_format(&vf_info, - IA_CSS_FRAME_FORMAT_YUV_LINE); + IA_CSS_FRAME_FORMAT_YUV_LINE); /* we build up the pipeline starting at the end */ /* Capture post-processing */ if (need_pp) { struct ia_css_binary_descr capture_pp_descr; - ia_css_pipe_get_capturepp_binarydesc(pipe, - &capture_pp_descr, &post_out_info, pipe_out_info, &vf_info); + ia_css_pipe_get_capturepp_binarydesc(pipe, &capture_pp_descr, + &post_out_info, + pipe_out_info, &vf_info); err = ia_css_binary_find(&capture_pp_descr, - &pipe->pipe_settings.capture.capture_pp_binary); + &pipe->pipe_settings.capture.capture_pp_binary); if (err) return err; } else { @@ -6537,7 +6393,7 @@ static int load_low_light_binaries( ia_css_pipe_get_post_anr_binarydesc(pipe, &post_anr_descr, &post_in_info, &post_out_info, &vf_info); err = ia_css_binary_find(&post_anr_descr, - &pipe->pipe_settings.capture.post_isp_binary); + &pipe->pipe_settings.capture.post_isp_binary); if (err) return err; } @@ -6547,9 +6403,9 @@ static int load_low_light_binaries( struct ia_css_binary_descr anr_descr; ia_css_pipe_get_anr_binarydesc(pipe, &anr_descr, &anr_in_info, - &pipe->pipe_settings.capture.post_isp_binary.in_frame_info); + &pipe->pipe_settings.capture.post_isp_binary.in_frame_info); err = ia_css_binary_find(&anr_descr, - &pipe->pipe_settings.capture.anr_gdc_binary); + &pipe->pipe_settings.capture.anr_gdc_binary); if (err) return err; } @@ -6561,9 +6417,9 @@ static int load_low_light_binaries( struct ia_css_binary_descr pre_anr_descr; ia_css_pipe_get_pre_anr_binarydesc(pipe, &pre_anr_descr, &pre_in_info, - &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info); + &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info); err = ia_css_binary_find(&pre_anr_descr, - &pipe->pipe_settings.capture.pre_isp_binary); + &pipe->pipe_settings.capture.pre_isp_binary); if (err) return err; } @@ -6582,10 +6438,10 @@ static int load_low_light_binaries( { struct ia_css_binary_descr vf_pp_descr; - ia_css_pipe_get_vfpp_binarydesc(pipe, - &vf_pp_descr, vf_pp_in_info, pipe_vf_out_info); + ia_css_pipe_get_vfpp_binarydesc(pipe, &vf_pp_descr, + vf_pp_in_info, pipe_vf_out_info); err = ia_css_binary_find(&vf_pp_descr, - &pipe->pipe_settings.capture.vf_pp_binary); + &pipe->pipe_settings.capture.vf_pp_binary); if (err) return err; } @@ -6597,8 +6453,8 @@ static int load_low_light_binaries( #endif if (need_isp_copy) err = load_copy_binary(pipe, - &pipe->pipe_settings.capture.copy_binary, - &pipe->pipe_settings.capture.pre_isp_binary); + &pipe->pipe_settings.capture.copy_binary, + &pipe->pipe_settings.capture.pre_isp_binary); return err; } @@ -6623,15 +6479,15 @@ static bool copy_on_sp(struct ia_css_pipe *pipe) return rval; } -static int load_capture_binaries( - struct ia_css_pipe *pipe) { +static int load_capture_binaries(struct ia_css_pipe *pipe) +{ int err = 0; bool must_be_raw; IA_CSS_ENTER_PRIVATE(""); assert(pipe); assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || - pipe->mode == IA_CSS_PIPE_ID_COPY); + pipe->mode == IA_CSS_PIPE_ID_COPY); if (pipe->pipe_settings.capture.primary_binary[0].info) { IA_CSS_LEAVE_ERR_PRIVATE(0); @@ -6692,13 +6548,14 @@ static int load_capture_binaries( } static int -unload_capture_binaries(struct ia_css_pipe *pipe) { +unload_capture_binaries(struct ia_css_pipe *pipe) +{ unsigned int i; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || ((pipe->mode != IA_CSS_PIPE_ID_CAPTURE) && (pipe->mode != IA_CSS_PIPE_ID_COPY))) - { + if (!pipe || (pipe->mode != IA_CSS_PIPE_ID_CAPTURE && + pipe->mode != IA_CSS_PIPE_ID_COPY)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -6726,7 +6583,8 @@ unload_capture_binaries(struct ia_css_pipe *pipe) { static bool need_downscaling(const struct ia_css_resolution in_res, - const struct ia_css_resolution out_res) { + const struct ia_css_resolution out_res) +{ if (in_res.width > out_res.width || in_res.height > out_res.height) return true; @@ -6734,7 +6592,8 @@ need_downscaling(const struct ia_css_resolution in_res, } static bool -need_yuv_scaler_stage(const struct ia_css_pipe *pipe) { +need_yuv_scaler_stage(const struct ia_css_pipe *pipe) +{ unsigned int i; struct ia_css_resolution in_res, out_res; @@ -6773,10 +6632,11 @@ need_yuv_scaler_stage(const struct ia_css_pipe *pipe) { /* which has some hard-coded knowledge which prevents reuse of the function. */ /* Later, merge this with ia_css_pipe_create_cas_scaler_desc */ static int ia_css_pipe_create_cas_scaler_desc_single_output( - struct ia_css_frame_info *cas_scaler_in_info, - struct ia_css_frame_info *cas_scaler_out_info, - struct ia_css_frame_info *cas_scaler_vf_info, - struct ia_css_cas_binary_descr *descr) { + struct ia_css_frame_info *cas_scaler_in_info, + struct ia_css_frame_info *cas_scaler_out_info, + struct ia_css_frame_info *cas_scaler_vf_info, + struct ia_css_cas_binary_descr *descr) +{ unsigned int i; unsigned int hor_ds_factor = 0, ver_ds_factor = 0; int err = 0; @@ -6794,9 +6654,9 @@ static int ia_css_pipe_create_cas_scaler_desc_single_output( descr->num_output_stage = 1; hor_ds_factor = CEIL_DIV(cas_scaler_in_info->res.width, - cas_scaler_out_info->res.width); + cas_scaler_out_info->res.width); ver_ds_factor = CEIL_DIV(cas_scaler_in_info->res.height, - cas_scaler_out_info->res.height); + cas_scaler_out_info->res.height); /* use the same horizontal and vertical downscaling factor for simplicity */ assert(hor_ds_factor == ver_ds_factor); @@ -6806,31 +6666,36 @@ static int ia_css_pipe_create_cas_scaler_desc_single_output( i *= max_scale_factor_per_stage; } - descr->in_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), - GFP_KERNEL); + descr->in_info = kmalloc(descr->num_stage * + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->in_info) { err = -ENOMEM; goto ERR; } - descr->internal_out_info = kmalloc(descr->num_stage * sizeof( - struct ia_css_frame_info), GFP_KERNEL); + descr->internal_out_info = kmalloc(descr->num_stage * + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->internal_out_info) { err = -ENOMEM; goto ERR; } - descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), - GFP_KERNEL); + descr->out_info = kmalloc(descr->num_stage * + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->out_info) { err = -ENOMEM; goto ERR; } - descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), - GFP_KERNEL); + descr->vf_info = kmalloc(descr->num_stage * + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->vf_info) { err = -ENOMEM; goto ERR; } - descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL); + descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), + GFP_KERNEL); if (!descr->is_output_stage) { err = -ENOMEM; goto ERR; @@ -6874,9 +6739,9 @@ static int ia_css_pipe_create_cas_scaler_desc_single_output( max_scale_factor_per_stage; descr->internal_out_info[i].format = IA_CSS_FRAME_FORMAT_YUV420; ia_css_frame_info_init(&descr->internal_out_info[i], - tmp_in_info.res.width / max_scale_factor_per_stage, - tmp_in_info.res.height / max_scale_factor_per_stage, - IA_CSS_FRAME_FORMAT_YUV420, 0); + tmp_in_info.res.width / max_scale_factor_per_stage, + tmp_in_info.res.height / max_scale_factor_per_stage, + IA_CSS_FRAME_FORMAT_YUV420, 0); descr->out_info[i].res.width = 0; descr->out_info[i].res.height = 0; descr->vf_info[i].res.width = 0; @@ -6892,9 +6757,10 @@ static int ia_css_pipe_create_cas_scaler_desc_single_output( } /* FIXME: merge most of this and single output version */ -static int ia_css_pipe_create_cas_scaler_desc( - struct ia_css_pipe *pipe, - struct ia_css_cas_binary_descr *descr) { +static int +ia_css_pipe_create_cas_scaler_desc(struct ia_css_pipe *pipe, + struct ia_css_cas_binary_descr *descr) +{ struct ia_css_frame_info in_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO; struct ia_css_frame_info *out_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE]; struct ia_css_frame_info *vf_out_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE]; @@ -6951,30 +6817,35 @@ static int ia_css_pipe_create_cas_scaler_desc( descr->num_stage = num_stages; descr->in_info = kmalloc_array(descr->num_stage, - sizeof(struct ia_css_frame_info), GFP_KERNEL); + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->in_info) { err = -ENOMEM; goto ERR; } - descr->internal_out_info = kmalloc(descr->num_stage * sizeof( - struct ia_css_frame_info), GFP_KERNEL); + descr->internal_out_info = kmalloc(descr->num_stage * + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->internal_out_info) { err = -ENOMEM; goto ERR; } - descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), - GFP_KERNEL); + descr->out_info = kmalloc(descr->num_stage * + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->out_info) { err = -ENOMEM; goto ERR; } - descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), - GFP_KERNEL); + descr->vf_info = kmalloc(descr->num_stage * + sizeof(struct ia_css_frame_info), + GFP_KERNEL); if (!descr->vf_info) { err = -ENOMEM; goto ERR; } - descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL); + descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), + GFP_KERNEL); if (!descr->is_output_stage) { err = -ENOMEM; goto ERR; @@ -6984,7 +6855,7 @@ static int ia_css_pipe_create_cas_scaler_desc( if (out_info[i]) { if (i > 0) { assert((out_info[i - 1]->res.width >= out_info[i]->res.width) && - (out_info[i - 1]->res.height >= out_info[i]->res.height)); + (out_info[i - 1]->res.height >= out_info[i]->res.height)); } } } @@ -7032,9 +6903,9 @@ static int ia_css_pipe_create_cas_scaler_desc( max_scale_factor_per_stage; descr->internal_out_info[i].format = IA_CSS_FRAME_FORMAT_YUV420; ia_css_frame_info_init(&descr->internal_out_info[i], - tmp_in_info.res.width / max_scale_factor_per_stage, - tmp_in_info.res.height / max_scale_factor_per_stage, - IA_CSS_FRAME_FORMAT_YUV420, 0); + tmp_in_info.res.width / max_scale_factor_per_stage, + tmp_in_info.res.height / max_scale_factor_per_stage, + IA_CSS_FRAME_FORMAT_YUV420, 0); descr->out_info[i].res.width = 0; descr->out_info[i].res.height = 0; descr->vf_info[i].res.width = 0; @@ -7050,7 +6921,8 @@ static int ia_css_pipe_create_cas_scaler_desc( } static void ia_css_pipe_destroy_cas_scaler_desc(struct ia_css_cas_binary_descr - *descr) { + *descr) +{ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "ia_css_pipe_destroy_cas_scaler_desc() enter:\n"); kfree(descr->in_info); @@ -7068,7 +6940,8 @@ static void ia_css_pipe_destroy_cas_scaler_desc(struct ia_css_cas_binary_descr } static int -load_yuvpp_binaries(struct ia_css_pipe *pipe) { +load_yuvpp_binaries(struct ia_css_pipe *pipe) +{ int err = 0; bool need_scaler = false; struct ia_css_frame_info *vf_pp_in_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE]; @@ -7093,8 +6966,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { mycs = &pipe->pipe_settings.yuvpp; - for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) - { + for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) { if (pipe->vf_output_info[i].res.width != 0) { err = ia_css_util_check_vf_out_info(&pipe->output_info[i], &pipe->vf_output_info[i]); @@ -7108,18 +6980,18 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { /* we build up the pipeline starting at the end */ /* Capture post-processing */ - if (need_scaler) - { + if (need_scaler) { struct ia_css_binary_descr yuv_scaler_descr; err = ia_css_pipe_create_cas_scaler_desc(pipe, - &cas_scaler_descr); + &cas_scaler_descr); if (err) goto ERR; mycs->num_output = cas_scaler_descr.num_output_stage; mycs->num_yuv_scaler = cas_scaler_descr.num_stage; mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage * - sizeof(struct ia_css_binary), GFP_KERNEL); + sizeof(struct ia_css_binary), + GFP_KERNEL); if (!mycs->yuv_scaler_binary) { err = -ENOMEM; goto ERR; @@ -7133,28 +7005,25 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { for (i = 0; i < cas_scaler_descr.num_stage; i++) { mycs->is_output_stage[i] = cas_scaler_descr.is_output_stage[i]; ia_css_pipe_get_yuvscaler_binarydesc(pipe, - &yuv_scaler_descr, &cas_scaler_descr.in_info[i], - &cas_scaler_descr.out_info[i], - &cas_scaler_descr.internal_out_info[i], - &cas_scaler_descr.vf_info[i]); + &yuv_scaler_descr, + &cas_scaler_descr.in_info[i], + &cas_scaler_descr.out_info[i], + &cas_scaler_descr.internal_out_info[i], + &cas_scaler_descr.vf_info[i]); err = ia_css_binary_find(&yuv_scaler_descr, - &mycs->yuv_scaler_binary[i]); + &mycs->yuv_scaler_binary[i]); if (err) goto ERR; } ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr); - } else - { + } else { mycs->num_output = 1; } if (need_scaler) - { next_binary = &mycs->yuv_scaler_binary[0]; - } else - { + else next_binary = NULL; - } #if defined(ISP2401) /* @@ -7180,11 +7049,10 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { need_isp_copy_binary = true; #endif /* ISP2401 */ - if (need_isp_copy_binary) - { + if (need_isp_copy_binary) { err = load_copy_binary(pipe, - &mycs->copy_binary, - next_binary); + &mycs->copy_binary, + next_binary); if (err) goto ERR; @@ -7211,8 +7079,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { } /* Viewfinder post-processing */ - if (need_scaler) - { + if (need_scaler) { for (i = 0, j = 0; i < mycs->num_yuv_scaler; i++) { if (mycs->is_output_stage[i]) { assert(j < 2); @@ -7222,19 +7089,18 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { } } mycs->num_vf_pp = j; - } else - { + } else { vf_pp_in_info[0] = &mycs->copy_binary.vf_frame_info; - for (i = 1; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) { + for (i = 1; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) vf_pp_in_info[i] = NULL; - } + mycs->num_vf_pp = 1; } - mycs->vf_pp_binary = kzalloc(mycs->num_vf_pp * sizeof(struct ia_css_binary), - GFP_KERNEL); - if (!mycs->vf_pp_binary) - { + mycs->vf_pp_binary = kzalloc(mycs->num_vf_pp * + sizeof(struct ia_css_binary), + GFP_KERNEL); + if (!mycs->vf_pp_binary) { err = -ENOMEM; goto ERR; } @@ -7242,8 +7108,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { { struct ia_css_binary_descr vf_pp_descr; - for (i = 0; i < mycs->num_vf_pp; i++) - { + for (i = 0; i < mycs->num_vf_pp; i++) { if (pipe->vf_output_info[i].res.width != 0) { ia_css_pipe_get_vfpp_binarydesc(pipe, &vf_pp_descr, vf_pp_in_info[i], &pipe->vf_output_info[i]); @@ -7259,34 +7124,31 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { ERR: if (need_scaler) - { ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr); - } + ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "load_yuvpp_binaries() leave, err=%d\n", err); return err; } static int -unload_yuvpp_binaries(struct ia_css_pipe *pipe) { +unload_yuvpp_binaries(struct ia_css_pipe *pipe) +{ unsigned int i; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) - { + if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } ia_css_binary_unload(&pipe->pipe_settings.yuvpp.copy_binary); for (i = 0; i < pipe->pipe_settings.yuvpp.num_yuv_scaler; i++) - { ia_css_binary_unload(&pipe->pipe_settings.yuvpp.yuv_scaler_binary[i]); - } + for (i = 0; i < pipe->pipe_settings.yuvpp.num_vf_pp; i++) - { ia_css_binary_unload(&pipe->pipe_settings.yuvpp.vf_pp_binary[i]); - } + kfree(pipe->pipe_settings.yuvpp.is_output_stage); pipe->pipe_settings.yuvpp.is_output_stage = NULL; kfree(pipe->pipe_settings.yuvpp.yuv_scaler_binary); @@ -7336,25 +7198,23 @@ static int yuvpp_start(struct ia_css_pipe *pipe) } static int -sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe) { +sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe) +{ int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if (!pipe) - { + if (!pipe) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } /* PIPE_MODE_COPY has no binaries, but has output frames to outside*/ - if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY) - { + if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY) { IA_CSS_LEAVE_ERR_PRIVATE(0); return 0; } - switch (pipe->mode) - { + switch (pipe->mode) { case IA_CSS_PIPE_ID_PREVIEW: err = unload_preview_binaries(pipe); break; @@ -7375,7 +7235,8 @@ sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe) { } static int -sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) { +sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) +{ int err = 0; assert(pipe); @@ -7385,8 +7246,7 @@ sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) { if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY) return err; - switch (pipe->mode) - { + switch (pipe->mode) { case IA_CSS_PIPE_ID_PREVIEW: err = load_preview_binaries(pipe); break; @@ -7405,8 +7265,7 @@ sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) { err = -EINVAL; break; } - if (err) - { + if (err) { if (sh_css_pipe_unload_binaries(pipe)) { /* currently css does not support multiple error returns in a single function, * using -EINVAL in this case */ @@ -7417,7 +7276,8 @@ sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) { } static int -create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { +create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) +{ struct ia_css_pipeline *me; int err = 0; struct ia_css_pipeline_stage *vf_pp_stage = NULL, @@ -7444,15 +7304,13 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { #endif IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); - if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) - { + if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } me = &pipe->pipeline; ia_css_pipeline_clean(me); - for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) - { + for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) { out_frame[i] = NULL; vf_frame[i] = NULL; } @@ -7480,8 +7338,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { /* the input frame can come from: * a) memory: connect yuvscaler to me->in_frame * b) sensor, via copy binary: connect yuvscaler to copy binary later on */ - if (need_in_frameinfo_memory) - { + if (need_in_frameinfo_memory) { /* TODO: improve for different input formats. */ /* @@ -7530,13 +7387,11 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { } in_frame = &me->in_frame; - } else - { + } else { in_frame = NULL; } - for (i = 0; i < num_output_stage; i++) - { + for (i = 0; i < num_output_stage; i++) { assert(i < IA_CSS_PIPE_MAX_OUTPUT_STAGE); if (pipe->output_info[i].res.width != 0) { err = init_out_frameinfo_defaults(pipe, &me->out_frame[i], i); @@ -7563,8 +7418,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { yuv_scaler_binary = pipe->pipe_settings.yuvpp.yuv_scaler_binary; need_scaler = need_yuv_scaler_stage(pipe); - if (pipe->pipe_settings.yuvpp.copy_binary.info) - { + if (pipe->pipe_settings.yuvpp.copy_binary.info) { struct ia_css_frame *in_frame_local = NULL; #ifdef ISP2401 @@ -7574,18 +7428,26 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { #endif if (need_scaler) { - ia_css_pipe_util_set_output_frames(bin_out_frame, 0, NULL); - ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, - bin_out_frame, in_frame_local, NULL); + ia_css_pipe_util_set_output_frames(bin_out_frame, + 0, NULL); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + copy_binary, + bin_out_frame, + in_frame_local, + NULL); } else { - ia_css_pipe_util_set_output_frames(bin_out_frame, 0, out_frame[0]); - ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, - bin_out_frame, in_frame_local, NULL); + ia_css_pipe_util_set_output_frames(bin_out_frame, + 0, out_frame[0]); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + copy_binary, + bin_out_frame, + in_frame_local, + NULL); } err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - ©_stage); + &stage_desc, + ©_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -7602,8 +7464,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { } } - if (need_scaler) - { + if (need_scaler) { struct ia_css_frame *tmp_out_frame = NULL; struct ia_css_frame *tmp_vf_frame = NULL; struct ia_css_frame *tmp_in_frame = in_frame; @@ -7618,10 +7479,11 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { tmp_vf_frame = NULL; } - err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, tmp_out_frame, - NULL, - &yuv_scaler_binary[i], - &yuv_scaler_stage); + err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, + tmp_out_frame, + NULL, + &yuv_scaler_binary[i], + &yuv_scaler_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -7632,8 +7494,10 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { if (pipe->pipe_settings.yuvpp.is_output_stage[i]) { if (tmp_vf_frame && (tmp_vf_frame->info.res.width != 0)) { in_frame = yuv_scaler_stage->args.out_vf_frame; - err = add_vf_pp_stage(pipe, in_frame, tmp_vf_frame, &vf_pp_binary[j], - &vf_pp_stage); + err = add_vf_pp_stage(pipe, in_frame, + tmp_vf_frame, + &vf_pp_binary[j], + &vf_pp_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -7643,12 +7507,11 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { j++; } } - } else if (copy_stage) - { + } else if (copy_stage) { if (vf_frame[0] && vf_frame[0]->info.res.width != 0) { in_frame = copy_stage->args.out_vf_frame; - err = add_vf_pp_stage(pipe, in_frame, vf_frame[0], &vf_pp_binary[0], - &vf_pp_stage); + err = add_vf_pp_stage(pipe, in_frame, vf_frame[0], + &vf_pp_binary[0], &vf_pp_stage); } if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -7656,7 +7519,8 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { } } - ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous); + ia_css_pipeline_finalize_stages(&pipe->pipeline, + pipe->stream->config.continuous); IA_CSS_LEAVE_ERR_PRIVATE(0); @@ -7665,8 +7529,9 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { static int create_host_copy_pipeline(struct ia_css_pipe *pipe, - unsigned int max_input_width, - struct ia_css_frame *out_frame) { + unsigned int max_input_width, + struct ia_css_frame *out_frame) +{ struct ia_css_pipeline *me; int err = 0; struct ia_css_pipeline_stage_desc stage_desc; @@ -7683,16 +7548,10 @@ create_host_copy_pipeline(struct ia_css_pipe *pipe, out_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE; if (copy_on_sp(pipe) && - pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) - { - ia_css_frame_info_init( - &out_frame->info, - JPEG_BYTES, - 1, - IA_CSS_FRAME_FORMAT_BINARY_8, - 0); - } else if (out_frame->info.format == IA_CSS_FRAME_FORMAT_RAW) - { + pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) { + ia_css_frame_info_init(&out_frame->info, JPEG_BYTES, 1, + IA_CSS_FRAME_FORMAT_BINARY_8, 0); + } else if (out_frame->info.format == IA_CSS_FRAME_FORMAT_RAW) { out_frame->info.raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe); } @@ -7702,12 +7561,12 @@ create_host_copy_pipeline(struct ia_css_pipe *pipe, pipe->mode = IA_CSS_PIPE_ID_COPY; ia_css_pipe_get_sp_func_stage_desc(&stage_desc, out_frame, - IA_CSS_PIPELINE_RAW_COPY, max_input_width); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - NULL); + IA_CSS_PIPELINE_RAW_COPY, + max_input_width); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, NULL); - ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous); + ia_css_pipeline_finalize_stages(&pipe->pipeline, + pipe->stream->config.continuous); ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "create_host_copy_pipeline() leave:\n"); @@ -7716,7 +7575,8 @@ create_host_copy_pipeline(struct ia_css_pipe *pipe, } static int -create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) { +create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) +{ struct ia_css_pipeline *me = &pipe->pipeline; int err = 0; struct ia_css_pipeline_stage_desc stage_desc; @@ -7745,9 +7605,10 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) { me->pipe_id = IA_CSS_PIPE_ID_CAPTURE; pipe->mode = IA_CSS_PIPE_ID_CAPTURE; ia_css_pipe_get_sp_func_stage_desc(&stage_desc, out_frame, - IA_CSS_PIPELINE_ISYS_COPY, max_input_width); + IA_CSS_PIPELINE_ISYS_COPY, + max_input_width); err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, &out_stage); + &stage_desc, &out_stage); if (err) return err; @@ -7760,7 +7621,8 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) { } static int -create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { +create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) +{ struct ia_css_pipeline *me; int err = 0; enum ia_css_capture_mode mode; @@ -7798,7 +7660,8 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { IA_CSS_ENTER_PRIVATE(""); assert(pipe); assert(pipe->stream); - assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY); + assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || + pipe->mode == IA_CSS_PIPE_ID_COPY); me = &pipe->pipeline; mode = pipe->config.default_capture_config.mode; @@ -7824,8 +7687,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { /* Construct in_frame info (only in case we have dynamic input */ need_in_frameinfo_memory = pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY; #endif - if (need_in_frameinfo_memory) - { + if (need_in_frameinfo_memory) { err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame, IA_CSS_FRAME_FORMAT_RAW); if (err) { @@ -7834,22 +7696,19 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { } in_frame = &me->in_frame; - } else - { + } else { in_frame = NULL; } err = init_out_frameinfo_defaults(pipe, &me->out_frame[0], 0); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } out_frame = &me->out_frame[0]; /* Construct vf_frame info (only in case we have VF) */ - if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) - { + if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) { if (mode == IA_CSS_CAPTURE_MODE_RAW || mode == IA_CSS_CAPTURE_MODE_BAYER) { /* These modes don't support viewfinder output */ vf_frame = NULL; @@ -7857,22 +7716,20 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { init_vf_frameinfo_defaults(pipe, &me->vf_frame[0], 0); vf_frame = &me->vf_frame[0]; } - } else - { + } else { vf_frame = NULL; } copy_binary = &pipe->pipe_settings.capture.copy_binary; num_primary_stage = pipe->pipe_settings.capture.num_primary_stage; - if ((num_primary_stage == 0) && (mode == IA_CSS_CAPTURE_MODE_PRIMARY)) - { + if ((num_primary_stage == 0) && (mode == IA_CSS_CAPTURE_MODE_PRIMARY)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } + for (i = 0; i < num_primary_stage; i++) - { primary_binary[i] = &pipe->pipe_settings.capture.primary_binary[i]; - } + vf_pp_binary = &pipe->pipe_settings.capture.vf_pp_binary; pre_isp_binary = &pipe->pipe_settings.capture.pre_isp_binary; anr_gdc_binary = &pipe->pipe_settings.capture.anr_gdc_binary; @@ -7889,43 +7746,51 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { need_yuv_pp = (yuv_scaler_binary && yuv_scaler_binary->info); need_ldc = (capture_ldc_binary && capture_ldc_binary->info); - if (pipe->pipe_settings.capture.copy_binary.info) - { + if (pipe->pipe_settings.capture.copy_binary.info) { if (raw) { ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame); #if defined(ISP2401) if (!continuous) { - ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, - out_frames, in_frame, NULL); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + copy_binary, + out_frames, + in_frame, + NULL); } else { in_frame = pipe->stream->last_pipe->continuous_frames[0]; - ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, - out_frames, in_frame, NULL); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + copy_binary, + out_frames, + in_frame, + NULL); } #else - ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, - out_frames, NULL, NULL); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + copy_binary, + out_frames, + NULL, NULL); #endif } else { - ia_css_pipe_util_set_output_frames(out_frames, 0, in_frame); - ia_css_pipe_get_generic_stage_desc(&stage_desc, copy_binary, - out_frames, NULL, NULL); + ia_css_pipe_util_set_output_frames(out_frames, 0, + in_frame); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + copy_binary, + out_frames, + NULL, NULL); } err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - ¤t_stage); + &stage_desc, + ¤t_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - } else if (pipe->stream->config.continuous) - { + } else if (pipe->stream->config.continuous) { in_frame = pipe->stream->last_pipe->continuous_frames[0]; } - if (mode == IA_CSS_CAPTURE_MODE_PRIMARY) - { + if (mode == IA_CSS_CAPTURE_MODE_PRIMARY) { struct ia_css_frame *local_in_frame = NULL; struct ia_css_frame *local_out_frame = NULL; @@ -7953,11 +7818,14 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { * Proper investigation should be done to come up with the clean * solution. * */ - ia_css_pipe_get_generic_stage_desc(&stage_desc, primary_binary[i], - out_frames, local_in_frame, NULL); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + primary_binary[i], + out_frames, + local_in_frame, + NULL); err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - ¤t_stage); + &stage_desc, + ¤t_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -7970,22 +7838,21 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { IA_CSS_BINARY_MODE_COPY; current_stage->args.copy_output = current_stage->args.copy_vf; } else if (mode == IA_CSS_CAPTURE_MODE_ADVANCED || - mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT) - { + mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT) { ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); ia_css_pipe_get_generic_stage_desc(&stage_desc, pre_isp_binary, - out_frames, in_frame, NULL); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, NULL); + out_frames, in_frame, NULL); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + NULL); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); ia_css_pipe_get_generic_stage_desc(&stage_desc, anr_gdc_binary, - out_frames, NULL, NULL); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, NULL); + out_frames, NULL, NULL); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + NULL); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -7993,28 +7860,31 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { if (need_pp) { ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); - ia_css_pipe_get_generic_stage_desc(&stage_desc, post_isp_binary, - out_frames, NULL, NULL); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + post_isp_binary, + out_frames, + NULL, NULL); } else { - ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame); - ia_css_pipe_get_generic_stage_desc(&stage_desc, post_isp_binary, - out_frames, NULL, NULL); + ia_css_pipe_util_set_output_frames(out_frames, 0, + out_frame); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + post_isp_binary, + out_frames, + NULL, NULL); } - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, ¤t_stage); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + ¤t_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - } else if (mode == IA_CSS_CAPTURE_MODE_BAYER) - { + } else if (mode == IA_CSS_CAPTURE_MODE_BAYER) { ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame); ia_css_pipe_get_generic_stage_desc(&stage_desc, pre_isp_binary, - out_frames, in_frame, NULL); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - NULL); + out_frames, in_frame, NULL); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + NULL); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -8022,49 +7892,48 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { } #ifndef ISP2401 - if (need_pp && current_stage) - { + if (need_pp && current_stage) { struct ia_css_frame *local_in_frame = NULL; local_in_frame = current_stage->args.out_frame[0]; if (need_ldc) { ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); - ia_css_pipe_get_generic_stage_desc(&stage_desc, capture_ldc_binary, - out_frames, local_in_frame, NULL); + ia_css_pipe_get_generic_stage_desc(&stage_desc, + capture_ldc_binary, + out_frames, + local_in_frame, + NULL); err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - ¤t_stage); + &stage_desc, + ¤t_stage); local_in_frame = current_stage->args.out_frame[0]; } err = add_capture_pp_stage(pipe, me, local_in_frame, - need_yuv_pp ? NULL : out_frame, + need_yuv_pp ? NULL : out_frame, #else /* ldc and capture_pp not supported in same pipeline */ - if (need_ldc && current_stage) - { + if (need_ldc && current_stage) { in_frame = current_stage->args.out_frame[0]; ia_css_pipe_util_set_output_frames(out_frames, 0, out_frame); ia_css_pipe_get_generic_stage_desc(&stage_desc, capture_ldc_binary, - out_frames, in_frame, NULL); - err = ia_css_pipeline_create_and_add_stage(me, - &stage_desc, - NULL); - } else if (need_pp && current_stage) - { + out_frames, in_frame, NULL); + err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, + NULL); + } else if (need_pp && current_stage) { in_frame = current_stage->args.out_frame[0]; - err = add_capture_pp_stage(pipe, me, in_frame, need_yuv_pp ? NULL : out_frame, + err = add_capture_pp_stage(pipe, me, in_frame, + need_yuv_pp ? NULL : out_frame, #endif - capture_pp_binary, - ¤t_stage); + capture_pp_binary, + ¤t_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } } - if (need_yuv_pp && current_stage) - { + if (need_yuv_pp && current_stage) { struct ia_css_frame *tmp_in_frame = current_stage->args.out_frame[0]; struct ia_css_frame *tmp_out_frame = NULL; @@ -8074,10 +7943,10 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { else tmp_out_frame = NULL; - err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, tmp_out_frame, - NULL, - &yuv_scaler_binary[i], - &yuv_scaler_stage); + err = add_yuv_scaler_stage(pipe, me, tmp_in_frame, + tmp_out_frame, NULL, + &yuv_scaler_binary[i], + &yuv_scaler_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -8096,11 +7965,12 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { * should not be considered as a clean solution. Proper * investigation should be done to come up with the clean solution. * */ - if (mode != IA_CSS_CAPTURE_MODE_RAW && mode != IA_CSS_CAPTURE_MODE_BAYER && current_stage && vf_frame) - { + if (mode != IA_CSS_CAPTURE_MODE_RAW && + mode != IA_CSS_CAPTURE_MODE_BAYER && + current_stage && vf_frame) { in_frame = current_stage->args.out_vf_frame; err = add_vf_pp_stage(pipe, in_frame, vf_frame, vf_pp_binary, - ¤t_stage); + ¤t_stage); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -8115,7 +7985,8 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { } static int -create_host_capture_pipeline(struct ia_css_pipe *pipe) { +create_host_capture_pipeline(struct ia_css_pipe *pipe) +{ int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); @@ -8124,8 +7995,7 @@ create_host_capture_pipeline(struct ia_css_pipe *pipe) { err = create_host_isyscopy_capture_pipeline(pipe); else err = create_host_regular_capture_pipeline(pipe); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8135,8 +8005,8 @@ create_host_capture_pipeline(struct ia_css_pipe *pipe) { return err; } -static int capture_start( - struct ia_css_pipe *pipe) { +static int capture_start(struct ia_css_pipe *pipe) +{ struct ia_css_pipeline *me; int err = 0; @@ -8151,7 +8021,7 @@ static int capture_start( me = &pipe->pipeline; if ((pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW || - pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER) && + pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER) && (pipe->config.mode != IA_CSS_PIPE_MODE_COPY)) { if (copy_on_sp(pipe)) { err = start_copy_on_sp(pipe, &me->out_frame[0]); @@ -8195,7 +8065,7 @@ static int capture_start( if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY && pipe->stream->reconfigure_css_rx) { ia_css_isys_rx_configure(&pipe->stream->csi_rx_config, - pipe->stream->config.mode); + pipe->stream->config.mode); pipe->stream->reconfigure_css_rx = false; } #endif @@ -8206,8 +8076,9 @@ static int capture_start( static int sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe, - struct ia_css_frame_info *info, - unsigned int idx) { + struct ia_css_frame_info *info, + unsigned int idx) +{ assert(pipe); assert(info); @@ -8216,8 +8087,7 @@ sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe, *info = pipe->output_info[idx]; if (copy_on_sp(pipe) && - pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) - { + pipe->stream->config.input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8) { ia_css_frame_info_init( info, JPEG_BYTES, @@ -8225,8 +8095,7 @@ sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe, IA_CSS_FRAME_FORMAT_BINARY_8, 0); } else if (info->format == IA_CSS_FRAME_FORMAT_RAW || - info->format == IA_CSS_FRAME_FORMAT_RAW_PACKED) - { + info->format == IA_CSS_FRAME_FORMAT_RAW_PACKED) { info->raw_bit_depth = ia_css_pipe_util_pipe_input_format_bpp(pipe); } @@ -8238,9 +8107,10 @@ sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe, void ia_css_stream_send_input_frame(const struct ia_css_stream *stream, - const unsigned short *data, - unsigned int width, - unsigned int height) { + const unsigned short *data, + unsigned int width, + unsigned int height) +{ assert(stream); ia_css_inputfifo_send_input_frame( @@ -8251,7 +8121,8 @@ ia_css_stream_send_input_frame(const struct ia_css_stream *stream, } void -ia_css_stream_start_input_frame(const struct ia_css_stream *stream) { +ia_css_stream_start_input_frame(const struct ia_css_stream *stream) +{ assert(stream); ia_css_inputfifo_start_frame( @@ -8262,21 +8133,23 @@ ia_css_stream_start_input_frame(const struct ia_css_stream *stream) { void ia_css_stream_send_input_line(const struct ia_css_stream *stream, - const unsigned short *data, - unsigned int width, - const unsigned short *data2, - unsigned int width2) { + const unsigned short *data, + unsigned int width, + const unsigned short *data2, + unsigned int width2) +{ assert(stream); ia_css_inputfifo_send_line(stream->config.channel_id, - data, width, data2, width2); + data, width, data2, width2); } void ia_css_stream_send_input_embedded_line(const struct ia_css_stream *stream, - enum atomisp_input_format format, - const unsigned short *data, - unsigned int width) { + enum atomisp_input_format format, + const unsigned short *data, + unsigned int width) +{ assert(stream); if (!data || width == 0) return; @@ -8285,14 +8158,16 @@ ia_css_stream_send_input_embedded_line(const struct ia_css_stream *stream, } void -ia_css_stream_end_input_frame(const struct ia_css_stream *stream) { +ia_css_stream_end_input_frame(const struct ia_css_stream *stream) +{ assert(stream); ia_css_inputfifo_end_frame(stream->config.channel_id); } static void -append_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) { +append_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) +{ IA_CSS_ENTER_PRIVATE("l = %p, firmware = %p", l, firmware); if (!l) { IA_CSS_ERROR("NULL fw_info"); @@ -8307,7 +8182,8 @@ append_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) { } static void -remove_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) { +remove_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) +{ assert(*l); assert(firmware); (void)l; @@ -8349,12 +8225,12 @@ static int upload_isp_code(struct ia_css_fw_info *firmware) } static int -acc_load_extension(struct ia_css_fw_info *firmware) { +acc_load_extension(struct ia_css_fw_info *firmware) +{ int err; struct ia_css_fw_info *hd = firmware; - while (hd) - { + while (hd) { err = upload_isp_code(hd); if (err) return err; @@ -8368,7 +8244,8 @@ acc_load_extension(struct ia_css_fw_info *firmware) { } static void -acc_unload_extension(struct ia_css_fw_info *firmware) { +acc_unload_extension(struct ia_css_fw_info *firmware) +{ struct ia_css_fw_info *hd = firmware; struct ia_css_fw_info *hdn = NULL; @@ -8392,13 +8269,13 @@ acc_unload_extension(struct ia_css_fw_info *firmware) { /* Load firmware for extension */ static int ia_css_pipe_load_extension(struct ia_css_pipe *pipe, - struct ia_css_fw_info *firmware) { + struct ia_css_fw_info *firmware) +{ int err = 0; IA_CSS_ENTER_PRIVATE("fw = %p pipe = %p", firmware, pipe); - if ((!firmware) || (!pipe)) - { + if ((!firmware) || (!pipe)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -8416,7 +8293,8 @@ ia_css_pipe_load_extension(struct ia_css_pipe *pipe, /* Unload firmware for extension */ static void ia_css_pipe_unload_extension(struct ia_css_pipe *pipe, - struct ia_css_fw_info *firmware) { + struct ia_css_fw_info *firmware) +{ IA_CSS_ENTER_PRIVATE("fw = %p pipe = %p", firmware, pipe); if ((!firmware) || (!pipe)) { @@ -8435,7 +8313,8 @@ ia_css_pipe_unload_extension(struct ia_css_pipe *pipe, } bool -ia_css_pipeline_uses_params(struct ia_css_pipeline *me) { +ia_css_pipeline_uses_params(struct ia_css_pipeline *me) +{ struct ia_css_pipeline_stage *stage; assert(me); @@ -8456,7 +8335,8 @@ ia_css_pipeline_uses_params(struct ia_css_pipeline *me) { static int sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, - const void *acc_fw) { + const void *acc_fw) +{ struct ia_css_fw_info *fw = (struct ia_css_fw_info *)acc_fw; /* In QoS case, load_extension already called, so skipping */ int err = 0; @@ -8468,14 +8348,13 @@ sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, "sh_css_pipeline_add_acc_stage() enter: pipeline=%p, acc_fw=%p\n", pipeline, acc_fw); - if (!err) - { + if (!err) { struct ia_css_pipeline_stage_desc stage_desc; ia_css_pipe_get_acc_stage_desc(&stage_desc, NULL, fw); err = ia_css_pipeline_create_and_add_stage(pipeline, - &stage_desc, - NULL); + &stage_desc, + NULL); } ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, @@ -8488,7 +8367,8 @@ sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, * Refer to "sh_css_internal.h" for details. */ int ia_css_stream_capture_frame(struct ia_css_stream *stream, - unsigned int exp_id) { + unsigned int exp_id) +{ struct sh_css_tag_descr tag_descr; u32 encoded_tag_descr; int err; @@ -8526,11 +8406,9 @@ int ia_css_stream_capture_frame(struct ia_css_stream *stream, * @brief Configure the continuous capture. * Refer to "sh_css_internal.h" for details. */ -int ia_css_stream_capture( - struct ia_css_stream *stream, - int num_captures, - unsigned int skip, - int offset) { +int ia_css_stream_capture(struct ia_css_stream *stream, int num_captures, + unsigned int skip, int offset) +{ struct sh_css_tag_descr tag_descr; unsigned int encoded_tag_descr; int return_err; @@ -8593,8 +8471,9 @@ void ia_css_stream_request_flash(struct ia_css_stream *stream) ia_css_debug_dump_sp_sw_debug_info(); ia_css_debug_dump_debug_info(NULL); } - } else + } else { IA_CSS_LOG("SP is not running!"); + } #endif ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, @@ -8602,7 +8481,8 @@ void ia_css_stream_request_flash(struct ia_css_stream *stream) } static void -sh_css_init_host_sp_control_vars(void) { +sh_css_init_host_sp_control_vars(void) +{ const struct ia_css_fw_info *fw; unsigned int HIVE_ADDR_ia_css_ispctrl_sp_isp_started; @@ -8643,22 +8523,22 @@ sh_css_init_host_sp_control_vars(void) { (void)HIVE_ADDR_host_sp_com; sp_dmem_store_uint32(SP0_ID, - (unsigned int)sp_address_of(ia_css_ispctrl_sp_isp_started), - (uint32_t)(0)); + (unsigned int)sp_address_of(ia_css_ispctrl_sp_isp_started), + (uint32_t)(0)); sp_dmem_store_uint32(SP0_ID, - (unsigned int)sp_address_of(host_sp_queues_initialized), - (uint32_t)(0)); + (unsigned int)sp_address_of(host_sp_queues_initialized), + (uint32_t)(0)); sp_dmem_store_uint32(SP0_ID, - (unsigned int)sp_address_of(sp_sleep_mode), - (uint32_t)(0)); + (unsigned int)sp_address_of(sp_sleep_mode), + (uint32_t)(0)); sp_dmem_store_uint32(SP0_ID, - (unsigned int)sp_address_of(ia_css_dmaproxy_sp_invalidate_tlb), - (uint32_t)(false)); + (unsigned int)sp_address_of(ia_css_dmaproxy_sp_invalidate_tlb), + (uint32_t)(false)); #ifndef ISP2401 sp_dmem_store_uint32(SP0_ID, - (unsigned int)sp_address_of(sp_stop_copy_preview), - my_css.stop_copy_preview ? (uint32_t)(1) : (uint32_t)(0)); + (unsigned int)sp_address_of(sp_stop_copy_preview), + my_css.stop_copy_preview ? (uint32_t)(1) : (uint32_t)(0)); #endif store_sp_array_uint(host_sp_com, o, host2sp_cmd_ready); @@ -8685,8 +8565,8 @@ void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config) } void -ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config - *extra_config) { +ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config *extra_config) +{ if (!extra_config) { IA_CSS_ERROR("NULL input parameter"); return; @@ -8716,11 +8596,11 @@ void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config) } static int -ia_css_acc_pipe_create(struct ia_css_pipe *pipe) { +ia_css_acc_pipe_create(struct ia_css_pipe *pipe) +{ int err = 0; - if (!pipe) - { + if (!pipe) { IA_CSS_ERROR("NULL input parameter"); return -EINVAL; } @@ -8730,9 +8610,7 @@ ia_css_acc_pipe_create(struct ia_css_pipe *pipe) { pipe->config.acc_num_execs = 1; if (pipe->config.acc_extension) - { err = ia_css_pipe_load_extension(pipe, pipe->config.acc_extension); - } return err; } @@ -8751,9 +8629,8 @@ int ia_css_pipe_create(const struct ia_css_pipe_config *config, err = ia_css_pipe_create_extra(config, NULL, pipe); - if (err == 0) { + if (err == 0) IA_CSS_LOG("pipe created successfully = %p", *pipe); - } IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -8762,8 +8639,9 @@ int ia_css_pipe_create(const struct ia_css_pipe_config *config, int ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, - const struct ia_css_pipe_extra_config *extra_config, - struct ia_css_pipe **pipe) { + const struct ia_css_pipe_extra_config *extra_config, + struct ia_css_pipe **pipe) +{ int err = -EINVAL; struct ia_css_pipe *internal_pipe = NULL; unsigned int i; @@ -8771,14 +8649,12 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, IA_CSS_ENTER_PRIVATE("config = %p, extra_config = %p and pipe = %p", config, extra_config, pipe); /* do not allow to create more than the maximum limit */ - if (my_css.pipe_counter >= IA_CSS_PIPELINE_NUM_MAX) - { + if (my_css.pipe_counter >= IA_CSS_PIPELINE_NUM_MAX) { IA_CSS_LEAVE_ERR_PRIVATE(-ENOSPC); return -EINVAL; } - if ((!pipe) || (!config)) - { + if ((!pipe) || (!config)) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } @@ -8787,8 +8663,7 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, ia_css_debug_dump_pipe_extra_config(extra_config); err = create_pipe(config->mode, &internal_pipe, false); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8800,8 +8675,7 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, else ia_css_pipe_extra_config_defaults(&internal_pipe->extra_config); - if (config->mode == IA_CSS_PIPE_MODE_ACC) - { + if (config->mode == IA_CSS_PIPE_MODE_ACC) { /* Temporary hack to migrate acceleration to CSS 2.0. * In the future the code for all pipe types should be * unified. */ @@ -8828,15 +8702,13 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, set bayer_ds_out_res equal to IF output resolution(IF may do cropping on sensor output) or use default decimation factor 1. */ if (internal_pipe->extra_config.enable_raw_binning && - internal_pipe->config.bayer_ds_out_res.width) - { + internal_pipe->config.bayer_ds_out_res.width) { /* fill some code here, if no code is needed, please remove it during integration */ } /* YUV downscaling */ if ((internal_pipe->config.vf_pp_in_res.width || - internal_pipe->config.capt_pp_in_res.width)) - { + internal_pipe->config.capt_pp_in_res.width)) { enum ia_css_frame_format format; if (internal_pipe->config.vf_pp_in_res.width) { @@ -8857,8 +8729,7 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, } } if (internal_pipe->config.vf_pp_in_res.width && - internal_pipe->config.mode == IA_CSS_PIPE_MODE_PREVIEW) - { + internal_pipe->config.mode == IA_CSS_PIPE_MODE_PREVIEW) { ia_css_frame_info_init( &internal_pipe->vf_yuv_ds_input_info, internal_pipe->config.vf_pp_in_res.width, @@ -8866,8 +8737,7 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, IA_CSS_FRAME_FORMAT_YUV_LINE, 0); } /* handle bayer downscaling output info */ - if (internal_pipe->config.bayer_ds_out_res.width) - { + if (internal_pipe->config.bayer_ds_out_res.width) { ia_css_frame_info_init( &internal_pipe->bds_output_info, internal_pipe->config.bayer_ds_out_res.width, @@ -8876,8 +8746,7 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, } /* handle output info, assume always needed */ - for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) - { + for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) { if (internal_pipe->config.output_info[i].res.width) { err = sh_css_pipe_configure_output( internal_pipe, @@ -8913,10 +8782,9 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, } } } - if (internal_pipe->config.acc_extension) - { + if (internal_pipe->config.acc_extension) { err = ia_css_pipe_load_extension(internal_pipe, - internal_pipe->config.acc_extension); + internal_pipe->config.acc_extension); if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); kvfree(internal_pipe); @@ -8934,18 +8802,16 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, int ia_css_pipe_get_info(const struct ia_css_pipe *pipe, - struct ia_css_pipe_info *pipe_info) { + struct ia_css_pipe_info *pipe_info) +{ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_get_info()\n"); - assert(pipe_info); - if (!pipe_info) - { + if (!pipe_info) { ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, "ia_css_pipe_get_info: pipe_info cannot be NULL\n"); return -EINVAL; } - if (!pipe || !pipe->stream) - { + if (!pipe || !pipe->stream) { ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, "ia_css_pipe_get_info: ia_css_stream_create needs to be called before ia_css_[stream/pipe]_get_info\n"); return -EINVAL; @@ -8972,41 +8838,37 @@ bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info) int ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe, - int pin_index, - enum ia_css_frame_format new_format) { + int pin_index, + enum ia_css_frame_format new_format) +{ int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p, pin_index = %d, new_formats = %d", pipe, pin_index, new_format); - if (!pipe) - { + if (!pipe) { IA_CSS_ERROR("pipe is not set"); err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - if (0 != pin_index && 1 != pin_index) - { + if (0 != pin_index && 1 != pin_index) { IA_CSS_ERROR("pin index is not valid"); err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - if (new_format != IA_CSS_FRAME_FORMAT_NV12_TILEY) - { + if (new_format != IA_CSS_FRAME_FORMAT_NV12_TILEY) { IA_CSS_ERROR("new format is not valid"); err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; - } else - { + } else { err = ia_css_pipe_check_format(pipe, new_format); if (!err) { - if (pin_index == 0) { + if (pin_index == 0) pipe->output_info[0].format = new_format; - } else { + else pipe->vf_output_info[0].format = new_format; - } } } IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -9016,7 +8878,8 @@ ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe, #if !defined(ISP2401) /* Configuration of INPUT_SYSTEM_VERSION_2401 is done on SP */ static int -ia_css_stream_configure_rx(struct ia_css_stream *stream) { +ia_css_stream_configure_rx(struct ia_css_stream *stream) +{ struct ia_css_input_port *config; assert(stream); @@ -9045,11 +8908,10 @@ ia_css_stream_configure_rx(struct ia_css_stream *stream) { if (config->compression.type == IA_CSS_CSI2_COMPRESSION_TYPE_NONE) stream->csi_rx_config.comp = MIPI_PREDICTOR_NONE; else - { /* not implemented yet, requires extension of the rx_cfg_t * struct */ return -EINVAL; - } + stream->csi_rx_config.is_two_ppc = (stream->config.pixels_per_clock == 2); stream->reconfigure_css_rx = true; return 0; @@ -9057,10 +8919,9 @@ ia_css_stream_configure_rx(struct ia_css_stream *stream) { #endif static struct ia_css_pipe * -find_pipe(struct ia_css_pipe *pipes[], - unsigned int num_pipes, - enum ia_css_pipe_mode mode, - bool copy_pipe) { +find_pipe(struct ia_css_pipe *pipes[], unsigned int num_pipes, + enum ia_css_pipe_mode mode, bool copy_pipe) +{ unsigned int i; assert(pipes); @@ -9076,24 +8937,21 @@ find_pipe(struct ia_css_pipe *pipes[], } static int -ia_css_acc_stream_create(struct ia_css_stream *stream) { +ia_css_acc_stream_create(struct ia_css_stream *stream) +{ int i; int err = 0; - assert(stream); IA_CSS_ENTER_PRIVATE("stream = %p", stream); - if (!stream) - { + if (!stream) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; } - for (i = 0; i < stream->num_pipes; i++) - { + for (i = 0; i < stream->num_pipes; i++) { struct ia_css_pipe *pipe = stream->pipes[i]; - assert(pipe); if (!pipe) { IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); return -EINVAL; @@ -9104,14 +8962,12 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) { /* Map SP threads before doing anything. */ err = map_sp_threads(stream, true); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - for (i = 0; i < stream->num_pipes; i++) - { + for (i = 0; i < stream->num_pipes; i++) { struct ia_css_pipe *pipe = stream->pipes[i]; assert(pipe); @@ -9119,8 +8975,7 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) { } err = create_host_pipeline_structure(stream); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -9134,7 +8989,8 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) { static int metadata_info_init(const struct ia_css_metadata_config *mdc, - struct ia_css_metadata_info *md) { + struct ia_css_metadata_info *md) +{ /* Either both width and height should be set or neither */ if ((mdc->resolution.height > 0) ^ (mdc->resolution.width > 0)) return -EINVAL; @@ -9161,7 +9017,7 @@ static int check_pipe_resolutions(const struct ia_css_pipe *pipe) } if (ia_css_util_check_res(pipe->config.input_effective_res.width, - pipe->config.input_effective_res.height) != 0) { + pipe->config.input_effective_res.height) != 0) { IA_CSS_ERROR("effective resolution not supported"); err = -EINVAL; goto EXIT; @@ -9169,7 +9025,7 @@ static int check_pipe_resolutions(const struct ia_css_pipe *pipe) if (!ia_css_util_resolution_is_zero( pipe->stream->config.input_config.input_res)) { if (!ia_css_util_res_leq(pipe->config.input_effective_res, - pipe->stream->config.input_config.input_res)) { + pipe->stream->config.input_config.input_res)) { IA_CSS_ERROR("effective resolution is larger than input resolution"); err = -EINVAL; goto EXIT; @@ -9192,9 +9048,10 @@ static int check_pipe_resolutions(const struct ia_css_pipe *pipe) int ia_css_stream_create(const struct ia_css_stream_config *stream_config, - int num_pipes, - struct ia_css_pipe *pipes[], - struct ia_css_stream **stream) { + int num_pipes, + struct ia_css_pipe *pipes[], + struct ia_css_stream **stream) +{ struct ia_css_pipe *curr_pipe; struct ia_css_stream *curr_stream = NULL; bool spcopyonly; @@ -9213,8 +9070,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* some checks */ if (num_pipes == 0 || !stream || - !pipes) - { + !pipes) { err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; @@ -9223,8 +9079,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, #if !defined(ISP2401) /* We don't support metadata for JPEG stream, since they both use str2mem */ if (stream_config->input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8 && - stream_config->metadata_config.resolution.height > 0) - { + stream_config->metadata_config.resolution.height > 0) { err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; @@ -9232,8 +9087,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, #endif #ifdef ISP2401 - if (stream_config->online && stream_config->pack_raw_pixels) - { + if (stream_config->online && stream_config->pack_raw_pixels) { IA_CSS_LOG("online and pack raw is invalid on input system 2401"); err = -EINVAL; IA_CSS_LEAVE_ERR(err); @@ -9288,16 +9142,14 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* Currently we only supported metadata up to a certain size. */ err = metadata_info_init(&stream_config->metadata_config, &md_info); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } /* allocate the stream instance */ curr_stream = kzalloc(sizeof(struct ia_css_stream), GFP_KERNEL); - if (!curr_stream) - { + if (!curr_stream) { err = -ENOMEM; IA_CSS_LEAVE_ERR(err); return err; @@ -9308,8 +9160,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* allocate pipes */ curr_stream->num_pipes = num_pipes; curr_stream->pipes = kcalloc(num_pipes, sizeof(struct ia_css_pipe *), GFP_KERNEL); - if (!curr_stream->pipes) - { + if (!curr_stream->pipes) { curr_stream->num_pipes = 0; kfree(curr_stream); curr_stream = NULL; @@ -9332,8 +9183,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, #endif #ifdef ISP2401 - if (curr_stream->config.online) - { + if (curr_stream->config.online) { curr_stream->config.source.port.num_lanes = stream_config->source.port.num_lanes; curr_stream->config.mode = IA_CSS_INPUT_MODE_BUFFERED_SENSOR; @@ -9351,8 +9201,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, curr_stream->config.lock_all); /* copy mode specific stuff */ - switch (curr_stream->config.mode) - { + switch (curr_stream->config.mode) { case IA_CSS_INPUT_MODE_SENSOR: case IA_CSS_INPUT_MODE_BUFFERED_SENSOR: #if !defined(ISP2401) @@ -9362,11 +9211,11 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, case IA_CSS_INPUT_MODE_TPG: #if !defined(ISP2401) IA_CSS_LOG("tpg_configuration: x_mask=%d, y_mask=%d, x_delta=%d, y_delta=%d, xy_mask=%d", - curr_stream->config.source.tpg.x_mask, - curr_stream->config.source.tpg.y_mask, - curr_stream->config.source.tpg.x_delta, - curr_stream->config.source.tpg.y_delta, - curr_stream->config.source.tpg.xy_mask); + curr_stream->config.source.tpg.x_mask, + curr_stream->config.source.tpg.y_mask, + curr_stream->config.source.tpg.x_delta, + curr_stream->config.source.tpg.y_delta, + curr_stream->config.source.tpg.xy_mask); sh_css_sp_configure_tpg( curr_stream->config.source.tpg.x_mask, @@ -9391,17 +9240,14 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, } #ifdef ISP2401 - err = aspect_ratio_crop_init(curr_stream, - pipes, - &aspect_ratio_crop_enabled); - if (err) - { + err = aspect_ratio_crop_init(curr_stream, pipes, + &aspect_ratio_crop_enabled); + if (err) { IA_CSS_LEAVE_ERR(err); goto ERR; } #endif - for (i = 0; i < num_pipes; i++) - { + for (i = 0; i < num_pipes; i++) { struct ia_css_resolution effective_res; curr_pipe = pipes[i]; @@ -9432,8 +9278,8 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, curr_pipe->config.input_effective_res = effective_res; } IA_CSS_LOG("effective_res=%dx%d", - effective_res.width, - effective_res.height); + effective_res.width, + effective_res.height); } if (IS_ISP2401) { @@ -9441,9 +9287,8 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, if (pipes[i]->config.mode != IA_CSS_PIPE_MODE_ACC && pipes[i]->config.mode != IA_CSS_PIPE_MODE_COPY) { err = check_pipe_resolutions(pipes[i]); - if (err) { + if (err) goto ERR; - } } } } @@ -9453,32 +9298,28 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, goto ERR; IA_CSS_LOG("isp_params_configs: %p", curr_stream->isp_params_configs); - if (num_pipes == 1 && pipes[0]->config.mode == IA_CSS_PIPE_MODE_ACC) - { + if (num_pipes == 1 && pipes[0]->config.mode == IA_CSS_PIPE_MODE_ACC) { *stream = curr_stream; err = ia_css_acc_stream_create(curr_stream); goto ERR; } /* sensor binning */ - if (!spcopyonly) - { + if (!spcopyonly) { sensor_binning_changed = sh_css_params_set_binning_factor(curr_stream, - curr_stream->config.sensor_binning_factor); - } else - { + curr_stream->config.sensor_binning_factor); + } else { sensor_binning_changed = false; } IA_CSS_LOG("sensor_binning=%d, changed=%d", - curr_stream->config.sensor_binning_factor, sensor_binning_changed); + curr_stream->config.sensor_binning_factor, sensor_binning_changed); /* loop over pipes */ IA_CSS_LOG("num_pipes=%d", num_pipes); curr_stream->cont_capt = false; /* Temporary hack: we give the preview pipe a reference to the capture * pipe in continuous capture mode. */ - if (curr_stream->config.continuous) - { + if (curr_stream->config.continuous) { /* Search for the preview pipe and create the copy pipe */ struct ia_css_pipe *preview_pipe; struct ia_css_pipe *video_pipe; @@ -9496,17 +9337,18 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* Create copy pipe here, since it may not be exposed to the driver */ preview_pipe = find_pipe(pipes, num_pipes, - IA_CSS_PIPE_MODE_PREVIEW, false); + IA_CSS_PIPE_MODE_PREVIEW, false); video_pipe = find_pipe(pipes, num_pipes, - IA_CSS_PIPE_MODE_VIDEO, false); - acc_pipe = find_pipe(pipes, num_pipes, - IA_CSS_PIPE_MODE_ACC, false); + IA_CSS_PIPE_MODE_VIDEO, false); + acc_pipe = find_pipe(pipes, num_pipes, IA_CSS_PIPE_MODE_ACC, + false); if (acc_pipe && num_pipes == 2 && curr_stream->cont_capt) curr_stream->cont_capt = false; /* preview + QoS case will not need cont_capt switch */ if (curr_stream->cont_capt) { capture_pipe = find_pipe(pipes, num_pipes, - IA_CSS_PIPE_MODE_CAPTURE, false); + IA_CSS_PIPE_MODE_CAPTURE, + false); if (!capture_pipe) { err = -EINVAL; goto ERR; @@ -9526,9 +9368,9 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, preview_pipe->pipe_settings.preview.copy_pipe = copy_pipe; copy_pipe->stream = curr_stream; } - if (preview_pipe && curr_stream->cont_capt) { + if (preview_pipe && curr_stream->cont_capt) preview_pipe->pipe_settings.preview.capture_pipe = capture_pipe; - } + if (video_pipe && !video_pipe->pipe_settings.video.copy_pipe) { err = create_pipe(IA_CSS_PIPE_MODE_CAPTURE, ©_pipe, true); if (err) @@ -9537,15 +9379,13 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, video_pipe->pipe_settings.video.copy_pipe = copy_pipe; copy_pipe->stream = curr_stream; } - if (video_pipe && curr_stream->cont_capt) { + if (video_pipe && curr_stream->cont_capt) video_pipe->pipe_settings.video.capture_pipe = capture_pipe; - } - if (preview_pipe && acc_pipe) { + + if (preview_pipe && acc_pipe) preview_pipe->pipe_settings.preview.acc_pipe = acc_pipe; - } } - for (i = 0; i < num_pipes; i++) - { + for (i = 0; i < num_pipes; i++) { curr_pipe = pipes[i]; /* set current stream */ curr_pipe->stream = curr_stream; @@ -9566,8 +9406,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, } /* now pipes have been configured, info should be available */ - for (i = 0; i < num_pipes; i++) - { + for (i = 0; i < num_pipes; i++) { struct ia_css_pipe_info *pipe_info = NULL; curr_pipe = pipes[i]; @@ -9591,10 +9430,12 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, if (!spcopyonly) { if (!IS_ISP2401) err = sh_css_pipe_get_shading_info(curr_pipe, - &pipe_info->shading_info, NULL); + &pipe_info->shading_info, + NULL); else err = sh_css_pipe_get_shading_info(curr_pipe, - &pipe_info->shading_info, &curr_pipe->config); + &pipe_info->shading_info, + &curr_pipe->config); if (err) goto ERR; @@ -9604,7 +9445,8 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, goto ERR; for (j = 0; j < IA_CSS_PIPE_MAX_OUTPUT_STAGE; j++) { sh_css_pipe_get_viewfinder_frame_info(curr_pipe, - &pipe_info->vf_output_info[j], j); + &pipe_info->vf_output_info[j], + j); if (err) goto ERR; } @@ -9617,22 +9459,19 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* Map SP threads before doing anything. */ err = map_sp_threads(curr_stream, true); - if (err) - { + if (err) { IA_CSS_LOG("map_sp_threads: return_err=%d", err); goto ERR; } - for (i = 0; i < num_pipes; i++) - { + for (i = 0; i < num_pipes; i++) { curr_pipe = pipes[i]; ia_css_pipe_map_queue(curr_pipe, true); } /* Create host side pipeline objects without stages */ err = create_host_pipeline_structure(curr_stream); - if (err) - { + if (err) { IA_CSS_LOG("create_host_pipeline_structure: return_err=%d", err); goto ERR; } @@ -9670,13 +9509,13 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, } int -ia_css_stream_destroy(struct ia_css_stream *stream) { +ia_css_stream_destroy(struct ia_css_stream *stream) +{ int i; int err = 0; IA_CSS_ENTER_PRIVATE("stream = %p", stream); - if (!stream) - { + if (!stream) { err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -9685,8 +9524,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { ia_css_stream_isp_parameters_uninit(stream); if ((stream->last_pipe) && - ia_css_pipeline_is_mapped(stream->last_pipe->pipe_num)) - { + ia_css_pipeline_is_mapped(stream->last_pipe->pipe_num)) { #if defined(ISP2401) bool free_mpi; @@ -9748,8 +9586,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { } /* remove references from pipes to stream */ - for (i = 0; i < stream->num_pipes; i++) - { + for (i = 0; i < stream->num_pipes; i++) { struct ia_css_pipe *entry = stream->pipes[i]; assert(entry); @@ -9778,8 +9615,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { /* working mode: take out of the seed list */ if (my_css_save.mode == sh_css_mode_working) { for (i = 0; i < MAX_ACTIVE_STREAMS; i++) { - if (my_css_save.stream_seeds[i].stream == stream) - { + if (my_css_save.stream_seeds[i].stream == stream) { IA_CSS_LOG("took out stream %d", i); my_css_save.stream_seeds[i].stream = NULL; break; @@ -9795,7 +9631,8 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { int ia_css_stream_get_info(const struct ia_css_stream *stream, - struct ia_css_stream_info *stream_info) { + struct ia_css_stream_info *stream_info) +{ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_info: enter/exit\n"); assert(stream); assert(stream_info); @@ -9811,59 +9648,58 @@ ia_css_stream_get_info(const struct ia_css_stream *stream, * The stream handle is used to identify the correct entry in the css_save struct */ int -ia_css_stream_load(struct ia_css_stream *stream) { - if (!IS_ISP2401) { - int i; - int err; +ia_css_stream_load(struct ia_css_stream *stream) +{ + int i, j, err; - assert(stream); - ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_load() enter,\n"); - for (i = 0; i < MAX_ACTIVE_STREAMS; i++) - { - if (my_css_save.stream_seeds[i].stream == stream) { - int j; - - for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) { - if ((err = ia_css_pipe_create(&my_css_save.stream_seeds[i].pipe_config[j], - &my_css_save.stream_seeds[i].pipes[j])) != 0) { - if (j) { - int k; - - for (k = 0; k < j; k++) - ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[k]); - } - return err; - } - } - err = ia_css_stream_create(&my_css_save.stream_seeds[i].stream_config, - my_css_save.stream_seeds[i].num_pipes, - my_css_save.stream_seeds[i].pipes, - &my_css_save.stream_seeds[i].stream); - if (err) { - ia_css_stream_destroy(stream); - for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) - ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[j]); - return err; - } - break; - } - } - ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_load() exit,\n"); - return 0; - } else { + if (IS_ISP2401) { /* TODO remove function - DEPRECATED */ (void)stream; return -ENOTSUPP; } + + assert(stream); + ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_load() enter,\n"); + for (i = 0; i < MAX_ACTIVE_STREAMS; i++) { + if (my_css_save.stream_seeds[i].stream != stream) + continue; + + for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) { + int k; + + err = ia_css_pipe_create(&my_css_save.stream_seeds[i].pipe_config[j], + &my_css_save.stream_seeds[i].pipes[j]); + if (!err) + continue; + + for (k = 0; k < j; k++) + ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[k]); + return err; + } + err = ia_css_stream_create(&my_css_save.stream_seeds[i].stream_config, + my_css_save.stream_seeds[i].num_pipes, + my_css_save.stream_seeds[i].pipes, + &my_css_save.stream_seeds[i].stream); + if (!err) + break; + + ia_css_stream_destroy(stream); + for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) + ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[j]); + return err; + } + + ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_load() exit,\n"); + return 0; } int -ia_css_stream_start(struct ia_css_stream *stream) { +ia_css_stream_start(struct ia_css_stream *stream) +{ int err = 0; IA_CSS_ENTER("stream = %p", stream); - if ((!stream) || (!stream->last_pipe)) - { + if ((!stream) || (!stream->last_pipe)) { IA_CSS_LEAVE_ERR(-EINVAL); return -EINVAL; } @@ -9873,8 +9709,7 @@ ia_css_stream_start(struct ia_css_stream *stream) { /* Create host side pipeline. */ err = create_host_pipeline(stream); - if (err) - { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } @@ -9887,8 +9722,7 @@ ia_css_stream_start(struct ia_css_stream *stream) { #if !defined(ISP2401) /* Initialize mipi size checks */ - if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) - { + if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) { unsigned int idx; unsigned int port = (unsigned int)(stream->config.source.port.port); @@ -9899,8 +9733,7 @@ ia_css_stream_start(struct ia_css_stream *stream) { } #endif - if (stream->config.mode != IA_CSS_INPUT_MODE_MEMORY) - { + if (stream->config.mode != IA_CSS_INPUT_MODE_MEMORY) { err = sh_css_config_input_network(stream); if (err) return err; @@ -9912,7 +9745,8 @@ ia_css_stream_start(struct ia_css_stream *stream) { } int -ia_css_stream_stop(struct ia_css_stream *stream) { +ia_css_stream_stop(struct ia_css_stream *stream) +{ int err = 0; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_stop() enter/exit\n"); @@ -9923,22 +9757,19 @@ ia_css_stream_stop(struct ia_css_stream *stream) { #if !defined(ISP2401) /* De-initialize mipi size checks */ - if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) - { + if (stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) { unsigned int idx; unsigned int port = (unsigned int)(stream->config.source.port.port); - for (idx = 0; idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT; idx++) { + for (idx = 0; idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT; idx++) sh_css_sp_group.config.mipi_sizes_for_check[port][idx] = 0; - } } #endif - if (!IS_ISP2401) { + if (!IS_ISP2401) err = ia_css_pipeline_request_stop(&stream->last_pipe->pipeline); - } else { + else err = sh_css_pipes_stop(stream); - } if (err) return err; @@ -9951,16 +9782,16 @@ ia_css_stream_stop(struct ia_css_stream *stream) { } bool -ia_css_stream_has_stopped(struct ia_css_stream *stream) { +ia_css_stream_has_stopped(struct ia_css_stream *stream) +{ bool stopped; assert(stream); - if (!IS_ISP2401) { + if (!IS_ISP2401) stopped = ia_css_pipeline_has_stopped(&stream->last_pipe->pipeline); - } else { + else stopped = sh_css_pipes_have_stopped(stream); - } return stopped; } @@ -9971,7 +9802,8 @@ ia_css_stream_has_stopped(struct ia_css_stream *stream) { * The stream handle is used to identify the correct entry in the css_save struct */ int -ia_css_stream_unload(struct ia_css_stream *stream) { +ia_css_stream_unload(struct ia_css_stream *stream) +{ int i; assert(stream); @@ -9979,8 +9811,7 @@ ia_css_stream_unload(struct ia_css_stream *stream) { /* some checks */ assert(stream); for (i = 0; i < MAX_ACTIVE_STREAMS; i++) - if (my_css_save.stream_seeds[i].stream == stream) - { + if (my_css_save.stream_seeds[i].stream == stream) { int j; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, @@ -10000,7 +9831,8 @@ ia_css_stream_unload(struct ia_css_stream *stream) { int ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe, - enum ia_css_pipe_id *pipe_id) { + enum ia_css_pipe_id *pipe_id) +{ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_temp_pipe_to_pipe_id() enter/exit\n"); if (pipe) *pipe_id = pipe->mode; @@ -10011,18 +9843,21 @@ ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe, } enum atomisp_input_format -ia_css_stream_get_format(const struct ia_css_stream *stream) { +ia_css_stream_get_format(const struct ia_css_stream *stream) +{ return stream->config.input_config.format; } bool -ia_css_stream_get_two_pixels_per_clock(const struct ia_css_stream *stream) { +ia_css_stream_get_two_pixels_per_clock(const struct ia_css_stream *stream) +{ return (stream->config.pixels_per_clock == 2); } struct ia_css_binary * ia_css_stream_get_shading_correction_binary(const struct ia_css_stream - *stream) { + *stream) +{ struct ia_css_pipe *pipe; assert(stream); @@ -10040,7 +9875,8 @@ ia_css_stream_get_shading_correction_binary(const struct ia_css_stream } struct ia_css_binary * -ia_css_stream_get_dvs_binary(const struct ia_css_stream *stream) { +ia_css_stream_get_dvs_binary(const struct ia_css_stream *stream) +{ int i; struct ia_css_pipe *video_pipe = NULL; @@ -10059,7 +9895,8 @@ ia_css_stream_get_dvs_binary(const struct ia_css_stream *stream) { } struct ia_css_binary * -ia_css_stream_get_3a_binary(const struct ia_css_stream *stream) { +ia_css_stream_get_3a_binary(const struct ia_css_stream *stream) +{ struct ia_css_pipe *pipe; struct ia_css_binary *s3a_binary = NULL; @@ -10081,7 +9918,8 @@ ia_css_stream_get_3a_binary(const struct ia_css_stream *stream) { int ia_css_stream_set_output_padded_width(struct ia_css_stream *stream, - unsigned int output_padded_width) { + unsigned int output_padded_width) +{ struct ia_css_pipe *pipe; assert(stream); @@ -10098,7 +9936,8 @@ ia_css_stream_set_output_padded_width(struct ia_css_stream *stream, } static struct ia_css_binary * -ia_css_pipe_get_shading_correction_binary(const struct ia_css_pipe *pipe) { +ia_css_pipe_get_shading_correction_binary(const struct ia_css_pipe *pipe) +{ struct ia_css_binary *binary = NULL; assert(pipe); @@ -10143,7 +9982,8 @@ ia_css_pipe_get_shading_correction_binary(const struct ia_css_pipe *pipe) { } static struct ia_css_binary * -ia_css_pipe_get_s3a_binary(const struct ia_css_pipe *pipe) { +ia_css_pipe_get_s3a_binary(const struct ia_css_pipe *pipe) +{ struct ia_css_binary *binary = NULL; assert(pipe); @@ -10166,9 +10006,9 @@ ia_css_pipe_get_s3a_binary(const struct ia_css_pipe *pipe) { } } } else if (pipe->config.default_capture_config.mode == - IA_CSS_CAPTURE_MODE_BAYER) + IA_CSS_CAPTURE_MODE_BAYER) { binary = (struct ia_css_binary *)&pipe->pipe_settings.capture.pre_isp_binary; - else if (pipe->config.default_capture_config.mode == + } else if (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_ADVANCED || pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT) { if (pipe->config.isp_pipe_version == IA_CSS_PIPE_VERSION_1) @@ -10190,7 +10030,8 @@ ia_css_pipe_get_s3a_binary(const struct ia_css_pipe *pipe) { } static struct ia_css_binary * -ia_css_pipe_get_sdis_binary(const struct ia_css_pipe *pipe) { +ia_css_pipe_get_sdis_binary(const struct ia_css_pipe *pipe) +{ struct ia_css_binary *binary = NULL; assert(pipe); @@ -10210,14 +10051,16 @@ ia_css_pipe_get_sdis_binary(const struct ia_css_pipe *pipe) { } struct ia_css_pipeline * -ia_css_pipe_get_pipeline(const struct ia_css_pipe *pipe) { +ia_css_pipe_get_pipeline(const struct ia_css_pipe *pipe) +{ assert(pipe); return (struct ia_css_pipeline *)&pipe->pipeline; } unsigned int -ia_css_pipe_get_pipe_num(const struct ia_css_pipe *pipe) { +ia_css_pipe_get_pipe_num(const struct ia_css_pipe *pipe) +{ assert(pipe); /* KW was not sure this function was not returning a value @@ -10234,7 +10077,8 @@ ia_css_pipe_get_pipe_num(const struct ia_css_pipe *pipe) { } unsigned int -ia_css_pipe_get_isp_pipe_version(const struct ia_css_pipe *pipe) { +ia_css_pipe_get_isp_pipe_version(const struct ia_css_pipe *pipe) +{ assert(pipe); return (unsigned int)pipe->config.isp_pipe_version; @@ -10243,7 +10087,8 @@ ia_css_pipe_get_isp_pipe_version(const struct ia_css_pipe *pipe) { #define SP_START_TIMEOUT_US 30000000 int -ia_css_start_sp(void) { +ia_css_start_sp(void) +{ unsigned long timeout; int err = 0; @@ -10252,13 +10097,11 @@ ia_css_start_sp(void) { /* waiting for the SP is completely started */ timeout = SP_START_TIMEOUT_US; - while ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_INITIALIZED) && timeout) - { + while ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_INITIALIZED) && timeout) { timeout--; udelay(1); } - if (timeout == 0) - { + if (timeout == 0) { IA_CSS_ERROR("timeout during SP initialization"); return -EINVAL; } @@ -10286,14 +10129,14 @@ ia_css_start_sp(void) { #define SP_SHUTDOWN_TIMEOUT_US 200000 int -ia_css_stop_sp(void) { +ia_css_stop_sp(void) +{ unsigned long timeout; int err = 0; IA_CSS_ENTER("void"); - if (!sh_css_sp_is_running()) - { + if (!sh_css_sp_is_running()) { err = -EINVAL; IA_CSS_LEAVE("SP already stopped : return_err=%d", err); @@ -10305,8 +10148,7 @@ ia_css_stop_sp(void) { if (!IS_ISP2401) { sh_css_write_host2sp_command(host2sp_cmd_terminate); } else { - if (!sh_css_write_host2sp_command(host2sp_cmd_terminate)) - { + if (!sh_css_write_host2sp_command(host2sp_cmd_terminate)) { IA_CSS_ERROR("Call to 'sh-css_write_host2sp_command()' failed"); ia_css_debug_dump_sp_sw_debug_info(); ia_css_debug_dump_debug_info(NULL); @@ -10316,27 +10158,23 @@ ia_css_stop_sp(void) { sh_css_sp_set_sp_running(false); timeout = SP_SHUTDOWN_TIMEOUT_US; - while (!ia_css_spctrl_is_idle(SP0_ID) && timeout) - { + while (!ia_css_spctrl_is_idle(SP0_ID) && timeout) { timeout--; udelay(1); } if ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_TERMINATED)) IA_CSS_WARNING("SP has not terminated (SW)"); - if (timeout == 0) - { + if (timeout == 0) { IA_CSS_WARNING("SP is not idle"); ia_css_debug_dump_sp_sw_debug_info(); } timeout = SP_SHUTDOWN_TIMEOUT_US; - while (!isp_ctrl_getbit(ISP0_ID, ISP_SC_REG, ISP_IDLE_BIT) && timeout) - { + while (!isp_ctrl_getbit(ISP0_ID, ISP_SC_REG, ISP_IDLE_BIT) && timeout) { timeout--; udelay(1); } - if (timeout == 0) - { + if (timeout == 0) { IA_CSS_WARNING("ISP is not idle"); ia_css_debug_dump_sp_sw_debug_info(); } @@ -10351,7 +10189,8 @@ ia_css_stop_sp(void) { } int -ia_css_update_continuous_frames(struct ia_css_stream *stream) { +ia_css_update_continuous_frames(struct ia_css_stream *stream) +{ struct ia_css_pipe *pipe; unsigned int i; @@ -10359,8 +10198,7 @@ ia_css_update_continuous_frames(struct ia_css_stream *stream) { IA_CSS_DEBUG_TRACE, "sh_css_update_continuous_frames() enter:\n"); - if (!stream) - { + if (!stream) { ia_css_debug_dtrace( IA_CSS_DEBUG_TRACE, "sh_css_update_continuous_frames() leave: invalid stream, return_void\n"); @@ -10371,10 +10209,9 @@ ia_css_update_continuous_frames(struct ia_css_stream *stream) { for (i = stream->config.init_num_cont_raw_buf; i < stream->config.target_num_cont_raw_buf; i++) - { sh_css_update_host2sp_offline_frame(i, pipe->continuous_frames[i], pipe->cont_md_buffers[i]); - } + sh_css_update_host2sp_cont_num_raw_frames (stream->config.target_num_cont_raw_buf, true); ia_css_debug_dtrace( @@ -10500,7 +10337,8 @@ void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map) #if CONFIG_ON_FRAME_ENQUEUE() static int set_config_on_frame_enqueue(struct ia_css_frame_info - *info, struct frame_data_wrapper *frame) { + *info, struct frame_data_wrapper *frame) +{ frame->config_on_frame_enqueue.padded_width = 0; /* currently we support configuration on frame enqueue only on YUV formats */ @@ -10508,11 +10346,11 @@ static int set_config_on_frame_enqueue(struct ia_css_frame_info switch (info->format) { case IA_CSS_FRAME_FORMAT_YUV420: case IA_CSS_FRAME_FORMAT_NV12: - if (info->padded_width > info->res.width) { + if (info->padded_width > info->res.width) frame->config_on_frame_enqueue.padded_width = info->padded_width; - } else if ((info->padded_width < info->res.width) && (info->padded_width > 0)) { + else if ((info->padded_width < info->res.width) && (info->padded_width > 0)) return -EINVAL; - } + /* nothing to do if width == padded width or padded width is zeroed (the same) */ break; default: @@ -10524,22 +10362,21 @@ static int set_config_on_frame_enqueue(struct ia_css_frame_info #endif int -ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) { +ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) +{ int ret; IA_CSS_ENTER(""); /* Only continuous streams have a tagger to which we can send the * unlock message. */ - if (!stream || !stream->config.continuous) - { + if (!stream || !stream->config.continuous) { IA_CSS_ERROR("invalid stream pointer"); return -EINVAL; } if (exp_id > IA_CSS_ISYS_MAX_EXPOSURE_ID || - exp_id < IA_CSS_ISYS_MIN_EXPOSURE_ID) - { + exp_id < IA_CSS_ISYS_MIN_EXPOSURE_ID) { IA_CSS_ERROR("invalid exposure ID: %d\n", exp_id); return -EINVAL; } @@ -10558,7 +10395,8 @@ ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) { */ int ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, - bool enable) { + bool enable) +{ unsigned int thread_id; struct ia_css_pipeline_stage *stage; int err = 0; @@ -10566,20 +10404,16 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, IA_CSS_ENTER(""); /* Parameter Check */ - if (!pipe || !pipe->stream) - { + if (!pipe || !pipe->stream) { IA_CSS_ERROR("Invalid Pipe."); err = -EINVAL; - } else if (!(pipe->config.acc_extension)) - { + } else if (!(pipe->config.acc_extension)) { IA_CSS_ERROR("Invalid Pipe(No Extension Firmware)"); err = -EINVAL; - } else if (!sh_css_sp_is_running()) - { + } else if (!sh_css_sp_is_running()) { IA_CSS_ERROR("Leaving: queue unavailable."); err = -EBUSY; - } else - { + } else { /* Query the threadid and stage_num for the Extension firmware*/ ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage); @@ -10607,7 +10441,8 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, */ int ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, - bool *enable) { + bool *enable) +{ struct ia_css_pipeline_stage *stage; unsigned int thread_id; int err = 0; @@ -10615,20 +10450,16 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, IA_CSS_ENTER(""); /* Parameter Check */ - if (!pipe || !pipe->stream) - { + if (!pipe || !pipe->stream) { IA_CSS_ERROR("Invalid Pipe."); err = -EINVAL; - } else if (!(pipe->config.acc_extension)) - { + } else if (!(pipe->config.acc_extension)) { IA_CSS_ERROR("Invalid Pipe (No Extension Firmware)."); err = -EINVAL; - } else if (!sh_css_sp_is_running()) - { + } else if (!sh_css_sp_is_running()) { IA_CSS_ERROR("Leaving: queue unavailable."); err = -EBUSY; - } else - { + } else { /* Query the threadid and stage_num corresponding to the Extension firmware*/ ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage); @@ -10646,9 +10477,10 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, /* ISP2401 */ int ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, - u32 fw_handle, - struct ia_css_isp_param_css_segments *css_seg, - struct ia_css_isp_param_isp_segments *isp_seg) { + u32 fw_handle, + struct ia_css_isp_param_css_segments *css_seg, + struct ia_css_isp_param_isp_segments *isp_seg) +{ unsigned int HIVE_ADDR_sp_group; static struct sh_css_sp_group sp_group; static struct sh_css_sp_stage sp_stage; @@ -10666,27 +10498,23 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, fw = &sh_css_sp_fw; /* Parameter Check */ - if (!pipe || !pipe->stream) - { + if (!pipe || !pipe->stream) { IA_CSS_ERROR("Invalid Pipe."); err = -EINVAL; - } else if (!(pipe->config.acc_extension)) - { + } else if (!(pipe->config.acc_extension)) { IA_CSS_ERROR("Invalid Pipe (No Extension Firmware)."); err = -EINVAL; - } else if (!sh_css_sp_is_running()) - { + } else if (!sh_css_sp_is_running()) { IA_CSS_ERROR("Leaving: queue unavailable."); err = -EBUSY; - } else - { + } else { /* Query the thread_id and stage_num corresponding to the Extension firmware */ ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage); if (!err) { /* Get the Extension State */ enabled = (SH_CSS_QOS_STAGE_IS_ENABLED(&sh_css_sp_group.pipe[thread_id], - stage->stage_num)) ? true : false; + stage->stage_num)) ? true : false; /* Update mapped arg only when extension stage is not enabled */ if (enabled) { IA_CSS_ERROR("Leaving: cannot update when stage is enabled."); @@ -10696,13 +10524,14 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, HIVE_ADDR_sp_group = fw->info.sp.group; sp_dmem_load(SP0_ID, - (unsigned int)sp_address_of(sp_group), - &sp_group, sizeof(struct sh_css_sp_group)); + (unsigned int)sp_address_of(sp_group), + &sp_group, + sizeof(struct sh_css_sp_group)); hmm_load(sp_group.pipe[thread_id].sp_stage_addr[stage_num], - &sp_stage, sizeof(struct sh_css_sp_stage)); + &sp_stage, sizeof(struct sh_css_sp_stage)); hmm_load(sp_stage.isp_stage_addr, - &isp_stage, sizeof(struct sh_css_isp_stage)); + &isp_stage, sizeof(struct sh_css_isp_stage)); for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++) { isp_stage.mem_initializers.params[IA_CSS_PARAM_CLASS_PARAM][mem].address = @@ -10718,7 +10547,8 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, } hmm_store(sp_stage.isp_stage_addr, - &isp_stage, sizeof(struct sh_css_isp_stage)); + &isp_stage, + sizeof(struct sh_css_isp_stage)); } } } @@ -10729,8 +10559,9 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, #ifdef ISP2401 static int aspect_ratio_crop_init(struct ia_css_stream *curr_stream, - struct ia_css_pipe *pipes[], - bool *do_crop_status) { + struct ia_css_pipe *pipes[], + bool *do_crop_status) +{ int err = 0; int i; struct ia_css_pipe *curr_pipe; @@ -10739,15 +10570,13 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream, if ((!curr_stream) || (curr_stream->num_pipes == 0) || (!pipes) || - (!do_crop_status)) - { + (!do_crop_status)) { err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } - for (i = 0; i < curr_stream->num_pipes; i++) - { + for (i = 0; i < curr_stream->num_pipes; i++) { curr_pipe = pipes[i]; pipe_mask |= (1 << curr_pipe->config.mode); } @@ -10761,7 +10590,8 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream, } static bool -aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe) { +aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe) +{ bool status = false; if ((curr_pipe) && enabled) { @@ -10776,7 +10606,8 @@ aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe) { static int aspect_ratio_crop(struct ia_css_pipe *curr_pipe, - struct ia_css_resolution *effective_res) { + struct ia_css_resolution *effective_res) +{ int err = 0; struct ia_css_resolution crop_res; struct ia_css_resolution *in_res = NULL; @@ -10786,8 +10617,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, bool use_capt_pp_in_res = false; if ((!curr_pipe) || - (!effective_res)) - { + (!effective_res)) { err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; @@ -10795,8 +10625,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, if ((curr_pipe->config.mode != IA_CSS_PIPE_MODE_PREVIEW) && (curr_pipe->config.mode != IA_CSS_PIPE_MODE_VIDEO) && - (curr_pipe->config.mode != IA_CSS_PIPE_MODE_CAPTURE)) - { + (curr_pipe->config.mode != IA_CSS_PIPE_MODE_CAPTURE)) { err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; @@ -10817,8 +10646,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, in_res = &curr_pipe->stream->config.input_config.effective_res; out_res = &curr_pipe->output_info[0].res; - switch (curr_pipe->config.mode) - { + switch (curr_pipe->config.mode) { case IA_CSS_PIPE_MODE_PREVIEW: if (use_bds_output_info) out_res = &curr_pipe->bds_output_info.res; @@ -10838,27 +10666,26 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, case IA_CSS_PIPE_MODE_YUVPP: default: IA_CSS_ERROR("aspect ratio cropping invalid args: mode[%d]\n", - curr_pipe->config.mode); + curr_pipe->config.mode); assert(0); break; } err = ia_css_frame_find_crop_resolution(in_res, out_res, &crop_res); if (!err) - { *effective_res = crop_res; - } else - { + else /* in case of error fallback to default * effective resolution from driver. */ IA_CSS_LOG("ia_css_frame_find_crop_resolution() failed with err(%d)", err); - } + return err; } #endif static void -sh_css_hmm_buffer_record_init(void) { +sh_css_hmm_buffer_record_init(void) +{ int i; for (i = 0; i < MAX_HMM_BUFFER_NUM; i++) @@ -10866,7 +10693,8 @@ sh_css_hmm_buffer_record_init(void) { } static void -sh_css_hmm_buffer_record_uninit(void) { +sh_css_hmm_buffer_record_uninit(void) +{ int i; struct sh_css_hmm_buffer_record *buffer_record = NULL; @@ -10882,7 +10710,8 @@ sh_css_hmm_buffer_record_uninit(void) { } static void -sh_css_hmm_buffer_record_reset(struct sh_css_hmm_buffer_record *buffer_record) { +sh_css_hmm_buffer_record_reset(struct sh_css_hmm_buffer_record *buffer_record) +{ assert(buffer_record); buffer_record->in_use = false; buffer_record->type = IA_CSS_BUFFER_TYPE_INVALID; @@ -10893,14 +10722,15 @@ sh_css_hmm_buffer_record_reset(struct sh_css_hmm_buffer_record *buffer_record) { static struct sh_css_hmm_buffer_record *sh_css_hmm_buffer_record_acquire(struct ia_css_rmgr_vbuf_handle *h_vbuf, enum ia_css_buffer_type type, - hrt_address kernel_ptr) { + hrt_address kernel_ptr) +{ int i; struct sh_css_hmm_buffer_record *buffer_record = NULL; struct sh_css_hmm_buffer_record *out_buffer_record = NULL; assert(h_vbuf); assert((type > IA_CSS_BUFFER_TYPE_INVALID) && - (type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE)); + (type < IA_CSS_NUM_DYNAMIC_BUFFER_TYPE)); assert(kernel_ptr != 0); buffer_record = &hmm_buffer_record[0]; @@ -10921,7 +10751,8 @@ static struct sh_css_hmm_buffer_record static struct sh_css_hmm_buffer_record *sh_css_hmm_buffer_record_validate(ia_css_ptr ddr_buffer_addr, - enum ia_css_buffer_type type) { + enum ia_css_buffer_type type) +{ int i; struct sh_css_hmm_buffer_record *buffer_record = NULL; bool found_record = false; diff --git a/drivers/staging/media/av7110/Kconfig b/drivers/staging/media/av7110/Kconfig new file mode 100644 index 000000000000..9faf9d2d4001 --- /dev/null +++ b/drivers/staging/media/av7110/Kconfig @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DVB_AV7110_IR + bool + depends on RC_CORE=y || RC_CORE = DVB_AV7110 + default DVB_AV7110 + +config DVB_AV7110 + tristate "AV7110 cards" + depends on DVB_CORE && PCI && I2C + select TTPCI_EEPROM + select VIDEO_SAA7146_VV + depends on VIDEO_DEV # dependencies of VIDEO_SAA7146_VV + select DVB_VES1820 if MEDIA_SUBDRV_AUTOSELECT + select DVB_VES1X93 if MEDIA_SUBDRV_AUTOSELECT + select DVB_STV0299 if MEDIA_SUBDRV_AUTOSELECT + select DVB_TDA8083 if MEDIA_SUBDRV_AUTOSELECT + select DVB_SP8870 if MEDIA_SUBDRV_AUTOSELECT + select DVB_STV0297 if MEDIA_SUBDRV_AUTOSELECT + select DVB_L64781 if MEDIA_SUBDRV_AUTOSELECT + select DVB_LNBP21 if MEDIA_SUBDRV_AUTOSELECT + help + Support for SAA7146 and AV7110 based DVB cards as produced + by Fujitsu-Siemens, Technotrend, Hauppauge and others. + + This driver only supports the fullfeatured cards with + onboard MPEG2 decoder. + + This driver needs an external firmware. Please use the script + "/scripts/get_dvb_firmware av7110" to + download/extract it, and then copy it to /usr/lib/hotplug/firmware + or /lib/firmware (depending on configuration of firmware hotplug). + + Alternatively, you can download the file and use the kernel's + EXTRA_FIRMWARE configuration option to build it into your + kernel image by adding the filename to the EXTRA_FIRMWARE + configuration option string. + + Say Y if you own such a card and want to use it. + +config DVB_AV7110_OSD + bool "AV7110 OSD support" + depends on DVB_AV7110 + default y if DVB_AV7110=y || DVB_AV7110=m + help + The AV7110 firmware provides some code to generate an OnScreenDisplay + on the video output. This is kind of nonstandard and not guaranteed to + be maintained. + + Anyway, some popular DVB software like VDR uses this OSD to render + its menus, so say Y if you want to use this software. + + All other people say N. + +config DVB_BUDGET_PATCH + tristate "AV7110 cards with Budget Patch" + depends on DVB_BUDGET_CORE && I2C + depends on DVB_AV7110 + select DVB_STV0299 if MEDIA_SUBDRV_AUTOSELECT + select DVB_VES1X93 if MEDIA_SUBDRV_AUTOSELECT + select DVB_TDA8083 if MEDIA_SUBDRV_AUTOSELECT + help + Support for Budget Patch (full TS) modification on + SAA7146+AV7110 based cards (DVB-S cards). This + driver doesn't use onboard MPEG2 decoder. The + card is driven in Budget-only mode. Card is + required to have loaded firmware to tune properly. + Firmware can be loaded by insertion and removal of + standard AV7110 driver prior to loading this + driver. + + Say Y if you own such a card and want to use it. + + To compile this driver as a module, choose M here: the + module will be called budget-patch. + +if DVB_AV7110 + +# Frontend driver that it is used only by AV7110 driver +# While technically independent, it doesn't make sense to keep +# it if we drop support for AV7110, as no other driver will use it. + +config DVB_SP8870 + tristate "Spase sp8870 based" + depends on DVB_CORE && I2C + default m if !MEDIA_SUBDRV_AUTOSELECT + help + A DVB-T tuner module. Say Y when you want to support this frontend. + + This driver needs external firmware. Please use the command + "/scripts/get_dvb_firmware sp8870" to + download/extract it, and then copy it to /usr/lib/hotplug/firmware + or /lib/firmware (depending on configuration of firmware hotplug). + +endif diff --git a/drivers/staging/media/av7110/Makefile b/drivers/staging/media/av7110/Makefile new file mode 100644 index 000000000000..307b267598ea --- /dev/null +++ b/drivers/staging/media/av7110/Makefile @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the AV7110 DVB device driver +# + +dvb-ttpci-objs := av7110_hw.o av7110_v4l.o av7110_av.o av7110_ca.o av7110.o \ + av7110_ipack.o dvb_filter.o + +ifdef CONFIG_DVB_AV7110_IR +dvb-ttpci-objs += av7110_ir.o +endif + +obj-$(CONFIG_DVB_BUDGET_PATCH) += budget-patch.o + +obj-$(CONFIG_DVB_AV7110) += dvb-ttpci.o + +obj-$(CONFIG_DVB_SP8870) += sp8870.o + +ccflags-y += -I $(srctree)/drivers/media/dvb-frontends +ccflags-y += -I $(srctree)/drivers/media/tuners +ccflags-y += -I $(srctree)/drivers/media/pci/ttpci +ccflags-y += -I $(srctree)/drivers/media/common diff --git a/drivers/staging/media/av7110/TODO b/drivers/staging/media/av7110/TODO new file mode 100644 index 000000000000..60062d8441b3 --- /dev/null +++ b/drivers/staging/media/av7110/TODO @@ -0,0 +1,3 @@ +- This driver is too old and relies on a different API. + Drop it from Kernel on a couple of versions. +- Cleanup patches for the drivers here won't be accepted. diff --git a/Documentation/userspace-api/media/dvb/audio-bilingual-channel-select.rst b/drivers/staging/media/av7110/audio-bilingual-channel-select.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-bilingual-channel-select.rst rename to drivers/staging/media/av7110/audio-bilingual-channel-select.rst diff --git a/Documentation/userspace-api/media/dvb/audio-channel-select.rst b/drivers/staging/media/av7110/audio-channel-select.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-channel-select.rst rename to drivers/staging/media/av7110/audio-channel-select.rst diff --git a/Documentation/userspace-api/media/dvb/audio-clear-buffer.rst b/drivers/staging/media/av7110/audio-clear-buffer.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-clear-buffer.rst rename to drivers/staging/media/av7110/audio-clear-buffer.rst diff --git a/Documentation/userspace-api/media/dvb/audio-continue.rst b/drivers/staging/media/av7110/audio-continue.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-continue.rst rename to drivers/staging/media/av7110/audio-continue.rst diff --git a/Documentation/userspace-api/media/dvb/audio-fclose.rst b/drivers/staging/media/av7110/audio-fclose.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-fclose.rst rename to drivers/staging/media/av7110/audio-fclose.rst diff --git a/Documentation/userspace-api/media/dvb/audio-fopen.rst b/drivers/staging/media/av7110/audio-fopen.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-fopen.rst rename to drivers/staging/media/av7110/audio-fopen.rst diff --git a/Documentation/userspace-api/media/dvb/audio-fwrite.rst b/drivers/staging/media/av7110/audio-fwrite.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-fwrite.rst rename to drivers/staging/media/av7110/audio-fwrite.rst diff --git a/Documentation/userspace-api/media/dvb/audio-get-capabilities.rst b/drivers/staging/media/av7110/audio-get-capabilities.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-get-capabilities.rst rename to drivers/staging/media/av7110/audio-get-capabilities.rst diff --git a/Documentation/userspace-api/media/dvb/audio-get-status.rst b/drivers/staging/media/av7110/audio-get-status.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-get-status.rst rename to drivers/staging/media/av7110/audio-get-status.rst diff --git a/Documentation/userspace-api/media/dvb/audio-pause.rst b/drivers/staging/media/av7110/audio-pause.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-pause.rst rename to drivers/staging/media/av7110/audio-pause.rst diff --git a/Documentation/userspace-api/media/dvb/audio-play.rst b/drivers/staging/media/av7110/audio-play.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-play.rst rename to drivers/staging/media/av7110/audio-play.rst diff --git a/Documentation/userspace-api/media/dvb/audio-select-source.rst b/drivers/staging/media/av7110/audio-select-source.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-select-source.rst rename to drivers/staging/media/av7110/audio-select-source.rst diff --git a/Documentation/userspace-api/media/dvb/audio-set-av-sync.rst b/drivers/staging/media/av7110/audio-set-av-sync.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-set-av-sync.rst rename to drivers/staging/media/av7110/audio-set-av-sync.rst diff --git a/Documentation/userspace-api/media/dvb/audio-set-bypass-mode.rst b/drivers/staging/media/av7110/audio-set-bypass-mode.rst similarity index 94% rename from Documentation/userspace-api/media/dvb/audio-set-bypass-mode.rst rename to drivers/staging/media/av7110/audio-set-bypass-mode.rst index ecac02f1b2fc..80d551a2053a 100644 --- a/Documentation/userspace-api/media/dvb/audio-set-bypass-mode.rst +++ b/drivers/staging/media/av7110/audio-set-bypass-mode.rst @@ -50,7 +50,7 @@ Description This ioctl call asks the Audio Device to bypass the Audio decoder and forward the stream without decoding. This mode shall be used if streams -that can’t be handled by the Digital TV system shall be decoded. Dolby +that can't be handled by the Digital TV system shall be decoded. Dolby DigitalTM streams are automatically forwarded by the Digital TV subsystem if the hardware can handle it. diff --git a/Documentation/userspace-api/media/dvb/audio-set-id.rst b/drivers/staging/media/av7110/audio-set-id.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-set-id.rst rename to drivers/staging/media/av7110/audio-set-id.rst diff --git a/Documentation/userspace-api/media/dvb/audio-set-mixer.rst b/drivers/staging/media/av7110/audio-set-mixer.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-set-mixer.rst rename to drivers/staging/media/av7110/audio-set-mixer.rst diff --git a/Documentation/userspace-api/media/dvb/audio-set-mute.rst b/drivers/staging/media/av7110/audio-set-mute.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-set-mute.rst rename to drivers/staging/media/av7110/audio-set-mute.rst diff --git a/Documentation/userspace-api/media/dvb/audio-set-streamtype.rst b/drivers/staging/media/av7110/audio-set-streamtype.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-set-streamtype.rst rename to drivers/staging/media/av7110/audio-set-streamtype.rst diff --git a/Documentation/userspace-api/media/dvb/audio-stop.rst b/drivers/staging/media/av7110/audio-stop.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio-stop.rst rename to drivers/staging/media/av7110/audio-stop.rst diff --git a/include/uapi/linux/dvb/audio.h b/drivers/staging/media/av7110/audio.h similarity index 100% rename from include/uapi/linux/dvb/audio.h rename to drivers/staging/media/av7110/audio.h diff --git a/Documentation/userspace-api/media/dvb/audio.rst b/drivers/staging/media/av7110/audio.rst similarity index 90% rename from Documentation/userspace-api/media/dvb/audio.rst rename to drivers/staging/media/av7110/audio.rst index eaae5675a47d..aa753336b31f 100644 --- a/Documentation/userspace-api/media/dvb/audio.rst +++ b/drivers/staging/media/av7110/audio.rst @@ -11,7 +11,7 @@ TV hardware. It can be accessed through ``/dev/dvb/adapter?/audio?``. Data types and ioctl definitions can be accessed by including ``linux/dvb/audio.h`` in your application. -Please note that some Digital TV cards don’t have their own MPEG decoder, which +Please note that some Digital TV cards don't have their own MPEG decoder, which results in the omission of the audio and video device. These ioctls were also used by V4L2 to control MPEG decoders implemented diff --git a/Documentation/userspace-api/media/dvb/audio_data_types.rst b/drivers/staging/media/av7110/audio_data_types.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio_data_types.rst rename to drivers/staging/media/av7110/audio_data_types.rst diff --git a/Documentation/userspace-api/media/dvb/audio_function_calls.rst b/drivers/staging/media/av7110/audio_function_calls.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/audio_function_calls.rst rename to drivers/staging/media/av7110/audio_function_calls.rst diff --git a/drivers/media/pci/ttpci/av7110.c b/drivers/staging/media/av7110/av7110.c similarity index 100% rename from drivers/media/pci/ttpci/av7110.c rename to drivers/staging/media/av7110/av7110.c diff --git a/drivers/media/pci/ttpci/av7110.h b/drivers/staging/media/av7110/av7110.h similarity index 98% rename from drivers/media/pci/ttpci/av7110.h rename to drivers/staging/media/av7110/av7110.h index 809d938ae166..b8e8fc8ddbe9 100644 --- a/drivers/media/pci/ttpci/av7110.h +++ b/drivers/staging/media/av7110/av7110.h @@ -9,11 +9,12 @@ #include #include -#include -#include +#include "video.h" +#include "audio.h" +#include "osd.h" + #include #include -#include #include #include diff --git a/drivers/media/pci/ttpci/av7110_av.c b/drivers/staging/media/av7110/av7110_av.c similarity index 100% rename from drivers/media/pci/ttpci/av7110_av.c rename to drivers/staging/media/av7110/av7110_av.c diff --git a/drivers/media/pci/ttpci/av7110_av.h b/drivers/staging/media/av7110/av7110_av.h similarity index 100% rename from drivers/media/pci/ttpci/av7110_av.h rename to drivers/staging/media/av7110/av7110_av.h diff --git a/drivers/media/pci/ttpci/av7110_ca.c b/drivers/staging/media/av7110/av7110_ca.c similarity index 100% rename from drivers/media/pci/ttpci/av7110_ca.c rename to drivers/staging/media/av7110/av7110_ca.c diff --git a/drivers/media/pci/ttpci/av7110_ca.h b/drivers/staging/media/av7110/av7110_ca.h similarity index 100% rename from drivers/media/pci/ttpci/av7110_ca.h rename to drivers/staging/media/av7110/av7110_ca.h diff --git a/drivers/media/pci/ttpci/av7110_hw.c b/drivers/staging/media/av7110/av7110_hw.c similarity index 100% rename from drivers/media/pci/ttpci/av7110_hw.c rename to drivers/staging/media/av7110/av7110_hw.c diff --git a/drivers/media/pci/ttpci/av7110_hw.h b/drivers/staging/media/av7110/av7110_hw.h similarity index 100% rename from drivers/media/pci/ttpci/av7110_hw.h rename to drivers/staging/media/av7110/av7110_hw.h diff --git a/drivers/media/pci/ttpci/av7110_ipack.c b/drivers/staging/media/av7110/av7110_ipack.c similarity index 100% rename from drivers/media/pci/ttpci/av7110_ipack.c rename to drivers/staging/media/av7110/av7110_ipack.c diff --git a/drivers/media/pci/ttpci/av7110_ipack.h b/drivers/staging/media/av7110/av7110_ipack.h similarity index 100% rename from drivers/media/pci/ttpci/av7110_ipack.h rename to drivers/staging/media/av7110/av7110_ipack.h diff --git a/drivers/media/pci/ttpci/av7110_ir.c b/drivers/staging/media/av7110/av7110_ir.c similarity index 100% rename from drivers/media/pci/ttpci/av7110_ir.c rename to drivers/staging/media/av7110/av7110_ir.c diff --git a/drivers/media/pci/ttpci/av7110_v4l.c b/drivers/staging/media/av7110/av7110_v4l.c similarity index 100% rename from drivers/media/pci/ttpci/av7110_v4l.c rename to drivers/staging/media/av7110/av7110_v4l.c diff --git a/drivers/media/pci/ttpci/budget-patch.c b/drivers/staging/media/av7110/budget-patch.c similarity index 100% rename from drivers/media/pci/ttpci/budget-patch.c rename to drivers/staging/media/av7110/budget-patch.c diff --git a/drivers/media/pci/ttpci/dvb_filter.c b/drivers/staging/media/av7110/dvb_filter.c similarity index 100% rename from drivers/media/pci/ttpci/dvb_filter.c rename to drivers/staging/media/av7110/dvb_filter.c diff --git a/drivers/media/pci/ttpci/dvb_filter.h b/drivers/staging/media/av7110/dvb_filter.h similarity index 100% rename from drivers/media/pci/ttpci/dvb_filter.h rename to drivers/staging/media/av7110/dvb_filter.h diff --git a/include/uapi/linux/dvb/osd.h b/drivers/staging/media/av7110/osd.h similarity index 100% rename from include/uapi/linux/dvb/osd.h rename to drivers/staging/media/av7110/osd.h diff --git a/drivers/media/dvb-frontends/sp8870.c b/drivers/staging/media/av7110/sp8870.c similarity index 100% rename from drivers/media/dvb-frontends/sp8870.c rename to drivers/staging/media/av7110/sp8870.c diff --git a/drivers/media/dvb-frontends/sp8870.h b/drivers/staging/media/av7110/sp8870.h similarity index 100% rename from drivers/media/dvb-frontends/sp8870.h rename to drivers/staging/media/av7110/sp8870.h diff --git a/Documentation/userspace-api/media/dvb/video-clear-buffer.rst b/drivers/staging/media/av7110/video-clear-buffer.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-clear-buffer.rst rename to drivers/staging/media/av7110/video-clear-buffer.rst diff --git a/Documentation/userspace-api/media/dvb/video-command.rst b/drivers/staging/media/av7110/video-command.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-command.rst rename to drivers/staging/media/av7110/video-command.rst diff --git a/Documentation/userspace-api/media/dvb/video-continue.rst b/drivers/staging/media/av7110/video-continue.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-continue.rst rename to drivers/staging/media/av7110/video-continue.rst diff --git a/Documentation/userspace-api/media/dvb/video-fast-forward.rst b/drivers/staging/media/av7110/video-fast-forward.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-fast-forward.rst rename to drivers/staging/media/av7110/video-fast-forward.rst diff --git a/Documentation/userspace-api/media/dvb/video-fclose.rst b/drivers/staging/media/av7110/video-fclose.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-fclose.rst rename to drivers/staging/media/av7110/video-fclose.rst diff --git a/Documentation/userspace-api/media/dvb/video-fopen.rst b/drivers/staging/media/av7110/video-fopen.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-fopen.rst rename to drivers/staging/media/av7110/video-fopen.rst diff --git a/Documentation/userspace-api/media/dvb/video-freeze.rst b/drivers/staging/media/av7110/video-freeze.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-freeze.rst rename to drivers/staging/media/av7110/video-freeze.rst diff --git a/Documentation/userspace-api/media/dvb/video-fwrite.rst b/drivers/staging/media/av7110/video-fwrite.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-fwrite.rst rename to drivers/staging/media/av7110/video-fwrite.rst diff --git a/Documentation/userspace-api/media/dvb/video-get-capabilities.rst b/drivers/staging/media/av7110/video-get-capabilities.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-get-capabilities.rst rename to drivers/staging/media/av7110/video-get-capabilities.rst diff --git a/Documentation/userspace-api/media/dvb/video-get-event.rst b/drivers/staging/media/av7110/video-get-event.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-get-event.rst rename to drivers/staging/media/av7110/video-get-event.rst diff --git a/Documentation/userspace-api/media/dvb/video-get-frame-count.rst b/drivers/staging/media/av7110/video-get-frame-count.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-get-frame-count.rst rename to drivers/staging/media/av7110/video-get-frame-count.rst diff --git a/Documentation/userspace-api/media/dvb/video-get-pts.rst b/drivers/staging/media/av7110/video-get-pts.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-get-pts.rst rename to drivers/staging/media/av7110/video-get-pts.rst diff --git a/Documentation/userspace-api/media/dvb/video-get-size.rst b/drivers/staging/media/av7110/video-get-size.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-get-size.rst rename to drivers/staging/media/av7110/video-get-size.rst diff --git a/Documentation/userspace-api/media/dvb/video-get-status.rst b/drivers/staging/media/av7110/video-get-status.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-get-status.rst rename to drivers/staging/media/av7110/video-get-status.rst diff --git a/Documentation/userspace-api/media/dvb/video-play.rst b/drivers/staging/media/av7110/video-play.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-play.rst rename to drivers/staging/media/av7110/video-play.rst diff --git a/Documentation/userspace-api/media/dvb/video-select-source.rst b/drivers/staging/media/av7110/video-select-source.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-select-source.rst rename to drivers/staging/media/av7110/video-select-source.rst diff --git a/Documentation/userspace-api/media/dvb/video-set-blank.rst b/drivers/staging/media/av7110/video-set-blank.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-set-blank.rst rename to drivers/staging/media/av7110/video-set-blank.rst diff --git a/Documentation/userspace-api/media/dvb/video-set-display-format.rst b/drivers/staging/media/av7110/video-set-display-format.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-set-display-format.rst rename to drivers/staging/media/av7110/video-set-display-format.rst diff --git a/Documentation/userspace-api/media/dvb/video-set-format.rst b/drivers/staging/media/av7110/video-set-format.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-set-format.rst rename to drivers/staging/media/av7110/video-set-format.rst diff --git a/Documentation/userspace-api/media/dvb/video-set-streamtype.rst b/drivers/staging/media/av7110/video-set-streamtype.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-set-streamtype.rst rename to drivers/staging/media/av7110/video-set-streamtype.rst diff --git a/Documentation/userspace-api/media/dvb/video-slowmotion.rst b/drivers/staging/media/av7110/video-slowmotion.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-slowmotion.rst rename to drivers/staging/media/av7110/video-slowmotion.rst diff --git a/Documentation/userspace-api/media/dvb/video-stillpicture.rst b/drivers/staging/media/av7110/video-stillpicture.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-stillpicture.rst rename to drivers/staging/media/av7110/video-stillpicture.rst diff --git a/Documentation/userspace-api/media/dvb/video-stop.rst b/drivers/staging/media/av7110/video-stop.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-stop.rst rename to drivers/staging/media/av7110/video-stop.rst diff --git a/Documentation/userspace-api/media/dvb/video-try-command.rst b/drivers/staging/media/av7110/video-try-command.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video-try-command.rst rename to drivers/staging/media/av7110/video-try-command.rst diff --git a/include/uapi/linux/dvb/video.h b/drivers/staging/media/av7110/video.h similarity index 100% rename from include/uapi/linux/dvb/video.h rename to drivers/staging/media/av7110/video.h diff --git a/Documentation/userspace-api/media/dvb/video.rst b/drivers/staging/media/av7110/video.rst similarity index 93% rename from Documentation/userspace-api/media/dvb/video.rst rename to drivers/staging/media/av7110/video.rst index 38a8d39a1d25..808705b769a1 100644 --- a/Documentation/userspace-api/media/dvb/video.rst +++ b/drivers/staging/media/av7110/video.rst @@ -16,7 +16,7 @@ stream, not its presentation on the TV or computer screen. On PCs this is typically handled by an associated video4linux device, e.g. **/dev/video**, which allows scaling and defining output windows. -Some Digital TV cards don’t have their own MPEG decoder, which results in the +Some Digital TV cards don't have their own MPEG decoder, which results in the omission of the audio and video device as well as the video4linux device. diff --git a/Documentation/userspace-api/media/dvb/video_function_calls.rst b/drivers/staging/media/av7110/video_function_calls.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video_function_calls.rst rename to drivers/staging/media/av7110/video_function_calls.rst diff --git a/Documentation/userspace-api/media/dvb/video_types.rst b/drivers/staging/media/av7110/video_types.rst similarity index 100% rename from Documentation/userspace-api/media/dvb/video_types.rst rename to drivers/staging/media/av7110/video_types.rst diff --git a/drivers/staging/media/hantro/Kconfig b/drivers/staging/media/hantro/Kconfig index 5b6cf9f62b1a..20b1f6d7b69c 100644 --- a/drivers/staging/media/hantro/Kconfig +++ b/drivers/staging/media/hantro/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 config VIDEO_HANTRO tristate "Hantro VPU driver" - depends on ARCH_MXC || ARCH_ROCKCHIP || COMPILE_TEST + depends on ARCH_MXC || ARCH_ROCKCHIP || ARCH_AT91 || COMPILE_TEST depends on VIDEO_DEV && VIDEO_V4L2 select MEDIA_CONTROLLER select MEDIA_CONTROLLER_REQUEST_API @@ -24,6 +24,14 @@ config VIDEO_HANTRO_IMX8M help Enable support for i.MX8M SoCs. +config VIDEO_HANTRO_SAMA5D4 + bool "Hantro VDEC SAMA5D4 support" + depends on VIDEO_HANTRO + depends on ARCH_AT91 || COMPILE_TEST + default y + help + Enable support for Microchip SAMA5D4 SoCs. + config VIDEO_HANTRO_ROCKCHIP bool "Hantro VPU Rockchip support" depends on VIDEO_HANTRO diff --git a/drivers/staging/media/hantro/Makefile b/drivers/staging/media/hantro/Makefile index 743ce08eb184..287370188d2a 100644 --- a/drivers/staging/media/hantro/Makefile +++ b/drivers/staging/media/hantro/Makefile @@ -7,20 +7,25 @@ hantro-vpu-y += \ hantro_v4l2.o \ hantro_postproc.o \ hantro_h1_jpeg_enc.o \ + hantro_g1.o \ hantro_g1_h264_dec.o \ hantro_g1_mpeg2_dec.o \ + hantro_g2_hevc_dec.o \ hantro_g1_vp8_dec.o \ - rk3399_vpu_hw_jpeg_enc.o \ - rk3399_vpu_hw_mpeg2_dec.o \ - rk3399_vpu_hw_vp8_dec.o \ + rockchip_vpu2_hw_jpeg_enc.o \ + rockchip_vpu2_hw_mpeg2_dec.o \ + rockchip_vpu2_hw_vp8_dec.o \ hantro_jpeg.o \ hantro_h264.o \ + hantro_hevc.o \ hantro_mpeg2.o \ hantro_vp8.o hantro-vpu-$(CONFIG_VIDEO_HANTRO_IMX8M) += \ imx8m_vpu_hw.o +hantro-vpu-$(CONFIG_VIDEO_HANTRO_SAMA5D4) += \ + sama5d4_vdec_hw.o + hantro-vpu-$(CONFIG_VIDEO_HANTRO_ROCKCHIP) += \ - rk3288_vpu_hw.o \ - rk3399_vpu_hw.o + rockchip_vpu_hw.o diff --git a/drivers/staging/media/hantro/hantro.h b/drivers/staging/media/hantro/hantro.h index 6c1b888abe75..a70c386de6f1 100644 --- a/drivers/staging/media/hantro/hantro.h +++ b/drivers/staging/media/hantro/hantro.h @@ -34,6 +34,7 @@ struct hantro_codec_ops; #define HANTRO_MPEG2_DECODER BIT(16) #define HANTRO_VP8_DECODER BIT(17) #define HANTRO_H264_DECODER BIT(18) +#define HANTRO_HEVC_DECODER BIT(19) #define HANTRO_DECODERS 0xffff0000 /** @@ -99,6 +100,7 @@ struct hantro_variant { * @HANTRO_MODE_H264_DEC: H264 decoder. * @HANTRO_MODE_MPEG2_DEC: MPEG-2 decoder. * @HANTRO_MODE_VP8_DEC: VP8 decoder. + * @HANTRO_MODE_HEVC_DEC: HEVC decoder. */ enum hantro_codec_mode { HANTRO_MODE_NONE = -1, @@ -106,6 +108,7 @@ enum hantro_codec_mode { HANTRO_MODE_H264_DEC, HANTRO_MODE_MPEG2_DEC, HANTRO_MODE_VP8_DEC, + HANTRO_MODE_HEVC_DEC, }; /* @@ -218,6 +221,7 @@ struct hantro_dev { * @jpeg_enc: JPEG-encoding context. * @mpeg2_dec: MPEG-2-decoding context. * @vp8_dec: VP8-decoding context. + * @hevc_dec: HEVC-decoding context. */ struct hantro_ctx { struct hantro_dev *dev; @@ -244,6 +248,7 @@ struct hantro_ctx { struct hantro_jpeg_enc_hw_ctx jpeg_enc; struct hantro_mpeg2_dec_hw_ctx mpeg2_dec; struct hantro_vp8_dec_hw_ctx vp8_dec; + struct hantro_hevc_dec_hw_ctx hevc_dec; }; }; @@ -410,12 +415,8 @@ hantro_get_dst_buf(struct hantro_ctx *ctx) return v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); } -static inline bool -hantro_needs_postproc(const struct hantro_ctx *ctx, - const struct hantro_fmt *fmt) -{ - return !ctx->is_encoder && fmt->fourcc != V4L2_PIX_FMT_NV12; -} +bool hantro_needs_postproc(const struct hantro_ctx *ctx, + const struct hantro_fmt *fmt); static inline dma_addr_t hantro_get_dec_buf_addr(struct hantro_ctx *ctx, struct vb2_buffer *vb) diff --git a/drivers/staging/media/hantro/hantro_drv.c b/drivers/staging/media/hantro/hantro_drv.c index 595e82a82728..31d8449ca1d2 100644 --- a/drivers/staging/media/hantro/hantro_drv.c +++ b/drivers/staging/media/hantro/hantro_drv.c @@ -56,16 +56,12 @@ dma_addr_t hantro_get_ref(struct hantro_ctx *ctx, u64 ts) return hantro_get_dec_buf_addr(ctx, buf); } -static void hantro_job_finish(struct hantro_dev *vpu, - struct hantro_ctx *ctx, - enum vb2_buffer_state result) +static void hantro_job_finish_no_pm(struct hantro_dev *vpu, + struct hantro_ctx *ctx, + enum vb2_buffer_state result) { struct vb2_v4l2_buffer *src, *dst; - pm_runtime_mark_last_busy(vpu->dev); - pm_runtime_put_autosuspend(vpu->dev); - clk_bulk_disable(vpu->variant->num_clocks, vpu->clocks); - src = v4l2_m2m_next_src_buf(ctx->fh.m2m_ctx); dst = v4l2_m2m_next_dst_buf(ctx->fh.m2m_ctx); @@ -81,6 +77,18 @@ static void hantro_job_finish(struct hantro_dev *vpu, result); } +static void hantro_job_finish(struct hantro_dev *vpu, + struct hantro_ctx *ctx, + enum vb2_buffer_state result) +{ + pm_runtime_mark_last_busy(vpu->dev); + pm_runtime_put_autosuspend(vpu->dev); + + clk_bulk_disable(vpu->variant->num_clocks, vpu->clocks); + + hantro_job_finish_no_pm(vpu, ctx, result); +} + void hantro_irq_done(struct hantro_dev *vpu, enum vb2_buffer_state result) { @@ -152,20 +160,23 @@ static void device_run(void *priv) src = hantro_get_src_buf(ctx); dst = hantro_get_dst_buf(ctx); + ret = pm_runtime_resume_and_get(ctx->dev->dev); + if (ret < 0) + goto err_cancel_job; + ret = clk_bulk_enable(ctx->dev->variant->num_clocks, ctx->dev->clocks); if (ret) goto err_cancel_job; - ret = pm_runtime_get_sync(ctx->dev->dev); - if (ret < 0) - goto err_cancel_job; v4l2_m2m_buf_copy_metadata(src, dst, true); - ctx->codec_ops->run(ctx); + if (ctx->codec_ops->run(ctx)) + goto err_cancel_job; + return; err_cancel_job: - hantro_job_finish(ctx->dev, ctx, VB2_BUF_STATE_ERROR); + hantro_job_finish_no_pm(ctx->dev, ctx, VB2_BUF_STATE_ERROR); } static struct v4l2_m2m_ops vpu_m2m_ops = { @@ -243,6 +254,18 @@ static int hantro_try_ctrl(struct v4l2_ctrl *ctrl) if (sps->bit_depth_luma_minus8 != 0) /* Only 8-bit is supported */ return -EINVAL; + } else if (ctrl->id == V4L2_CID_MPEG_VIDEO_HEVC_SPS) { + const struct v4l2_ctrl_hevc_sps *sps = ctrl->p_new.p_hevc_sps; + + if (sps->bit_depth_luma_minus8 != sps->bit_depth_chroma_minus8) + /* Luma and chroma bit depth mismatch */ + return -EINVAL; + if (sps->bit_depth_luma_minus8 != 0) + /* Only 8-bit is supported */ + return -EINVAL; + if (sps->flags & V4L2_HEVC_SPS_FLAG_SCALING_LIST_ENABLED) + /* No scaling support */ + return -EINVAL; } return 0; } @@ -267,6 +290,26 @@ static int hantro_jpeg_s_ctrl(struct v4l2_ctrl *ctrl) return 0; } +static int hantro_hevc_s_ctrl(struct v4l2_ctrl *ctrl) +{ + struct hantro_ctx *ctx; + + ctx = container_of(ctrl->handler, + struct hantro_ctx, ctrl_handler); + + vpu_debug(1, "s_ctrl: id = %d, val = %d\n", ctrl->id, ctrl->val); + + switch (ctrl->id) { + case V4L2_CID_HANTRO_HEVC_SLICE_HEADER_SKIP: + ctx->hevc_dec.ctrls.hevc_hdr_skip_length = ctrl->val; + break; + default: + return -EINVAL; + } + + return 0; +} + static const struct v4l2_ctrl_ops hantro_ctrl_ops = { .try_ctrl = hantro_try_ctrl, }; @@ -275,6 +318,10 @@ static const struct v4l2_ctrl_ops hantro_jpeg_ctrl_ops = { .s_ctrl = hantro_jpeg_s_ctrl, }; +static const struct v4l2_ctrl_ops hantro_hevc_ctrl_ops = { + .s_ctrl = hantro_hevc_s_ctrl, +}; + static const struct hantro_ctrl controls[] = { { .codec = HANTRO_JPEG_ENCODER, @@ -289,12 +336,17 @@ static const struct hantro_ctrl controls[] = { }, { .codec = HANTRO_MPEG2_DECODER, .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS, + .id = V4L2_CID_STATELESS_MPEG2_SEQUENCE, }, }, { .codec = HANTRO_MPEG2_DECODER, .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION, + .id = V4L2_CID_STATELESS_MPEG2_PICTURE, + }, + }, { + .codec = HANTRO_MPEG2_DECODER, + .cfg = { + .id = V4L2_CID_STATELESS_MPEG2_QUANTISATION, }, }, { .codec = HANTRO_VP8_DECODER, @@ -349,6 +401,64 @@ static const struct hantro_ctrl controls[] = { .def = V4L2_MPEG_VIDEO_H264_PROFILE_MAIN, } }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE, + .min = V4L2_MPEG_VIDEO_HEVC_DECODE_MODE_FRAME_BASED, + .max = V4L2_MPEG_VIDEO_HEVC_DECODE_MODE_FRAME_BASED, + .def = V4L2_MPEG_VIDEO_HEVC_DECODE_MODE_FRAME_BASED, + }, + }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_START_CODE, + .min = V4L2_MPEG_VIDEO_HEVC_START_CODE_ANNEX_B, + .max = V4L2_MPEG_VIDEO_HEVC_START_CODE_ANNEX_B, + .def = V4L2_MPEG_VIDEO_HEVC_START_CODE_ANNEX_B, + }, + }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_PROFILE, + .min = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, + .max = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN_10, + .def = V4L2_MPEG_VIDEO_HEVC_PROFILE_MAIN, + }, + }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_LEVEL, + .min = V4L2_MPEG_VIDEO_HEVC_LEVEL_1, + .max = V4L2_MPEG_VIDEO_HEVC_LEVEL_5_1, + }, + }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_SPS, + .ops = &hantro_ctrl_ops, + }, + }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_PPS, + }, + }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS, + }, + }, { + .codec = HANTRO_HEVC_DECODER, + .cfg = { + .id = V4L2_CID_HANTRO_HEVC_SLICE_HEADER_SKIP, + .name = "Hantro HEVC slice header skip bytes", + .type = V4L2_CTRL_TYPE_INTEGER, + .min = 0, + .def = 0, + .max = 0x100, + .step = 1, + .ops = &hantro_hevc_ctrl_ops, + }, }, }; @@ -472,12 +582,18 @@ static const struct v4l2_file_operations hantro_fops = { static const struct of_device_id of_hantro_match[] = { #ifdef CONFIG_VIDEO_HANTRO_ROCKCHIP - { .compatible = "rockchip,rk3399-vpu", .data = &rk3399_vpu_variant, }, - { .compatible = "rockchip,rk3328-vpu", .data = &rk3328_vpu_variant, }, + { .compatible = "rockchip,rk3036-vpu", .data = &rk3036_vpu_variant, }, + { .compatible = "rockchip,rk3066-vpu", .data = &rk3066_vpu_variant, }, { .compatible = "rockchip,rk3288-vpu", .data = &rk3288_vpu_variant, }, + { .compatible = "rockchip,rk3328-vpu", .data = &rk3328_vpu_variant, }, + { .compatible = "rockchip,rk3399-vpu", .data = &rk3399_vpu_variant, }, #endif #ifdef CONFIG_VIDEO_HANTRO_IMX8M { .compatible = "nxp,imx8mq-vpu", .data = &imx8mq_vpu_variant, }, + { .compatible = "nxp,imx8mq-vpu-g2", .data = &imx8mq_vpu_g2_variant }, +#endif +#ifdef CONFIG_VIDEO_HANTRO_SAMA5D4 + { .compatible = "microchip,sama5d4-vdec", .data = &sama5d4_vdec_variant, }, #endif { /* sentinel */ } }; @@ -752,12 +868,23 @@ static int hantro_probe(struct platform_device *pdev) if (!vpu->clocks) return -ENOMEM; - for (i = 0; i < vpu->variant->num_clocks; i++) - vpu->clocks[i].id = vpu->variant->clk_names[i]; - ret = devm_clk_bulk_get(&pdev->dev, vpu->variant->num_clocks, - vpu->clocks); - if (ret) - return ret; + if (vpu->variant->num_clocks > 1) { + for (i = 0; i < vpu->variant->num_clocks; i++) + vpu->clocks[i].id = vpu->variant->clk_names[i]; + + ret = devm_clk_bulk_get(&pdev->dev, vpu->variant->num_clocks, + vpu->clocks); + if (ret) + return ret; + } else { + /* + * If the driver has a single clk, chances are there will be no + * actual name in the DT bindings. + */ + vpu->clocks[0].clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(vpu->clocks[0].clk)) + return PTR_ERR(vpu->clocks[0].clk); + } num_bases = vpu->variant->num_regs ?: 1; vpu->reg_bases = devm_kcalloc(&pdev->dev, num_bases, @@ -785,13 +912,23 @@ static int hantro_probe(struct platform_device *pdev) vb2_dma_contig_set_max_seg_size(&pdev->dev, DMA_BIT_MASK(32)); for (i = 0; i < vpu->variant->num_irqs; i++) { - const char *irq_name = vpu->variant->irqs[i].name; + const char *irq_name; int irq; if (!vpu->variant->irqs[i].handler) continue; - irq = platform_get_irq_byname(vpu->pdev, irq_name); + if (vpu->variant->num_clocks > 1) { + irq_name = vpu->variant->irqs[i].name; + irq = platform_get_irq_byname(vpu->pdev, irq_name); + } else { + /* + * If the driver has a single IRQ, chances are there + * will be no actual name in the DT bindings. + */ + irq_name = "default"; + irq = platform_get_irq(vpu->pdev, 0); + } if (irq <= 0) return -ENXIO; diff --git a/drivers/staging/media/hantro/hantro_g1.c b/drivers/staging/media/hantro/hantro_g1.c new file mode 100644 index 000000000000..0ab1cee62218 --- /dev/null +++ b/drivers/staging/media/hantro/hantro_g1.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hantro VPU codec driver + * + * Copyright (C) 2018 Rockchip Electronics Co., Ltd. + * Jeffy Chen + * Copyright (C) 2019 Pengutronix, Philipp Zabel + * Copyright (C) 2021 Collabora Ltd, Emil Velikov + */ + +#include "hantro.h" +#include "hantro_g1_regs.h" + +irqreturn_t hantro_g1_irq(int irq, void *dev_id) +{ + struct hantro_dev *vpu = dev_id; + enum vb2_buffer_state state; + u32 status; + + status = vdpu_read(vpu, G1_REG_INTERRUPT); + state = (status & G1_REG_INTERRUPT_DEC_RDY_INT) ? + VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; + + vdpu_write(vpu, 0, G1_REG_INTERRUPT); + vdpu_write(vpu, G1_REG_CONFIG_DEC_CLK_GATE_E, G1_REG_CONFIG); + + hantro_irq_done(vpu, state); + + return IRQ_HANDLED; +} + +void hantro_g1_reset(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + + vdpu_write(vpu, G1_REG_INTERRUPT_DEC_IRQ_DIS, G1_REG_INTERRUPT); + vdpu_write(vpu, G1_REG_CONFIG_DEC_CLK_GATE_E, G1_REG_CONFIG); + vdpu_write(vpu, 1, G1_REG_SOFT_RESET); +} diff --git a/drivers/staging/media/hantro/hantro_g1_h264_dec.c b/drivers/staging/media/hantro/hantro_g1_h264_dec.c index 845bef73d218..5c792b7bcb79 100644 --- a/drivers/staging/media/hantro/hantro_g1_h264_dec.c +++ b/drivers/staging/media/hantro/hantro_g1_h264_dec.c @@ -273,13 +273,15 @@ static void set_buffers(struct hantro_ctx *ctx) vdpu_write_relaxed(vpu, ctx->h264_dec.priv.dma, G1_REG_ADDR_QTABLE); } -void hantro_g1_h264_dec_run(struct hantro_ctx *ctx) +int hantro_g1_h264_dec_run(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; + int ret; /* Prepare the H264 decoder context. */ - if (hantro_h264_dec_prepare_run(ctx)) - return; + ret = hantro_h264_dec_prepare_run(ctx); + if (ret) + return ret; /* Configure hardware registers. */ set_params(ctx); @@ -301,4 +303,6 @@ void hantro_g1_h264_dec_run(struct hantro_ctx *ctx) G1_REG_CONFIG_DEC_CLK_GATE_E, G1_REG_CONFIG); vdpu_write(vpu, G1_REG_INTERRUPT_DEC_E, G1_REG_INTERRUPT); + + return 0; } diff --git a/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c b/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c index 6386a3989bfe..9aea331e1a3c 100644 --- a/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c +++ b/drivers/staging/media/hantro/hantro_g1_mpeg2_dec.c @@ -10,6 +10,7 @@ #include #include "hantro.h" #include "hantro_hw.h" +#include "hantro_g1_regs.h" #define G1_SWREG(nr) ((nr) * 4) @@ -20,7 +21,6 @@ #define G1_REG_REFER2_BASE G1_SWREG(16) #define G1_REG_REFER3_BASE G1_SWREG(17) #define G1_REG_QTABLE_BASE G1_SWREG(40) -#define G1_REG_DEC_E(v) ((v) ? BIT(0) : 0) #define G1_REG_DEC_AXI_RD_ID(v) (((v) << 24) & GENMASK(31, 24)) #define G1_REG_DEC_TIMEOUT_E(v) ((v) ? BIT(23) : 0) @@ -77,43 +77,33 @@ #define G1_REG_APF_THRESHOLD(v) (((v) << 0) & GENMASK(13, 0)) -#define PICT_TOP_FIELD 1 -#define PICT_BOTTOM_FIELD 2 -#define PICT_FRAME 3 - static void -hantro_g1_mpeg2_dec_set_quantization(struct hantro_dev *vpu, +hantro_g1_mpeg2_dec_set_quantisation(struct hantro_dev *vpu, struct hantro_ctx *ctx) { - struct v4l2_ctrl_mpeg2_quantization *quantization; + struct v4l2_ctrl_mpeg2_quantisation *q; - quantization = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION); - hantro_mpeg2_dec_copy_qtable(ctx->mpeg2_dec.qtable.cpu, - quantization); - vdpu_write_relaxed(vpu, ctx->mpeg2_dec.qtable.dma, - G1_REG_QTABLE_BASE); + q = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_MPEG2_QUANTISATION); + hantro_mpeg2_dec_copy_qtable(ctx->mpeg2_dec.qtable.cpu, q); + vdpu_write_relaxed(vpu, ctx->mpeg2_dec.qtable.dma, G1_REG_QTABLE_BASE); } static void hantro_g1_mpeg2_dec_set_buffers(struct hantro_dev *vpu, struct hantro_ctx *ctx, struct vb2_buffer *src_buf, struct vb2_buffer *dst_buf, - const struct v4l2_mpeg2_sequence *sequence, - const struct v4l2_mpeg2_picture *picture, - const struct v4l2_ctrl_mpeg2_slice_params *slice_params) + const struct v4l2_ctrl_mpeg2_sequence *seq, + const struct v4l2_ctrl_mpeg2_picture *pic) { dma_addr_t forward_addr = 0, backward_addr = 0; dma_addr_t current_addr, addr; - switch (picture->picture_coding_type) { - case V4L2_MPEG2_PICTURE_CODING_TYPE_B: - backward_addr = hantro_get_ref(ctx, - slice_params->backward_ref_ts); + switch (pic->picture_coding_type) { + case V4L2_MPEG2_PIC_CODING_TYPE_B: + backward_addr = hantro_get_ref(ctx, pic->backward_ref_ts); fallthrough; - case V4L2_MPEG2_PICTURE_CODING_TYPE_P: - forward_addr = hantro_get_ref(ctx, - slice_params->forward_ref_ts); + case V4L2_MPEG2_PIC_CODING_TYPE_P: + forward_addr = hantro_get_ref(ctx, pic->forward_ref_ts); } /* Source bitstream buffer */ @@ -124,7 +114,7 @@ hantro_g1_mpeg2_dec_set_buffers(struct hantro_dev *vpu, struct hantro_ctx *ctx, addr = hantro_get_dec_buf_addr(ctx, dst_buf); current_addr = addr; - if (picture->picture_structure == PICT_BOTTOM_FIELD) + if (pic->picture_structure == V4L2_MPEG2_PIC_BOTTOM_FIELD) addr += ALIGN(ctx->dst_fmt.width, 16); vdpu_write_relaxed(vpu, addr, G1_REG_DEC_OUT_BASE); @@ -134,18 +124,18 @@ hantro_g1_mpeg2_dec_set_buffers(struct hantro_dev *vpu, struct hantro_ctx *ctx, backward_addr = current_addr; /* Set forward ref frame (top/bottom field) */ - if (picture->picture_structure == PICT_FRAME || - picture->picture_coding_type == V4L2_MPEG2_PICTURE_CODING_TYPE_B || - (picture->picture_structure == PICT_TOP_FIELD && - picture->top_field_first) || - (picture->picture_structure == PICT_BOTTOM_FIELD && - !picture->top_field_first)) { + if (pic->picture_structure == V4L2_MPEG2_PIC_FRAME || + pic->picture_coding_type == V4L2_MPEG2_PIC_CODING_TYPE_B || + (pic->picture_structure == V4L2_MPEG2_PIC_TOP_FIELD && + pic->flags & V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST) || + (pic->picture_structure == V4L2_MPEG2_PIC_BOTTOM_FIELD && + !(pic->flags & V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST))) { vdpu_write_relaxed(vpu, forward_addr, G1_REG_REFER0_BASE); vdpu_write_relaxed(vpu, forward_addr, G1_REG_REFER1_BASE); - } else if (picture->picture_structure == PICT_TOP_FIELD) { + } else if (pic->picture_structure == V4L2_MPEG2_PIC_TOP_FIELD) { vdpu_write_relaxed(vpu, forward_addr, G1_REG_REFER0_BASE); vdpu_write_relaxed(vpu, current_addr, G1_REG_REFER1_BASE); - } else if (picture->picture_structure == PICT_BOTTOM_FIELD) { + } else if (pic->picture_structure == V4L2_MPEG2_PIC_BOTTOM_FIELD) { vdpu_write_relaxed(vpu, current_addr, G1_REG_REFER0_BASE); vdpu_write_relaxed(vpu, forward_addr, G1_REG_REFER1_BASE); } @@ -155,13 +145,12 @@ hantro_g1_mpeg2_dec_set_buffers(struct hantro_dev *vpu, struct hantro_ctx *ctx, vdpu_write_relaxed(vpu, backward_addr, G1_REG_REFER3_BASE); } -void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) +int hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; struct vb2_v4l2_buffer *src_buf, *dst_buf; - const struct v4l2_ctrl_mpeg2_slice_params *slice_params; - const struct v4l2_mpeg2_sequence *sequence; - const struct v4l2_mpeg2_picture *picture; + const struct v4l2_ctrl_mpeg2_sequence *seq; + const struct v4l2_ctrl_mpeg2_picture *pic; u32 reg; src_buf = hantro_get_src_buf(ctx); @@ -170,10 +159,10 @@ void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) /* Apply request controls if any */ hantro_start_prepare_run(ctx); - slice_params = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS); - sequence = &slice_params->sequence; - picture = &slice_params->picture; + seq = hantro_get_ctrl(ctx, + V4L2_CID_STATELESS_MPEG2_SEQUENCE); + pic = hantro_get_ctrl(ctx, + V4L2_CID_STATELESS_MPEG2_PICTURE); reg = G1_REG_DEC_AXI_RD_ID(0) | G1_REG_DEC_TIMEOUT_E(1) | @@ -193,11 +182,11 @@ void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) reg = G1_REG_DEC_MODE(5) | G1_REG_RLC_MODE_E(0) | - G1_REG_PIC_INTERLACE_E(!sequence->progressive_sequence) | - G1_REG_PIC_FIELDMODE_E(picture->picture_structure != PICT_FRAME) | - G1_REG_PIC_B_E(picture->picture_coding_type == V4L2_MPEG2_PICTURE_CODING_TYPE_B) | - G1_REG_PIC_INTER_E(picture->picture_coding_type != V4L2_MPEG2_PICTURE_CODING_TYPE_I) | - G1_REG_PIC_TOPFIELD_E(picture->picture_structure == PICT_TOP_FIELD) | + G1_REG_PIC_INTERLACE_E(!(seq->flags & V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE)) | + G1_REG_PIC_FIELDMODE_E(pic->picture_structure != V4L2_MPEG2_PIC_FRAME) | + G1_REG_PIC_B_E(pic->picture_coding_type == V4L2_MPEG2_PIC_CODING_TYPE_B) | + G1_REG_PIC_INTER_E(pic->picture_coding_type != V4L2_MPEG2_PIC_CODING_TYPE_I) | + G1_REG_PIC_TOPFIELD_E(pic->picture_structure == V4L2_MPEG2_PIC_TOP_FIELD) | G1_REG_FWD_INTERLACE_E(0) | G1_REG_FILTERING_DIS(1) | G1_REG_WRITE_MVS_E(0) | @@ -206,27 +195,27 @@ void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) reg = G1_REG_PIC_MB_WIDTH(MB_WIDTH(ctx->dst_fmt.width)) | G1_REG_PIC_MB_HEIGHT_P(MB_HEIGHT(ctx->dst_fmt.height)) | - G1_REG_ALT_SCAN_E(picture->alternate_scan) | - G1_REG_TOPFIELDFIRST_E(picture->top_field_first); + G1_REG_ALT_SCAN_E(pic->flags & V4L2_MPEG2_PIC_FLAG_ALT_SCAN) | + G1_REG_TOPFIELDFIRST_E(pic->flags & V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST); vdpu_write_relaxed(vpu, reg, G1_SWREG(4)); - reg = G1_REG_STRM_START_BIT(slice_params->data_bit_offset) | - G1_REG_QSCALE_TYPE(picture->q_scale_type) | - G1_REG_CON_MV_E(picture->concealment_motion_vectors) | - G1_REG_INTRA_DC_PREC(picture->intra_dc_precision) | - G1_REG_INTRA_VLC_TAB(picture->intra_vlc_format) | - G1_REG_FRAME_PRED_DCT(picture->frame_pred_frame_dct); + reg = G1_REG_STRM_START_BIT(0) | + G1_REG_QSCALE_TYPE(pic->flags & V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE) | + G1_REG_CON_MV_E(pic->flags & V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV) | + G1_REG_INTRA_DC_PREC(pic->intra_dc_precision) | + G1_REG_INTRA_VLC_TAB(pic->flags & V4L2_MPEG2_PIC_FLAG_INTRA_VLC) | + G1_REG_FRAME_PRED_DCT(pic->flags & V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT); vdpu_write_relaxed(vpu, reg, G1_SWREG(5)); reg = G1_REG_INIT_QP(1) | - G1_REG_STREAM_LEN(slice_params->bit_size >> 3); + G1_REG_STREAM_LEN(vb2_get_plane_payload(&src_buf->vb2_buf, 0)); vdpu_write_relaxed(vpu, reg, G1_SWREG(6)); - reg = G1_REG_ALT_SCAN_FLAG_E(picture->alternate_scan) | - G1_REG_FCODE_FWD_HOR(picture->f_code[0][0]) | - G1_REG_FCODE_FWD_VER(picture->f_code[0][1]) | - G1_REG_FCODE_BWD_HOR(picture->f_code[1][0]) | - G1_REG_FCODE_BWD_VER(picture->f_code[1][1]) | + reg = G1_REG_ALT_SCAN_FLAG_E(pic->flags & V4L2_MPEG2_PIC_FLAG_ALT_SCAN) | + G1_REG_FCODE_FWD_HOR(pic->f_code[0][0]) | + G1_REG_FCODE_FWD_VER(pic->f_code[0][1]) | + G1_REG_FCODE_BWD_HOR(pic->f_code[1][0]) | + G1_REG_FCODE_BWD_VER(pic->f_code[1][1]) | G1_REG_MV_ACCURACY_FWD(1) | G1_REG_MV_ACCURACY_BWD(1); vdpu_write_relaxed(vpu, reg, G1_SWREG(18)); @@ -238,14 +227,14 @@ void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx) reg = G1_REG_APF_THRESHOLD(8); vdpu_write_relaxed(vpu, reg, G1_SWREG(55)); - hantro_g1_mpeg2_dec_set_quantization(vpu, ctx); - + hantro_g1_mpeg2_dec_set_quantisation(vpu, ctx); hantro_g1_mpeg2_dec_set_buffers(vpu, ctx, &src_buf->vb2_buf, &dst_buf->vb2_buf, - sequence, picture, slice_params); + seq, pic); hantro_end_prepare_run(ctx); - reg = G1_REG_DEC_E(1); - vdpu_write(vpu, reg, G1_SWREG(1)); + vdpu_write(vpu, G1_REG_INTERRUPT_DEC_E, G1_REG_INTERRUPT); + + return 0; } diff --git a/drivers/staging/media/hantro/hantro_g1_vp8_dec.c b/drivers/staging/media/hantro/hantro_g1_vp8_dec.c index 57002ba70176..96622a7f8279 100644 --- a/drivers/staging/media/hantro/hantro_g1_vp8_dec.c +++ b/drivers/staging/media/hantro/hantro_g1_vp8_dec.c @@ -425,7 +425,7 @@ static void cfg_buffers(struct hantro_ctx *ctx, vdpu_write_relaxed(vpu, dst_dma, G1_REG_ADDR_DST); } -void hantro_g1_vp8_dec_run(struct hantro_ctx *ctx) +int hantro_g1_vp8_dec_run(struct hantro_ctx *ctx) { const struct v4l2_ctrl_vp8_frame *hdr; struct hantro_dev *vpu = ctx->dev; @@ -438,7 +438,7 @@ void hantro_g1_vp8_dec_run(struct hantro_ctx *ctx) hdr = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_VP8_FRAME); if (WARN_ON(!hdr)) - return; + return -EINVAL; /* Reset segment_map buffer in keyframe */ if (V4L2_VP8_FRAME_IS_KEY_FRAME(hdr) && ctx->vp8_dec.segment_map.cpu) @@ -498,4 +498,6 @@ void hantro_g1_vp8_dec_run(struct hantro_ctx *ctx) hantro_end_prepare_run(ctx); vdpu_write(vpu, G1_REG_INTERRUPT_DEC_E, G1_REG_INTERRUPT); + + return 0; } diff --git a/drivers/staging/media/hantro/hantro_g2_hevc_dec.c b/drivers/staging/media/hantro/hantro_g2_hevc_dec.c new file mode 100644 index 000000000000..340efb57fd18 --- /dev/null +++ b/drivers/staging/media/hantro/hantro_g2_hevc_dec.c @@ -0,0 +1,586 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hantro VPU HEVC codec driver + * + * Copyright (C) 2020 Safran Passenger Innovations LLC + */ + +#include "hantro_hw.h" +#include "hantro_g2_regs.h" + +#define HEVC_DEC_MODE 0xC + +#define BUS_WIDTH_32 0 +#define BUS_WIDTH_64 1 +#define BUS_WIDTH_128 2 +#define BUS_WIDTH_256 3 + +static inline void hantro_write_addr(struct hantro_dev *vpu, + unsigned long offset, + dma_addr_t addr) +{ + vdpu_write(vpu, addr & 0xffffffff, offset); +} + +static void prepare_tile_info_buffer(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; + const struct v4l2_ctrl_hevc_pps *pps = ctrls->pps; + const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; + u16 *p = (u16 *)((u8 *)ctx->hevc_dec.tile_sizes.cpu); + unsigned int num_tile_rows = pps->num_tile_rows_minus1 + 1; + unsigned int num_tile_cols = pps->num_tile_columns_minus1 + 1; + unsigned int pic_width_in_ctbs, pic_height_in_ctbs; + unsigned int max_log2_ctb_size, ctb_size; + bool tiles_enabled, uniform_spacing; + u32 no_chroma = 0; + + tiles_enabled = !!(pps->flags & V4L2_HEVC_PPS_FLAG_TILES_ENABLED); + uniform_spacing = !!(pps->flags & V4L2_HEVC_PPS_FLAG_UNIFORM_SPACING); + + hantro_reg_write(vpu, &g2_tile_e, tiles_enabled); + + max_log2_ctb_size = sps->log2_min_luma_coding_block_size_minus3 + 3 + + sps->log2_diff_max_min_luma_coding_block_size; + pic_width_in_ctbs = (sps->pic_width_in_luma_samples + + (1 << max_log2_ctb_size) - 1) >> max_log2_ctb_size; + pic_height_in_ctbs = (sps->pic_height_in_luma_samples + (1 << max_log2_ctb_size) - 1) + >> max_log2_ctb_size; + ctb_size = 1 << max_log2_ctb_size; + + vpu_debug(1, "Preparing tile sizes buffer for %dx%d CTBs (CTB size %d)\n", + pic_width_in_ctbs, pic_height_in_ctbs, ctb_size); + + if (tiles_enabled) { + unsigned int i, j, h; + + vpu_debug(1, "Tiles enabled! %dx%d\n", num_tile_cols, num_tile_rows); + + hantro_reg_write(vpu, &g2_num_tile_rows, num_tile_rows); + hantro_reg_write(vpu, &g2_num_tile_cols, num_tile_cols); + + /* write width + height for each tile in pic */ + if (!uniform_spacing) { + u32 tmp_w = 0, tmp_h = 0; + + for (i = 0; i < num_tile_rows; i++) { + if (i == num_tile_rows - 1) + h = pic_height_in_ctbs - tmp_h; + else + h = pps->row_height_minus1[i] + 1; + tmp_h += h; + if (i == 0 && h == 1 && ctb_size == 16) + no_chroma = 1; + for (j = 0, tmp_w = 0; j < num_tile_cols - 1; j++) { + tmp_w += pps->column_width_minus1[j] + 1; + *p++ = pps->column_width_minus1[j + 1]; + *p++ = h; + if (i == 0 && h == 1 && ctb_size == 16) + no_chroma = 1; + } + /* last column */ + *p++ = pic_width_in_ctbs - tmp_w; + *p++ = h; + } + } else { /* uniform spacing */ + u32 tmp, prev_h, prev_w; + + for (i = 0, prev_h = 0; i < num_tile_rows; i++) { + tmp = (i + 1) * pic_height_in_ctbs / num_tile_rows; + h = tmp - prev_h; + prev_h = tmp; + if (i == 0 && h == 1 && ctb_size == 16) + no_chroma = 1; + for (j = 0, prev_w = 0; j < num_tile_cols; j++) { + tmp = (j + 1) * pic_width_in_ctbs / num_tile_cols; + *p++ = tmp - prev_w; + *p++ = h; + if (j == 0 && + (pps->column_width_minus1[0] + 1) == 1 && + ctb_size == 16) + no_chroma = 1; + prev_w = tmp; + } + } + } + } else { + hantro_reg_write(vpu, &g2_num_tile_rows, 1); + hantro_reg_write(vpu, &g2_num_tile_cols, 1); + + /* There's one tile, with dimensions equal to pic size. */ + p[0] = pic_width_in_ctbs; + p[1] = pic_height_in_ctbs; + } + + if (no_chroma) + vpu_debug(1, "%s: no chroma!\n", __func__); +} + +static void set_params(struct hantro_ctx *ctx) +{ + const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; + const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; + const struct v4l2_ctrl_hevc_pps *pps = ctrls->pps; + const struct v4l2_ctrl_hevc_decode_params *decode_params = ctrls->decode_params; + struct hantro_dev *vpu = ctx->dev; + u32 min_log2_cb_size, max_log2_ctb_size, min_cb_size, max_ctb_size; + u32 pic_width_in_min_cbs, pic_height_in_min_cbs; + u32 pic_width_aligned, pic_height_aligned; + u32 partial_ctb_x, partial_ctb_y; + + hantro_reg_write(vpu, &g2_bit_depth_y_minus8, sps->bit_depth_luma_minus8); + hantro_reg_write(vpu, &g2_bit_depth_c_minus8, sps->bit_depth_chroma_minus8); + + hantro_reg_write(vpu, &g2_output_8_bits, 0); + + hantro_reg_write(vpu, &g2_hdr_skip_length, ctrls->hevc_hdr_skip_length); + + min_log2_cb_size = sps->log2_min_luma_coding_block_size_minus3 + 3; + max_log2_ctb_size = min_log2_cb_size + sps->log2_diff_max_min_luma_coding_block_size; + + hantro_reg_write(vpu, &g2_min_cb_size, min_log2_cb_size); + hantro_reg_write(vpu, &g2_max_cb_size, max_log2_ctb_size); + + min_cb_size = 1 << min_log2_cb_size; + max_ctb_size = 1 << max_log2_ctb_size; + + pic_width_in_min_cbs = sps->pic_width_in_luma_samples / min_cb_size; + pic_height_in_min_cbs = sps->pic_height_in_luma_samples / min_cb_size; + pic_width_aligned = ALIGN(sps->pic_width_in_luma_samples, max_ctb_size); + pic_height_aligned = ALIGN(sps->pic_height_in_luma_samples, max_ctb_size); + + partial_ctb_x = !!(sps->pic_width_in_luma_samples != pic_width_aligned); + partial_ctb_y = !!(sps->pic_height_in_luma_samples != pic_height_aligned); + + hantro_reg_write(vpu, &g2_partial_ctb_x, partial_ctb_x); + hantro_reg_write(vpu, &g2_partial_ctb_y, partial_ctb_y); + + hantro_reg_write(vpu, &g2_pic_width_in_cbs, pic_width_in_min_cbs); + hantro_reg_write(vpu, &g2_pic_height_in_cbs, pic_height_in_min_cbs); + + hantro_reg_write(vpu, &g2_pic_width_4x4, + (pic_width_in_min_cbs * min_cb_size) / 4); + hantro_reg_write(vpu, &g2_pic_height_4x4, + (pic_height_in_min_cbs * min_cb_size) / 4); + + hantro_reg_write(vpu, &hevc_max_inter_hierdepth, + sps->max_transform_hierarchy_depth_inter); + hantro_reg_write(vpu, &hevc_max_intra_hierdepth, + sps->max_transform_hierarchy_depth_intra); + hantro_reg_write(vpu, &hevc_min_trb_size, + sps->log2_min_luma_transform_block_size_minus2 + 2); + hantro_reg_write(vpu, &hevc_max_trb_size, + sps->log2_min_luma_transform_block_size_minus2 + 2 + + sps->log2_diff_max_min_luma_transform_block_size); + + hantro_reg_write(vpu, &g2_tempor_mvp_e, + !!(sps->flags & V4L2_HEVC_SPS_FLAG_SPS_TEMPORAL_MVP_ENABLED) && + !(decode_params->flags & V4L2_HEVC_DECODE_PARAM_FLAG_IDR_PIC)); + hantro_reg_write(vpu, &g2_strong_smooth_e, + !!(sps->flags & V4L2_HEVC_SPS_FLAG_STRONG_INTRA_SMOOTHING_ENABLED)); + hantro_reg_write(vpu, &g2_asym_pred_e, + !!(sps->flags & V4L2_HEVC_SPS_FLAG_AMP_ENABLED)); + hantro_reg_write(vpu, &g2_sao_e, + !!(sps->flags & V4L2_HEVC_SPS_FLAG_SAMPLE_ADAPTIVE_OFFSET)); + hantro_reg_write(vpu, &g2_sign_data_hide, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_SIGN_DATA_HIDING_ENABLED)); + + if (pps->flags & V4L2_HEVC_PPS_FLAG_CU_QP_DELTA_ENABLED) { + hantro_reg_write(vpu, &g2_cu_qpd_e, 1); + hantro_reg_write(vpu, &g2_max_cu_qpd_depth, pps->diff_cu_qp_delta_depth); + } else { + hantro_reg_write(vpu, &g2_cu_qpd_e, 0); + hantro_reg_write(vpu, &g2_max_cu_qpd_depth, 0); + } + + if (pps->flags & V4L2_HEVC_PPS_FLAG_PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT) { + hantro_reg_write(vpu, &g2_cb_qp_offset, pps->pps_cb_qp_offset); + hantro_reg_write(vpu, &g2_cr_qp_offset, pps->pps_cr_qp_offset); + } else { + hantro_reg_write(vpu, &g2_cb_qp_offset, 0); + hantro_reg_write(vpu, &g2_cr_qp_offset, 0); + } + + hantro_reg_write(vpu, &g2_filt_offset_beta, pps->pps_beta_offset_div2); + hantro_reg_write(vpu, &g2_filt_offset_tc, pps->pps_tc_offset_div2); + hantro_reg_write(vpu, &g2_slice_hdr_ext_e, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_SLICE_SEGMENT_HEADER_EXTENSION_PRESENT)); + hantro_reg_write(vpu, &g2_slice_hdr_ext_bits, pps->num_extra_slice_header_bits); + hantro_reg_write(vpu, &g2_slice_chqp_present, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_PPS_SLICE_CHROMA_QP_OFFSETS_PRESENT)); + hantro_reg_write(vpu, &g2_weight_bipr_idc, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_WEIGHTED_BIPRED)); + hantro_reg_write(vpu, &g2_transq_bypass, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_TRANSQUANT_BYPASS_ENABLED)); + hantro_reg_write(vpu, &g2_list_mod_e, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_LISTS_MODIFICATION_PRESENT)); + hantro_reg_write(vpu, &g2_entropy_sync_e, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_ENTROPY_CODING_SYNC_ENABLED)); + hantro_reg_write(vpu, &g2_cabac_init_present, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_CABAC_INIT_PRESENT)); + hantro_reg_write(vpu, &g2_idr_pic_e, + !!(decode_params->flags & V4L2_HEVC_DECODE_PARAM_FLAG_IRAP_PIC)); + hantro_reg_write(vpu, &hevc_parallel_merge, + pps->log2_parallel_merge_level_minus2 + 2); + hantro_reg_write(vpu, &g2_pcm_filt_d, + !!(sps->flags & V4L2_HEVC_SPS_FLAG_PCM_LOOP_FILTER_DISABLED)); + hantro_reg_write(vpu, &g2_pcm_e, + !!(sps->flags & V4L2_HEVC_SPS_FLAG_PCM_ENABLED)); + if (sps->flags & V4L2_HEVC_SPS_FLAG_PCM_ENABLED) { + hantro_reg_write(vpu, &g2_max_pcm_size, + sps->log2_diff_max_min_pcm_luma_coding_block_size + + sps->log2_min_pcm_luma_coding_block_size_minus3 + 3); + hantro_reg_write(vpu, &g2_min_pcm_size, + sps->log2_min_pcm_luma_coding_block_size_minus3 + 3); + hantro_reg_write(vpu, &g2_bit_depth_pcm_y, + sps->pcm_sample_bit_depth_luma_minus1 + 1); + hantro_reg_write(vpu, &g2_bit_depth_pcm_c, + sps->pcm_sample_bit_depth_chroma_minus1 + 1); + } else { + hantro_reg_write(vpu, &g2_max_pcm_size, 0); + hantro_reg_write(vpu, &g2_min_pcm_size, 0); + hantro_reg_write(vpu, &g2_bit_depth_pcm_y, 0); + hantro_reg_write(vpu, &g2_bit_depth_pcm_c, 0); + } + + hantro_reg_write(vpu, &g2_start_code_e, 1); + hantro_reg_write(vpu, &g2_init_qp, pps->init_qp_minus26 + 26); + hantro_reg_write(vpu, &g2_weight_pred_e, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_WEIGHTED_PRED)); + hantro_reg_write(vpu, &g2_cabac_init_present, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_CABAC_INIT_PRESENT)); + hantro_reg_write(vpu, &g2_const_intra_e, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_CONSTRAINED_INTRA_PRED)); + hantro_reg_write(vpu, &g2_transform_skip, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_TRANSFORM_SKIP_ENABLED)); + hantro_reg_write(vpu, &g2_out_filtering_dis, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER)); + hantro_reg_write(vpu, &g2_filt_ctrl_pres, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT)); + hantro_reg_write(vpu, &g2_dependent_slice, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT_ENABLED)); + hantro_reg_write(vpu, &g2_filter_override, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_OVERRIDE_ENABLED)); + hantro_reg_write(vpu, &g2_refidx0_active, + pps->num_ref_idx_l0_default_active_minus1 + 1); + hantro_reg_write(vpu, &g2_refidx1_active, + pps->num_ref_idx_l1_default_active_minus1 + 1); + hantro_reg_write(vpu, &g2_apf_threshold, 8); +} + +static int find_ref_pic_index(const struct v4l2_hevc_dpb_entry *dpb, int pic_order_cnt) +{ + int i; + + for (i = 0; i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) { + if (dpb[i].pic_order_cnt[0] == pic_order_cnt) + return i; + } + + return 0x0; +} + +static void set_ref_pic_list(struct hantro_ctx *ctx) +{ + const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; + struct hantro_dev *vpu = ctx->dev; + const struct v4l2_ctrl_hevc_decode_params *decode_params = ctrls->decode_params; + const struct v4l2_hevc_dpb_entry *dpb = decode_params->dpb; + u32 list0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX] = {}; + u32 list1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX] = {}; + static const struct hantro_reg ref_pic_regs0[] = { + hevc_rlist_f0, + hevc_rlist_f1, + hevc_rlist_f2, + hevc_rlist_f3, + hevc_rlist_f4, + hevc_rlist_f5, + hevc_rlist_f6, + hevc_rlist_f7, + hevc_rlist_f8, + hevc_rlist_f9, + hevc_rlist_f10, + hevc_rlist_f11, + hevc_rlist_f12, + hevc_rlist_f13, + hevc_rlist_f14, + hevc_rlist_f15, + }; + static const struct hantro_reg ref_pic_regs1[] = { + hevc_rlist_b0, + hevc_rlist_b1, + hevc_rlist_b2, + hevc_rlist_b3, + hevc_rlist_b4, + hevc_rlist_b5, + hevc_rlist_b6, + hevc_rlist_b7, + hevc_rlist_b8, + hevc_rlist_b9, + hevc_rlist_b10, + hevc_rlist_b11, + hevc_rlist_b12, + hevc_rlist_b13, + hevc_rlist_b14, + hevc_rlist_b15, + }; + unsigned int i, j; + + /* List 0 contains: short term before, short term after and long term */ + j = 0; + for (i = 0; i < decode_params->num_poc_st_curr_before && j < ARRAY_SIZE(list0); i++) + list0[j++] = find_ref_pic_index(dpb, decode_params->poc_st_curr_before[i]); + for (i = 0; i < decode_params->num_poc_st_curr_after && j < ARRAY_SIZE(list0); i++) + list0[j++] = find_ref_pic_index(dpb, decode_params->poc_st_curr_after[i]); + for (i = 0; i < decode_params->num_poc_lt_curr && j < ARRAY_SIZE(list0); i++) + list0[j++] = find_ref_pic_index(dpb, decode_params->poc_lt_curr[i]); + + /* Fill the list, copying over and over */ + i = 0; + while (j < ARRAY_SIZE(list0)) + list0[j++] = list0[i++]; + + j = 0; + for (i = 0; i < decode_params->num_poc_st_curr_after && j < ARRAY_SIZE(list1); i++) + list1[j++] = find_ref_pic_index(dpb, decode_params->poc_st_curr_after[i]); + for (i = 0; i < decode_params->num_poc_st_curr_before && j < ARRAY_SIZE(list1); i++) + list1[j++] = find_ref_pic_index(dpb, decode_params->poc_st_curr_before[i]); + for (i = 0; i < decode_params->num_poc_lt_curr && j < ARRAY_SIZE(list1); i++) + list1[j++] = find_ref_pic_index(dpb, decode_params->poc_lt_curr[i]); + + i = 0; + while (j < ARRAY_SIZE(list1)) + list1[j++] = list1[i++]; + + for (i = 0; i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) { + hantro_reg_write(vpu, &ref_pic_regs0[i], list0[i]); + hantro_reg_write(vpu, &ref_pic_regs1[i], list1[i]); + } +} + +static int set_ref(struct hantro_ctx *ctx) +{ + const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; + const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; + const struct v4l2_ctrl_hevc_pps *pps = ctrls->pps; + const struct v4l2_ctrl_hevc_decode_params *decode_params = ctrls->decode_params; + const struct v4l2_hevc_dpb_entry *dpb = decode_params->dpb; + dma_addr_t luma_addr, chroma_addr, mv_addr = 0; + struct hantro_dev *vpu = ctx->dev; + size_t cr_offset = hantro_hevc_chroma_offset(sps); + size_t mv_offset = hantro_hevc_motion_vectors_offset(sps); + u32 max_ref_frames; + u16 dpb_longterm_e; + static const struct hantro_reg cur_poc[] = { + hevc_cur_poc_00, + hevc_cur_poc_01, + hevc_cur_poc_02, + hevc_cur_poc_03, + hevc_cur_poc_04, + hevc_cur_poc_05, + hevc_cur_poc_06, + hevc_cur_poc_07, + hevc_cur_poc_08, + hevc_cur_poc_09, + hevc_cur_poc_10, + hevc_cur_poc_11, + hevc_cur_poc_12, + hevc_cur_poc_13, + hevc_cur_poc_14, + hevc_cur_poc_15, + }; + unsigned int i; + + max_ref_frames = decode_params->num_poc_lt_curr + + decode_params->num_poc_st_curr_before + + decode_params->num_poc_st_curr_after; + /* + * Set max_ref_frames to non-zero to avoid HW hang when decoding + * badly marked I-frames. + */ + max_ref_frames = max_ref_frames ? max_ref_frames : 1; + hantro_reg_write(vpu, &g2_num_ref_frames, max_ref_frames); + hantro_reg_write(vpu, &g2_filter_over_slices, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED)); + hantro_reg_write(vpu, &g2_filter_over_tiles, + !!(pps->flags & V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED)); + + /* + * Write POC count diff from current pic. For frame decoding only compute + * pic_order_cnt[0] and ignore pic_order_cnt[1] used in field-coding. + */ + for (i = 0; i < decode_params->num_active_dpb_entries && i < ARRAY_SIZE(cur_poc); i++) { + char poc_diff = decode_params->pic_order_cnt_val - dpb[i].pic_order_cnt[0]; + + hantro_reg_write(vpu, &cur_poc[i], poc_diff); + } + + if (i < ARRAY_SIZE(cur_poc)) { + /* + * After the references, fill one entry pointing to itself, + * i.e. difference is zero. + */ + hantro_reg_write(vpu, &cur_poc[i], 0); + i++; + } + + /* Fill the rest with the current picture */ + for (; i < ARRAY_SIZE(cur_poc); i++) + hantro_reg_write(vpu, &cur_poc[i], decode_params->pic_order_cnt_val); + + set_ref_pic_list(ctx); + + /* We will only keep the references picture that are still used */ + ctx->hevc_dec.ref_bufs_used = 0; + + /* Set up addresses of DPB buffers */ + dpb_longterm_e = 0; + for (i = 0; i < decode_params->num_active_dpb_entries && + i < (V4L2_HEVC_DPB_ENTRIES_NUM_MAX - 1); i++) { + luma_addr = hantro_hevc_get_ref_buf(ctx, dpb[i].pic_order_cnt[0]); + if (!luma_addr) + return -ENOMEM; + + chroma_addr = luma_addr + cr_offset; + mv_addr = luma_addr + mv_offset; + + if (dpb[i].rps == V4L2_HEVC_DPB_ENTRY_RPS_LT_CURR) + dpb_longterm_e |= BIT(V4L2_HEVC_DPB_ENTRIES_NUM_MAX - 1 - i); + + hantro_write_addr(vpu, G2_REG_ADDR_REF(i), luma_addr); + hantro_write_addr(vpu, G2_REG_CHR_REF(i), chroma_addr); + hantro_write_addr(vpu, G2_REG_DMV_REF(i), mv_addr); + } + + luma_addr = hantro_hevc_get_ref_buf(ctx, decode_params->pic_order_cnt_val); + if (!luma_addr) + return -ENOMEM; + + chroma_addr = luma_addr + cr_offset; + mv_addr = luma_addr + mv_offset; + + hantro_write_addr(vpu, G2_REG_ADDR_REF(i), luma_addr); + hantro_write_addr(vpu, G2_REG_CHR_REF(i), chroma_addr); + hantro_write_addr(vpu, G2_REG_DMV_REF(i++), mv_addr); + + hantro_write_addr(vpu, G2_ADDR_DST, luma_addr); + hantro_write_addr(vpu, G2_ADDR_DST_CHR, chroma_addr); + hantro_write_addr(vpu, G2_ADDR_DST_MV, mv_addr); + + hantro_hevc_ref_remove_unused(ctx); + + for (; i < V4L2_HEVC_DPB_ENTRIES_NUM_MAX; i++) { + hantro_write_addr(vpu, G2_REG_ADDR_REF(i), 0); + hantro_write_addr(vpu, G2_REG_CHR_REF(i), 0); + hantro_write_addr(vpu, G2_REG_DMV_REF(i), 0); + } + + hantro_reg_write(vpu, &g2_refer_lterm_e, dpb_longterm_e); + + return 0; +} + +static void set_buffers(struct hantro_ctx *ctx) +{ + struct vb2_v4l2_buffer *src_buf, *dst_buf; + struct hantro_dev *vpu = ctx->dev; + const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; + const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; + size_t cr_offset = hantro_hevc_chroma_offset(sps); + dma_addr_t src_dma, dst_dma; + u32 src_len, src_buf_len; + + src_buf = hantro_get_src_buf(ctx); + dst_buf = hantro_get_dst_buf(ctx); + + /* Source (stream) buffer. */ + src_dma = vb2_dma_contig_plane_dma_addr(&src_buf->vb2_buf, 0); + src_len = vb2_get_plane_payload(&src_buf->vb2_buf, 0); + src_buf_len = vb2_plane_size(&src_buf->vb2_buf, 0); + + hantro_write_addr(vpu, G2_ADDR_STR, src_dma); + hantro_reg_write(vpu, &g2_stream_len, src_len); + hantro_reg_write(vpu, &g2_strm_buffer_len, src_buf_len); + hantro_reg_write(vpu, &g2_strm_start_offset, 0); + hantro_reg_write(vpu, &g2_write_mvs_e, 1); + + /* Destination (decoded frame) buffer. */ + dst_dma = hantro_get_dec_buf_addr(ctx, &dst_buf->vb2_buf); + + hantro_write_addr(vpu, G2_RASTER_SCAN, dst_dma); + hantro_write_addr(vpu, G2_RASTER_SCAN_CHR, dst_dma + cr_offset); + hantro_write_addr(vpu, G2_ADDR_TILE_SIZE, ctx->hevc_dec.tile_sizes.dma); + hantro_write_addr(vpu, G2_TILE_FILTER, ctx->hevc_dec.tile_filter.dma); + hantro_write_addr(vpu, G2_TILE_SAO, ctx->hevc_dec.tile_sao.dma); + hantro_write_addr(vpu, G2_TILE_BSD, ctx->hevc_dec.tile_bsd.dma); +} + +static void hantro_g2_check_idle(struct hantro_dev *vpu) +{ + int i; + + for (i = 0; i < 3; i++) { + u32 status; + + /* Make sure the VPU is idle */ + status = vdpu_read(vpu, G2_REG_INTERRUPT); + if (status & G2_REG_INTERRUPT_DEC_E) { + dev_warn(vpu->dev, "device still running, aborting"); + status |= G2_REG_INTERRUPT_DEC_ABORT_E | G2_REG_INTERRUPT_DEC_IRQ_DIS; + vdpu_write(vpu, status, G2_REG_INTERRUPT); + } + } +} + +int hantro_g2_hevc_dec_run(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + int ret; + + hantro_g2_check_idle(vpu); + + /* Prepare HEVC decoder context. */ + ret = hantro_hevc_dec_prepare_run(ctx); + if (ret) + return ret; + + /* Configure hardware registers. */ + set_params(ctx); + + /* set reference pictures */ + ret = set_ref(ctx); + if (ret) + return ret; + + set_buffers(ctx); + prepare_tile_info_buffer(ctx); + + hantro_end_prepare_run(ctx); + + hantro_reg_write(vpu, &g2_mode, HEVC_DEC_MODE); + hantro_reg_write(vpu, &g2_clk_gate_e, 1); + + /* Don't disable output */ + hantro_reg_write(vpu, &g2_out_dis, 0); + + /* Don't compress buffers */ + hantro_reg_write(vpu, &g2_ref_compress_bypass, 1); + + /* use NV12 as output format */ + hantro_reg_write(vpu, &g2_out_rs_e, 1); + + /* Bus width and max burst */ + hantro_reg_write(vpu, &g2_buswidth, BUS_WIDTH_128); + hantro_reg_write(vpu, &g2_max_burst, 16); + + /* Swap */ + hantro_reg_write(vpu, &g2_strm_swap, 0xf); + hantro_reg_write(vpu, &g2_dirmv_swap, 0xf); + hantro_reg_write(vpu, &g2_compress_swap, 0xf); + + /* Start decoding! */ + vdpu_write(vpu, G2_REG_INTERRUPT_DEC_E, G2_REG_INTERRUPT); + + return 0; +} diff --git a/drivers/staging/media/hantro/hantro_g2_regs.h b/drivers/staging/media/hantro/hantro_g2_regs.h new file mode 100644 index 000000000000..bb22fa921914 --- /dev/null +++ b/drivers/staging/media/hantro/hantro_g2_regs.h @@ -0,0 +1,198 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2021, Collabora + * + * Author: Benjamin Gaignard + */ + +#ifndef HANTRO_G2_REGS_H_ +#define HANTRO_G2_REGS_H_ + +#include "hantro.h" + +#define G2_SWREG(nr) ((nr) * 4) + +#define G2_DEC_REG(b, s, m) \ + ((const struct hantro_reg) { \ + .base = G2_SWREG(b), \ + .shift = s, \ + .mask = m, \ + }) + +#define G2_REG_VERSION G2_SWREG(0) + +#define G2_REG_INTERRUPT G2_SWREG(1) +#define G2_REG_INTERRUPT_DEC_RDY_INT BIT(12) +#define G2_REG_INTERRUPT_DEC_ABORT_E BIT(5) +#define G2_REG_INTERRUPT_DEC_IRQ_DIS BIT(4) +#define G2_REG_INTERRUPT_DEC_E BIT(0) + +#define g2_strm_swap G2_DEC_REG(2, 28, 0xf) +#define g2_dirmv_swap G2_DEC_REG(2, 20, 0xf) + +#define g2_mode G2_DEC_REG(3, 27, 0x1f) +#define g2_compress_swap G2_DEC_REG(3, 20, 0xf) +#define g2_ref_compress_bypass G2_DEC_REG(3, 17, 0x1) +#define g2_out_rs_e G2_DEC_REG(3, 16, 0x1) +#define g2_out_dis G2_DEC_REG(3, 15, 0x1) +#define g2_out_filtering_dis G2_DEC_REG(3, 14, 0x1) +#define g2_write_mvs_e G2_DEC_REG(3, 12, 0x1) + +#define g2_pic_width_in_cbs G2_DEC_REG(4, 19, 0x1fff) +#define g2_pic_height_in_cbs G2_DEC_REG(4, 6, 0x1fff) +#define g2_num_ref_frames G2_DEC_REG(4, 0, 0x1f) + +#define g2_scaling_list_e G2_DEC_REG(5, 24, 0x1) +#define g2_cb_qp_offset G2_DEC_REG(5, 19, 0x1f) +#define g2_cr_qp_offset G2_DEC_REG(5, 14, 0x1f) +#define g2_sign_data_hide G2_DEC_REG(5, 12, 0x1) +#define g2_tempor_mvp_e G2_DEC_REG(5, 11, 0x1) +#define g2_max_cu_qpd_depth G2_DEC_REG(5, 5, 0x3f) +#define g2_cu_qpd_e G2_DEC_REG(5, 4, 0x1) + +#define g2_stream_len G2_DEC_REG(6, 0, 0xffffffff) + +#define g2_cabac_init_present G2_DEC_REG(7, 31, 0x1) +#define g2_weight_pred_e G2_DEC_REG(7, 28, 0x1) +#define g2_weight_bipr_idc G2_DEC_REG(7, 26, 0x3) +#define g2_filter_over_slices G2_DEC_REG(7, 25, 0x1) +#define g2_filter_over_tiles G2_DEC_REG(7, 24, 0x1) +#define g2_asym_pred_e G2_DEC_REG(7, 23, 0x1) +#define g2_sao_e G2_DEC_REG(7, 22, 0x1) +#define g2_pcm_filt_d G2_DEC_REG(7, 21, 0x1) +#define g2_slice_chqp_present G2_DEC_REG(7, 20, 0x1) +#define g2_dependent_slice G2_DEC_REG(7, 19, 0x1) +#define g2_filter_override G2_DEC_REG(7, 18, 0x1) +#define g2_strong_smooth_e G2_DEC_REG(7, 17, 0x1) +#define g2_filt_offset_beta G2_DEC_REG(7, 12, 0x1f) +#define g2_filt_offset_tc G2_DEC_REG(7, 7, 0x1f) +#define g2_slice_hdr_ext_e G2_DEC_REG(7, 6, 0x1) +#define g2_slice_hdr_ext_bits G2_DEC_REG(7, 3, 0x7) + +#define g2_const_intra_e G2_DEC_REG(8, 31, 0x1) +#define g2_filt_ctrl_pres G2_DEC_REG(8, 30, 0x1) +#define g2_idr_pic_e G2_DEC_REG(8, 16, 0x1) +#define g2_bit_depth_pcm_y G2_DEC_REG(8, 12, 0xf) +#define g2_bit_depth_pcm_c G2_DEC_REG(8, 8, 0xf) +#define g2_bit_depth_y_minus8 G2_DEC_REG(8, 6, 0x3) +#define g2_bit_depth_c_minus8 G2_DEC_REG(8, 4, 0x3) +#define g2_output_8_bits G2_DEC_REG(8, 3, 0x1) + +#define g2_refidx1_active G2_DEC_REG(9, 19, 0x1f) +#define g2_refidx0_active G2_DEC_REG(9, 14, 0x1f) +#define g2_hdr_skip_length G2_DEC_REG(9, 0, 0x3fff) + +#define g2_start_code_e G2_DEC_REG(10, 31, 0x1) +#define g2_init_qp G2_DEC_REG(10, 24, 0x3f) +#define g2_num_tile_cols G2_DEC_REG(10, 19, 0x1f) +#define g2_num_tile_rows G2_DEC_REG(10, 14, 0x1f) +#define g2_tile_e G2_DEC_REG(10, 1, 0x1) +#define g2_entropy_sync_e G2_DEC_REG(10, 0, 0x1) + +#define g2_refer_lterm_e G2_DEC_REG(12, 16, 0xffff) +#define g2_min_cb_size G2_DEC_REG(12, 13, 0x7) +#define g2_max_cb_size G2_DEC_REG(12, 10, 0x7) +#define g2_min_pcm_size G2_DEC_REG(12, 7, 0x7) +#define g2_max_pcm_size G2_DEC_REG(12, 4, 0x7) +#define g2_pcm_e G2_DEC_REG(12, 3, 0x1) +#define g2_transform_skip G2_DEC_REG(12, 2, 0x1) +#define g2_transq_bypass G2_DEC_REG(12, 1, 0x1) +#define g2_list_mod_e G2_DEC_REG(12, 0, 0x1) + +#define hevc_min_trb_size G2_DEC_REG(13, 13, 0x7) +#define hevc_max_trb_size G2_DEC_REG(13, 10, 0x7) +#define hevc_max_intra_hierdepth G2_DEC_REG(13, 7, 0x7) +#define hevc_max_inter_hierdepth G2_DEC_REG(13, 4, 0x7) +#define hevc_parallel_merge G2_DEC_REG(13, 0, 0xf) + +#define hevc_rlist_f0 G2_DEC_REG(14, 0, 0x1f) +#define hevc_rlist_f1 G2_DEC_REG(14, 10, 0x1f) +#define hevc_rlist_f2 G2_DEC_REG(14, 20, 0x1f) +#define hevc_rlist_b0 G2_DEC_REG(14, 5, 0x1f) +#define hevc_rlist_b1 G2_DEC_REG(14, 15, 0x1f) +#define hevc_rlist_b2 G2_DEC_REG(14, 25, 0x1f) + +#define hevc_rlist_f3 G2_DEC_REG(15, 0, 0x1f) +#define hevc_rlist_f4 G2_DEC_REG(15, 10, 0x1f) +#define hevc_rlist_f5 G2_DEC_REG(15, 20, 0x1f) +#define hevc_rlist_b3 G2_DEC_REG(15, 5, 0x1f) +#define hevc_rlist_b4 G2_DEC_REG(15, 15, 0x1f) +#define hevc_rlist_b5 G2_DEC_REG(15, 25, 0x1f) + +#define hevc_rlist_f6 G2_DEC_REG(16, 0, 0x1f) +#define hevc_rlist_f7 G2_DEC_REG(16, 10, 0x1f) +#define hevc_rlist_f8 G2_DEC_REG(16, 20, 0x1f) +#define hevc_rlist_b6 G2_DEC_REG(16, 5, 0x1f) +#define hevc_rlist_b7 G2_DEC_REG(16, 15, 0x1f) +#define hevc_rlist_b8 G2_DEC_REG(16, 25, 0x1f) + +#define hevc_rlist_f9 G2_DEC_REG(17, 0, 0x1f) +#define hevc_rlist_f10 G2_DEC_REG(17, 10, 0x1f) +#define hevc_rlist_f11 G2_DEC_REG(17, 20, 0x1f) +#define hevc_rlist_b9 G2_DEC_REG(17, 5, 0x1f) +#define hevc_rlist_b10 G2_DEC_REG(17, 15, 0x1f) +#define hevc_rlist_b11 G2_DEC_REG(17, 25, 0x1f) + +#define hevc_rlist_f12 G2_DEC_REG(18, 0, 0x1f) +#define hevc_rlist_f13 G2_DEC_REG(18, 10, 0x1f) +#define hevc_rlist_f14 G2_DEC_REG(18, 20, 0x1f) +#define hevc_rlist_b12 G2_DEC_REG(18, 5, 0x1f) +#define hevc_rlist_b13 G2_DEC_REG(18, 15, 0x1f) +#define hevc_rlist_b14 G2_DEC_REG(18, 25, 0x1f) + +#define hevc_rlist_f15 G2_DEC_REG(19, 0, 0x1f) +#define hevc_rlist_b15 G2_DEC_REG(19, 5, 0x1f) + +#define g2_partial_ctb_x G2_DEC_REG(20, 31, 0x1) +#define g2_partial_ctb_y G2_DEC_REG(20, 30, 0x1) +#define g2_pic_width_4x4 G2_DEC_REG(20, 16, 0xfff) +#define g2_pic_height_4x4 G2_DEC_REG(20, 0, 0xfff) +#define hevc_cur_poc_00 G2_DEC_REG(46, 24, 0xff) +#define hevc_cur_poc_01 G2_DEC_REG(46, 16, 0xff) +#define hevc_cur_poc_02 G2_DEC_REG(46, 8, 0xff) +#define hevc_cur_poc_03 G2_DEC_REG(46, 0, 0xff) + +#define hevc_cur_poc_04 G2_DEC_REG(47, 24, 0xff) +#define hevc_cur_poc_05 G2_DEC_REG(47, 16, 0xff) +#define hevc_cur_poc_06 G2_DEC_REG(47, 8, 0xff) +#define hevc_cur_poc_07 G2_DEC_REG(47, 0, 0xff) + +#define hevc_cur_poc_08 G2_DEC_REG(48, 24, 0xff) +#define hevc_cur_poc_09 G2_DEC_REG(48, 16, 0xff) +#define hevc_cur_poc_10 G2_DEC_REG(48, 8, 0xff) +#define hevc_cur_poc_11 G2_DEC_REG(48, 0, 0xff) + +#define hevc_cur_poc_12 G2_DEC_REG(49, 24, 0xff) +#define hevc_cur_poc_13 G2_DEC_REG(49, 16, 0xff) +#define hevc_cur_poc_14 G2_DEC_REG(49, 8, 0xff) +#define hevc_cur_poc_15 G2_DEC_REG(49, 0, 0xff) + +#define g2_apf_threshold G2_DEC_REG(55, 0, 0xffff) + +#define g2_clk_gate_e G2_DEC_REG(58, 16, 0x1) +#define g2_buswidth G2_DEC_REG(58, 8, 0x7) +#define g2_max_burst G2_DEC_REG(58, 0, 0xff) + +#define G2_REG_CONFIG G2_SWREG(58) +#define G2_REG_CONFIG_DEC_CLK_GATE_E BIT(16) +#define G2_REG_CONFIG_DEC_CLK_GATE_IDLE_E BIT(17) + +#define G2_ADDR_DST (G2_SWREG(65)) +#define G2_REG_ADDR_REF(i) (G2_SWREG(67) + ((i) * 0x8)) +#define G2_ADDR_DST_CHR (G2_SWREG(99)) +#define G2_REG_CHR_REF(i) (G2_SWREG(101) + ((i) * 0x8)) +#define G2_ADDR_DST_MV (G2_SWREG(133)) +#define G2_REG_DMV_REF(i) (G2_SWREG(135) + ((i) * 0x8)) +#define G2_ADDR_TILE_SIZE (G2_SWREG(167)) +#define G2_ADDR_STR (G2_SWREG(169)) +#define HEVC_SCALING_LIST (G2_SWREG(171)) +#define G2_RASTER_SCAN (G2_SWREG(175)) +#define G2_RASTER_SCAN_CHR (G2_SWREG(177)) +#define G2_TILE_FILTER (G2_SWREG(179)) +#define G2_TILE_SAO (G2_SWREG(181)) +#define G2_TILE_BSD (G2_SWREG(183)) + +#define g2_strm_buffer_len G2_DEC_REG(258, 0, 0xffffffff) +#define g2_strm_start_offset G2_DEC_REG(259, 0, 0xffffffff) + +#endif diff --git a/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c b/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c index b88dc4ed06db..56cf261a8e95 100644 --- a/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c +++ b/drivers/staging/media/hantro/hantro_h1_jpeg_enc.c @@ -88,7 +88,7 @@ hantro_h1_jpeg_enc_set_qtable(struct hantro_dev *vpu, } } -void hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx) +int hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; struct vb2_v4l2_buffer *src_buf, *dst_buf; @@ -136,6 +136,8 @@ void hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx) hantro_end_prepare_run(ctx); vepu_write(vpu, reg, H1_REG_ENC_CTRL); + + return 0; } void hantro_jpeg_enc_done(struct hantro_ctx *ctx) diff --git a/drivers/staging/media/hantro/hantro_hevc.c b/drivers/staging/media/hantro/hantro_hevc.c new file mode 100644 index 000000000000..5347f5a41c2a --- /dev/null +++ b/drivers/staging/media/hantro/hantro_hevc.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hantro VPU HEVC codec driver + * + * Copyright (C) 2020 Safran Passenger Innovations LLC + */ + +#include +#include + +#include "hantro.h" +#include "hantro_hw.h" + +#define VERT_FILTER_RAM_SIZE 8 /* bytes per pixel row */ +/* + * BSD control data of current picture at tile border + * 128 bits per 4x4 tile = 128/(8*4) bytes per row + */ +#define BSD_CTRL_RAM_SIZE 4 /* bytes per pixel row */ +/* tile border coefficients of filter */ +#define VERT_SAO_RAM_SIZE 48 /* bytes per pixel */ + +#define MAX_TILE_COLS 20 +#define MAX_TILE_ROWS 22 + +#define UNUSED_REF -1 + +#define G2_ALIGN 16 + +size_t hantro_hevc_chroma_offset(const struct v4l2_ctrl_hevc_sps *sps) +{ + int bytes_per_pixel = sps->bit_depth_luma_minus8 == 0 ? 1 : 2; + + return sps->pic_width_in_luma_samples * + sps->pic_height_in_luma_samples * bytes_per_pixel; +} + +size_t hantro_hevc_motion_vectors_offset(const struct v4l2_ctrl_hevc_sps *sps) +{ + size_t cr_offset = hantro_hevc_chroma_offset(sps); + + return ALIGN((cr_offset * 3) / 2, G2_ALIGN); +} + +static size_t hantro_hevc_mv_size(const struct v4l2_ctrl_hevc_sps *sps) +{ + u32 min_cb_log2_size_y = sps->log2_min_luma_coding_block_size_minus3 + 3; + u32 ctb_log2_size_y = min_cb_log2_size_y + sps->log2_diff_max_min_luma_coding_block_size; + u32 pic_width_in_ctbs_y = (sps->pic_width_in_luma_samples + (1 << ctb_log2_size_y) - 1) + >> ctb_log2_size_y; + u32 pic_height_in_ctbs_y = (sps->pic_height_in_luma_samples + (1 << ctb_log2_size_y) - 1) + >> ctb_log2_size_y; + size_t mv_size; + + mv_size = pic_width_in_ctbs_y * pic_height_in_ctbs_y * + (1 << (2 * (ctb_log2_size_y - 4))) * 16; + + vpu_debug(4, "%dx%d (CTBs) %zu MV bytes\n", + pic_width_in_ctbs_y, pic_height_in_ctbs_y, mv_size); + + return mv_size; +} + +static size_t hantro_hevc_ref_size(struct hantro_ctx *ctx) +{ + const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; + const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; + + return hantro_hevc_motion_vectors_offset(sps) + hantro_hevc_mv_size(sps); +} + +static void hantro_hevc_ref_free(struct hantro_ctx *ctx) +{ + struct hantro_hevc_dec_hw_ctx *hevc_dec = &ctx->hevc_dec; + struct hantro_dev *vpu = ctx->dev; + int i; + + for (i = 0; i < NUM_REF_PICTURES; i++) { + if (hevc_dec->ref_bufs[i].cpu) + dma_free_coherent(vpu->dev, hevc_dec->ref_bufs[i].size, + hevc_dec->ref_bufs[i].cpu, + hevc_dec->ref_bufs[i].dma); + } +} + +static void hantro_hevc_ref_init(struct hantro_ctx *ctx) +{ + struct hantro_hevc_dec_hw_ctx *hevc_dec = &ctx->hevc_dec; + int i; + + for (i = 0; i < NUM_REF_PICTURES; i++) + hevc_dec->ref_bufs_poc[i] = UNUSED_REF; +} + +dma_addr_t hantro_hevc_get_ref_buf(struct hantro_ctx *ctx, + int poc) +{ + struct hantro_hevc_dec_hw_ctx *hevc_dec = &ctx->hevc_dec; + int i; + + /* Find the reference buffer in already know ones */ + for (i = 0; i < NUM_REF_PICTURES; i++) { + if (hevc_dec->ref_bufs_poc[i] == poc) { + hevc_dec->ref_bufs_used |= 1 << i; + return hevc_dec->ref_bufs[i].dma; + } + } + + /* Allocate a new reference buffer */ + for (i = 0; i < NUM_REF_PICTURES; i++) { + if (hevc_dec->ref_bufs_poc[i] == UNUSED_REF) { + if (!hevc_dec->ref_bufs[i].cpu) { + struct hantro_dev *vpu = ctx->dev; + + /* + * Allocate the space needed for the raw data + + * motion vector data. Optimizations could be to + * allocate raw data in non coherent memory and only + * clear the motion vector data. + */ + hevc_dec->ref_bufs[i].cpu = + dma_alloc_coherent(vpu->dev, + hantro_hevc_ref_size(ctx), + &hevc_dec->ref_bufs[i].dma, + GFP_KERNEL); + if (!hevc_dec->ref_bufs[i].cpu) + return 0; + + hevc_dec->ref_bufs[i].size = hantro_hevc_ref_size(ctx); + } + hevc_dec->ref_bufs_used |= 1 << i; + memset(hevc_dec->ref_bufs[i].cpu, 0, hantro_hevc_ref_size(ctx)); + hevc_dec->ref_bufs_poc[i] = poc; + + return hevc_dec->ref_bufs[i].dma; + } + } + + return 0; +} + +void hantro_hevc_ref_remove_unused(struct hantro_ctx *ctx) +{ + struct hantro_hevc_dec_hw_ctx *hevc_dec = &ctx->hevc_dec; + int i; + + /* Just tag buffer as unused, do not free them */ + for (i = 0; i < NUM_REF_PICTURES; i++) { + if (hevc_dec->ref_bufs_poc[i] == UNUSED_REF) + continue; + + if (hevc_dec->ref_bufs_used & (1 << i)) + continue; + + hevc_dec->ref_bufs_poc[i] = UNUSED_REF; + } +} + +static int tile_buffer_reallocate(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + struct hantro_hevc_dec_hw_ctx *hevc_dec = &ctx->hevc_dec; + const struct hantro_hevc_dec_ctrls *ctrls = &ctx->hevc_dec.ctrls; + const struct v4l2_ctrl_hevc_pps *pps = ctrls->pps; + const struct v4l2_ctrl_hevc_sps *sps = ctrls->sps; + unsigned int num_tile_cols = pps->num_tile_columns_minus1 + 1; + unsigned int height64 = (sps->pic_height_in_luma_samples + 63) & ~63; + unsigned int size; + + if (num_tile_cols <= 1 || + num_tile_cols <= hevc_dec->num_tile_cols_allocated) + return 0; + + /* Need to reallocate due to tiles passed via PPS */ + if (hevc_dec->tile_filter.cpu) { + dma_free_coherent(vpu->dev, hevc_dec->tile_filter.size, + hevc_dec->tile_filter.cpu, + hevc_dec->tile_filter.dma); + hevc_dec->tile_filter.cpu = NULL; + } + + if (hevc_dec->tile_sao.cpu) { + dma_free_coherent(vpu->dev, hevc_dec->tile_sao.size, + hevc_dec->tile_sao.cpu, + hevc_dec->tile_sao.dma); + hevc_dec->tile_sao.cpu = NULL; + } + + if (hevc_dec->tile_bsd.cpu) { + dma_free_coherent(vpu->dev, hevc_dec->tile_bsd.size, + hevc_dec->tile_bsd.cpu, + hevc_dec->tile_bsd.dma); + hevc_dec->tile_bsd.cpu = NULL; + } + + size = VERT_FILTER_RAM_SIZE * height64 * (num_tile_cols - 1); + hevc_dec->tile_filter.cpu = dma_alloc_coherent(vpu->dev, size, + &hevc_dec->tile_filter.dma, + GFP_KERNEL); + if (!hevc_dec->tile_filter.cpu) + goto err_free_tile_buffers; + hevc_dec->tile_filter.size = size; + + size = VERT_SAO_RAM_SIZE * height64 * (num_tile_cols - 1); + hevc_dec->tile_sao.cpu = dma_alloc_coherent(vpu->dev, size, + &hevc_dec->tile_sao.dma, + GFP_KERNEL); + if (!hevc_dec->tile_sao.cpu) + goto err_free_tile_buffers; + hevc_dec->tile_sao.size = size; + + size = BSD_CTRL_RAM_SIZE * height64 * (num_tile_cols - 1); + hevc_dec->tile_bsd.cpu = dma_alloc_coherent(vpu->dev, size, + &hevc_dec->tile_bsd.dma, + GFP_KERNEL); + if (!hevc_dec->tile_bsd.cpu) + goto err_free_tile_buffers; + hevc_dec->tile_bsd.size = size; + + hevc_dec->num_tile_cols_allocated = num_tile_cols; + + return 0; + +err_free_tile_buffers: + if (hevc_dec->tile_filter.cpu) + dma_free_coherent(vpu->dev, hevc_dec->tile_filter.size, + hevc_dec->tile_filter.cpu, + hevc_dec->tile_filter.dma); + hevc_dec->tile_filter.cpu = NULL; + + if (hevc_dec->tile_sao.cpu) + dma_free_coherent(vpu->dev, hevc_dec->tile_sao.size, + hevc_dec->tile_sao.cpu, + hevc_dec->tile_sao.dma); + hevc_dec->tile_sao.cpu = NULL; + + if (hevc_dec->tile_bsd.cpu) + dma_free_coherent(vpu->dev, hevc_dec->tile_bsd.size, + hevc_dec->tile_bsd.cpu, + hevc_dec->tile_bsd.dma); + hevc_dec->tile_bsd.cpu = NULL; + + return -ENOMEM; +} + +int hantro_hevc_dec_prepare_run(struct hantro_ctx *ctx) +{ + struct hantro_hevc_dec_hw_ctx *hevc_ctx = &ctx->hevc_dec; + struct hantro_hevc_dec_ctrls *ctrls = &hevc_ctx->ctrls; + int ret; + + hantro_start_prepare_run(ctx); + + ctrls->decode_params = + hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS); + if (WARN_ON(!ctrls->decode_params)) + return -EINVAL; + + ctrls->sps = + hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_HEVC_SPS); + if (WARN_ON(!ctrls->sps)) + return -EINVAL; + + ctrls->pps = + hantro_get_ctrl(ctx, V4L2_CID_MPEG_VIDEO_HEVC_PPS); + if (WARN_ON(!ctrls->pps)) + return -EINVAL; + + ret = tile_buffer_reallocate(ctx); + if (ret) + return ret; + + return 0; +} + +void hantro_hevc_dec_exit(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + struct hantro_hevc_dec_hw_ctx *hevc_dec = &ctx->hevc_dec; + + if (hevc_dec->tile_sizes.cpu) + dma_free_coherent(vpu->dev, hevc_dec->tile_sizes.size, + hevc_dec->tile_sizes.cpu, + hevc_dec->tile_sizes.dma); + hevc_dec->tile_sizes.cpu = NULL; + + if (hevc_dec->tile_filter.cpu) + dma_free_coherent(vpu->dev, hevc_dec->tile_filter.size, + hevc_dec->tile_filter.cpu, + hevc_dec->tile_filter.dma); + hevc_dec->tile_filter.cpu = NULL; + + if (hevc_dec->tile_sao.cpu) + dma_free_coherent(vpu->dev, hevc_dec->tile_sao.size, + hevc_dec->tile_sao.cpu, + hevc_dec->tile_sao.dma); + hevc_dec->tile_sao.cpu = NULL; + + if (hevc_dec->tile_bsd.cpu) + dma_free_coherent(vpu->dev, hevc_dec->tile_bsd.size, + hevc_dec->tile_bsd.cpu, + hevc_dec->tile_bsd.dma); + hevc_dec->tile_bsd.cpu = NULL; + + hantro_hevc_ref_free(ctx); +} + +int hantro_hevc_dec_init(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + struct hantro_hevc_dec_hw_ctx *hevc_dec = &ctx->hevc_dec; + unsigned int size; + + memset(hevc_dec, 0, sizeof(*hevc_dec)); + + /* + * Maximum number of tiles times width and height (2 bytes each), + * rounding up to next 16 bytes boundary + one extra 16 byte + * chunk (HW guys wanted to have this). + */ + size = round_up(MAX_TILE_COLS * MAX_TILE_ROWS * 4 * sizeof(u16) + 16, 16); + hevc_dec->tile_sizes.cpu = dma_alloc_coherent(vpu->dev, size, + &hevc_dec->tile_sizes.dma, + GFP_KERNEL); + if (!hevc_dec->tile_sizes.cpu) + return -ENOMEM; + + hevc_dec->tile_sizes.size = size; + + hantro_hevc_ref_init(ctx); + + return 0; +} diff --git a/drivers/staging/media/hantro/hantro_hw.h b/drivers/staging/media/hantro/hantro_hw.h index 83b3e42b63a3..5dcf65805396 100644 --- a/drivers/staging/media/hantro/hantro_hw.h +++ b/drivers/staging/media/hantro/hantro_hw.h @@ -20,6 +20,8 @@ #define MB_WIDTH(w) DIV_ROUND_UP(w, MB_DIM) #define MB_HEIGHT(h) DIV_ROUND_UP(h, MB_DIM) +#define NUM_REF_PICTURES (V4L2_HEVC_DPB_ENTRIES_NUM_MAX + 1) + struct hantro_dev; struct hantro_ctx; struct hantro_buf; @@ -95,6 +97,46 @@ struct hantro_h264_dec_hw_ctx { struct hantro_h264_dec_ctrls ctrls; }; +/** + * struct hantro_hevc_dec_ctrls + * @decode_params: Decode params + * @sps: SPS info + * @pps: PPS info + * @hevc_hdr_skip_length: the number of data (in bits) to skip in the + * slice segment header syntax after 'slice type' + * token + */ +struct hantro_hevc_dec_ctrls { + const struct v4l2_ctrl_hevc_decode_params *decode_params; + const struct v4l2_ctrl_hevc_sps *sps; + const struct v4l2_ctrl_hevc_pps *pps; + u32 hevc_hdr_skip_length; +}; + +/** + * struct hantro_hevc_dec_hw_ctx + * @tile_sizes: Tile sizes buffer + * @tile_filter: Tile vertical filter buffer + * @tile_sao: Tile SAO buffer + * @tile_bsd: Tile BSD control buffer + * @ref_bufs: Internal reference buffers + * @ref_bufs_poc: Internal reference buffers picture order count + * @ref_bufs_used: Bitfield of used reference buffers + * @ctrls: V4L2 controls attached to a run + * @num_tile_cols_allocated: number of allocated tiles + */ +struct hantro_hevc_dec_hw_ctx { + struct hantro_aux_buf tile_sizes; + struct hantro_aux_buf tile_filter; + struct hantro_aux_buf tile_sao; + struct hantro_aux_buf tile_bsd; + struct hantro_aux_buf ref_bufs[NUM_REF_PICTURES]; + int ref_bufs_poc[NUM_REF_PICTURES]; + u32 ref_bufs_used; + struct hantro_hevc_dec_ctrls ctrls; + unsigned int num_tile_cols_allocated; +}; + /** * struct hantro_mpeg2_dec_hw_ctx * @@ -133,14 +175,15 @@ struct hantro_postproc_ctx { * Optional and called from process context. * @run: Start single {en,de)coding job. Called from atomic context * to indicate that a pair of buffers is ready and the hardware - * should be programmed and started. + * should be programmed and started. Returns zero if OK, a + * negative value in error cases. * @done: Read back processing results and additional data from hardware. * @reset: Reset the hardware in case of a timeout. */ struct hantro_codec_ops { int (*init)(struct hantro_ctx *ctx); void (*exit)(struct hantro_ctx *ctx); - void (*run)(struct hantro_ctx *ctx); + int (*run)(struct hantro_ctx *ctx); void (*done)(struct hantro_ctx *ctx); void (*reset)(struct hantro_ctx *ctx); }; @@ -148,22 +191,26 @@ struct hantro_codec_ops { /** * enum hantro_enc_fmt - source format ID for hardware registers. * - * @RK3288_VPU_ENC_FMT_YUV420P: Y/CbCr 4:2:0 planar format - * @RK3288_VPU_ENC_FMT_YUV420SP: Y/CbCr 4:2:0 semi-planar format - * @RK3288_VPU_ENC_FMT_YUYV422: YUV 4:2:2 packed format (YUYV) - * @RK3288_VPU_ENC_FMT_UYVY422: YUV 4:2:2 packed format (UYVY) + * @ROCKCHIP_VPU_ENC_FMT_YUV420P: Y/CbCr 4:2:0 planar format + * @ROCKCHIP_VPU_ENC_FMT_YUV420SP: Y/CbCr 4:2:0 semi-planar format + * @ROCKCHIP_VPU_ENC_FMT_YUYV422: YUV 4:2:2 packed format (YUYV) + * @ROCKCHIP_VPU_ENC_FMT_UYVY422: YUV 4:2:2 packed format (UYVY) */ enum hantro_enc_fmt { - RK3288_VPU_ENC_FMT_YUV420P = 0, - RK3288_VPU_ENC_FMT_YUV420SP = 1, - RK3288_VPU_ENC_FMT_YUYV422 = 2, - RK3288_VPU_ENC_FMT_UYVY422 = 3, + ROCKCHIP_VPU_ENC_FMT_YUV420P = 0, + ROCKCHIP_VPU_ENC_FMT_YUV420SP = 1, + ROCKCHIP_VPU_ENC_FMT_YUYV422 = 2, + ROCKCHIP_VPU_ENC_FMT_UYVY422 = 3, }; -extern const struct hantro_variant rk3399_vpu_variant; -extern const struct hantro_variant rk3328_vpu_variant; -extern const struct hantro_variant rk3288_vpu_variant; +extern const struct hantro_variant imx8mq_vpu_g2_variant; extern const struct hantro_variant imx8mq_vpu_variant; +extern const struct hantro_variant rk3036_vpu_variant; +extern const struct hantro_variant rk3066_vpu_variant; +extern const struct hantro_variant rk3288_vpu_variant; +extern const struct hantro_variant rk3328_vpu_variant; +extern const struct hantro_variant rk3399_vpu_variant; +extern const struct hantro_variant sama5d4_vdec_variant; extern const struct hantro_postproc_regs hantro_g1_postproc_regs; @@ -176,8 +223,11 @@ void hantro_irq_done(struct hantro_dev *vpu, void hantro_start_prepare_run(struct hantro_ctx *ctx); void hantro_end_prepare_run(struct hantro_ctx *ctx); -void hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx); -void rk3399_vpu_jpeg_enc_run(struct hantro_ctx *ctx); +irqreturn_t hantro_g1_irq(int irq, void *dev_id); +void hantro_g1_reset(struct hantro_ctx *ctx); + +int hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx); +int rockchip_vpu2_jpeg_enc_run(struct hantro_ctx *ctx); int hantro_jpeg_enc_init(struct hantro_ctx *ctx); void hantro_jpeg_enc_exit(struct hantro_ctx *ctx); void hantro_jpeg_enc_done(struct hantro_ctx *ctx); @@ -185,10 +235,19 @@ void hantro_jpeg_enc_done(struct hantro_ctx *ctx); dma_addr_t hantro_h264_get_ref_buf(struct hantro_ctx *ctx, unsigned int dpb_idx); int hantro_h264_dec_prepare_run(struct hantro_ctx *ctx); -void hantro_g1_h264_dec_run(struct hantro_ctx *ctx); +int hantro_g1_h264_dec_run(struct hantro_ctx *ctx); int hantro_h264_dec_init(struct hantro_ctx *ctx); void hantro_h264_dec_exit(struct hantro_ctx *ctx); +int hantro_hevc_dec_init(struct hantro_ctx *ctx); +void hantro_hevc_dec_exit(struct hantro_ctx *ctx); +int hantro_g2_hevc_dec_run(struct hantro_ctx *ctx); +int hantro_hevc_dec_prepare_run(struct hantro_ctx *ctx); +dma_addr_t hantro_hevc_get_ref_buf(struct hantro_ctx *ctx, int poc); +void hantro_hevc_ref_remove_unused(struct hantro_ctx *ctx); +size_t hantro_hevc_chroma_offset(const struct v4l2_ctrl_hevc_sps *sps); +size_t hantro_hevc_motion_vectors_offset(const struct v4l2_ctrl_hevc_sps *sps); + static inline size_t hantro_h264_mv_size(unsigned int width, unsigned int height) { @@ -216,15 +275,15 @@ hantro_h264_mv_size(unsigned int width, unsigned int height) return 64 * MB_WIDTH(width) * MB_WIDTH(height) + 32; } -void hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx); -void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx); +int hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx); +int rockchip_vpu2_mpeg2_dec_run(struct hantro_ctx *ctx); void hantro_mpeg2_dec_copy_qtable(u8 *qtable, - const struct v4l2_ctrl_mpeg2_quantization *ctrl); + const struct v4l2_ctrl_mpeg2_quantisation *ctrl); int hantro_mpeg2_dec_init(struct hantro_ctx *ctx); void hantro_mpeg2_dec_exit(struct hantro_ctx *ctx); -void hantro_g1_vp8_dec_run(struct hantro_ctx *ctx); -void rk3399_vpu_vp8_dec_run(struct hantro_ctx *ctx); +int hantro_g1_vp8_dec_run(struct hantro_ctx *ctx); +int rockchip_vpu2_vp8_dec_run(struct hantro_ctx *ctx); int hantro_vp8_dec_init(struct hantro_ctx *ctx); void hantro_vp8_dec_exit(struct hantro_ctx *ctx); void hantro_vp8_prob_update(struct hantro_ctx *ctx, diff --git a/drivers/staging/media/hantro/hantro_mpeg2.c b/drivers/staging/media/hantro/hantro_mpeg2.c index 1d334e6fcd06..04e545eb0a83 100644 --- a/drivers/staging/media/hantro/hantro_mpeg2.c +++ b/drivers/staging/media/hantro/hantro_mpeg2.c @@ -19,7 +19,7 @@ static const u8 zigzag[64] = { }; void hantro_mpeg2_dec_copy_qtable(u8 *qtable, - const struct v4l2_ctrl_mpeg2_quantization *ctrl) + const struct v4l2_ctrl_mpeg2_quantisation *ctrl) { int i, n; diff --git a/drivers/staging/media/hantro/hantro_postproc.c b/drivers/staging/media/hantro/hantro_postproc.c index 6d2a8f2a8f0b..ed8916c950a4 100644 --- a/drivers/staging/media/hantro/hantro_postproc.c +++ b/drivers/staging/media/hantro/hantro_postproc.c @@ -50,6 +50,20 @@ const struct hantro_postproc_regs hantro_g1_postproc_regs = { .display_width = {G1_REG_PP_DISPLAY_WIDTH, 0, 0xfff}, }; +bool hantro_needs_postproc(const struct hantro_ctx *ctx, + const struct hantro_fmt *fmt) +{ + struct hantro_dev *vpu = ctx->dev; + + if (ctx->is_encoder) + return false; + + if (!vpu->variant->postproc_fmts) + return false; + + return fmt->fourcc != V4L2_PIX_FMT_NV12; +} + void hantro_postproc_enable(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; diff --git a/drivers/staging/media/hantro/hantro_v4l2.c b/drivers/staging/media/hantro/hantro_v4l2.c index 1bc118e375a1..bcb0bdff4a9a 100644 --- a/drivers/staging/media/hantro/hantro_v4l2.c +++ b/drivers/staging/media/hantro/hantro_v4l2.c @@ -55,7 +55,9 @@ static const struct hantro_fmt * hantro_get_postproc_formats(const struct hantro_ctx *ctx, unsigned int *num_fmts) { - if (ctx->is_encoder) { + struct hantro_dev *vpu = ctx->dev; + + if (ctx->is_encoder || !vpu->variant->postproc_fmts) { *num_fmts = 0; return NULL; } @@ -390,6 +392,7 @@ hantro_update_requires_request(struct hantro_ctx *ctx, u32 fourcc) case V4L2_PIX_FMT_MPEG2_SLICE: case V4L2_PIX_FMT_VP8_FRAME: case V4L2_PIX_FMT_H264_SLICE: + case V4L2_PIX_FMT_HEVC_SLICE: ctx->fh.m2m_ctx->out_q_ctx.q.requires_requests = true; break; default: @@ -639,7 +642,14 @@ static int hantro_buf_prepare(struct vb2_buffer *vb) ret = hantro_buf_plane_check(vb, pix_fmt); if (ret) return ret; - vb2_set_plane_payload(vb, 0, pix_fmt->plane_fmt[0].sizeimage); + /* + * Buffer's bytesused must be written by driver for CAPTURE buffers. + * (for OUTPUT buffers, if userspace passes 0 bytesused, v4l2-core sets + * it to buffer length). + */ + if (V4L2_TYPE_IS_CAPTURE(vq->type)) + vb2_set_plane_payload(vb, 0, pix_fmt->plane_fmt[0].sizeimage); + return 0; } diff --git a/drivers/staging/media/hantro/imx8m_vpu_hw.c b/drivers/staging/media/hantro/imx8m_vpu_hw.c index c222de075ef4..ea919bfb9891 100644 --- a/drivers/staging/media/hantro/imx8m_vpu_hw.c +++ b/drivers/staging/media/hantro/imx8m_vpu_hw.c @@ -11,6 +11,7 @@ #include "hantro.h" #include "hantro_jpeg.h" #include "hantro_g1_regs.h" +#include "hantro_g2_regs.h" #define CTRL_SOFT_RESET 0x00 #define RESET_G1 BIT(1) @@ -109,10 +110,10 @@ static const struct hantro_fmt imx8m_vpu_dec_fmts[] = { .frmsize = { .min_width = 48, .max_width = 3840, - .step_width = 16, + .step_width = MB_DIM, .min_height = 48, .max_height = 2160, - .step_height = 16, + .step_height = MB_DIM, }, }, { @@ -130,6 +131,26 @@ static const struct hantro_fmt imx8m_vpu_dec_fmts[] = { }, }; +static const struct hantro_fmt imx8m_vpu_g2_dec_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_NV12, + .codec_mode = HANTRO_MODE_NONE, + }, + { + .fourcc = V4L2_PIX_FMT_HEVC_SLICE, + .codec_mode = HANTRO_MODE_HEVC_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 3840, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 2160, + .step_height = MB_DIM, + }, + }, +}; + static irqreturn_t imx8m_vpu_g1_irq(int irq, void *dev_id) { struct hantro_dev *vpu = dev_id; @@ -148,9 +169,26 @@ static irqreturn_t imx8m_vpu_g1_irq(int irq, void *dev_id) return IRQ_HANDLED; } +static irqreturn_t imx8m_vpu_g2_irq(int irq, void *dev_id) +{ + struct hantro_dev *vpu = dev_id; + enum vb2_buffer_state state; + u32 status; + + status = vdpu_read(vpu, G2_REG_INTERRUPT); + state = (status & G2_REG_INTERRUPT_DEC_RDY_INT) ? + VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; + + vdpu_write(vpu, 0, G2_REG_INTERRUPT); + vdpu_write(vpu, G2_REG_CONFIG_DEC_CLK_GATE_E, G2_REG_CONFIG); + + hantro_irq_done(vpu, state); + + return IRQ_HANDLED; +} + static int imx8mq_vpu_hw_init(struct hantro_dev *vpu) { - vpu->dec_base = vpu->reg_bases[0]; vpu->ctrl_base = vpu->reg_bases[vpu->variant->num_regs - 1]; return 0; @@ -163,6 +201,13 @@ static void imx8m_vpu_g1_reset(struct hantro_ctx *ctx) imx8m_soft_reset(vpu, RESET_G1); } +static void imx8m_vpu_g2_reset(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + + imx8m_soft_reset(vpu, RESET_G2); +} + /* * Supported codec ops. */ @@ -188,13 +233,25 @@ static const struct hantro_codec_ops imx8mq_vpu_codec_ops[] = { }, }; +static const struct hantro_codec_ops imx8mq_vpu_g2_codec_ops[] = { + [HANTRO_MODE_HEVC_DEC] = { + .run = hantro_g2_hevc_dec_run, + .reset = imx8m_vpu_g2_reset, + .init = hantro_hevc_dec_init, + .exit = hantro_hevc_dec_exit, + }, +}; + /* * VPU variants. */ static const struct hantro_irq imx8mq_irqs[] = { { "g1", imx8m_vpu_g1_irq }, - { "g2", NULL /* TODO: imx8m_vpu_g2_irq */ }, +}; + +static const struct hantro_irq imx8mq_g2_irqs[] = { + { "g2", imx8m_vpu_g2_irq }, }; static const char * const imx8mq_clk_names[] = { "g1", "g2", "bus" }; @@ -218,3 +275,17 @@ const struct hantro_variant imx8mq_vpu_variant = { .reg_names = imx8mq_reg_names, .num_regs = ARRAY_SIZE(imx8mq_reg_names) }; + +const struct hantro_variant imx8mq_vpu_g2_variant = { + .dec_offset = 0x0, + .dec_fmts = imx8m_vpu_g2_dec_fmts, + .num_dec_fmts = ARRAY_SIZE(imx8m_vpu_g2_dec_fmts), + .codec = HANTRO_HEVC_DECODER, + .codec_ops = imx8mq_vpu_g2_codec_ops, + .init = imx8mq_vpu_hw_init, + .runtime_resume = imx8mq_runtime_resume, + .irqs = imx8mq_g2_irqs, + .num_irqs = ARRAY_SIZE(imx8mq_g2_irqs), + .clk_names = imx8mq_clk_names, + .num_clocks = ARRAY_SIZE(imx8mq_clk_names), +}; diff --git a/drivers/staging/media/hantro/rk3288_vpu_hw.c b/drivers/staging/media/hantro/rk3288_vpu_hw.c deleted file mode 100644 index 7b299ee3e93d..000000000000 --- a/drivers/staging/media/hantro/rk3288_vpu_hw.c +++ /dev/null @@ -1,236 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Hantro VPU codec driver - * - * Copyright (C) 2018 Rockchip Electronics Co., Ltd. - * Jeffy Chen - */ - -#include - -#include "hantro.h" -#include "hantro_jpeg.h" -#include "hantro_g1_regs.h" -#include "hantro_h1_regs.h" - -#define RK3288_ACLK_MAX_FREQ (400 * 1000 * 1000) - -/* - * Supported formats. - */ - -static const struct hantro_fmt rk3288_vpu_enc_fmts[] = { - { - .fourcc = V4L2_PIX_FMT_YUV420M, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_YUV420P, - }, - { - .fourcc = V4L2_PIX_FMT_NV12M, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_YUV420SP, - }, - { - .fourcc = V4L2_PIX_FMT_YUYV, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_YUYV422, - }, - { - .fourcc = V4L2_PIX_FMT_UYVY, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_UYVY422, - }, - { - .fourcc = V4L2_PIX_FMT_JPEG, - .codec_mode = HANTRO_MODE_JPEG_ENC, - .max_depth = 2, - .header_size = JPEG_HEADER_SIZE, - .frmsize = { - .min_width = 96, - .max_width = 8192, - .step_width = MB_DIM, - .min_height = 32, - .max_height = 8192, - .step_height = MB_DIM, - }, - }, -}; - -static const struct hantro_fmt rk3288_vpu_postproc_fmts[] = { - { - .fourcc = V4L2_PIX_FMT_YUYV, - .codec_mode = HANTRO_MODE_NONE, - }, -}; - -static const struct hantro_fmt rk3288_vpu_dec_fmts[] = { - { - .fourcc = V4L2_PIX_FMT_NV12, - .codec_mode = HANTRO_MODE_NONE, - }, - { - .fourcc = V4L2_PIX_FMT_H264_SLICE, - .codec_mode = HANTRO_MODE_H264_DEC, - .max_depth = 2, - .frmsize = { - .min_width = 48, - .max_width = 4096, - .step_width = MB_DIM, - .min_height = 48, - .max_height = 2304, - .step_height = MB_DIM, - }, - }, - { - .fourcc = V4L2_PIX_FMT_MPEG2_SLICE, - .codec_mode = HANTRO_MODE_MPEG2_DEC, - .max_depth = 2, - .frmsize = { - .min_width = 48, - .max_width = 1920, - .step_width = MB_DIM, - .min_height = 48, - .max_height = 1088, - .step_height = MB_DIM, - }, - }, - { - .fourcc = V4L2_PIX_FMT_VP8_FRAME, - .codec_mode = HANTRO_MODE_VP8_DEC, - .max_depth = 2, - .frmsize = { - .min_width = 48, - .max_width = 3840, - .step_width = MB_DIM, - .min_height = 48, - .max_height = 2160, - .step_height = MB_DIM, - }, - }, -}; - -static irqreturn_t rk3288_vepu_irq(int irq, void *dev_id) -{ - struct hantro_dev *vpu = dev_id; - enum vb2_buffer_state state; - u32 status; - - status = vepu_read(vpu, H1_REG_INTERRUPT); - state = (status & H1_REG_INTERRUPT_FRAME_RDY) ? - VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; - - vepu_write(vpu, 0, H1_REG_INTERRUPT); - vepu_write(vpu, 0, H1_REG_AXI_CTRL); - - hantro_irq_done(vpu, state); - - return IRQ_HANDLED; -} - -static irqreturn_t rk3288_vdpu_irq(int irq, void *dev_id) -{ - struct hantro_dev *vpu = dev_id; - enum vb2_buffer_state state; - u32 status; - - status = vdpu_read(vpu, G1_REG_INTERRUPT); - state = (status & G1_REG_INTERRUPT_DEC_RDY_INT) ? - VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; - - vdpu_write(vpu, 0, G1_REG_INTERRUPT); - vdpu_write(vpu, G1_REG_CONFIG_DEC_CLK_GATE_E, G1_REG_CONFIG); - - hantro_irq_done(vpu, state); - - return IRQ_HANDLED; -} - -static int rk3288_vpu_hw_init(struct hantro_dev *vpu) -{ - /* Bump ACLK to max. possible freq. to improve performance. */ - clk_set_rate(vpu->clocks[0].clk, RK3288_ACLK_MAX_FREQ); - return 0; -} - -static void rk3288_vpu_enc_reset(struct hantro_ctx *ctx) -{ - struct hantro_dev *vpu = ctx->dev; - - vepu_write(vpu, H1_REG_INTERRUPT_DIS_BIT, H1_REG_INTERRUPT); - vepu_write(vpu, 0, H1_REG_ENC_CTRL); - vepu_write(vpu, 0, H1_REG_AXI_CTRL); -} - -static void rk3288_vpu_dec_reset(struct hantro_ctx *ctx) -{ - struct hantro_dev *vpu = ctx->dev; - - vdpu_write(vpu, G1_REG_INTERRUPT_DEC_IRQ_DIS, G1_REG_INTERRUPT); - vdpu_write(vpu, G1_REG_CONFIG_DEC_CLK_GATE_E, G1_REG_CONFIG); - vdpu_write(vpu, 1, G1_REG_SOFT_RESET); -} - -/* - * Supported codec ops. - */ - -static const struct hantro_codec_ops rk3288_vpu_codec_ops[] = { - [HANTRO_MODE_JPEG_ENC] = { - .run = hantro_h1_jpeg_enc_run, - .reset = rk3288_vpu_enc_reset, - .init = hantro_jpeg_enc_init, - .done = hantro_jpeg_enc_done, - .exit = hantro_jpeg_enc_exit, - }, - [HANTRO_MODE_H264_DEC] = { - .run = hantro_g1_h264_dec_run, - .reset = rk3288_vpu_dec_reset, - .init = hantro_h264_dec_init, - .exit = hantro_h264_dec_exit, - }, - [HANTRO_MODE_MPEG2_DEC] = { - .run = hantro_g1_mpeg2_dec_run, - .reset = rk3288_vpu_dec_reset, - .init = hantro_mpeg2_dec_init, - .exit = hantro_mpeg2_dec_exit, - }, - [HANTRO_MODE_VP8_DEC] = { - .run = hantro_g1_vp8_dec_run, - .reset = rk3288_vpu_dec_reset, - .init = hantro_vp8_dec_init, - .exit = hantro_vp8_dec_exit, - }, -}; - -/* - * VPU variant. - */ - -static const struct hantro_irq rk3288_irqs[] = { - { "vepu", rk3288_vepu_irq }, - { "vdpu", rk3288_vdpu_irq }, -}; - -static const char * const rk3288_clk_names[] = { - "aclk", "hclk" -}; - -const struct hantro_variant rk3288_vpu_variant = { - .enc_offset = 0x0, - .enc_fmts = rk3288_vpu_enc_fmts, - .num_enc_fmts = ARRAY_SIZE(rk3288_vpu_enc_fmts), - .dec_offset = 0x400, - .dec_fmts = rk3288_vpu_dec_fmts, - .num_dec_fmts = ARRAY_SIZE(rk3288_vpu_dec_fmts), - .postproc_fmts = rk3288_vpu_postproc_fmts, - .num_postproc_fmts = ARRAY_SIZE(rk3288_vpu_postproc_fmts), - .postproc_regs = &hantro_g1_postproc_regs, - .codec = HANTRO_JPEG_ENCODER | HANTRO_MPEG2_DECODER | - HANTRO_VP8_DECODER | HANTRO_H264_DECODER, - .codec_ops = rk3288_vpu_codec_ops, - .irqs = rk3288_irqs, - .num_irqs = ARRAY_SIZE(rk3288_irqs), - .init = rk3288_vpu_hw_init, - .clk_names = rk3288_clk_names, - .num_clocks = ARRAY_SIZE(rk3288_clk_names) -}; diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw.c b/drivers/staging/media/hantro/rk3399_vpu_hw.c deleted file mode 100644 index 7a7962cf771e..000000000000 --- a/drivers/staging/media/hantro/rk3399_vpu_hw.c +++ /dev/null @@ -1,222 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Hantro VPU codec driver - * - * Copyright (C) 2018 Rockchip Electronics Co., Ltd. - * Jeffy Chen - */ - -#include - -#include "hantro.h" -#include "hantro_jpeg.h" -#include "rk3399_vpu_regs.h" - -#define RK3399_ACLK_MAX_FREQ (400 * 1000 * 1000) - -/* - * Supported formats. - */ - -static const struct hantro_fmt rk3399_vpu_enc_fmts[] = { - { - .fourcc = V4L2_PIX_FMT_YUV420M, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_YUV420P, - }, - { - .fourcc = V4L2_PIX_FMT_NV12M, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_YUV420SP, - }, - { - .fourcc = V4L2_PIX_FMT_YUYV, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_YUYV422, - }, - { - .fourcc = V4L2_PIX_FMT_UYVY, - .codec_mode = HANTRO_MODE_NONE, - .enc_fmt = RK3288_VPU_ENC_FMT_UYVY422, - }, - { - .fourcc = V4L2_PIX_FMT_JPEG, - .codec_mode = HANTRO_MODE_JPEG_ENC, - .max_depth = 2, - .header_size = JPEG_HEADER_SIZE, - .frmsize = { - .min_width = 96, - .max_width = 8192, - .step_width = MB_DIM, - .min_height = 32, - .max_height = 8192, - .step_height = MB_DIM, - }, - }, -}; - -static const struct hantro_fmt rk3399_vpu_dec_fmts[] = { - { - .fourcc = V4L2_PIX_FMT_NV12, - .codec_mode = HANTRO_MODE_NONE, - }, - { - .fourcc = V4L2_PIX_FMT_MPEG2_SLICE, - .codec_mode = HANTRO_MODE_MPEG2_DEC, - .max_depth = 2, - .frmsize = { - .min_width = 48, - .max_width = 1920, - .step_width = MB_DIM, - .min_height = 48, - .max_height = 1088, - .step_height = MB_DIM, - }, - }, - { - .fourcc = V4L2_PIX_FMT_VP8_FRAME, - .codec_mode = HANTRO_MODE_VP8_DEC, - .max_depth = 2, - .frmsize = { - .min_width = 48, - .max_width = 3840, - .step_width = MB_DIM, - .min_height = 48, - .max_height = 2160, - .step_height = MB_DIM, - }, - }, -}; - -static irqreturn_t rk3399_vepu_irq(int irq, void *dev_id) -{ - struct hantro_dev *vpu = dev_id; - enum vb2_buffer_state state; - u32 status; - - status = vepu_read(vpu, VEPU_REG_INTERRUPT); - state = (status & VEPU_REG_INTERRUPT_FRAME_READY) ? - VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; - - vepu_write(vpu, 0, VEPU_REG_INTERRUPT); - vepu_write(vpu, 0, VEPU_REG_AXI_CTRL); - - hantro_irq_done(vpu, state); - - return IRQ_HANDLED; -} - -static irqreturn_t rk3399_vdpu_irq(int irq, void *dev_id) -{ - struct hantro_dev *vpu = dev_id; - enum vb2_buffer_state state; - u32 status; - - status = vdpu_read(vpu, VDPU_REG_INTERRUPT); - state = (status & VDPU_REG_INTERRUPT_DEC_IRQ) ? - VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; - - vdpu_write(vpu, 0, VDPU_REG_INTERRUPT); - vdpu_write(vpu, 0, VDPU_REG_AXI_CTRL); - - hantro_irq_done(vpu, state); - - return IRQ_HANDLED; -} - -static int rk3399_vpu_hw_init(struct hantro_dev *vpu) -{ - /* Bump ACLK to max. possible freq. to improve performance. */ - clk_set_rate(vpu->clocks[0].clk, RK3399_ACLK_MAX_FREQ); - return 0; -} - -static void rk3399_vpu_enc_reset(struct hantro_ctx *ctx) -{ - struct hantro_dev *vpu = ctx->dev; - - vepu_write(vpu, VEPU_REG_INTERRUPT_DIS_BIT, VEPU_REG_INTERRUPT); - vepu_write(vpu, 0, VEPU_REG_ENCODE_START); - vepu_write(vpu, 0, VEPU_REG_AXI_CTRL); -} - -static void rk3399_vpu_dec_reset(struct hantro_ctx *ctx) -{ - struct hantro_dev *vpu = ctx->dev; - - vdpu_write(vpu, VDPU_REG_INTERRUPT_DEC_IRQ_DIS, VDPU_REG_INTERRUPT); - vdpu_write(vpu, 0, VDPU_REG_EN_FLAGS); - vdpu_write(vpu, 1, VDPU_REG_SOFT_RESET); -} - -/* - * Supported codec ops. - */ - -static const struct hantro_codec_ops rk3399_vpu_codec_ops[] = { - [HANTRO_MODE_JPEG_ENC] = { - .run = rk3399_vpu_jpeg_enc_run, - .reset = rk3399_vpu_enc_reset, - .init = hantro_jpeg_enc_init, - .exit = hantro_jpeg_enc_exit, - }, - [HANTRO_MODE_MPEG2_DEC] = { - .run = rk3399_vpu_mpeg2_dec_run, - .reset = rk3399_vpu_dec_reset, - .init = hantro_mpeg2_dec_init, - .exit = hantro_mpeg2_dec_exit, - }, - [HANTRO_MODE_VP8_DEC] = { - .run = rk3399_vpu_vp8_dec_run, - .reset = rk3399_vpu_dec_reset, - .init = hantro_vp8_dec_init, - .exit = hantro_vp8_dec_exit, - }, -}; - -/* - * VPU variant. - */ - -static const struct hantro_irq rk3399_irqs[] = { - { "vepu", rk3399_vepu_irq }, - { "vdpu", rk3399_vdpu_irq }, -}; - -static const char * const rk3399_clk_names[] = { - "aclk", "hclk" -}; - -const struct hantro_variant rk3399_vpu_variant = { - .enc_offset = 0x0, - .enc_fmts = rk3399_vpu_enc_fmts, - .num_enc_fmts = ARRAY_SIZE(rk3399_vpu_enc_fmts), - .dec_offset = 0x400, - .dec_fmts = rk3399_vpu_dec_fmts, - .num_dec_fmts = ARRAY_SIZE(rk3399_vpu_dec_fmts), - .codec = HANTRO_JPEG_ENCODER | HANTRO_MPEG2_DECODER | - HANTRO_VP8_DECODER, - .codec_ops = rk3399_vpu_codec_ops, - .irqs = rk3399_irqs, - .num_irqs = ARRAY_SIZE(rk3399_irqs), - .init = rk3399_vpu_hw_init, - .clk_names = rk3399_clk_names, - .num_clocks = ARRAY_SIZE(rk3399_clk_names) -}; - -static const struct hantro_irq rk3328_irqs[] = { - { "vdpu", rk3399_vdpu_irq }, -}; - -const struct hantro_variant rk3328_vpu_variant = { - .dec_offset = 0x400, - .dec_fmts = rk3399_vpu_dec_fmts, - .num_dec_fmts = ARRAY_SIZE(rk3399_vpu_dec_fmts), - .codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER, - .codec_ops = rk3399_vpu_codec_ops, - .irqs = rk3328_irqs, - .num_irqs = ARRAY_SIZE(rk3328_irqs), - .init = rk3399_vpu_hw_init, - .clk_names = rk3399_clk_names, - .num_clocks = ARRAY_SIZE(rk3399_clk_names), -}; diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c b/drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c similarity index 87% rename from drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c rename to drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c index 3498e6124acd..991213ce1610 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_hw_jpeg_enc.c +++ b/drivers/staging/media/hantro/rockchip_vpu2_hw_jpeg_enc.c @@ -28,12 +28,12 @@ #include "hantro.h" #include "hantro_v4l2.h" #include "hantro_hw.h" -#include "rk3399_vpu_regs.h" +#include "rockchip_vpu2_regs.h" #define VEPU_JPEG_QUANT_TABLE_COUNT 16 -static void rk3399_vpu_set_src_img_ctrl(struct hantro_dev *vpu, - struct hantro_ctx *ctx) +static void rockchip_vpu2_set_src_img_ctrl(struct hantro_dev *vpu, + struct hantro_ctx *ctx) { struct v4l2_pix_format_mplane *pix_fmt = &ctx->src_fmt; u32 reg; @@ -59,9 +59,9 @@ static void rk3399_vpu_set_src_img_ctrl(struct hantro_dev *vpu, vepu_write_relaxed(vpu, reg, VEPU_REG_ENC_CTRL1); } -static void rk3399_vpu_jpeg_enc_set_buffers(struct hantro_dev *vpu, - struct hantro_ctx *ctx, - struct vb2_buffer *src_buf) +static void rockchip_vpu2_jpeg_enc_set_buffers(struct hantro_dev *vpu, + struct hantro_ctx *ctx, + struct vb2_buffer *src_buf) { struct v4l2_pix_format_mplane *pix_fmt = &ctx->src_fmt; dma_addr_t src[3]; @@ -92,9 +92,9 @@ static void rk3399_vpu_jpeg_enc_set_buffers(struct hantro_dev *vpu, } static void -rk3399_vpu_jpeg_enc_set_qtable(struct hantro_dev *vpu, - unsigned char *luma_qtable, - unsigned char *chroma_qtable) +rockchip_vpu2_jpeg_enc_set_qtable(struct hantro_dev *vpu, + unsigned char *luma_qtable, + unsigned char *chroma_qtable) { u32 reg, i; __be32 *luma_qtable_p; @@ -118,7 +118,7 @@ rk3399_vpu_jpeg_enc_set_qtable(struct hantro_dev *vpu, } } -void rk3399_vpu_jpeg_enc_run(struct hantro_ctx *ctx) +int rockchip_vpu2_jpeg_enc_run(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; struct vb2_v4l2_buffer *src_buf, *dst_buf; @@ -141,11 +141,11 @@ void rk3399_vpu_jpeg_enc_run(struct hantro_ctx *ctx) vepu_write_relaxed(vpu, VEPU_REG_ENCODE_FORMAT_JPEG, VEPU_REG_ENCODE_START); - rk3399_vpu_set_src_img_ctrl(vpu, ctx); - rk3399_vpu_jpeg_enc_set_buffers(vpu, ctx, &src_buf->vb2_buf); - rk3399_vpu_jpeg_enc_set_qtable(vpu, - hantro_jpeg_get_qtable(0), - hantro_jpeg_get_qtable(1)); + rockchip_vpu2_set_src_img_ctrl(vpu, ctx); + rockchip_vpu2_jpeg_enc_set_buffers(vpu, ctx, &src_buf->vb2_buf); + rockchip_vpu2_jpeg_enc_set_qtable(vpu, + hantro_jpeg_get_qtable(0), + hantro_jpeg_get_qtable(1)); reg = VEPU_REG_OUTPUT_SWAP32 | VEPU_REG_OUTPUT_SWAP16 @@ -168,4 +168,6 @@ void rk3399_vpu_jpeg_enc_run(struct hantro_ctx *ctx) /* Kick the watchdog and start encoding */ hantro_end_prepare_run(ctx); vepu_write(vpu, reg, VEPU_REG_ENCODE_START); + + return 0; } diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c b/drivers/staging/media/hantro/rockchip_vpu2_hw_mpeg2_dec.c similarity index 66% rename from drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c rename to drivers/staging/media/hantro/rockchip_vpu2_hw_mpeg2_dec.c index f610fa5b4335..b66737fab46b 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_hw_mpeg2_dec.c +++ b/drivers/staging/media/hantro/rockchip_vpu2_hw_mpeg2_dec.c @@ -79,43 +79,34 @@ #define VDPU_REG_MV_ACCURACY_FWD(v) ((v) ? BIT(2) : 0) #define VDPU_REG_MV_ACCURACY_BWD(v) ((v) ? BIT(1) : 0) -#define PICT_TOP_FIELD 1 -#define PICT_BOTTOM_FIELD 2 -#define PICT_FRAME 3 - static void -rk3399_vpu_mpeg2_dec_set_quantization(struct hantro_dev *vpu, - struct hantro_ctx *ctx) +rockchip_vpu2_mpeg2_dec_set_quantisation(struct hantro_dev *vpu, + struct hantro_ctx *ctx) { - struct v4l2_ctrl_mpeg2_quantization *quantization; + struct v4l2_ctrl_mpeg2_quantisation *q; - quantization = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION); - hantro_mpeg2_dec_copy_qtable(ctx->mpeg2_dec.qtable.cpu, quantization); - vdpu_write_relaxed(vpu, ctx->mpeg2_dec.qtable.dma, - VDPU_REG_QTABLE_BASE); + q = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_MPEG2_QUANTISATION); + hantro_mpeg2_dec_copy_qtable(ctx->mpeg2_dec.qtable.cpu, q); + vdpu_write_relaxed(vpu, ctx->mpeg2_dec.qtable.dma, VDPU_REG_QTABLE_BASE); } static void -rk3399_vpu_mpeg2_dec_set_buffers(struct hantro_dev *vpu, - struct hantro_ctx *ctx, - struct vb2_buffer *src_buf, - struct vb2_buffer *dst_buf, - const struct v4l2_mpeg2_sequence *sequence, - const struct v4l2_mpeg2_picture *picture, - const struct v4l2_ctrl_mpeg2_slice_params *slice_params) +rockchip_vpu2_mpeg2_dec_set_buffers(struct hantro_dev *vpu, + struct hantro_ctx *ctx, + struct vb2_buffer *src_buf, + struct vb2_buffer *dst_buf, + const struct v4l2_ctrl_mpeg2_sequence *seq, + const struct v4l2_ctrl_mpeg2_picture *pic) { dma_addr_t forward_addr = 0, backward_addr = 0; dma_addr_t current_addr, addr; - switch (picture->picture_coding_type) { - case V4L2_MPEG2_PICTURE_CODING_TYPE_B: - backward_addr = hantro_get_ref(ctx, - slice_params->backward_ref_ts); + switch (pic->picture_coding_type) { + case V4L2_MPEG2_PIC_CODING_TYPE_B: + backward_addr = hantro_get_ref(ctx, pic->backward_ref_ts); fallthrough; - case V4L2_MPEG2_PICTURE_CODING_TYPE_P: - forward_addr = hantro_get_ref(ctx, - slice_params->forward_ref_ts); + case V4L2_MPEG2_PIC_CODING_TYPE_P: + forward_addr = hantro_get_ref(ctx, pic->forward_ref_ts); } /* Source bitstream buffer */ @@ -126,7 +117,7 @@ rk3399_vpu_mpeg2_dec_set_buffers(struct hantro_dev *vpu, addr = vb2_dma_contig_plane_dma_addr(dst_buf, 0); current_addr = addr; - if (picture->picture_structure == PICT_BOTTOM_FIELD) + if (pic->picture_structure == V4L2_MPEG2_PIC_BOTTOM_FIELD) addr += ALIGN(ctx->dst_fmt.width, 16); vdpu_write_relaxed(vpu, addr, VDPU_REG_DEC_OUT_BASE); @@ -136,18 +127,18 @@ rk3399_vpu_mpeg2_dec_set_buffers(struct hantro_dev *vpu, backward_addr = current_addr; /* Set forward ref frame (top/bottom field) */ - if (picture->picture_structure == PICT_FRAME || - picture->picture_coding_type == V4L2_MPEG2_PICTURE_CODING_TYPE_B || - (picture->picture_structure == PICT_TOP_FIELD && - picture->top_field_first) || - (picture->picture_structure == PICT_BOTTOM_FIELD && - !picture->top_field_first)) { + if (pic->picture_structure == V4L2_MPEG2_PIC_FRAME || + pic->picture_coding_type == V4L2_MPEG2_PIC_CODING_TYPE_B || + (pic->picture_structure == V4L2_MPEG2_PIC_TOP_FIELD && + pic->flags & V4L2_MPEG2_PIC_TOP_FIELD) || + (pic->picture_structure == V4L2_MPEG2_PIC_BOTTOM_FIELD && + !(pic->flags & V4L2_MPEG2_PIC_TOP_FIELD))) { vdpu_write_relaxed(vpu, forward_addr, VDPU_REG_REFER0_BASE); vdpu_write_relaxed(vpu, forward_addr, VDPU_REG_REFER1_BASE); - } else if (picture->picture_structure == PICT_TOP_FIELD) { + } else if (pic->picture_structure == V4L2_MPEG2_PIC_TOP_FIELD) { vdpu_write_relaxed(vpu, forward_addr, VDPU_REG_REFER0_BASE); vdpu_write_relaxed(vpu, current_addr, VDPU_REG_REFER1_BASE); - } else if (picture->picture_structure == PICT_BOTTOM_FIELD) { + } else if (pic->picture_structure == V4L2_MPEG2_PIC_BOTTOM_FIELD) { vdpu_write_relaxed(vpu, current_addr, VDPU_REG_REFER0_BASE); vdpu_write_relaxed(vpu, forward_addr, VDPU_REG_REFER1_BASE); } @@ -157,13 +148,12 @@ rk3399_vpu_mpeg2_dec_set_buffers(struct hantro_dev *vpu, vdpu_write_relaxed(vpu, backward_addr, VDPU_REG_REFER3_BASE); } -void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) +int rockchip_vpu2_mpeg2_dec_run(struct hantro_ctx *ctx) { struct hantro_dev *vpu = ctx->dev; struct vb2_v4l2_buffer *src_buf, *dst_buf; - const struct v4l2_ctrl_mpeg2_slice_params *slice_params; - const struct v4l2_mpeg2_sequence *sequence; - const struct v4l2_mpeg2_picture *picture; + const struct v4l2_ctrl_mpeg2_sequence *seq; + const struct v4l2_ctrl_mpeg2_picture *pic; u32 reg; src_buf = hantro_get_src_buf(ctx); @@ -171,10 +161,10 @@ void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) hantro_start_prepare_run(ctx); - slice_params = hantro_get_ctrl(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS); - sequence = &slice_params->sequence; - picture = &slice_params->picture; + seq = hantro_get_ctrl(ctx, + V4L2_CID_STATELESS_MPEG2_SEQUENCE); + pic = hantro_get_ctrl(ctx, + V4L2_CID_STATELESS_MPEG2_PICTURE); reg = VDPU_REG_DEC_ADV_PRE_DIS(0) | VDPU_REG_DEC_SCMD_DIS(0) | @@ -183,7 +173,7 @@ void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) vdpu_write_relaxed(vpu, reg, VDPU_SWREG(50)); reg = VDPU_REG_INIT_QP(1) | - VDPU_REG_STREAM_LEN(slice_params->bit_size >> 3); + VDPU_REG_STREAM_LEN(vb2_get_plane_payload(&src_buf->vb2_buf, 0)); vdpu_write_relaxed(vpu, reg, VDPU_SWREG(51)); reg = VDPU_REG_APF_THRESHOLD(8) | @@ -209,11 +199,11 @@ void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) vdpu_write_relaxed(vpu, reg, VDPU_SWREG(56)); reg = VDPU_REG_RLC_MODE_E(0) | - VDPU_REG_PIC_INTERLACE_E(!sequence->progressive_sequence) | - VDPU_REG_PIC_FIELDMODE_E(picture->picture_structure != PICT_FRAME) | - VDPU_REG_PIC_B_E(picture->picture_coding_type == V4L2_MPEG2_PICTURE_CODING_TYPE_B) | - VDPU_REG_PIC_INTER_E(picture->picture_coding_type != V4L2_MPEG2_PICTURE_CODING_TYPE_I) | - VDPU_REG_PIC_TOPFIELD_E(picture->picture_structure == PICT_TOP_FIELD) | + VDPU_REG_PIC_INTERLACE_E(!(seq->flags & V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE)) | + VDPU_REG_PIC_FIELDMODE_E(pic->picture_structure != V4L2_MPEG2_PIC_FRAME) | + VDPU_REG_PIC_B_E(pic->picture_coding_type == V4L2_MPEG2_PIC_CODING_TYPE_B) | + VDPU_REG_PIC_INTER_E(pic->picture_coding_type != V4L2_MPEG2_PIC_CODING_TYPE_I) | + VDPU_REG_PIC_TOPFIELD_E(pic->picture_structure == V4L2_MPEG2_PIC_TOP_FIELD) | VDPU_REG_FWD_INTERLACE_E(0) | VDPU_REG_WRITE_MVS_E(0) | VDPU_REG_DEC_TIMEOUT_E(1) | @@ -222,36 +212,37 @@ void rk3399_vpu_mpeg2_dec_run(struct hantro_ctx *ctx) reg = VDPU_REG_PIC_MB_WIDTH(MB_WIDTH(ctx->dst_fmt.width)) | VDPU_REG_PIC_MB_HEIGHT_P(MB_HEIGHT(ctx->dst_fmt.height)) | - VDPU_REG_ALT_SCAN_E(picture->alternate_scan) | - VDPU_REG_TOPFIELDFIRST_E(picture->top_field_first); + VDPU_REG_ALT_SCAN_E(pic->flags & V4L2_MPEG2_PIC_FLAG_ALT_SCAN) | + VDPU_REG_TOPFIELDFIRST_E(pic->flags & V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST); vdpu_write_relaxed(vpu, reg, VDPU_SWREG(120)); - reg = VDPU_REG_STRM_START_BIT(slice_params->data_bit_offset) | - VDPU_REG_QSCALE_TYPE(picture->q_scale_type) | - VDPU_REG_CON_MV_E(picture->concealment_motion_vectors) | - VDPU_REG_INTRA_DC_PREC(picture->intra_dc_precision) | - VDPU_REG_INTRA_VLC_TAB(picture->intra_vlc_format) | - VDPU_REG_FRAME_PRED_DCT(picture->frame_pred_frame_dct); + reg = VDPU_REG_STRM_START_BIT(0) | + VDPU_REG_QSCALE_TYPE(pic->flags & V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE) | + VDPU_REG_CON_MV_E(pic->flags & V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV) | + VDPU_REG_INTRA_DC_PREC(pic->intra_dc_precision) | + VDPU_REG_INTRA_VLC_TAB(pic->flags & V4L2_MPEG2_PIC_FLAG_INTRA_VLC) | + VDPU_REG_FRAME_PRED_DCT(pic->flags & V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT); vdpu_write_relaxed(vpu, reg, VDPU_SWREG(122)); - reg = VDPU_REG_ALT_SCAN_FLAG_E(picture->alternate_scan) | - VDPU_REG_FCODE_FWD_HOR(picture->f_code[0][0]) | - VDPU_REG_FCODE_FWD_VER(picture->f_code[0][1]) | - VDPU_REG_FCODE_BWD_HOR(picture->f_code[1][0]) | - VDPU_REG_FCODE_BWD_VER(picture->f_code[1][1]) | + reg = VDPU_REG_ALT_SCAN_FLAG_E(pic->flags & V4L2_MPEG2_PIC_FLAG_ALT_SCAN) | + VDPU_REG_FCODE_FWD_HOR(pic->f_code[0][0]) | + VDPU_REG_FCODE_FWD_VER(pic->f_code[0][1]) | + VDPU_REG_FCODE_BWD_HOR(pic->f_code[1][0]) | + VDPU_REG_FCODE_BWD_VER(pic->f_code[1][1]) | VDPU_REG_MV_ACCURACY_FWD(1) | VDPU_REG_MV_ACCURACY_BWD(1); vdpu_write_relaxed(vpu, reg, VDPU_SWREG(136)); - rk3399_vpu_mpeg2_dec_set_quantization(vpu, ctx); + rockchip_vpu2_mpeg2_dec_set_quantisation(vpu, ctx); - rk3399_vpu_mpeg2_dec_set_buffers(vpu, ctx, &src_buf->vb2_buf, - &dst_buf->vb2_buf, - sequence, picture, slice_params); + rockchip_vpu2_mpeg2_dec_set_buffers(vpu, ctx, &src_buf->vb2_buf, + &dst_buf->vb2_buf, seq, pic); /* Kick the watchdog and start decoding */ hantro_end_prepare_run(ctx); reg = vdpu_read(vpu, VDPU_SWREG(57)) | VDPU_REG_DEC_E(1); vdpu_write(vpu, reg, VDPU_SWREG(57)); + + return 0; } diff --git a/drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c b/drivers/staging/media/hantro/rockchip_vpu2_hw_vp8_dec.c similarity index 99% rename from drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c rename to drivers/staging/media/hantro/rockchip_vpu2_hw_vp8_dec.c index 8661a3cc1e6b..951b55f58a61 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_hw_vp8_dec.c +++ b/drivers/staging/media/hantro/rockchip_vpu2_hw_vp8_dec.c @@ -503,7 +503,7 @@ static void cfg_buffers(struct hantro_ctx *ctx, vdpu_write_relaxed(vpu, dst_dma, VDPU_REG_ADDR_DST); } -void rk3399_vpu_vp8_dec_run(struct hantro_ctx *ctx) +int rockchip_vpu2_vp8_dec_run(struct hantro_ctx *ctx) { const struct v4l2_ctrl_vp8_frame *hdr; struct hantro_dev *vpu = ctx->dev; @@ -516,7 +516,7 @@ void rk3399_vpu_vp8_dec_run(struct hantro_ctx *ctx) hdr = hantro_get_ctrl(ctx, V4L2_CID_STATELESS_VP8_FRAME); if (WARN_ON(!hdr)) - return; + return -EINVAL; /* Reset segment_map buffer in keyframe */ if (V4L2_VP8_FRAME_IS_KEY_FRAME(hdr) && ctx->vp8_dec.segment_map.cpu) @@ -589,4 +589,6 @@ void rk3399_vpu_vp8_dec_run(struct hantro_ctx *ctx) hantro_end_prepare_run(ctx); hantro_reg_write(vpu, &vp8_dec_start_dec, 1); + + return 0; } diff --git a/drivers/staging/media/hantro/rk3399_vpu_regs.h b/drivers/staging/media/hantro/rockchip_vpu2_regs.h similarity index 99% rename from drivers/staging/media/hantro/rk3399_vpu_regs.h rename to drivers/staging/media/hantro/rockchip_vpu2_regs.h index 88d096920f30..49e40889545b 100644 --- a/drivers/staging/media/hantro/rk3399_vpu_regs.h +++ b/drivers/staging/media/hantro/rockchip_vpu2_regs.h @@ -6,8 +6,8 @@ * Alpha Lin */ -#ifndef RK3399_VPU_REGS_H_ -#define RK3399_VPU_REGS_H_ +#ifndef ROCKCHIP_VPU2_REGS_H_ +#define ROCKCHIP_VPU2_REGS_H_ /* Encoder registers. */ #define VEPU_REG_VP8_QUT_1ST(i) (0x000 + ((i) * 0x24)) @@ -597,4 +597,4 @@ #define VDPU_REG_PRED_FLT_PRED_BC_TAP_4_3(x) (((x) & 0x3ff) << 12) #define VDPU_REG_PRED_FLT_PRED_BC_TAP_5_0(x) (((x) & 0x3ff) << 2) -#endif /* RK3399_VPU_REGS_H_ */ +#endif /* ROCKCHIP_VPU2_REGS_H_ */ diff --git a/drivers/staging/media/hantro/rockchip_vpu_hw.c b/drivers/staging/media/hantro/rockchip_vpu_hw.c new file mode 100644 index 000000000000..3ccc16413f42 --- /dev/null +++ b/drivers/staging/media/hantro/rockchip_vpu_hw.c @@ -0,0 +1,526 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hantro VPU codec driver + * + * Copyright (C) 2018 Rockchip Electronics Co., Ltd. + * Jeffy Chen + */ + +#include + +#include "hantro.h" +#include "hantro_jpeg.h" +#include "hantro_g1_regs.h" +#include "hantro_h1_regs.h" +#include "rockchip_vpu2_regs.h" + +#define RK3066_ACLK_MAX_FREQ (300 * 1000 * 1000) +#define RK3288_ACLK_MAX_FREQ (400 * 1000 * 1000) + +/* + * Supported formats. + */ + +static const struct hantro_fmt rockchip_vpu_enc_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_YUV420M, + .codec_mode = HANTRO_MODE_NONE, + .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUV420P, + }, + { + .fourcc = V4L2_PIX_FMT_NV12M, + .codec_mode = HANTRO_MODE_NONE, + .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUV420SP, + }, + { + .fourcc = V4L2_PIX_FMT_YUYV, + .codec_mode = HANTRO_MODE_NONE, + .enc_fmt = ROCKCHIP_VPU_ENC_FMT_YUYV422, + }, + { + .fourcc = V4L2_PIX_FMT_UYVY, + .codec_mode = HANTRO_MODE_NONE, + .enc_fmt = ROCKCHIP_VPU_ENC_FMT_UYVY422, + }, + { + .fourcc = V4L2_PIX_FMT_JPEG, + .codec_mode = HANTRO_MODE_JPEG_ENC, + .max_depth = 2, + .header_size = JPEG_HEADER_SIZE, + .frmsize = { + .min_width = 96, + .max_width = 8192, + .step_width = MB_DIM, + .min_height = 32, + .max_height = 8192, + .step_height = MB_DIM, + }, + }, +}; + +static const struct hantro_fmt rockchip_vpu1_postproc_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_YUYV, + .codec_mode = HANTRO_MODE_NONE, + }, +}; + +static const struct hantro_fmt rk3066_vpu_dec_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_NV12, + .codec_mode = HANTRO_MODE_NONE, + }, + { + .fourcc = V4L2_PIX_FMT_H264_SLICE, + .codec_mode = HANTRO_MODE_H264_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1920, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 1088, + .step_height = MB_DIM, + }, + }, + { + .fourcc = V4L2_PIX_FMT_MPEG2_SLICE, + .codec_mode = HANTRO_MODE_MPEG2_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1920, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 1088, + .step_height = MB_DIM, + }, + }, + { + .fourcc = V4L2_PIX_FMT_VP8_FRAME, + .codec_mode = HANTRO_MODE_VP8_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1920, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 1088, + .step_height = MB_DIM, + }, + }, +}; + +static const struct hantro_fmt rk3288_vpu_dec_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_NV12, + .codec_mode = HANTRO_MODE_NONE, + }, + { + .fourcc = V4L2_PIX_FMT_H264_SLICE, + .codec_mode = HANTRO_MODE_H264_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 4096, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 2304, + .step_height = MB_DIM, + }, + }, + { + .fourcc = V4L2_PIX_FMT_MPEG2_SLICE, + .codec_mode = HANTRO_MODE_MPEG2_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1920, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 1088, + .step_height = MB_DIM, + }, + }, + { + .fourcc = V4L2_PIX_FMT_VP8_FRAME, + .codec_mode = HANTRO_MODE_VP8_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 3840, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 2160, + .step_height = MB_DIM, + }, + }, +}; + +static const struct hantro_fmt rk3399_vpu_dec_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_NV12, + .codec_mode = HANTRO_MODE_NONE, + }, + { + .fourcc = V4L2_PIX_FMT_MPEG2_SLICE, + .codec_mode = HANTRO_MODE_MPEG2_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1920, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 1088, + .step_height = MB_DIM, + }, + }, + { + .fourcc = V4L2_PIX_FMT_VP8_FRAME, + .codec_mode = HANTRO_MODE_VP8_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 3840, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 2160, + .step_height = MB_DIM, + }, + }, +}; + +static irqreturn_t rockchip_vpu1_vepu_irq(int irq, void *dev_id) +{ + struct hantro_dev *vpu = dev_id; + enum vb2_buffer_state state; + u32 status; + + status = vepu_read(vpu, H1_REG_INTERRUPT); + state = (status & H1_REG_INTERRUPT_FRAME_RDY) ? + VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; + + vepu_write(vpu, 0, H1_REG_INTERRUPT); + vepu_write(vpu, 0, H1_REG_AXI_CTRL); + + hantro_irq_done(vpu, state); + + return IRQ_HANDLED; +} + +static irqreturn_t rockchip_vpu2_vdpu_irq(int irq, void *dev_id) +{ + struct hantro_dev *vpu = dev_id; + enum vb2_buffer_state state; + u32 status; + + status = vdpu_read(vpu, VDPU_REG_INTERRUPT); + state = (status & VDPU_REG_INTERRUPT_DEC_IRQ) ? + VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; + + vdpu_write(vpu, 0, VDPU_REG_INTERRUPT); + vdpu_write(vpu, 0, VDPU_REG_AXI_CTRL); + + hantro_irq_done(vpu, state); + + return IRQ_HANDLED; +} + +static irqreturn_t rockchip_vpu2_vepu_irq(int irq, void *dev_id) +{ + struct hantro_dev *vpu = dev_id; + enum vb2_buffer_state state; + u32 status; + + status = vepu_read(vpu, VEPU_REG_INTERRUPT); + state = (status & VEPU_REG_INTERRUPT_FRAME_READY) ? + VB2_BUF_STATE_DONE : VB2_BUF_STATE_ERROR; + + vepu_write(vpu, 0, VEPU_REG_INTERRUPT); + vepu_write(vpu, 0, VEPU_REG_AXI_CTRL); + + hantro_irq_done(vpu, state); + + return IRQ_HANDLED; +} + +static int rk3036_vpu_hw_init(struct hantro_dev *vpu) +{ + /* Bump ACLK to max. possible freq. to improve performance. */ + clk_set_rate(vpu->clocks[0].clk, RK3066_ACLK_MAX_FREQ); + return 0; +} + +static int rk3066_vpu_hw_init(struct hantro_dev *vpu) +{ + /* Bump ACLKs to max. possible freq. to improve performance. */ + clk_set_rate(vpu->clocks[0].clk, RK3066_ACLK_MAX_FREQ); + clk_set_rate(vpu->clocks[2].clk, RK3066_ACLK_MAX_FREQ); + return 0; +} + +static int rockchip_vpu_hw_init(struct hantro_dev *vpu) +{ + /* Bump ACLK to max. possible freq. to improve performance. */ + clk_set_rate(vpu->clocks[0].clk, RK3288_ACLK_MAX_FREQ); + return 0; +} + +static void rk3066_vpu_dec_reset(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + + vdpu_write(vpu, G1_REG_INTERRUPT_DEC_IRQ_DIS, G1_REG_INTERRUPT); + vdpu_write(vpu, G1_REG_CONFIG_DEC_CLK_GATE_E, G1_REG_CONFIG); +} + +static void rockchip_vpu1_enc_reset(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + + vepu_write(vpu, H1_REG_INTERRUPT_DIS_BIT, H1_REG_INTERRUPT); + vepu_write(vpu, 0, H1_REG_ENC_CTRL); + vepu_write(vpu, 0, H1_REG_AXI_CTRL); +} + +static void rockchip_vpu2_dec_reset(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + + vdpu_write(vpu, VDPU_REG_INTERRUPT_DEC_IRQ_DIS, VDPU_REG_INTERRUPT); + vdpu_write(vpu, 0, VDPU_REG_EN_FLAGS); + vdpu_write(vpu, 1, VDPU_REG_SOFT_RESET); +} + +static void rockchip_vpu2_enc_reset(struct hantro_ctx *ctx) +{ + struct hantro_dev *vpu = ctx->dev; + + vepu_write(vpu, VEPU_REG_INTERRUPT_DIS_BIT, VEPU_REG_INTERRUPT); + vepu_write(vpu, 0, VEPU_REG_ENCODE_START); + vepu_write(vpu, 0, VEPU_REG_AXI_CTRL); +} + +/* + * Supported codec ops. + */ +static const struct hantro_codec_ops rk3036_vpu_codec_ops[] = { + [HANTRO_MODE_H264_DEC] = { + .run = hantro_g1_h264_dec_run, + .reset = hantro_g1_reset, + .init = hantro_h264_dec_init, + .exit = hantro_h264_dec_exit, + }, + [HANTRO_MODE_MPEG2_DEC] = { + .run = hantro_g1_mpeg2_dec_run, + .reset = hantro_g1_reset, + .init = hantro_mpeg2_dec_init, + .exit = hantro_mpeg2_dec_exit, + }, + [HANTRO_MODE_VP8_DEC] = { + .run = hantro_g1_vp8_dec_run, + .reset = hantro_g1_reset, + .init = hantro_vp8_dec_init, + .exit = hantro_vp8_dec_exit, + }, +}; + +static const struct hantro_codec_ops rk3066_vpu_codec_ops[] = { + [HANTRO_MODE_JPEG_ENC] = { + .run = hantro_h1_jpeg_enc_run, + .reset = rockchip_vpu1_enc_reset, + .init = hantro_jpeg_enc_init, + .done = hantro_jpeg_enc_done, + .exit = hantro_jpeg_enc_exit, + }, + [HANTRO_MODE_H264_DEC] = { + .run = hantro_g1_h264_dec_run, + .reset = rk3066_vpu_dec_reset, + .init = hantro_h264_dec_init, + .exit = hantro_h264_dec_exit, + }, + [HANTRO_MODE_MPEG2_DEC] = { + .run = hantro_g1_mpeg2_dec_run, + .reset = rk3066_vpu_dec_reset, + .init = hantro_mpeg2_dec_init, + .exit = hantro_mpeg2_dec_exit, + }, + [HANTRO_MODE_VP8_DEC] = { + .run = hantro_g1_vp8_dec_run, + .reset = rk3066_vpu_dec_reset, + .init = hantro_vp8_dec_init, + .exit = hantro_vp8_dec_exit, + }, +}; + +static const struct hantro_codec_ops rk3288_vpu_codec_ops[] = { + [HANTRO_MODE_JPEG_ENC] = { + .run = hantro_h1_jpeg_enc_run, + .reset = rockchip_vpu1_enc_reset, + .init = hantro_jpeg_enc_init, + .done = hantro_jpeg_enc_done, + .exit = hantro_jpeg_enc_exit, + }, + [HANTRO_MODE_H264_DEC] = { + .run = hantro_g1_h264_dec_run, + .reset = hantro_g1_reset, + .init = hantro_h264_dec_init, + .exit = hantro_h264_dec_exit, + }, + [HANTRO_MODE_MPEG2_DEC] = { + .run = hantro_g1_mpeg2_dec_run, + .reset = hantro_g1_reset, + .init = hantro_mpeg2_dec_init, + .exit = hantro_mpeg2_dec_exit, + }, + [HANTRO_MODE_VP8_DEC] = { + .run = hantro_g1_vp8_dec_run, + .reset = hantro_g1_reset, + .init = hantro_vp8_dec_init, + .exit = hantro_vp8_dec_exit, + }, +}; + +static const struct hantro_codec_ops rk3399_vpu_codec_ops[] = { + [HANTRO_MODE_JPEG_ENC] = { + .run = rockchip_vpu2_jpeg_enc_run, + .reset = rockchip_vpu2_enc_reset, + .init = hantro_jpeg_enc_init, + .exit = hantro_jpeg_enc_exit, + }, + [HANTRO_MODE_MPEG2_DEC] = { + .run = rockchip_vpu2_mpeg2_dec_run, + .reset = rockchip_vpu2_dec_reset, + .init = hantro_mpeg2_dec_init, + .exit = hantro_mpeg2_dec_exit, + }, + [HANTRO_MODE_VP8_DEC] = { + .run = rockchip_vpu2_vp8_dec_run, + .reset = rockchip_vpu2_dec_reset, + .init = hantro_vp8_dec_init, + .exit = hantro_vp8_dec_exit, + }, +}; + +/* + * VPU variant. + */ + +static const struct hantro_irq rockchip_vdpu1_irqs[] = { + { "vdpu", hantro_g1_irq }, +}; + +static const struct hantro_irq rockchip_vpu1_irqs[] = { + { "vepu", rockchip_vpu1_vepu_irq }, + { "vdpu", hantro_g1_irq }, +}; + +static const struct hantro_irq rockchip_vdpu2_irqs[] = { + { "vdpu", rockchip_vpu2_vdpu_irq }, +}; + +static const struct hantro_irq rockchip_vpu2_irqs[] = { + { "vepu", rockchip_vpu2_vepu_irq }, + { "vdpu", rockchip_vpu2_vdpu_irq }, +}; + +static const char * const rk3066_vpu_clk_names[] = { + "aclk_vdpu", "hclk_vdpu", + "aclk_vepu", "hclk_vepu" +}; + +static const char * const rockchip_vpu_clk_names[] = { + "aclk", "hclk" +}; + +const struct hantro_variant rk3036_vpu_variant = { + .dec_offset = 0x400, + .dec_fmts = rk3066_vpu_dec_fmts, + .num_dec_fmts = ARRAY_SIZE(rk3066_vpu_dec_fmts), + .postproc_fmts = rockchip_vpu1_postproc_fmts, + .num_postproc_fmts = ARRAY_SIZE(rockchip_vpu1_postproc_fmts), + .postproc_regs = &hantro_g1_postproc_regs, + .codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER | + HANTRO_H264_DECODER, + .codec_ops = rk3036_vpu_codec_ops, + .irqs = rockchip_vdpu1_irqs, + .num_irqs = ARRAY_SIZE(rockchip_vdpu1_irqs), + .init = rk3036_vpu_hw_init, + .clk_names = rockchip_vpu_clk_names, + .num_clocks = ARRAY_SIZE(rockchip_vpu_clk_names) +}; + +/* + * Despite this variant has separate clocks for decoder and encoder, + * it's still required to enable all four of them for either decoding + * or encoding and we can't split it in separate g1/h1 variants. + */ +const struct hantro_variant rk3066_vpu_variant = { + .enc_offset = 0x0, + .enc_fmts = rockchip_vpu_enc_fmts, + .num_enc_fmts = ARRAY_SIZE(rockchip_vpu_enc_fmts), + .dec_offset = 0x400, + .dec_fmts = rk3066_vpu_dec_fmts, + .num_dec_fmts = ARRAY_SIZE(rk3066_vpu_dec_fmts), + .postproc_fmts = rockchip_vpu1_postproc_fmts, + .num_postproc_fmts = ARRAY_SIZE(rockchip_vpu1_postproc_fmts), + .postproc_regs = &hantro_g1_postproc_regs, + .codec = HANTRO_JPEG_ENCODER | HANTRO_MPEG2_DECODER | + HANTRO_VP8_DECODER | HANTRO_H264_DECODER, + .codec_ops = rk3066_vpu_codec_ops, + .irqs = rockchip_vpu1_irqs, + .num_irqs = ARRAY_SIZE(rockchip_vpu1_irqs), + .init = rk3066_vpu_hw_init, + .clk_names = rk3066_vpu_clk_names, + .num_clocks = ARRAY_SIZE(rk3066_vpu_clk_names) +}; + +const struct hantro_variant rk3288_vpu_variant = { + .enc_offset = 0x0, + .enc_fmts = rockchip_vpu_enc_fmts, + .num_enc_fmts = ARRAY_SIZE(rockchip_vpu_enc_fmts), + .dec_offset = 0x400, + .dec_fmts = rk3288_vpu_dec_fmts, + .num_dec_fmts = ARRAY_SIZE(rk3288_vpu_dec_fmts), + .postproc_fmts = rockchip_vpu1_postproc_fmts, + .num_postproc_fmts = ARRAY_SIZE(rockchip_vpu1_postproc_fmts), + .postproc_regs = &hantro_g1_postproc_regs, + .codec = HANTRO_JPEG_ENCODER | HANTRO_MPEG2_DECODER | + HANTRO_VP8_DECODER | HANTRO_H264_DECODER, + .codec_ops = rk3288_vpu_codec_ops, + .irqs = rockchip_vpu1_irqs, + .num_irqs = ARRAY_SIZE(rockchip_vpu1_irqs), + .init = rockchip_vpu_hw_init, + .clk_names = rockchip_vpu_clk_names, + .num_clocks = ARRAY_SIZE(rockchip_vpu_clk_names) +}; + +const struct hantro_variant rk3328_vpu_variant = { + .dec_offset = 0x400, + .dec_fmts = rk3399_vpu_dec_fmts, + .num_dec_fmts = ARRAY_SIZE(rk3399_vpu_dec_fmts), + .codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER, + .codec_ops = rk3399_vpu_codec_ops, + .irqs = rockchip_vdpu2_irqs, + .num_irqs = ARRAY_SIZE(rockchip_vdpu2_irqs), + .init = rockchip_vpu_hw_init, + .clk_names = rockchip_vpu_clk_names, + .num_clocks = ARRAY_SIZE(rockchip_vpu_clk_names), +}; + +const struct hantro_variant rk3399_vpu_variant = { + .enc_offset = 0x0, + .enc_fmts = rockchip_vpu_enc_fmts, + .num_enc_fmts = ARRAY_SIZE(rockchip_vpu_enc_fmts), + .dec_offset = 0x400, + .dec_fmts = rk3399_vpu_dec_fmts, + .num_dec_fmts = ARRAY_SIZE(rk3399_vpu_dec_fmts), + .codec = HANTRO_JPEG_ENCODER | HANTRO_MPEG2_DECODER | + HANTRO_VP8_DECODER, + .codec_ops = rk3399_vpu_codec_ops, + .irqs = rockchip_vpu2_irqs, + .num_irqs = ARRAY_SIZE(rockchip_vpu2_irqs), + .init = rockchip_vpu_hw_init, + .clk_names = rockchip_vpu_clk_names, + .num_clocks = ARRAY_SIZE(rockchip_vpu_clk_names) +}; diff --git a/drivers/staging/media/hantro/sama5d4_vdec_hw.c b/drivers/staging/media/hantro/sama5d4_vdec_hw.c new file mode 100644 index 000000000000..58ae72c2b723 --- /dev/null +++ b/drivers/staging/media/hantro/sama5d4_vdec_hw.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Hantro VDEC driver + * + * Copyright (C) 2021 Collabora Ltd, Emil Velikov + */ + +#include "hantro.h" + +/* + * Supported formats. + */ + +static const struct hantro_fmt sama5d4_vdec_postproc_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_YUYV, + .codec_mode = HANTRO_MODE_NONE, + }, +}; + +static const struct hantro_fmt sama5d4_vdec_fmts[] = { + { + .fourcc = V4L2_PIX_FMT_NV12, + .codec_mode = HANTRO_MODE_NONE, + }, + { + .fourcc = V4L2_PIX_FMT_MPEG2_SLICE, + .codec_mode = HANTRO_MODE_MPEG2_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1280, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 720, + .step_height = MB_DIM, + }, + }, + { + .fourcc = V4L2_PIX_FMT_VP8_FRAME, + .codec_mode = HANTRO_MODE_VP8_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1280, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 720, + .step_height = MB_DIM, + }, + }, + { + .fourcc = V4L2_PIX_FMT_H264_SLICE, + .codec_mode = HANTRO_MODE_H264_DEC, + .max_depth = 2, + .frmsize = { + .min_width = 48, + .max_width = 1280, + .step_width = MB_DIM, + .min_height = 48, + .max_height = 720, + .step_height = MB_DIM, + }, + }, +}; + +static int sama5d4_hw_init(struct hantro_dev *vpu) +{ + return 0; +} + +/* + * Supported codec ops. + */ + +static const struct hantro_codec_ops sama5d4_vdec_codec_ops[] = { + [HANTRO_MODE_MPEG2_DEC] = { + .run = hantro_g1_mpeg2_dec_run, + .reset = hantro_g1_reset, + .init = hantro_mpeg2_dec_init, + .exit = hantro_mpeg2_dec_exit, + }, + [HANTRO_MODE_VP8_DEC] = { + .run = hantro_g1_vp8_dec_run, + .reset = hantro_g1_reset, + .init = hantro_vp8_dec_init, + .exit = hantro_vp8_dec_exit, + }, + [HANTRO_MODE_H264_DEC] = { + .run = hantro_g1_h264_dec_run, + .reset = hantro_g1_reset, + .init = hantro_h264_dec_init, + .exit = hantro_h264_dec_exit, + }, +}; + +static const struct hantro_irq sama5d4_irqs[] = { + { "vdec", hantro_g1_irq }, +}; + +static const char * const sama5d4_clk_names[] = { "vdec_clk" }; + +const struct hantro_variant sama5d4_vdec_variant = { + .dec_fmts = sama5d4_vdec_fmts, + .num_dec_fmts = ARRAY_SIZE(sama5d4_vdec_fmts), + .postproc_fmts = sama5d4_vdec_postproc_fmts, + .num_postproc_fmts = ARRAY_SIZE(sama5d4_vdec_postproc_fmts), + .postproc_regs = &hantro_g1_postproc_regs, + .codec = HANTRO_MPEG2_DECODER | HANTRO_VP8_DECODER | + HANTRO_H264_DECODER, + .codec_ops = sama5d4_vdec_codec_ops, + .init = sama5d4_hw_init, + .irqs = sama5d4_irqs, + .num_irqs = ARRAY_SIZE(sama5d4_irqs), + .clk_names = sama5d4_clk_names, + .num_clocks = ARRAY_SIZE(sama5d4_clk_names), +}; diff --git a/drivers/staging/media/imx/imx-ic-prp.c b/drivers/staging/media/imx/imx-ic-prp.c index f21ed881295f..ac5fb332088e 100644 --- a/drivers/staging/media/imx/imx-ic-prp.c +++ b/drivers/staging/media/imx/imx-ic-prp.c @@ -79,13 +79,13 @@ static void prp_stop(struct prp_priv *priv) } static struct v4l2_mbus_framefmt * -__prp_get_fmt(struct prp_priv *priv, struct v4l2_subdev_pad_config *cfg, +__prp_get_fmt(struct prp_priv *priv, struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { struct imx_ic_priv *ic_priv = priv->ic_priv; if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&ic_priv->sd, cfg, pad); + return v4l2_subdev_get_try_format(&ic_priv->sd, sd_state, pad); else return &priv->format_mbus; } @@ -95,7 +95,7 @@ __prp_get_fmt(struct prp_priv *priv, struct v4l2_subdev_pad_config *cfg, */ static int prp_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct prp_priv *priv = sd_to_priv(sd); @@ -115,7 +115,8 @@ static int prp_enum_mbus_code(struct v4l2_subdev *sd, ret = -EINVAL; goto out; } - infmt = __prp_get_fmt(priv, cfg, PRP_SINK_PAD, code->which); + infmt = __prp_get_fmt(priv, sd_state, PRP_SINK_PAD, + code->which); code->code = infmt->code; break; default: @@ -127,7 +128,7 @@ static int prp_enum_mbus_code(struct v4l2_subdev *sd, } static int prp_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct prp_priv *priv = sd_to_priv(sd); @@ -139,7 +140,7 @@ static int prp_get_fmt(struct v4l2_subdev *sd, mutex_lock(&priv->lock); - fmt = __prp_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __prp_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); if (!fmt) { ret = -EINVAL; goto out; @@ -152,7 +153,7 @@ static int prp_get_fmt(struct v4l2_subdev *sd, } static int prp_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct prp_priv *priv = sd_to_priv(sd); @@ -171,7 +172,7 @@ static int prp_set_fmt(struct v4l2_subdev *sd, goto out; } - infmt = __prp_get_fmt(priv, cfg, PRP_SINK_PAD, sdformat->which); + infmt = __prp_get_fmt(priv, sd_state, PRP_SINK_PAD, sdformat->which); switch (sdformat->pad) { case PRP_SINK_PAD: @@ -201,7 +202,7 @@ static int prp_set_fmt(struct v4l2_subdev *sd, imx_media_try_colorimetry(&sdformat->format, true); - fmt = __prp_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __prp_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); *fmt = sdformat->format; out: mutex_unlock(&priv->lock); diff --git a/drivers/staging/media/imx/imx-ic-prpencvf.c b/drivers/staging/media/imx/imx-ic-prpencvf.c index d990553de87b..9b81cfbcd777 100644 --- a/drivers/staging/media/imx/imx-ic-prpencvf.c +++ b/drivers/staging/media/imx/imx-ic-prpencvf.c @@ -787,13 +787,13 @@ static void prp_stop(struct prp_priv *priv) } static struct v4l2_mbus_framefmt * -__prp_get_fmt(struct prp_priv *priv, struct v4l2_subdev_pad_config *cfg, +__prp_get_fmt(struct prp_priv *priv, struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { struct imx_ic_priv *ic_priv = priv->ic_priv; if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&ic_priv->sd, cfg, pad); + return v4l2_subdev_get_try_format(&ic_priv->sd, sd_state, pad); else return &priv->format_mbus[pad]; } @@ -841,7 +841,7 @@ static bool prp_bound_align_output(struct v4l2_mbus_framefmt *outfmt, */ static int prp_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad >= PRPENCVF_NUM_PADS) @@ -852,7 +852,7 @@ static int prp_enum_mbus_code(struct v4l2_subdev *sd, } static int prp_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct prp_priv *priv = sd_to_priv(sd); @@ -864,7 +864,7 @@ static int prp_get_fmt(struct v4l2_subdev *sd, mutex_lock(&priv->lock); - fmt = __prp_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __prp_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); if (!fmt) { ret = -EINVAL; goto out; @@ -877,7 +877,7 @@ static int prp_get_fmt(struct v4l2_subdev *sd, } static void prp_try_fmt(struct prp_priv *priv, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat, const struct imx_media_pixfmt **cc) { @@ -894,7 +894,8 @@ static void prp_try_fmt(struct prp_priv *priv, sdformat->format.code = (*cc)->codes[0]; } - infmt = __prp_get_fmt(priv, cfg, PRPENCVF_SINK_PAD, sdformat->which); + infmt = __prp_get_fmt(priv, sd_state, PRPENCVF_SINK_PAD, + sdformat->which); if (sdformat->pad == PRPENCVF_SRC_PAD) { sdformat->format.field = infmt->field; @@ -920,7 +921,7 @@ static void prp_try_fmt(struct prp_priv *priv, } static int prp_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct prp_priv *priv = sd_to_priv(sd); @@ -938,9 +939,9 @@ static int prp_set_fmt(struct v4l2_subdev *sd, goto out; } - prp_try_fmt(priv, cfg, sdformat, &cc); + prp_try_fmt(priv, sd_state, sdformat, &cc); - fmt = __prp_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __prp_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); *fmt = sdformat->format; /* propagate a default format to source pad */ @@ -952,9 +953,9 @@ static int prp_set_fmt(struct v4l2_subdev *sd, format.pad = PRPENCVF_SRC_PAD; format.which = sdformat->which; format.format = sdformat->format; - prp_try_fmt(priv, cfg, &format, &outcc); + prp_try_fmt(priv, sd_state, &format, &outcc); - outfmt = __prp_get_fmt(priv, cfg, PRPENCVF_SRC_PAD, + outfmt = __prp_get_fmt(priv, sd_state, PRPENCVF_SRC_PAD, sdformat->which); *outfmt = format.format; if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) @@ -970,7 +971,7 @@ static int prp_set_fmt(struct v4l2_subdev *sd, } static int prp_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct prp_priv *priv = sd_to_priv(sd); @@ -988,7 +989,7 @@ static int prp_enum_frame_size(struct v4l2_subdev *sd, format.format.code = fse->code; format.format.width = 1; format.format.height = 1; - prp_try_fmt(priv, cfg, &format, &cc); + prp_try_fmt(priv, sd_state, &format, &cc); fse->min_width = format.format.width; fse->min_height = format.format.height; @@ -1000,7 +1001,7 @@ static int prp_enum_frame_size(struct v4l2_subdev *sd, format.format.code = fse->code; format.format.width = -1; format.format.height = -1; - prp_try_fmt(priv, cfg, &format, &cc); + prp_try_fmt(priv, sd_state, &format, &cc); fse->max_width = format.format.width; fse->max_height = format.format.height; out: diff --git a/drivers/staging/media/imx/imx-media-csi.c b/drivers/staging/media/imx/imx-media-csi.c index e3bfd635a89a..bb1305c9daaf 100644 --- a/drivers/staging/media/imx/imx-media-csi.c +++ b/drivers/staging/media/imx/imx-media-csi.c @@ -750,9 +750,10 @@ static int csi_setup(struct csi_priv *priv) static int csi_start(struct csi_priv *priv) { - struct v4l2_fract *output_fi; + struct v4l2_fract *input_fi, *output_fi; int ret; + input_fi = &priv->frame_interval[CSI_SINK_PAD]; output_fi = &priv->frame_interval[priv->active_output_pad]; /* start upstream */ @@ -761,6 +762,17 @@ static int csi_start(struct csi_priv *priv) if (ret) return ret; + /* Skip first few frames from a BT.656 source */ + if (priv->upstream_ep.bus_type == V4L2_MBUS_BT656) { + u32 delay_usec, bad_frames = 20; + + delay_usec = DIV_ROUND_UP_ULL((u64)USEC_PER_SEC * + input_fi->numerator * bad_frames, + input_fi->denominator); + + usleep_range(delay_usec, delay_usec + 1000); + } + if (priv->dest == IPU_CSI_DEST_IDMAC) { ret = csi_idmac_start(priv); if (ret) @@ -1139,31 +1151,32 @@ static int csi_link_validate(struct v4l2_subdev *sd, } static struct v4l2_mbus_framefmt * -__csi_get_fmt(struct csi_priv *priv, struct v4l2_subdev_pad_config *cfg, +__csi_get_fmt(struct csi_priv *priv, struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&priv->sd, cfg, pad); + return v4l2_subdev_get_try_format(&priv->sd, sd_state, pad); else return &priv->format_mbus[pad]; } static struct v4l2_rect * -__csi_get_crop(struct csi_priv *priv, struct v4l2_subdev_pad_config *cfg, +__csi_get_crop(struct csi_priv *priv, struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_crop(&priv->sd, cfg, CSI_SINK_PAD); + return v4l2_subdev_get_try_crop(&priv->sd, sd_state, + CSI_SINK_PAD); else return &priv->crop; } static struct v4l2_rect * -__csi_get_compose(struct csi_priv *priv, struct v4l2_subdev_pad_config *cfg, +__csi_get_compose(struct csi_priv *priv, struct v4l2_subdev_state *sd_state, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_compose(&priv->sd, cfg, + return v4l2_subdev_get_try_compose(&priv->sd, sd_state, CSI_SINK_PAD); else return &priv->compose; @@ -1171,7 +1184,7 @@ __csi_get_compose(struct csi_priv *priv, struct v4l2_subdev_pad_config *cfg, static void csi_try_crop(struct csi_priv *priv, struct v4l2_rect *crop, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_mbus_framefmt *infmt, struct v4l2_fwnode_endpoint *upstream_ep) { @@ -1210,7 +1223,7 @@ static void csi_try_crop(struct csi_priv *priv, } static int csi_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct csi_priv *priv = v4l2_get_subdevdata(sd); @@ -1221,7 +1234,7 @@ static int csi_enum_mbus_code(struct v4l2_subdev *sd, mutex_lock(&priv->lock); - infmt = __csi_get_fmt(priv, cfg, CSI_SINK_PAD, code->which); + infmt = __csi_get_fmt(priv, sd_state, CSI_SINK_PAD, code->which); incc = imx_media_find_mbus_format(infmt->code, PIXFMT_SEL_ANY); switch (code->pad) { @@ -1263,7 +1276,7 @@ static int csi_enum_mbus_code(struct v4l2_subdev *sd, } static int csi_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct csi_priv *priv = v4l2_get_subdevdata(sd); @@ -1282,7 +1295,7 @@ static int csi_enum_frame_size(struct v4l2_subdev *sd, fse->min_height = MIN_H; fse->max_height = MAX_H; } else { - crop = __csi_get_crop(priv, cfg, fse->which); + crop = __csi_get_crop(priv, sd_state, fse->which); fse->min_width = fse->index & 1 ? crop->width / 2 : crop->width; @@ -1297,7 +1310,7 @@ static int csi_enum_frame_size(struct v4l2_subdev *sd, } static int csi_enum_frame_interval(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct csi_priv *priv = v4l2_get_subdevdata(sd); @@ -1313,7 +1326,7 @@ static int csi_enum_frame_interval(struct v4l2_subdev *sd, mutex_lock(&priv->lock); input_fi = &priv->frame_interval[CSI_SINK_PAD]; - crop = __csi_get_crop(priv, cfg, fie->which); + crop = __csi_get_crop(priv, sd_state, fie->which); if ((fie->width != crop->width && fie->width != crop->width / 2) || (fie->height != crop->height && fie->height != crop->height / 2)) { @@ -1333,7 +1346,7 @@ static int csi_enum_frame_interval(struct v4l2_subdev *sd, } static int csi_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct csi_priv *priv = v4l2_get_subdevdata(sd); @@ -1345,7 +1358,7 @@ static int csi_get_fmt(struct v4l2_subdev *sd, mutex_lock(&priv->lock); - fmt = __csi_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __csi_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); if (!fmt) { ret = -EINVAL; goto out; @@ -1358,11 +1371,11 @@ static int csi_get_fmt(struct v4l2_subdev *sd, } static void csi_try_field(struct csi_priv *priv, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct v4l2_mbus_framefmt *infmt = - __csi_get_fmt(priv, cfg, CSI_SINK_PAD, sdformat->which); + __csi_get_fmt(priv, sd_state, CSI_SINK_PAD, sdformat->which); /* * no restrictions on sink pad field type except must @@ -1408,7 +1421,7 @@ static void csi_try_field(struct csi_priv *priv, static void csi_try_fmt(struct csi_priv *priv, struct v4l2_fwnode_endpoint *upstream_ep, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat, struct v4l2_rect *crop, struct v4l2_rect *compose, @@ -1418,7 +1431,7 @@ static void csi_try_fmt(struct csi_priv *priv, struct v4l2_mbus_framefmt *infmt; u32 code; - infmt = __csi_get_fmt(priv, cfg, CSI_SINK_PAD, sdformat->which); + infmt = __csi_get_fmt(priv, sd_state, CSI_SINK_PAD, sdformat->which); switch (sdformat->pad) { case CSI_SRC_PAD_DIRECT: @@ -1445,7 +1458,7 @@ static void csi_try_fmt(struct csi_priv *priv, } } - csi_try_field(priv, cfg, sdformat); + csi_try_field(priv, sd_state, sdformat); /* propagate colorimetry from sink */ sdformat->format.colorspace = infmt->colorspace; @@ -1469,7 +1482,7 @@ static void csi_try_fmt(struct csi_priv *priv, sdformat->format.code = (*cc)->codes[0]; } - csi_try_field(priv, cfg, sdformat); + csi_try_field(priv, sd_state, sdformat); /* Reset crop and compose rectangles */ crop->left = 0; @@ -1478,7 +1491,8 @@ static void csi_try_fmt(struct csi_priv *priv, crop->height = sdformat->format.height; if (sdformat->format.field == V4L2_FIELD_ALTERNATE) crop->height *= 2; - csi_try_crop(priv, crop, cfg, &sdformat->format, upstream_ep); + csi_try_crop(priv, crop, sd_state, &sdformat->format, + upstream_ep); compose->left = 0; compose->top = 0; compose->width = crop->width; @@ -1492,7 +1506,7 @@ static void csi_try_fmt(struct csi_priv *priv, } static int csi_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct csi_priv *priv = v4l2_get_subdevdata(sd); @@ -1518,12 +1532,13 @@ static int csi_set_fmt(struct v4l2_subdev *sd, goto out; } - crop = __csi_get_crop(priv, cfg, sdformat->which); - compose = __csi_get_compose(priv, cfg, sdformat->which); + crop = __csi_get_crop(priv, sd_state, sdformat->which); + compose = __csi_get_compose(priv, sd_state, sdformat->which); - csi_try_fmt(priv, &upstream_ep, cfg, sdformat, crop, compose, &cc); + csi_try_fmt(priv, &upstream_ep, sd_state, sdformat, crop, compose, + &cc); - fmt = __csi_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __csi_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); *fmt = sdformat->format; if (sdformat->pad == CSI_SINK_PAD) { @@ -1538,10 +1553,11 @@ static int csi_set_fmt(struct v4l2_subdev *sd, format.pad = pad; format.which = sdformat->which; format.format = sdformat->format; - csi_try_fmt(priv, &upstream_ep, cfg, &format, + csi_try_fmt(priv, &upstream_ep, sd_state, &format, NULL, compose, &outcc); - outfmt = __csi_get_fmt(priv, cfg, pad, sdformat->which); + outfmt = __csi_get_fmt(priv, sd_state, pad, + sdformat->which); *outfmt = format.format; if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) @@ -1558,7 +1574,7 @@ static int csi_set_fmt(struct v4l2_subdev *sd, } static int csi_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct csi_priv *priv = v4l2_get_subdevdata(sd); @@ -1571,9 +1587,9 @@ static int csi_get_selection(struct v4l2_subdev *sd, mutex_lock(&priv->lock); - infmt = __csi_get_fmt(priv, cfg, CSI_SINK_PAD, sel->which); - crop = __csi_get_crop(priv, cfg, sel->which); - compose = __csi_get_compose(priv, cfg, sel->which); + infmt = __csi_get_fmt(priv, sd_state, CSI_SINK_PAD, sel->which); + crop = __csi_get_crop(priv, sd_state, sel->which); + compose = __csi_get_compose(priv, sd_state, sel->which); switch (sel->target) { case V4L2_SEL_TGT_CROP_BOUNDS: @@ -1622,7 +1638,7 @@ static int csi_set_scale(u32 *compose, u32 crop, u32 flags) } static int csi_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct csi_priv *priv = v4l2_get_subdevdata(sd); @@ -1647,9 +1663,9 @@ static int csi_set_selection(struct v4l2_subdev *sd, goto out; } - infmt = __csi_get_fmt(priv, cfg, CSI_SINK_PAD, sel->which); - crop = __csi_get_crop(priv, cfg, sel->which); - compose = __csi_get_compose(priv, cfg, sel->which); + infmt = __csi_get_fmt(priv, sd_state, CSI_SINK_PAD, sel->which); + crop = __csi_get_crop(priv, sd_state, sel->which); + compose = __csi_get_compose(priv, sd_state, sel->which); switch (sel->target) { case V4L2_SEL_TGT_CROP: @@ -1665,7 +1681,7 @@ static int csi_set_selection(struct v4l2_subdev *sd, goto out; } - csi_try_crop(priv, &sel->r, cfg, infmt, &upstream_ep); + csi_try_crop(priv, &sel->r, sd_state, infmt, &upstream_ep); *crop = sel->r; @@ -1706,7 +1722,7 @@ static int csi_set_selection(struct v4l2_subdev *sd, for (pad = CSI_SINK_PAD + 1; pad < CSI_NUM_PADS; pad++) { struct v4l2_mbus_framefmt *outfmt; - outfmt = __csi_get_fmt(priv, cfg, pad, sel->which); + outfmt = __csi_get_fmt(priv, sd_state, pad, sel->which); outfmt->width = compose->width; outfmt->height = compose->height; } diff --git a/drivers/staging/media/imx/imx-media-utils.c b/drivers/staging/media/imx/imx-media-utils.c index 5128915a5d6f..6f90acf9c725 100644 --- a/drivers/staging/media/imx/imx-media-utils.c +++ b/drivers/staging/media/imx/imx-media-utils.c @@ -429,7 +429,7 @@ EXPORT_SYMBOL_GPL(imx_media_init_mbus_fmt); * of a subdev. Can be used as the .init_cfg pad operation. */ int imx_media_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct v4l2_mbus_framefmt *mf_try; struct v4l2_subdev_format format; @@ -445,7 +445,7 @@ int imx_media_init_cfg(struct v4l2_subdev *sd, if (ret) continue; - mf_try = v4l2_subdev_get_try_format(sd, cfg, pad); + mf_try = v4l2_subdev_get_try_format(sd, sd_state, pad); *mf_try = format.format; } diff --git a/drivers/staging/media/imx/imx-media-vdic.c b/drivers/staging/media/imx/imx-media-vdic.c index abf290bda98d..3c2093c520ba 100644 --- a/drivers/staging/media/imx/imx-media-vdic.c +++ b/drivers/staging/media/imx/imx-media-vdic.c @@ -532,17 +532,17 @@ static int vdic_s_stream(struct v4l2_subdev *sd, int enable) } static struct v4l2_mbus_framefmt * -__vdic_get_fmt(struct vdic_priv *priv, struct v4l2_subdev_pad_config *cfg, +__vdic_get_fmt(struct vdic_priv *priv, struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&priv->sd, cfg, pad); + return v4l2_subdev_get_try_format(&priv->sd, sd_state, pad); else return &priv->format_mbus[pad]; } static int vdic_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (code->pad >= VDIC_NUM_PADS) @@ -553,7 +553,7 @@ static int vdic_enum_mbus_code(struct v4l2_subdev *sd, } static int vdic_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct vdic_priv *priv = v4l2_get_subdevdata(sd); @@ -565,7 +565,7 @@ static int vdic_get_fmt(struct v4l2_subdev *sd, mutex_lock(&priv->lock); - fmt = __vdic_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __vdic_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); if (!fmt) { ret = -EINVAL; goto out; @@ -578,7 +578,7 @@ static int vdic_get_fmt(struct v4l2_subdev *sd, } static void vdic_try_fmt(struct vdic_priv *priv, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat, const struct imx_media_pixfmt **cc) { @@ -594,7 +594,7 @@ static void vdic_try_fmt(struct vdic_priv *priv, sdformat->format.code = (*cc)->codes[0]; } - infmt = __vdic_get_fmt(priv, cfg, priv->active_input_pad, + infmt = __vdic_get_fmt(priv, sd_state, priv->active_input_pad, sdformat->which); switch (sdformat->pad) { @@ -620,7 +620,7 @@ static void vdic_try_fmt(struct vdic_priv *priv, } static int vdic_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct vdic_priv *priv = v4l2_get_subdevdata(sd); @@ -638,9 +638,9 @@ static int vdic_set_fmt(struct v4l2_subdev *sd, goto out; } - vdic_try_fmt(priv, cfg, sdformat, &cc); + vdic_try_fmt(priv, sd_state, sdformat, &cc); - fmt = __vdic_get_fmt(priv, cfg, sdformat->pad, sdformat->which); + fmt = __vdic_get_fmt(priv, sd_state, sdformat->pad, sdformat->which); *fmt = sdformat->format; /* propagate format to source pad */ @@ -653,9 +653,9 @@ static int vdic_set_fmt(struct v4l2_subdev *sd, format.pad = VDIC_SRC_PAD_DIRECT; format.which = sdformat->which; format.format = sdformat->format; - vdic_try_fmt(priv, cfg, &format, &outcc); + vdic_try_fmt(priv, sd_state, &format, &outcc); - outfmt = __vdic_get_fmt(priv, cfg, VDIC_SRC_PAD_DIRECT, + outfmt = __vdic_get_fmt(priv, sd_state, VDIC_SRC_PAD_DIRECT, sdformat->which); *outfmt = format.format; if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) diff --git a/drivers/staging/media/imx/imx-media.h b/drivers/staging/media/imx/imx-media.h index 492d9a64e704..6740e7917458 100644 --- a/drivers/staging/media/imx/imx-media.h +++ b/drivers/staging/media/imx/imx-media.h @@ -193,7 +193,7 @@ int imx_media_init_mbus_fmt(struct v4l2_mbus_framefmt *mbus, u32 width, u32 height, u32 code, u32 field, const struct imx_media_pixfmt **cc); int imx_media_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg); + struct v4l2_subdev_state *sd_state); void imx_media_try_colorimetry(struct v4l2_mbus_framefmt *tryfmt, bool ic_route); int imx_media_mbus_fmt_to_pix_fmt(struct v4l2_pix_format *pix, diff --git a/drivers/staging/media/imx/imx6-mipi-csi2.c b/drivers/staging/media/imx/imx6-mipi-csi2.c index fc2378ac04b7..9de0ebd439dc 100644 --- a/drivers/staging/media/imx/imx6-mipi-csi2.c +++ b/drivers/staging/media/imx/imx6-mipi-csi2.c @@ -508,17 +508,17 @@ static int csi2_link_setup(struct media_entity *entity, } static struct v4l2_mbus_framefmt * -__csi2_get_fmt(struct csi2_dev *csi2, struct v4l2_subdev_pad_config *cfg, +__csi2_get_fmt(struct csi2_dev *csi2, struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&csi2->sd, cfg, pad); + return v4l2_subdev_get_try_format(&csi2->sd, sd_state, pad); else return &csi2->format_mbus; } static int csi2_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct csi2_dev *csi2 = sd_to_dev(sd); @@ -526,7 +526,7 @@ static int csi2_get_fmt(struct v4l2_subdev *sd, mutex_lock(&csi2->lock); - fmt = __csi2_get_fmt(csi2, cfg, sdformat->pad, sdformat->which); + fmt = __csi2_get_fmt(csi2, sd_state, sdformat->pad, sdformat->which); sdformat->format = *fmt; @@ -536,7 +536,7 @@ static int csi2_get_fmt(struct v4l2_subdev *sd, } static int csi2_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct csi2_dev *csi2 = sd_to_dev(sd); @@ -557,7 +557,7 @@ static int csi2_set_fmt(struct v4l2_subdev *sd, if (sdformat->pad != CSI2_SINK_PAD) sdformat->format = csi2->format_mbus; - fmt = __csi2_get_fmt(csi2, cfg, sdformat->pad, sdformat->which); + fmt = __csi2_get_fmt(csi2, sd_state, sdformat->pad, sdformat->which); *fmt = sdformat->format; out: diff --git a/drivers/staging/media/imx/imx7-media-csi.c b/drivers/staging/media/imx/imx7-media-csi.c index f85a2f5f1413..894c4de31790 100644 --- a/drivers/staging/media/imx/imx7-media-csi.c +++ b/drivers/staging/media/imx/imx7-media-csi.c @@ -724,7 +724,7 @@ static int imx7_csi_s_stream(struct v4l2_subdev *sd, int enable) } static int imx7_csi_init_cfg(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg) + struct v4l2_subdev_state *sd_state) { struct imx7_csi *csi = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *mf; @@ -732,7 +732,7 @@ static int imx7_csi_init_cfg(struct v4l2_subdev *sd, int i; for (i = 0; i < IMX7_CSI_PADS_NUM; i++) { - mf = v4l2_subdev_get_try_format(sd, cfg, i); + mf = v4l2_subdev_get_try_format(sd, sd_state, i); ret = imx_media_init_mbus_fmt(mf, 800, 600, 0, V4L2_FIELD_NONE, &csi->cc[i]); @@ -745,18 +745,18 @@ static int imx7_csi_init_cfg(struct v4l2_subdev *sd, static struct v4l2_mbus_framefmt * imx7_csi_get_format(struct imx7_csi *csi, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&csi->sd, cfg, pad); + return v4l2_subdev_get_try_format(&csi->sd, sd_state, pad); return &csi->format_mbus[pad]; } static int imx7_csi_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct imx7_csi *csi = v4l2_get_subdevdata(sd); @@ -765,7 +765,8 @@ static int imx7_csi_enum_mbus_code(struct v4l2_subdev *sd, mutex_lock(&csi->lock); - in_fmt = imx7_csi_get_format(csi, cfg, IMX7_CSI_PAD_SINK, code->which); + in_fmt = imx7_csi_get_format(csi, sd_state, IMX7_CSI_PAD_SINK, + code->which); switch (code->pad) { case IMX7_CSI_PAD_SINK: @@ -791,7 +792,7 @@ static int imx7_csi_enum_mbus_code(struct v4l2_subdev *sd, } static int imx7_csi_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct imx7_csi *csi = v4l2_get_subdevdata(sd); @@ -800,7 +801,8 @@ static int imx7_csi_get_fmt(struct v4l2_subdev *sd, mutex_lock(&csi->lock); - fmt = imx7_csi_get_format(csi, cfg, sdformat->pad, sdformat->which); + fmt = imx7_csi_get_format(csi, sd_state, sdformat->pad, + sdformat->which); if (!fmt) { ret = -EINVAL; goto out_unlock; @@ -815,7 +817,7 @@ static int imx7_csi_get_fmt(struct v4l2_subdev *sd, } static int imx7_csi_try_fmt(struct imx7_csi *csi, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat, const struct imx_media_pixfmt **cc) { @@ -823,7 +825,7 @@ static int imx7_csi_try_fmt(struct imx7_csi *csi, struct v4l2_mbus_framefmt *in_fmt; u32 code; - in_fmt = imx7_csi_get_format(csi, cfg, IMX7_CSI_PAD_SINK, + in_fmt = imx7_csi_get_format(csi, sd_state, IMX7_CSI_PAD_SINK, sdformat->which); if (!in_fmt) return -EINVAL; @@ -868,7 +870,7 @@ static int imx7_csi_try_fmt(struct imx7_csi *csi, } static int imx7_csi_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *sdformat) { struct imx7_csi *csi = v4l2_get_subdevdata(sd); @@ -889,11 +891,12 @@ static int imx7_csi_set_fmt(struct v4l2_subdev *sd, goto out_unlock; } - ret = imx7_csi_try_fmt(csi, cfg, sdformat, &cc); + ret = imx7_csi_try_fmt(csi, sd_state, sdformat, &cc); if (ret < 0) goto out_unlock; - fmt = imx7_csi_get_format(csi, cfg, sdformat->pad, sdformat->which); + fmt = imx7_csi_get_format(csi, sd_state, sdformat->pad, + sdformat->which); if (!fmt) { ret = -EINVAL; goto out_unlock; @@ -906,11 +909,11 @@ static int imx7_csi_set_fmt(struct v4l2_subdev *sd, format.pad = IMX7_CSI_PAD_SRC; format.which = sdformat->which; format.format = sdformat->format; - if (imx7_csi_try_fmt(csi, cfg, &format, &outcc)) { + if (imx7_csi_try_fmt(csi, sd_state, &format, &outcc)) { ret = -EINVAL; goto out_unlock; } - outfmt = imx7_csi_get_format(csi, cfg, IMX7_CSI_PAD_SRC, + outfmt = imx7_csi_get_format(csi, sd_state, IMX7_CSI_PAD_SRC, sdformat->which); *outfmt = format.format; diff --git a/drivers/staging/media/imx/imx7-mipi-csis.c b/drivers/staging/media/imx/imx7-mipi-csis.c index 025fdc488bd6..ead696eb4610 100644 --- a/drivers/staging/media/imx/imx7-mipi-csis.c +++ b/drivers/staging/media/imx/imx7-mipi-csis.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -166,15 +167,11 @@ #define MIPI_CSIS_ISP_CONFIG_CH(n) (0x40 + (n) * 0x10) #define MIPI_CSIS_ISPCFG_MEM_FULL_GAP_MSK (0xff << 24) #define MIPI_CSIS_ISPCFG_MEM_FULL_GAP(x) ((x) << 24) -#define MIPI_CSIS_ISPCFG_DOUBLE_CMPNT BIT(12) +#define MIPI_CSIS_ISPCFG_PIXEL_MODE_SINGLE (0 << 12) +#define MIPI_CSIS_ISPCFG_PIXEL_MODE_DUAL (1 << 12) +#define MIPI_CSIS_ISPCFG_PIXEL_MODE_QUAD (2 << 12) /* i.MX8M[MNP] only */ #define MIPI_CSIS_ISPCFG_ALIGN_32BIT BIT(11) -#define MIPI_CSIS_ISPCFG_FMT_YCBCR422_8BIT (0x1e << 2) -#define MIPI_CSIS_ISPCFG_FMT_RAW8 (0x2a << 2) -#define MIPI_CSIS_ISPCFG_FMT_RAW10 (0x2b << 2) -#define MIPI_CSIS_ISPCFG_FMT_RAW12 (0x2c << 2) -#define MIPI_CSIS_ISPCFG_FMT_RAW14 (0x2d << 2) -/* User defined formats, x = 1...4 */ -#define MIPI_CSIS_ISPCFG_FMT_USER(x) ((0x30 + (x) - 1) << 2) +#define MIPI_CSIS_ISPCFG_FMT(fmt) ((fmt) << 2) #define MIPI_CSIS_ISPCFG_FMT_MASK (0x3f << 2) /* ISP Image Resolution register */ @@ -195,6 +192,24 @@ /* Debug control register */ #define MIPI_CSIS_DBG_CTRL 0xc0 +#define MIPI_CSIS_DBG_INTR_MSK 0xc4 +#define MIPI_CSIS_DBG_INTR_MSK_DT_NOT_SUPPORT BIT(25) +#define MIPI_CSIS_DBG_INTR_MSK_DT_IGNORE BIT(24) +#define MIPI_CSIS_DBG_INTR_MSK_ERR_FRAME_SIZE BIT(20) +#define MIPI_CSIS_DBG_INTR_MSK_TRUNCATED_FRAME BIT(16) +#define MIPI_CSIS_DBG_INTR_MSK_EARLY_FE BIT(12) +#define MIPI_CSIS_DBG_INTR_MSK_EARLY_FS BIT(8) +#define MIPI_CSIS_DBG_INTR_MSK_CAM_VSYNC_FALL BIT(4) +#define MIPI_CSIS_DBG_INTR_MSK_CAM_VSYNC_RISE BIT(0) +#define MIPI_CSIS_DBG_INTR_SRC 0xc8 +#define MIPI_CSIS_DBG_INTR_SRC_DT_NOT_SUPPORT BIT(25) +#define MIPI_CSIS_DBG_INTR_SRC_DT_IGNORE BIT(24) +#define MIPI_CSIS_DBG_INTR_SRC_ERR_FRAME_SIZE BIT(20) +#define MIPI_CSIS_DBG_INTR_SRC_TRUNCATED_FRAME BIT(16) +#define MIPI_CSIS_DBG_INTR_SRC_EARLY_FE BIT(12) +#define MIPI_CSIS_DBG_INTR_SRC_EARLY_FS BIT(8) +#define MIPI_CSIS_DBG_INTR_SRC_CAM_VSYNC_FALL BIT(4) +#define MIPI_CSIS_DBG_INTR_SRC_CAM_VSYNC_RISE BIT(0) /* Non-image packet data buffers */ #define MIPI_CSIS_PKTDATA_ODD 0x2000 @@ -203,6 +218,25 @@ #define DEFAULT_SCLK_CSIS_FREQ 166000000UL +/* MIPI CSI-2 Data Types */ +#define MIPI_CSI2_DATA_TYPE_YUV420_8 0x18 +#define MIPI_CSI2_DATA_TYPE_YUV420_10 0x19 +#define MIPI_CSI2_DATA_TYPE_LE_YUV420_8 0x1a +#define MIPI_CSI2_DATA_TYPE_CS_YUV420_8 0x1c +#define MIPI_CSI2_DATA_TYPE_CS_YUV420_10 0x1d +#define MIPI_CSI2_DATA_TYPE_YUV422_8 0x1e +#define MIPI_CSI2_DATA_TYPE_YUV422_10 0x1f +#define MIPI_CSI2_DATA_TYPE_RGB565 0x22 +#define MIPI_CSI2_DATA_TYPE_RGB666 0x23 +#define MIPI_CSI2_DATA_TYPE_RGB888 0x24 +#define MIPI_CSI2_DATA_TYPE_RAW6 0x28 +#define MIPI_CSI2_DATA_TYPE_RAW7 0x29 +#define MIPI_CSI2_DATA_TYPE_RAW8 0x2a +#define MIPI_CSI2_DATA_TYPE_RAW10 0x2b +#define MIPI_CSI2_DATA_TYPE_RAW12 0x2c +#define MIPI_CSI2_DATA_TYPE_RAW14 0x2d +#define MIPI_CSI2_DATA_TYPE_USER(x) (0x30 + (x)) + enum { ST_POWERED = 1, ST_STREAMING = 2, @@ -210,6 +244,7 @@ enum { }; struct mipi_csis_event { + bool debug; u32 mask; const char * const name; unsigned int counter; @@ -217,22 +252,30 @@ struct mipi_csis_event { static const struct mipi_csis_event mipi_csis_events[] = { /* Errors */ - { MIPI_CSIS_INT_SRC_ERR_SOT_HS, "SOT Error" }, - { MIPI_CSIS_INT_SRC_ERR_LOST_FS, "Lost Frame Start Error" }, - { MIPI_CSIS_INT_SRC_ERR_LOST_FE, "Lost Frame End Error" }, - { MIPI_CSIS_INT_SRC_ERR_OVER, "FIFO Overflow Error" }, - { MIPI_CSIS_INT_SRC_ERR_WRONG_CFG, "Wrong Configuration Error" }, - { MIPI_CSIS_INT_SRC_ERR_ECC, "ECC Error" }, - { MIPI_CSIS_INT_SRC_ERR_CRC, "CRC Error" }, - { MIPI_CSIS_INT_SRC_ERR_UNKNOWN, "Unknown Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_SOT_HS, "SOT Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_LOST_FS, "Lost Frame Start Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_LOST_FE, "Lost Frame End Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_OVER, "FIFO Overflow Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_WRONG_CFG, "Wrong Configuration Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_ECC, "ECC Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_CRC, "CRC Error" }, + { false, MIPI_CSIS_INT_SRC_ERR_UNKNOWN, "Unknown Error" }, + { true, MIPI_CSIS_DBG_INTR_SRC_DT_NOT_SUPPORT, "Data Type Not Supported" }, + { true, MIPI_CSIS_DBG_INTR_SRC_DT_IGNORE, "Data Type Ignored" }, + { true, MIPI_CSIS_DBG_INTR_SRC_ERR_FRAME_SIZE, "Frame Size Error" }, + { true, MIPI_CSIS_DBG_INTR_SRC_TRUNCATED_FRAME, "Truncated Frame" }, + { true, MIPI_CSIS_DBG_INTR_SRC_EARLY_FE, "Early Frame End" }, + { true, MIPI_CSIS_DBG_INTR_SRC_EARLY_FS, "Early Frame Start" }, /* Non-image data receive events */ - { MIPI_CSIS_INT_SRC_EVEN_BEFORE, "Non-image data before even frame" }, - { MIPI_CSIS_INT_SRC_EVEN_AFTER, "Non-image data after even frame" }, - { MIPI_CSIS_INT_SRC_ODD_BEFORE, "Non-image data before odd frame" }, - { MIPI_CSIS_INT_SRC_ODD_AFTER, "Non-image data after odd frame" }, + { false, MIPI_CSIS_INT_SRC_EVEN_BEFORE, "Non-image data before even frame" }, + { false, MIPI_CSIS_INT_SRC_EVEN_AFTER, "Non-image data after even frame" }, + { false, MIPI_CSIS_INT_SRC_ODD_BEFORE, "Non-image data before odd frame" }, + { false, MIPI_CSIS_INT_SRC_ODD_AFTER, "Non-image data after odd frame" }, /* Frame start/end */ - { MIPI_CSIS_INT_SRC_FRAME_START, "Frame Start" }, - { MIPI_CSIS_INT_SRC_FRAME_END, "Frame End" }, + { false, MIPI_CSIS_INT_SRC_FRAME_START, "Frame Start" }, + { false, MIPI_CSIS_INT_SRC_FRAME_END, "Frame End" }, + { true, MIPI_CSIS_DBG_INTR_SRC_CAM_VSYNC_FALL, "VSYNC Falling Edge" }, + { true, MIPI_CSIS_DBG_INTR_SRC_CAM_VSYNC_RISE, "VSYNC Rising Edge" }, }; #define MIPI_CSIS_NUM_EVENTS ARRAY_SIZE(mipi_csis_events) @@ -241,63 +284,63 @@ enum mipi_csis_clk { MIPI_CSIS_CLK_PCLK, MIPI_CSIS_CLK_WRAP, MIPI_CSIS_CLK_PHY, + MIPI_CSIS_CLK_AXI, }; static const char * const mipi_csis_clk_id[] = { "pclk", "wrap", "phy", + "axi", }; -struct csis_hw_reset { - struct regmap *src; - u8 req_src; - u8 rst_bit; +enum mipi_csis_version { + MIPI_CSIS_V3_3, + MIPI_CSIS_V3_6_3, +}; + +struct mipi_csis_info { + enum mipi_csis_version version; + unsigned int num_clocks; }; struct csi_state { - /* lock elements below */ - struct mutex lock; - /* lock for event handler */ - spinlock_t slock; struct device *dev; + void __iomem *regs; + struct clk_bulk_data *clks; + struct reset_control *mrst; + struct regulator *mipi_phy_regulator; + const struct mipi_csis_info *info; + u8 index; + + struct v4l2_subdev sd; struct media_pad pads[CSIS_PADS_NUM]; - struct v4l2_subdev mipi_sd; struct v4l2_async_notifier notifier; struct v4l2_subdev *src_sd; - u8 index; - struct platform_device *pdev; - struct phy *phy; - void __iomem *regs; - int irq; - u32 flags; - - struct dentry *debugfs_root; - bool debug; - - int num_clks; - struct clk_bulk_data *clks; - + struct v4l2_fwnode_bus_mipi_csi2 bus; u32 clk_frequency; u32 hs_settle; + u32 clk_settle; - struct reset_control *mrst; - + struct mutex lock; /* Protect csis_fmt, format_mbus and state */ const struct csis_pix_format *csis_fmt; struct v4l2_mbus_framefmt format_mbus; + u32 state; - struct v4l2_fwnode_bus_mipi_csi2 bus; - + spinlock_t slock; /* Protect events */ struct mipi_csis_event events[MIPI_CSIS_NUM_EVENTS]; - - struct csis_hw_reset hw_reset; - struct regulator *mipi_phy_regulator; + struct dentry *debugfs_root; + bool debug; }; +/* ----------------------------------------------------------------------------- + * Format helpers + */ + struct csis_pix_format { u32 code; - u32 fmt_reg; + u32 data_type; u8 width; }; @@ -305,143 +348,89 @@ static const struct csis_pix_format mipi_csis_formats[] = { /* YUV formats. */ { .code = MEDIA_BUS_FMT_UYVY8_1X16, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_YCBCR422_8BIT, + .data_type = MIPI_CSI2_DATA_TYPE_YUV422_8, .width = 16, }, /* RAW (Bayer and greyscale) formats. */ { .code = MEDIA_BUS_FMT_SBGGR8_1X8, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8, + .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SGBRG8_1X8, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8, + .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SGRBG8_1X8, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8, + .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SRGGB8_1X8, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8, + .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_Y8_1X8, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW8, + .data_type = MIPI_CSI2_DATA_TYPE_RAW8, .width = 8, }, { .code = MEDIA_BUS_FMT_SBGGR10_1X10, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW10, + .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SGBRG10_1X10, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW10, + .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SGRBG10_1X10, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW10, + .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SRGGB10_1X10, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW10, + .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_Y10_1X10, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW10, + .data_type = MIPI_CSI2_DATA_TYPE_RAW10, .width = 10, }, { .code = MEDIA_BUS_FMT_SBGGR12_1X12, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW12, + .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SGBRG12_1X12, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW12, + .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SGRBG12_1X12, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW12, + .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SRGGB12_1X12, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW12, + .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_Y12_1X12, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW12, + .data_type = MIPI_CSI2_DATA_TYPE_RAW12, .width = 12, }, { .code = MEDIA_BUS_FMT_SBGGR14_1X14, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW14, + .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, }, { .code = MEDIA_BUS_FMT_SGBRG14_1X14, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW14, + .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, }, { .code = MEDIA_BUS_FMT_SGRBG14_1X14, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW14, + .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, }, { .code = MEDIA_BUS_FMT_SRGGB14_1X14, - .fmt_reg = MIPI_CSIS_ISPCFG_FMT_RAW14, + .data_type = MIPI_CSI2_DATA_TYPE_RAW14, .width = 14, } }; -static inline void mipi_csis_write(struct csi_state *state, u32 reg, u32 val) -{ - writel(val, state->regs + reg); -} - -static inline u32 mipi_csis_read(struct csi_state *state, u32 reg) -{ - return readl(state->regs + reg); -} - -static int mipi_csis_dump_regs(struct csi_state *state) -{ - struct device *dev = &state->pdev->dev; - unsigned int i; - u32 cfg; - static const struct { - u32 offset; - const char * const name; - } registers[] = { - { MIPI_CSIS_CMN_CTRL, "CMN_CTRL" }, - { MIPI_CSIS_CLK_CTRL, "CLK_CTRL" }, - { MIPI_CSIS_INT_MSK, "INT_MSK" }, - { MIPI_CSIS_DPHY_STATUS, "DPHY_STATUS" }, - { MIPI_CSIS_DPHY_CMN_CTRL, "DPHY_CMN_CTRL" }, - { MIPI_CSIS_DPHY_SCTRL_L, "DPHY_SCTRL_L" }, - { MIPI_CSIS_DPHY_SCTRL_H, "DPHY_SCTRL_H" }, - { MIPI_CSIS_ISP_CONFIG_CH(0), "ISP_CONFIG_CH0" }, - { MIPI_CSIS_ISP_RESOL_CH(0), "ISP_RESOL_CH0" }, - { MIPI_CSIS_SDW_CONFIG_CH(0), "SDW_CONFIG_CH0" }, - { MIPI_CSIS_SDW_RESOL_CH(0), "SDW_RESOL_CH0" }, - { MIPI_CSIS_DBG_CTRL, "DBG_CTRL" }, - }; - - dev_info(dev, "--- REGISTERS ---\n"); - - for (i = 0; i < ARRAY_SIZE(registers); i++) { - cfg = mipi_csis_read(state, registers[i].offset); - dev_info(dev, "%14s: 0x%08x\n", registers[i].name, cfg); - } - - return 0; -} - -static struct csi_state * -mipi_notifier_to_csis_state(struct v4l2_async_notifier *n) -{ - return container_of(n, struct csi_state, notifier); -} - -static struct csi_state *mipi_sd_to_csis_state(struct v4l2_subdev *sdev) -{ - return container_of(sdev, struct csi_state, mipi_sd); -} - static const struct csis_pix_format *find_csis_format(u32 code) { unsigned int i; @@ -452,9 +441,24 @@ static const struct csis_pix_format *find_csis_format(u32 code) return NULL; } +/* ----------------------------------------------------------------------------- + * Hardware configuration + */ + +static inline u32 mipi_csis_read(struct csi_state *state, u32 reg) +{ + return readl(state->regs + reg); +} + +static inline void mipi_csis_write(struct csi_state *state, u32 reg, u32 val) +{ + writel(val, state->regs + reg); +} + static void mipi_csis_enable_interrupts(struct csi_state *state, bool on) { mipi_csis_write(state, MIPI_CSIS_INT_MSK, on ? 0xffffffff : 0); + mipi_csis_write(state, MIPI_CSIS_DBG_INTR_MSK, on ? 0xffffffff : 0); } static void mipi_csis_sw_reset(struct csi_state *state) @@ -466,25 +470,6 @@ static void mipi_csis_sw_reset(struct csi_state *state) usleep_range(10, 20); } -static int mipi_csis_phy_init(struct csi_state *state) -{ - state->mipi_phy_regulator = devm_regulator_get(state->dev, "phy"); - if (IS_ERR(state->mipi_phy_regulator)) - return PTR_ERR(state->mipi_phy_regulator); - - return regulator_set_voltage(state->mipi_phy_regulator, 1000000, - 1000000); -} - -static void mipi_csis_phy_reset(struct csi_state *state) -{ - reset_control_assert(state->mrst); - - msleep(20); - - reset_control_deassert(state->mrst); -} - static void mipi_csis_system_enable(struct csi_state *state, int on) { u32 val, mask; @@ -514,7 +499,7 @@ static void __mipi_csis_set_format(struct csi_state *state) /* Color format */ val = mipi_csis_read(state, MIPI_CSIS_ISP_CONFIG_CH(0)); val &= ~(MIPI_CSIS_ISPCFG_ALIGN_32BIT | MIPI_CSIS_ISPCFG_FMT_MASK); - val |= state->csis_fmt->fmt_reg; + val |= MIPI_CSIS_ISPCFG_FMT(state->csis_fmt->data_type); mipi_csis_write(state, MIPI_CSIS_ISP_CONFIG_CH(0), val); /* Pixel resolution */ @@ -546,11 +531,15 @@ static int mipi_csis_calculate_params(struct csi_state *state) /* * The HSSETTLE counter value is document in a table, but can also - * easily be calculated. + * easily be calculated. Hardcode the CLKSETTLE value to 0 for now + * (which is documented as corresponding to CSI-2 v0.87 to v1.00) until + * we figure out how to compute it correctly. */ state->hs_settle = (lane_rate - 5000000) / 45000000; - dev_dbg(state->dev, "lane rate %u, Ths_settle %u\n", - lane_rate, state->hs_settle); + state->clk_settle = 0; + + dev_dbg(state->dev, "lane rate %u, Tclk_settle %u, Ths_settle %u\n", + lane_rate, state->clk_settle, state->hs_settle); return 0; } @@ -563,13 +552,15 @@ static void mipi_csis_set_params(struct csi_state *state) val = mipi_csis_read(state, MIPI_CSIS_CMN_CTRL); val &= ~MIPI_CSIS_CMN_CTRL_LANE_NR_MASK; val |= (lanes - 1) << MIPI_CSIS_CMN_CTRL_LANE_NR_OFFSET; - val |= MIPI_CSIS_CMN_CTRL_INTER_MODE; + if (state->info->version == MIPI_CSIS_V3_3) + val |= MIPI_CSIS_CMN_CTRL_INTER_MODE; mipi_csis_write(state, MIPI_CSIS_CMN_CTRL, val); __mipi_csis_set_format(state); mipi_csis_write(state, MIPI_CSIS_DPHY_CMN_CTRL, - MIPI_CSIS_DPHY_CMN_CTRL_HSSETTLE(state->hs_settle)); + MIPI_CSIS_DPHY_CMN_CTRL_HSSETTLE(state->hs_settle) | + MIPI_CSIS_DPHY_CMN_CTRL_CLKSETTLE(state->clk_settle)); val = (0 << MIPI_CSIS_ISP_SYNC_HSYNC_LINTV_OFFSET) | (0 << MIPI_CSIS_ISP_SYNC_VSYNC_SINTV_OFFSET) @@ -601,31 +592,30 @@ static void mipi_csis_set_params(struct csi_state *state) static int mipi_csis_clk_enable(struct csi_state *state) { - return clk_bulk_prepare_enable(state->num_clks, state->clks); + return clk_bulk_prepare_enable(state->info->num_clocks, state->clks); } static void mipi_csis_clk_disable(struct csi_state *state) { - clk_bulk_disable_unprepare(state->num_clks, state->clks); + clk_bulk_disable_unprepare(state->info->num_clocks, state->clks); } static int mipi_csis_clk_get(struct csi_state *state) { - struct device *dev = &state->pdev->dev; unsigned int i; int ret; - state->num_clks = ARRAY_SIZE(mipi_csis_clk_id); - state->clks = devm_kcalloc(dev, state->num_clks, sizeof(*state->clks), - GFP_KERNEL); + state->clks = devm_kcalloc(state->dev, state->info->num_clocks, + sizeof(*state->clks), GFP_KERNEL); if (!state->clks) return -ENOMEM; - for (i = 0; i < state->num_clks; i++) + for (i = 0; i < state->info->num_clocks; i++) state->clks[i].id = mipi_csis_clk_id[i]; - ret = devm_clk_bulk_get(dev, state->num_clks, state->clks); + ret = devm_clk_bulk_get(state->dev, state->info->num_clocks, + state->clks); if (ret < 0) return ret; @@ -633,8 +623,8 @@ static int mipi_csis_clk_get(struct csi_state *state) ret = clk_set_rate(state->clks[MIPI_CSIS_CLK_WRAP].clk, state->clk_frequency); if (ret < 0) - dev_err(dev, "set rate=%d failed: %d\n", state->clk_frequency, - ret); + dev_err(state->dev, "set rate=%d failed: %d\n", + state->clk_frequency, ret); return ret; } @@ -653,6 +643,89 @@ static void mipi_csis_stop_stream(struct csi_state *state) mipi_csis_system_enable(state, false); } +static irqreturn_t mipi_csis_irq_handler(int irq, void *dev_id) +{ + struct csi_state *state = dev_id; + unsigned long flags; + unsigned int i; + u32 status; + u32 dbg_status; + + status = mipi_csis_read(state, MIPI_CSIS_INT_SRC); + dbg_status = mipi_csis_read(state, MIPI_CSIS_DBG_INTR_SRC); + + spin_lock_irqsave(&state->slock, flags); + + /* Update the event/error counters */ + if ((status & MIPI_CSIS_INT_SRC_ERRORS) || state->debug) { + for (i = 0; i < MIPI_CSIS_NUM_EVENTS; i++) { + struct mipi_csis_event *event = &state->events[i]; + + if ((!event->debug && (status & event->mask)) || + (event->debug && (dbg_status & event->mask))) + event->counter++; + } + } + spin_unlock_irqrestore(&state->slock, flags); + + mipi_csis_write(state, MIPI_CSIS_INT_SRC, status); + mipi_csis_write(state, MIPI_CSIS_DBG_INTR_SRC, dbg_status); + + return IRQ_HANDLED; +} + +/* ----------------------------------------------------------------------------- + * PHY regulator and reset + */ + +static int mipi_csis_phy_enable(struct csi_state *state) +{ + if (state->info->version != MIPI_CSIS_V3_3) + return 0; + + return regulator_enable(state->mipi_phy_regulator); +} + +static int mipi_csis_phy_disable(struct csi_state *state) +{ + if (state->info->version != MIPI_CSIS_V3_3) + return 0; + + return regulator_disable(state->mipi_phy_regulator); +} + +static void mipi_csis_phy_reset(struct csi_state *state) +{ + if (state->info->version != MIPI_CSIS_V3_3) + return; + + reset_control_assert(state->mrst); + msleep(20); + reset_control_deassert(state->mrst); +} + +static int mipi_csis_phy_init(struct csi_state *state) +{ + if (state->info->version != MIPI_CSIS_V3_3) + return 0; + + /* Get MIPI PHY reset and regulator. */ + state->mrst = devm_reset_control_get_exclusive(state->dev, NULL); + if (IS_ERR(state->mrst)) + return PTR_ERR(state->mrst); + + state->mipi_phy_regulator = devm_regulator_get(state->dev, "phy"); + if (IS_ERR(state->mipi_phy_regulator)) + return PTR_ERR(state->mipi_phy_regulator); + + return regulator_set_voltage(state->mipi_phy_regulator, 1000000, + 1000000); +} + +/* ----------------------------------------------------------------------------- + * Debug + */ + static void mipi_csis_clear_counters(struct csi_state *state) { unsigned long flags; @@ -666,460 +739,53 @@ static void mipi_csis_clear_counters(struct csi_state *state) static void mipi_csis_log_counters(struct csi_state *state, bool non_errors) { - int i = non_errors ? MIPI_CSIS_NUM_EVENTS : MIPI_CSIS_NUM_EVENTS - 4; - struct device *dev = &state->pdev->dev; + unsigned int num_events = non_errors ? MIPI_CSIS_NUM_EVENTS + : MIPI_CSIS_NUM_EVENTS - 8; unsigned long flags; + unsigned int i; spin_lock_irqsave(&state->slock, flags); - for (i--; i >= 0; i--) { + for (i = 0; i < num_events; ++i) { if (state->events[i].counter > 0 || state->debug) - dev_info(dev, "%s events: %d\n", state->events[i].name, + dev_info(state->dev, "%s events: %d\n", + state->events[i].name, state->events[i].counter); } spin_unlock_irqrestore(&state->slock, flags); } -/* - * V4L2 subdev operations - */ -static int mipi_csis_s_stream(struct v4l2_subdev *mipi_sd, int enable) +static int mipi_csis_dump_regs(struct csi_state *state) { - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - int ret; - - if (enable) { - ret = mipi_csis_calculate_params(state); - if (ret < 0) - return ret; - - mipi_csis_clear_counters(state); - - ret = pm_runtime_get_sync(&state->pdev->dev); - if (ret < 0) { - pm_runtime_put_noidle(&state->pdev->dev); - return ret; - } - ret = v4l2_subdev_call(state->src_sd, core, s_power, 1); - if (ret < 0 && ret != -ENOIOCTLCMD) - goto done; - } - - mutex_lock(&state->lock); - - if (enable) { - if (state->flags & ST_SUSPENDED) { - ret = -EBUSY; - goto unlock; - } - - mipi_csis_start_stream(state); - ret = v4l2_subdev_call(state->src_sd, video, s_stream, 1); - if (ret < 0) - goto unlock; - - mipi_csis_log_counters(state, true); - - state->flags |= ST_STREAMING; - } else { - v4l2_subdev_call(state->src_sd, video, s_stream, 0); - ret = v4l2_subdev_call(state->src_sd, core, s_power, 0); - if (ret == -ENOIOCTLCMD) - ret = 0; - mipi_csis_stop_stream(state); - state->flags &= ~ST_STREAMING; - if (state->debug) - mipi_csis_log_counters(state, true); - } - -unlock: - mutex_unlock(&state->lock); - -done: - if (!enable || ret < 0) - pm_runtime_put(&state->pdev->dev); - - return ret; -} - -static int mipi_csis_link_setup(struct media_entity *entity, - const struct media_pad *local_pad, - const struct media_pad *remote_pad, u32 flags) -{ - struct v4l2_subdev *mipi_sd = media_entity_to_v4l2_subdev(entity); - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - struct v4l2_subdev *remote_sd; - - dev_dbg(state->dev, "link setup %s -> %s", remote_pad->entity->name, - local_pad->entity->name); - - /* We only care about the link to the source. */ - if (!(local_pad->flags & MEDIA_PAD_FL_SINK)) - return 0; - - remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity); - - if (flags & MEDIA_LNK_FL_ENABLED) { - if (state->src_sd) - return -EBUSY; - - state->src_sd = remote_sd; - } else { - state->src_sd = NULL; - } - - return 0; -} - -static struct v4l2_mbus_framefmt * -mipi_csis_get_format(struct csi_state *state, - struct v4l2_subdev_pad_config *cfg, - enum v4l2_subdev_format_whence which, - unsigned int pad) -{ - if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&state->mipi_sd, cfg, pad); - - return &state->format_mbus; -} - -static int mipi_csis_init_cfg(struct v4l2_subdev *mipi_sd, - struct v4l2_subdev_pad_config *cfg) -{ - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - struct v4l2_mbus_framefmt *fmt_sink; - struct v4l2_mbus_framefmt *fmt_source; - enum v4l2_subdev_format_whence which; - - which = cfg ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; - fmt_sink = mipi_csis_get_format(state, cfg, which, CSIS_PAD_SINK); - - fmt_sink->code = MEDIA_BUS_FMT_UYVY8_1X16; - fmt_sink->width = MIPI_CSIS_DEF_PIX_WIDTH; - fmt_sink->height = MIPI_CSIS_DEF_PIX_HEIGHT; - fmt_sink->field = V4L2_FIELD_NONE; - - fmt_sink->colorspace = V4L2_COLORSPACE_SMPTE170M; - fmt_sink->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(fmt_sink->colorspace); - fmt_sink->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(fmt_sink->colorspace); - fmt_sink->quantization = - V4L2_MAP_QUANTIZATION_DEFAULT(false, fmt_sink->colorspace, - fmt_sink->ycbcr_enc); - - /* - * When called from mipi_csis_subdev_init() to initialize the active - * configuration, cfg is NULL, which indicates there's no source pad - * configuration to set. - */ - if (!cfg) - return 0; - - fmt_source = mipi_csis_get_format(state, cfg, which, CSIS_PAD_SOURCE); - *fmt_source = *fmt_sink; - - return 0; -} - -static int mipi_csis_get_fmt(struct v4l2_subdev *mipi_sd, - struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *sdformat) -{ - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - struct v4l2_mbus_framefmt *fmt; - - mutex_lock(&state->lock); - fmt = mipi_csis_get_format(state, cfg, sdformat->which, sdformat->pad); - sdformat->format = *fmt; - mutex_unlock(&state->lock); - - return 0; -} - -static int mipi_csis_enum_mbus_code(struct v4l2_subdev *mipi_sd, - struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_mbus_code_enum *code) -{ - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - - /* - * The CSIS can't transcode in any way, the source format is identical - * to the sink format. - */ - if (code->pad == CSIS_PAD_SOURCE) { - struct v4l2_mbus_framefmt *fmt; - - if (code->index > 0) - return -EINVAL; - - fmt = mipi_csis_get_format(state, cfg, code->which, code->pad); - code->code = fmt->code; - return 0; - } - - if (code->pad != CSIS_PAD_SINK) - return -EINVAL; - - if (code->index >= ARRAY_SIZE(mipi_csis_formats)) - return -EINVAL; - - code->code = mipi_csis_formats[code->index].code; - - return 0; -} - -static int mipi_csis_set_fmt(struct v4l2_subdev *mipi_sd, - struct v4l2_subdev_pad_config *cfg, - struct v4l2_subdev_format *sdformat) -{ - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - struct csis_pix_format const *csis_fmt; - struct v4l2_mbus_framefmt *fmt; - unsigned int align; - - /* - * The CSIS can't transcode in any way, the source format can't be - * modified. - */ - if (sdformat->pad == CSIS_PAD_SOURCE) - return mipi_csis_get_fmt(mipi_sd, cfg, sdformat); - - if (sdformat->pad != CSIS_PAD_SINK) - return -EINVAL; - - fmt = mipi_csis_get_format(state, cfg, sdformat->which, sdformat->pad); - - mutex_lock(&state->lock); - - /* Validate the media bus code and clamp the size. */ - csis_fmt = find_csis_format(sdformat->format.code); - if (!csis_fmt) - csis_fmt = &mipi_csis_formats[0]; - - fmt->code = csis_fmt->code; - fmt->width = sdformat->format.width; - fmt->height = sdformat->format.height; - - /* - * The total number of bits per line must be a multiple of 8. We thus - * need to align the width for formats that are not multiples of 8 - * bits. - */ - switch (csis_fmt->width % 8) { - case 0: - align = 0; - break; - case 4: - align = 1; - break; - case 2: - case 6: - align = 2; - break; - case 1: - case 3: - case 5: - case 7: - align = 3; - break; - } - - v4l_bound_align_image(&fmt->width, 1, CSIS_MAX_PIX_WIDTH, align, - &fmt->height, 1, CSIS_MAX_PIX_HEIGHT, 0, 0); - - sdformat->format = *fmt; - - /* Propagate the format from sink to source. */ - fmt = mipi_csis_get_format(state, cfg, sdformat->which, - CSIS_PAD_SOURCE); - *fmt = sdformat->format; - - /* Store the CSIS format descriptor for active formats. */ - if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) - state->csis_fmt = csis_fmt; - - mutex_unlock(&state->lock); - - return 0; -} - -static int mipi_csis_log_status(struct v4l2_subdev *mipi_sd) -{ - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - - mutex_lock(&state->lock); - mipi_csis_log_counters(state, true); - if (state->debug && (state->flags & ST_POWERED)) - mipi_csis_dump_regs(state); - mutex_unlock(&state->lock); - - return 0; -} - -static irqreturn_t mipi_csis_irq_handler(int irq, void *dev_id) -{ - struct csi_state *state = dev_id; - unsigned long flags; - unsigned int i; - u32 status; - - status = mipi_csis_read(state, MIPI_CSIS_INT_SRC); - - spin_lock_irqsave(&state->slock, flags); - - /* Update the event/error counters */ - if ((status & MIPI_CSIS_INT_SRC_ERRORS) || state->debug) { - for (i = 0; i < MIPI_CSIS_NUM_EVENTS; i++) { - if (!(status & state->events[i].mask)) - continue; - state->events[i].counter++; - } - } - spin_unlock_irqrestore(&state->slock, flags); - - mipi_csis_write(state, MIPI_CSIS_INT_SRC, status); - - return IRQ_HANDLED; -} - -static const struct v4l2_subdev_core_ops mipi_csis_core_ops = { - .log_status = mipi_csis_log_status, -}; - -static const struct media_entity_operations mipi_csis_entity_ops = { - .link_setup = mipi_csis_link_setup, - .link_validate = v4l2_subdev_link_validate, - .get_fwnode_pad = v4l2_subdev_get_fwnode_pad_1_to_1, -}; - -static const struct v4l2_subdev_video_ops mipi_csis_video_ops = { - .s_stream = mipi_csis_s_stream, -}; - -static const struct v4l2_subdev_pad_ops mipi_csis_pad_ops = { - .init_cfg = mipi_csis_init_cfg, - .enum_mbus_code = mipi_csis_enum_mbus_code, - .get_fmt = mipi_csis_get_fmt, - .set_fmt = mipi_csis_set_fmt, -}; - -static const struct v4l2_subdev_ops mipi_csis_subdev_ops = { - .core = &mipi_csis_core_ops, - .video = &mipi_csis_video_ops, - .pad = &mipi_csis_pad_ops, -}; - -static int mipi_csis_parse_dt(struct platform_device *pdev, - struct csi_state *state) -{ - struct device_node *node = pdev->dev.of_node; - - if (of_property_read_u32(node, "clock-frequency", - &state->clk_frequency)) - state->clk_frequency = DEFAULT_SCLK_CSIS_FREQ; - - /* Get MIPI PHY resets */ - state->mrst = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(state->mrst)) - return PTR_ERR(state->mrst); - - return 0; -} - -static int mipi_csis_pm_resume(struct device *dev, bool runtime); - -static int mipi_csis_notify_bound(struct v4l2_async_notifier *notifier, - struct v4l2_subdev *sd, - struct v4l2_async_subdev *asd) -{ - struct csi_state *state = mipi_notifier_to_csis_state(notifier); - struct media_pad *sink = &state->mipi_sd.entity.pads[CSIS_PAD_SINK]; - - return v4l2_create_fwnode_links_to_pad(sd, sink, 0); -} - -static const struct v4l2_async_notifier_operations mipi_csis_notify_ops = { - .bound = mipi_csis_notify_bound, -}; - -static int mipi_csis_subdev_init(struct v4l2_subdev *mipi_sd, - struct platform_device *pdev, - const struct v4l2_subdev_ops *ops) -{ - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - - v4l2_subdev_init(mipi_sd, ops); - mipi_sd->owner = THIS_MODULE; - snprintf(mipi_sd->name, sizeof(mipi_sd->name), "%s.%d", - CSIS_SUBDEV_NAME, state->index); - - mipi_sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; - mipi_sd->ctrl_handler = NULL; - - mipi_sd->entity.function = MEDIA_ENT_F_VID_IF_BRIDGE; - mipi_sd->entity.ops = &mipi_csis_entity_ops; - - mipi_sd->dev = &pdev->dev; - - state->csis_fmt = &mipi_csis_formats[0]; - mipi_csis_init_cfg(mipi_sd, NULL); - - v4l2_set_subdevdata(mipi_sd, &pdev->dev); - - state->pads[CSIS_PAD_SINK].flags = MEDIA_PAD_FL_SINK - | MEDIA_PAD_FL_MUST_CONNECT; - state->pads[CSIS_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE - | MEDIA_PAD_FL_MUST_CONNECT; - return media_entity_pads_init(&mipi_sd->entity, CSIS_PADS_NUM, - state->pads); -} - -static int mipi_csis_async_register(struct csi_state *state) -{ - struct v4l2_fwnode_endpoint vep = { - .bus_type = V4L2_MBUS_CSI2_DPHY, + static const struct { + u32 offset; + const char * const name; + } registers[] = { + { MIPI_CSIS_CMN_CTRL, "CMN_CTRL" }, + { MIPI_CSIS_CLK_CTRL, "CLK_CTRL" }, + { MIPI_CSIS_INT_MSK, "INT_MSK" }, + { MIPI_CSIS_DPHY_STATUS, "DPHY_STATUS" }, + { MIPI_CSIS_DPHY_CMN_CTRL, "DPHY_CMN_CTRL" }, + { MIPI_CSIS_DPHY_SCTRL_L, "DPHY_SCTRL_L" }, + { MIPI_CSIS_DPHY_SCTRL_H, "DPHY_SCTRL_H" }, + { MIPI_CSIS_ISP_CONFIG_CH(0), "ISP_CONFIG_CH0" }, + { MIPI_CSIS_ISP_RESOL_CH(0), "ISP_RESOL_CH0" }, + { MIPI_CSIS_SDW_CONFIG_CH(0), "SDW_CONFIG_CH0" }, + { MIPI_CSIS_SDW_RESOL_CH(0), "SDW_RESOL_CH0" }, + { MIPI_CSIS_DBG_CTRL, "DBG_CTRL" }, }; - struct v4l2_async_subdev *asd; - struct fwnode_handle *ep; - int ret; - v4l2_async_notifier_init(&state->notifier); + unsigned int i; + u32 cfg; - ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(state->dev), 0, 0, - FWNODE_GRAPH_ENDPOINT_NEXT); - if (!ep) - return -ENOTCONN; + dev_info(state->dev, "--- REGISTERS ---\n"); - ret = v4l2_fwnode_endpoint_parse(ep, &vep); - if (ret) - goto err_parse; - - state->bus = vep.bus.mipi_csi2; - - dev_dbg(state->dev, "data lanes: %d\n", state->bus.num_data_lanes); - dev_dbg(state->dev, "flags: 0x%08x\n", state->bus.flags); - - asd = v4l2_async_notifier_add_fwnode_remote_subdev( - &state->notifier, ep, struct v4l2_async_subdev); - if (IS_ERR(asd)) { - ret = PTR_ERR(asd); - goto err_parse; + for (i = 0; i < ARRAY_SIZE(registers); i++) { + cfg = mipi_csis_read(state, registers[i].offset); + dev_info(state->dev, "%14s: 0x%08x\n", registers[i].name, cfg); } - fwnode_handle_put(ep); - - state->notifier.ops = &mipi_csis_notify_ops; - - ret = v4l2_async_subdev_notifier_register(&state->mipi_sd, - &state->notifier); - if (ret) - return ret; - - return v4l2_async_register_subdev(&state->mipi_sd); - -err_parse: - fwnode_handle_put(ep); - - return ret; + return 0; } static int mipi_csis_dump_regs_show(struct seq_file *m, void *private) @@ -1145,117 +811,427 @@ static void mipi_csis_debugfs_exit(struct csi_state *state) debugfs_remove_recursive(state->debugfs_root); } -static int mipi_csis_probe(struct platform_device *pdev) +/* ----------------------------------------------------------------------------- + * V4L2 subdev operations + */ + +static struct csi_state *mipi_sd_to_csis_state(struct v4l2_subdev *sdev) { - struct device *dev = &pdev->dev; - struct csi_state *state; + return container_of(sdev, struct csi_state, sd); +} + +static int mipi_csis_s_stream(struct v4l2_subdev *sd, int enable) +{ + struct csi_state *state = mipi_sd_to_csis_state(sd); int ret; - state = devm_kzalloc(dev, sizeof(*state), GFP_KERNEL); - if (!state) - return -ENOMEM; - - spin_lock_init(&state->slock); - - state->pdev = pdev; - state->dev = dev; - - ret = mipi_csis_parse_dt(pdev, state); - if (ret < 0) { - dev_err(dev, "Failed to parse device tree: %d\n", ret); - return ret; - } - - ret = mipi_csis_phy_init(state); - if (ret < 0) - return ret; - - mipi_csis_phy_reset(state); - - state->regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(state->regs)) - return PTR_ERR(state->regs); - - state->irq = platform_get_irq(pdev, 0); - if (state->irq < 0) - return state->irq; - - ret = mipi_csis_clk_get(state); - if (ret < 0) - return ret; - - ret = mipi_csis_clk_enable(state); - if (ret < 0) { - dev_err(state->dev, "failed to enable clocks: %d\n", ret); - return ret; - } - - ret = devm_request_irq(dev, state->irq, mipi_csis_irq_handler, - 0, dev_name(dev), state); - if (ret) { - dev_err(dev, "Interrupt request failed\n"); - goto disable_clock; - } - - platform_set_drvdata(pdev, &state->mipi_sd); - - mutex_init(&state->lock); - ret = mipi_csis_subdev_init(&state->mipi_sd, pdev, - &mipi_csis_subdev_ops); - if (ret < 0) - goto disable_clock; - - ret = mipi_csis_async_register(state); - if (ret < 0) { - dev_err(&pdev->dev, "async register failed: %d\n", ret); - goto cleanup; - } - - memcpy(state->events, mipi_csis_events, sizeof(state->events)); - - mipi_csis_debugfs_init(state); - pm_runtime_enable(dev); - if (!pm_runtime_enabled(dev)) { - ret = mipi_csis_pm_resume(dev, true); + if (enable) { + ret = mipi_csis_calculate_params(state); if (ret < 0) - goto unregister_all; + return ret; + + mipi_csis_clear_counters(state); + + ret = pm_runtime_resume_and_get(state->dev); + if (ret < 0) + return ret; + + ret = v4l2_subdev_call(state->src_sd, core, s_power, 1); + if (ret < 0 && ret != -ENOIOCTLCMD) + goto done; } - dev_info(&pdev->dev, "lanes: %d, freq: %u\n", - state->bus.num_data_lanes, state->clk_frequency); + mutex_lock(&state->lock); - return 0; + if (enable) { + if (state->state & ST_SUSPENDED) { + ret = -EBUSY; + goto unlock; + } -unregister_all: - mipi_csis_debugfs_exit(state); -cleanup: - media_entity_cleanup(&state->mipi_sd.entity); - v4l2_async_notifier_unregister(&state->notifier); - v4l2_async_notifier_cleanup(&state->notifier); - v4l2_async_unregister_subdev(&state->mipi_sd); -disable_clock: - mipi_csis_clk_disable(state); - mutex_destroy(&state->lock); + mipi_csis_start_stream(state); + ret = v4l2_subdev_call(state->src_sd, video, s_stream, 1); + if (ret < 0) + goto unlock; + + mipi_csis_log_counters(state, true); + + state->state |= ST_STREAMING; + } else { + v4l2_subdev_call(state->src_sd, video, s_stream, 0); + ret = v4l2_subdev_call(state->src_sd, core, s_power, 0); + if (ret == -ENOIOCTLCMD) + ret = 0; + mipi_csis_stop_stream(state); + state->state &= ~ST_STREAMING; + if (state->debug) + mipi_csis_log_counters(state, true); + } + +unlock: + mutex_unlock(&state->lock); + +done: + if (!enable || ret < 0) + pm_runtime_put(state->dev); return ret; } +static struct v4l2_mbus_framefmt * +mipi_csis_get_format(struct csi_state *state, + struct v4l2_subdev_state *sd_state, + enum v4l2_subdev_format_whence which, + unsigned int pad) +{ + if (which == V4L2_SUBDEV_FORMAT_TRY) + return v4l2_subdev_get_try_format(&state->sd, sd_state, pad); + + return &state->format_mbus; +} + +static int mipi_csis_init_cfg(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state) +{ + struct csi_state *state = mipi_sd_to_csis_state(sd); + struct v4l2_mbus_framefmt *fmt_sink; + struct v4l2_mbus_framefmt *fmt_source; + enum v4l2_subdev_format_whence which; + + which = sd_state ? V4L2_SUBDEV_FORMAT_TRY : V4L2_SUBDEV_FORMAT_ACTIVE; + fmt_sink = mipi_csis_get_format(state, sd_state, which, CSIS_PAD_SINK); + + fmt_sink->code = MEDIA_BUS_FMT_UYVY8_1X16; + fmt_sink->width = MIPI_CSIS_DEF_PIX_WIDTH; + fmt_sink->height = MIPI_CSIS_DEF_PIX_HEIGHT; + fmt_sink->field = V4L2_FIELD_NONE; + + fmt_sink->colorspace = V4L2_COLORSPACE_SMPTE170M; + fmt_sink->xfer_func = V4L2_MAP_XFER_FUNC_DEFAULT(fmt_sink->colorspace); + fmt_sink->ycbcr_enc = V4L2_MAP_YCBCR_ENC_DEFAULT(fmt_sink->colorspace); + fmt_sink->quantization = + V4L2_MAP_QUANTIZATION_DEFAULT(false, fmt_sink->colorspace, + fmt_sink->ycbcr_enc); + + /* + * When called from mipi_csis_subdev_init() to initialize the active + * configuration, cfg is NULL, which indicates there's no source pad + * configuration to set. + */ + if (!sd_state) + return 0; + + fmt_source = mipi_csis_get_format(state, sd_state, which, + CSIS_PAD_SOURCE); + *fmt_source = *fmt_sink; + + return 0; +} + +static int mipi_csis_get_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *sdformat) +{ + struct csi_state *state = mipi_sd_to_csis_state(sd); + struct v4l2_mbus_framefmt *fmt; + + fmt = mipi_csis_get_format(state, sd_state, sdformat->which, + sdformat->pad); + + mutex_lock(&state->lock); + sdformat->format = *fmt; + mutex_unlock(&state->lock); + + return 0; +} + +static int mipi_csis_enum_mbus_code(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_mbus_code_enum *code) +{ + struct csi_state *state = mipi_sd_to_csis_state(sd); + + /* + * The CSIS can't transcode in any way, the source format is identical + * to the sink format. + */ + if (code->pad == CSIS_PAD_SOURCE) { + struct v4l2_mbus_framefmt *fmt; + + if (code->index > 0) + return -EINVAL; + + fmt = mipi_csis_get_format(state, sd_state, code->which, + code->pad); + code->code = fmt->code; + return 0; + } + + if (code->pad != CSIS_PAD_SINK) + return -EINVAL; + + if (code->index >= ARRAY_SIZE(mipi_csis_formats)) + return -EINVAL; + + code->code = mipi_csis_formats[code->index].code; + + return 0; +} + +static int mipi_csis_set_fmt(struct v4l2_subdev *sd, + struct v4l2_subdev_state *sd_state, + struct v4l2_subdev_format *sdformat) +{ + struct csi_state *state = mipi_sd_to_csis_state(sd); + struct csis_pix_format const *csis_fmt; + struct v4l2_mbus_framefmt *fmt; + unsigned int align; + + /* + * The CSIS can't transcode in any way, the source format can't be + * modified. + */ + if (sdformat->pad == CSIS_PAD_SOURCE) + return mipi_csis_get_fmt(sd, sd_state, sdformat); + + if (sdformat->pad != CSIS_PAD_SINK) + return -EINVAL; + + /* + * Validate the media bus code and clamp and align the size. + * + * The total number of bits per line must be a multiple of 8. We thus + * need to align the width for formats that are not multiples of 8 + * bits. + */ + csis_fmt = find_csis_format(sdformat->format.code); + if (!csis_fmt) + csis_fmt = &mipi_csis_formats[0]; + + switch (csis_fmt->width % 8) { + case 0: + align = 0; + break; + case 4: + align = 1; + break; + case 2: + case 6: + align = 2; + break; + case 1: + case 3: + case 5: + case 7: + align = 3; + break; + } + + v4l_bound_align_image(&sdformat->format.width, 1, + CSIS_MAX_PIX_WIDTH, align, + &sdformat->format.height, 1, + CSIS_MAX_PIX_HEIGHT, 0, 0); + + fmt = mipi_csis_get_format(state, sd_state, sdformat->which, + sdformat->pad); + + mutex_lock(&state->lock); + + fmt->code = csis_fmt->code; + fmt->width = sdformat->format.width; + fmt->height = sdformat->format.height; + + sdformat->format = *fmt; + + /* Propagate the format from sink to source. */ + fmt = mipi_csis_get_format(state, sd_state, sdformat->which, + CSIS_PAD_SOURCE); + *fmt = sdformat->format; + + /* Store the CSIS format descriptor for active formats. */ + if (sdformat->which == V4L2_SUBDEV_FORMAT_ACTIVE) + state->csis_fmt = csis_fmt; + + mutex_unlock(&state->lock); + + return 0; +} + +static int mipi_csis_log_status(struct v4l2_subdev *sd) +{ + struct csi_state *state = mipi_sd_to_csis_state(sd); + + mutex_lock(&state->lock); + mipi_csis_log_counters(state, true); + if (state->debug && (state->state & ST_POWERED)) + mipi_csis_dump_regs(state); + mutex_unlock(&state->lock); + + return 0; +} + +static const struct v4l2_subdev_core_ops mipi_csis_core_ops = { + .log_status = mipi_csis_log_status, +}; + +static const struct v4l2_subdev_video_ops mipi_csis_video_ops = { + .s_stream = mipi_csis_s_stream, +}; + +static const struct v4l2_subdev_pad_ops mipi_csis_pad_ops = { + .init_cfg = mipi_csis_init_cfg, + .enum_mbus_code = mipi_csis_enum_mbus_code, + .get_fmt = mipi_csis_get_fmt, + .set_fmt = mipi_csis_set_fmt, +}; + +static const struct v4l2_subdev_ops mipi_csis_subdev_ops = { + .core = &mipi_csis_core_ops, + .video = &mipi_csis_video_ops, + .pad = &mipi_csis_pad_ops, +}; + +/* ----------------------------------------------------------------------------- + * Media entity operations + */ + +static int mipi_csis_link_setup(struct media_entity *entity, + const struct media_pad *local_pad, + const struct media_pad *remote_pad, u32 flags) +{ + struct v4l2_subdev *sd = media_entity_to_v4l2_subdev(entity); + struct csi_state *state = mipi_sd_to_csis_state(sd); + struct v4l2_subdev *remote_sd; + + dev_dbg(state->dev, "link setup %s -> %s", remote_pad->entity->name, + local_pad->entity->name); + + /* We only care about the link to the source. */ + if (!(local_pad->flags & MEDIA_PAD_FL_SINK)) + return 0; + + remote_sd = media_entity_to_v4l2_subdev(remote_pad->entity); + + if (flags & MEDIA_LNK_FL_ENABLED) { + if (state->src_sd) + return -EBUSY; + + state->src_sd = remote_sd; + } else { + state->src_sd = NULL; + } + + return 0; +} + +static const struct media_entity_operations mipi_csis_entity_ops = { + .link_setup = mipi_csis_link_setup, + .link_validate = v4l2_subdev_link_validate, + .get_fwnode_pad = v4l2_subdev_get_fwnode_pad_1_to_1, +}; + +/* ----------------------------------------------------------------------------- + * Async subdev notifier + */ + +static struct csi_state * +mipi_notifier_to_csis_state(struct v4l2_async_notifier *n) +{ + return container_of(n, struct csi_state, notifier); +} + +static int mipi_csis_notify_bound(struct v4l2_async_notifier *notifier, + struct v4l2_subdev *sd, + struct v4l2_async_subdev *asd) +{ + struct csi_state *state = mipi_notifier_to_csis_state(notifier); + struct media_pad *sink = &state->sd.entity.pads[CSIS_PAD_SINK]; + + return v4l2_create_fwnode_links_to_pad(sd, sink, 0); +} + +static const struct v4l2_async_notifier_operations mipi_csis_notify_ops = { + .bound = mipi_csis_notify_bound, +}; + +static int mipi_csis_async_register(struct csi_state *state) +{ + struct v4l2_fwnode_endpoint vep = { + .bus_type = V4L2_MBUS_CSI2_DPHY, + }; + struct v4l2_async_subdev *asd; + struct fwnode_handle *ep; + unsigned int i; + int ret; + + v4l2_async_notifier_init(&state->notifier); + + ep = fwnode_graph_get_endpoint_by_id(dev_fwnode(state->dev), 0, 0, + FWNODE_GRAPH_ENDPOINT_NEXT); + if (!ep) + return -ENOTCONN; + + ret = v4l2_fwnode_endpoint_parse(ep, &vep); + if (ret) + goto err_parse; + + for (i = 0; i < vep.bus.mipi_csi2.num_data_lanes; ++i) { + if (vep.bus.mipi_csi2.data_lanes[i] != i + 1) { + dev_err(state->dev, + "data lanes reordering is not supported"); + ret = -EINVAL; + goto err_parse; + } + } + + state->bus = vep.bus.mipi_csi2; + + dev_dbg(state->dev, "data lanes: %d\n", state->bus.num_data_lanes); + dev_dbg(state->dev, "flags: 0x%08x\n", state->bus.flags); + + asd = v4l2_async_notifier_add_fwnode_remote_subdev( + &state->notifier, ep, struct v4l2_async_subdev); + if (IS_ERR(asd)) { + ret = PTR_ERR(asd); + goto err_parse; + } + + fwnode_handle_put(ep); + + state->notifier.ops = &mipi_csis_notify_ops; + + ret = v4l2_async_subdev_notifier_register(&state->sd, &state->notifier); + if (ret) + return ret; + + return v4l2_async_register_subdev(&state->sd); + +err_parse: + fwnode_handle_put(ep); + + return ret; +} + +/* ----------------------------------------------------------------------------- + * Suspend/resume + */ + static int mipi_csis_pm_suspend(struct device *dev, bool runtime) { - struct v4l2_subdev *mipi_sd = dev_get_drvdata(dev); - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); + struct v4l2_subdev *sd = dev_get_drvdata(dev); + struct csi_state *state = mipi_sd_to_csis_state(sd); int ret = 0; mutex_lock(&state->lock); - if (state->flags & ST_POWERED) { + if (state->state & ST_POWERED) { mipi_csis_stop_stream(state); - ret = regulator_disable(state->mipi_phy_regulator); + ret = mipi_csis_phy_disable(state); if (ret) goto unlock; mipi_csis_clk_disable(state); - state->flags &= ~ST_POWERED; + state->state &= ~ST_POWERED; if (!runtime) - state->flags |= ST_SUSPENDED; + state->state |= ST_SUSPENDED; } unlock: @@ -1266,26 +1242,26 @@ static int mipi_csis_pm_suspend(struct device *dev, bool runtime) static int mipi_csis_pm_resume(struct device *dev, bool runtime) { - struct v4l2_subdev *mipi_sd = dev_get_drvdata(dev); - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); + struct v4l2_subdev *sd = dev_get_drvdata(dev); + struct csi_state *state = mipi_sd_to_csis_state(sd); int ret = 0; mutex_lock(&state->lock); - if (!runtime && !(state->flags & ST_SUSPENDED)) + if (!runtime && !(state->state & ST_SUSPENDED)) goto unlock; - if (!(state->flags & ST_POWERED)) { - ret = regulator_enable(state->mipi_phy_regulator); + if (!(state->state & ST_POWERED)) { + ret = mipi_csis_phy_enable(state); if (ret) goto unlock; - state->flags |= ST_POWERED; + state->state |= ST_POWERED; mipi_csis_clk_enable(state); } - if (state->flags & ST_STREAMING) + if (state->state & ST_STREAMING) mipi_csis_start_stream(state); - state->flags &= ~ST_SUSPENDED; + state->state &= ~ST_SUSPENDED; unlock: mutex_unlock(&state->lock); @@ -1313,34 +1289,192 @@ static int __maybe_unused mipi_csis_runtime_resume(struct device *dev) return mipi_csis_pm_resume(dev, true); } -static int mipi_csis_remove(struct platform_device *pdev) -{ - struct v4l2_subdev *mipi_sd = platform_get_drvdata(pdev); - struct csi_state *state = mipi_sd_to_csis_state(mipi_sd); - - mipi_csis_debugfs_exit(state); - v4l2_async_notifier_unregister(&state->notifier); - v4l2_async_notifier_cleanup(&state->notifier); - v4l2_async_unregister_subdev(&state->mipi_sd); - - pm_runtime_disable(&pdev->dev); - mipi_csis_pm_suspend(&pdev->dev, true); - mipi_csis_clk_disable(state); - media_entity_cleanup(&state->mipi_sd.entity); - mutex_destroy(&state->lock); - pm_runtime_set_suspended(&pdev->dev); - - return 0; -} - static const struct dev_pm_ops mipi_csis_pm_ops = { SET_RUNTIME_PM_OPS(mipi_csis_runtime_suspend, mipi_csis_runtime_resume, NULL) SET_SYSTEM_SLEEP_PM_OPS(mipi_csis_suspend, mipi_csis_resume) }; +/* ----------------------------------------------------------------------------- + * Probe/remove & platform driver + */ + +static int mipi_csis_subdev_init(struct csi_state *state) +{ + struct v4l2_subdev *sd = &state->sd; + + v4l2_subdev_init(sd, &mipi_csis_subdev_ops); + sd->owner = THIS_MODULE; + snprintf(sd->name, sizeof(sd->name), "%s.%d", + CSIS_SUBDEV_NAME, state->index); + + sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; + sd->ctrl_handler = NULL; + + sd->entity.function = MEDIA_ENT_F_VID_IF_BRIDGE; + sd->entity.ops = &mipi_csis_entity_ops; + + sd->dev = state->dev; + + state->csis_fmt = &mipi_csis_formats[0]; + mipi_csis_init_cfg(sd, NULL); + + state->pads[CSIS_PAD_SINK].flags = MEDIA_PAD_FL_SINK + | MEDIA_PAD_FL_MUST_CONNECT; + state->pads[CSIS_PAD_SOURCE].flags = MEDIA_PAD_FL_SOURCE + | MEDIA_PAD_FL_MUST_CONNECT; + return media_entity_pads_init(&sd->entity, CSIS_PADS_NUM, + state->pads); +} + +static int mipi_csis_parse_dt(struct csi_state *state) +{ + struct device_node *node = state->dev->of_node; + + if (of_property_read_u32(node, "clock-frequency", + &state->clk_frequency)) + state->clk_frequency = DEFAULT_SCLK_CSIS_FREQ; + + return 0; +} + +static int mipi_csis_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct csi_state *state; + int irq; + int ret; + + state = devm_kzalloc(dev, sizeof(*state), GFP_KERNEL); + if (!state) + return -ENOMEM; + + mutex_init(&state->lock); + spin_lock_init(&state->slock); + + state->dev = dev; + state->info = of_device_get_match_data(dev); + + memcpy(state->events, mipi_csis_events, sizeof(state->events)); + + /* Parse DT properties. */ + ret = mipi_csis_parse_dt(state); + if (ret < 0) { + dev_err(dev, "Failed to parse device tree: %d\n", ret); + return ret; + } + + /* Acquire resources. */ + state->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(state->regs)) + return PTR_ERR(state->regs); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = mipi_csis_phy_init(state); + if (ret < 0) + return ret; + + ret = mipi_csis_clk_get(state); + if (ret < 0) + return ret; + + /* Reset PHY and enable the clocks. */ + mipi_csis_phy_reset(state); + + ret = mipi_csis_clk_enable(state); + if (ret < 0) { + dev_err(state->dev, "failed to enable clocks: %d\n", ret); + return ret; + } + + /* Now that the hardware is initialized, request the interrupt. */ + ret = devm_request_irq(dev, irq, mipi_csis_irq_handler, 0, + dev_name(dev), state); + if (ret) { + dev_err(dev, "Interrupt request failed\n"); + goto disable_clock; + } + + /* Initialize and register the subdev. */ + ret = mipi_csis_subdev_init(state); + if (ret < 0) + goto disable_clock; + + platform_set_drvdata(pdev, &state->sd); + + ret = mipi_csis_async_register(state); + if (ret < 0) { + dev_err(dev, "async register failed: %d\n", ret); + goto cleanup; + } + + /* Initialize debugfs. */ + mipi_csis_debugfs_init(state); + + /* Enable runtime PM. */ + pm_runtime_enable(dev); + if (!pm_runtime_enabled(dev)) { + ret = mipi_csis_pm_resume(dev, true); + if (ret < 0) + goto unregister_all; + } + + dev_info(dev, "lanes: %d, freq: %u\n", + state->bus.num_data_lanes, state->clk_frequency); + + return 0; + +unregister_all: + mipi_csis_debugfs_exit(state); +cleanup: + media_entity_cleanup(&state->sd.entity); + v4l2_async_notifier_unregister(&state->notifier); + v4l2_async_notifier_cleanup(&state->notifier); + v4l2_async_unregister_subdev(&state->sd); +disable_clock: + mipi_csis_clk_disable(state); + mutex_destroy(&state->lock); + + return ret; +} + +static int mipi_csis_remove(struct platform_device *pdev) +{ + struct v4l2_subdev *sd = platform_get_drvdata(pdev); + struct csi_state *state = mipi_sd_to_csis_state(sd); + + mipi_csis_debugfs_exit(state); + v4l2_async_notifier_unregister(&state->notifier); + v4l2_async_notifier_cleanup(&state->notifier); + v4l2_async_unregister_subdev(&state->sd); + + pm_runtime_disable(&pdev->dev); + mipi_csis_pm_suspend(&pdev->dev, true); + mipi_csis_clk_disable(state); + media_entity_cleanup(&state->sd.entity); + mutex_destroy(&state->lock); + pm_runtime_set_suspended(&pdev->dev); + + return 0; +} + static const struct of_device_id mipi_csis_of_match[] = { - { .compatible = "fsl,imx7-mipi-csi2", }, + { + .compatible = "fsl,imx7-mipi-csi2", + .data = &(const struct mipi_csis_info){ + .version = MIPI_CSIS_V3_3, + .num_clocks = 3, + }, + }, { + .compatible = "fsl,imx8mm-mipi-csi2", + .data = &(const struct mipi_csis_info){ + .version = MIPI_CSIS_V3_6_3, + .num_clocks = 4, + }, + }, { /* sentinel */ }, }; MODULE_DEVICE_TABLE(of, mipi_csis_of_match); @@ -1357,6 +1491,6 @@ static struct platform_driver mipi_csis_driver = { module_platform_driver(mipi_csis_driver); -MODULE_DESCRIPTION("i.MX7 MIPI CSI-2 Receiver driver"); +MODULE_DESCRIPTION("i.MX7 & i.MX8 MIPI CSI-2 receiver driver"); MODULE_LICENSE("GPL v2"); MODULE_ALIAS("platform:imx7-mipi-csi2"); diff --git a/drivers/staging/media/ipu3/include/intel-ipu3.h b/drivers/staging/media/ipu3/include/uapi/intel-ipu3.h similarity index 99% rename from drivers/staging/media/ipu3/include/intel-ipu3.h rename to drivers/staging/media/ipu3/include/uapi/intel-ipu3.h index 9b644fb23dde..fa3d6ee5adf2 100644 --- a/drivers/staging/media/ipu3/include/intel-ipu3.h +++ b/drivers/staging/media/ipu3/include/uapi/intel-ipu3.h @@ -9,8 +9,10 @@ /* from /drivers/staging/media/ipu3/include/videodev2.h */ /* Vendor specific - used for IPU3 camera sub-system */ -#define V4L2_META_FMT_IPU3_PARAMS v4l2_fourcc('i', 'p', '3', 'p') /* IPU3 processing parameters */ -#define V4L2_META_FMT_IPU3_STAT_3A v4l2_fourcc('i', 'p', '3', 's') /* IPU3 3A statistics */ +/* IPU3 processing parameters */ +#define V4L2_META_FMT_IPU3_PARAMS v4l2_fourcc('i', 'p', '3', 'p') +/* IPU3 3A statistics */ +#define V4L2_META_FMT_IPU3_STAT_3A v4l2_fourcc('i', 'p', '3', 's') /* from include/uapi/linux/v4l2-controls.h */ #define V4L2_CID_INTEL_IPU3_BASE (V4L2_CID_USER_BASE + 0x10c0) @@ -74,7 +76,6 @@ struct ipu3_uapi_grid_config { (IPU3_UAPI_AWB_MAX_SETS * \ (IPU3_UAPI_AWB_SET_SIZE + IPU3_UAPI_AWB_SPARE_FOR_BUBBLES)) - /** * struct ipu3_uapi_awb_raw_buffer - AWB raw buffer * @@ -244,8 +245,8 @@ struct ipu3_uapi_ae_ccm { */ struct ipu3_uapi_ae_config { struct ipu3_uapi_ae_grid_config grid_cfg __attribute__((aligned(32))); - struct ipu3_uapi_ae_weight_elem weights[ - IPU3_UAPI_AE_WEIGHTS] __attribute__((aligned(32))); + struct ipu3_uapi_ae_weight_elem weights[IPU3_UAPI_AE_WEIGHTS] + __attribute__((aligned(32))); struct ipu3_uapi_ae_ccm ae_ccm __attribute__((aligned(32))); } __packed; @@ -630,7 +631,7 @@ struct ipu3_uapi_bnr_static_config_wb_gains_thr_config { * @cg: Gain coefficient for threshold calculation, [0, 31], default 8. * @ci: Intensity coefficient for threshold calculation. range [0, 0x1f] * default 6. - * format: u3.2 (3 most significant bits represent whole number, + * format: u3.2 (3 most significant bits represent whole number, * 2 least significant bits represent the fractional part * with each count representing 0.25) * e.g. 6 in binary format is 00110, that translates to 1.5 diff --git a/drivers/staging/media/ipu3/ipu3-abi.h b/drivers/staging/media/ipu3/ipu3-abi.h index e1185602c7fd..c76935b436d7 100644 --- a/drivers/staging/media/ipu3/ipu3-abi.h +++ b/drivers/staging/media/ipu3/ipu3-abi.h @@ -4,7 +4,7 @@ #ifndef __IPU3_ABI_H #define __IPU3_ABI_H -#include "include/intel-ipu3.h" +#include "include/uapi/intel-ipu3.h" /******************* IMGU Hardware information *******************/ diff --git a/drivers/staging/media/ipu3/ipu3-css-pool.h b/drivers/staging/media/ipu3/ipu3-css-pool.h index 35519a08c08c..3f9e32e0e9a7 100644 --- a/drivers/staging/media/ipu3/ipu3-css-pool.h +++ b/drivers/staging/media/ipu3/ipu3-css-pool.h @@ -15,6 +15,7 @@ struct imgu_device; * @size: size of the buffer in bytes. * @vaddr: kernel virtual address. * @daddr: iova dma address to access IPU3. + * @pages: pages mapped to this buffer */ struct imgu_css_map { size_t size; diff --git a/drivers/staging/media/ipu3/ipu3-v4l2.c b/drivers/staging/media/ipu3/ipu3-v4l2.c index 6d9c49b39531..38a240764509 100644 --- a/drivers/staging/media/ipu3/ipu3-v4l2.c +++ b/drivers/staging/media/ipu3/ipu3-v4l2.c @@ -36,7 +36,7 @@ static int imgu_subdev_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) /* Initialize try_fmt */ for (i = 0; i < IMGU_NODE_NUM; i++) { struct v4l2_mbus_framefmt *try_fmt = - v4l2_subdev_get_try_format(sd, fh->pad, i); + v4l2_subdev_get_try_format(sd, fh->state, i); try_fmt->width = try_crop.width; try_fmt->height = try_crop.height; @@ -44,8 +44,8 @@ static int imgu_subdev_open(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) try_fmt->field = V4L2_FIELD_NONE; } - *v4l2_subdev_get_try_crop(sd, fh->pad, IMGU_NODE_IN) = try_crop; - *v4l2_subdev_get_try_compose(sd, fh->pad, IMGU_NODE_IN) = try_crop; + *v4l2_subdev_get_try_crop(sd, fh->state, IMGU_NODE_IN) = try_crop; + *v4l2_subdev_get_try_compose(sd, fh->state, IMGU_NODE_IN) = try_crop; return 0; } @@ -120,7 +120,7 @@ static int imgu_subdev_s_stream(struct v4l2_subdev *sd, int enable) } static int imgu_subdev_get_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imgu_device *imgu = v4l2_get_subdevdata(sd); @@ -136,7 +136,7 @@ static int imgu_subdev_get_fmt(struct v4l2_subdev *sd, if (fmt->which == V4L2_SUBDEV_FORMAT_ACTIVE) { fmt->format = imgu_pipe->nodes[pad].pad_fmt; } else { - mf = v4l2_subdev_get_try_format(sd, cfg, pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, pad); fmt->format = *mf; } @@ -144,7 +144,7 @@ static int imgu_subdev_get_fmt(struct v4l2_subdev *sd, } static int imgu_subdev_set_fmt(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct imgu_media_pipe *imgu_pipe; @@ -161,7 +161,7 @@ static int imgu_subdev_set_fmt(struct v4l2_subdev *sd, imgu_pipe = &imgu->imgu_pipe[pipe]; if (fmt->which == V4L2_SUBDEV_FORMAT_TRY) - mf = v4l2_subdev_get_try_format(sd, cfg, pad); + mf = v4l2_subdev_get_try_format(sd, sd_state, pad); else mf = &imgu_pipe->nodes[pad].pad_fmt; @@ -189,7 +189,7 @@ static int imgu_subdev_set_fmt(struct v4l2_subdev *sd, } static int imgu_subdev_get_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct v4l2_rect *try_sel, *r; @@ -202,11 +202,11 @@ static int imgu_subdev_get_selection(struct v4l2_subdev *sd, switch (sel->target) { case V4L2_SEL_TGT_CROP: - try_sel = v4l2_subdev_get_try_crop(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_crop(sd, sd_state, sel->pad); r = &imgu_sd->rect.eff; break; case V4L2_SEL_TGT_COMPOSE: - try_sel = v4l2_subdev_get_try_compose(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_compose(sd, sd_state, sel->pad); r = &imgu_sd->rect.bds; break; default: @@ -222,7 +222,7 @@ static int imgu_subdev_get_selection(struct v4l2_subdev *sd, } static int imgu_subdev_set_selection(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_selection *sel) { struct imgu_device *imgu = v4l2_get_subdevdata(sd); @@ -241,11 +241,11 @@ static int imgu_subdev_set_selection(struct v4l2_subdev *sd, switch (sel->target) { case V4L2_SEL_TGT_CROP: - try_sel = v4l2_subdev_get_try_crop(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_crop(sd, sd_state, sel->pad); rect = &imgu_sd->rect.eff; break; case V4L2_SEL_TGT_COMPOSE: - try_sel = v4l2_subdev_get_try_compose(sd, cfg, sel->pad); + try_sel = v4l2_subdev_get_try_compose(sd, sd_state, sel->pad); rect = &imgu_sd->rect.bds; break; default: diff --git a/drivers/staging/media/ipu3/ipu3.c b/drivers/staging/media/ipu3/ipu3.c index ee1bba6bdcac..8e1e9e46e604 100644 --- a/drivers/staging/media/ipu3/ipu3.c +++ b/drivers/staging/media/ipu3/ipu3.c @@ -392,10 +392,9 @@ int imgu_s_stream(struct imgu_device *imgu, int enable) } /* Set Power */ - r = pm_runtime_get_sync(dev); + r = pm_runtime_resume_and_get(dev); if (r < 0) { dev_err(dev, "failed to set imgu power\n"); - pm_runtime_put(dev); return r; } diff --git a/drivers/staging/media/meson/vdec/vdec_helpers.c b/drivers/staging/media/meson/vdec/vdec_helpers.c index 7f07a9175815..b9125c295d1d 100644 --- a/drivers/staging/media/meson/vdec/vdec_helpers.c +++ b/drivers/staging/media/meson/vdec/vdec_helpers.c @@ -183,7 +183,7 @@ int amvdec_set_canvases(struct amvdec_session *sess, u32 pixfmt = sess->pixfmt_cap; u32 width = ALIGN(sess->width, 32); u32 height = ALIGN(sess->height, 32); - u32 reg_cur = reg_base[0]; + u32 reg_cur; u32 reg_num_cur = 0; u32 reg_base_cur = 0; int i = 0; diff --git a/drivers/staging/media/omap4iss/iss.h b/drivers/staging/media/omap4iss/iss.h index b88f9529683c..3f587e000729 100644 --- a/drivers/staging/media/omap4iss/iss.h +++ b/drivers/staging/media/omap4iss/iss.h @@ -119,9 +119,6 @@ struct iss_device { unsigned int isp_subclk_resources; }; -#define v4l2_dev_to_iss_device(dev) \ - container_of(dev, struct iss_device, v4l2_dev) - int omap4iss_get_external_info(struct iss_pipeline *pipe, struct media_link *link); diff --git a/drivers/staging/media/omap4iss/iss_csi2.c b/drivers/staging/media/omap4iss/iss_csi2.c index a6dc2d2b1228..124ab2f44fbf 100644 --- a/drivers/staging/media/omap4iss/iss_csi2.c +++ b/drivers/staging/media/omap4iss/iss_csi2.c @@ -825,19 +825,20 @@ static const struct iss_video_operations csi2_issvideo_ops = { static struct v4l2_mbus_framefmt * __csi2_get_format(struct iss_csi2_device *csi2, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&csi2->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&csi2->subdev, sd_state, + pad); return &csi2->formats[pad]; } static void csi2_try_format(struct iss_csi2_device *csi2, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) @@ -868,7 +869,8 @@ csi2_try_format(struct iss_csi2_device *csi2, * compression. */ pixelcode = fmt->code; - format = __csi2_get_format(csi2, cfg, CSI2_PAD_SINK, which); + format = __csi2_get_format(csi2, sd_state, CSI2_PAD_SINK, + which); memcpy(fmt, format, sizeof(*fmt)); /* @@ -894,7 +896,7 @@ csi2_try_format(struct iss_csi2_device *csi2, * return -EINVAL or zero on success */ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct iss_csi2_device *csi2 = v4l2_get_subdevdata(sd); @@ -907,7 +909,7 @@ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, code->code = csi2_input_fmts[code->index]; } else { - format = __csi2_get_format(csi2, cfg, CSI2_PAD_SINK, + format = __csi2_get_format(csi2, sd_state, CSI2_PAD_SINK, code->which); switch (code->index) { case 0: @@ -931,7 +933,7 @@ static int csi2_enum_mbus_code(struct v4l2_subdev *sd, } static int csi2_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct iss_csi2_device *csi2 = v4l2_get_subdevdata(sd); @@ -943,7 +945,7 @@ static int csi2_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - csi2_try_format(csi2, cfg, fse->pad, &format, fse->which); + csi2_try_format(csi2, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -953,7 +955,7 @@ static int csi2_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - csi2_try_format(csi2, cfg, fse->pad, &format, fse->which); + csi2_try_format(csi2, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -968,13 +970,13 @@ static int csi2_enum_frame_size(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int csi2_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_csi2_device *csi2 = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csi2_get_format(csi2, cfg, fmt->pad, fmt->which); + format = __csi2_get_format(csi2, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; @@ -990,25 +992,26 @@ static int csi2_get_format(struct v4l2_subdev *sd, * return -EINVAL or zero on success */ static int csi2_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_csi2_device *csi2 = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __csi2_get_format(csi2, cfg, fmt->pad, fmt->which); + format = __csi2_get_format(csi2, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; - csi2_try_format(csi2, cfg, fmt->pad, &fmt->format, fmt->which); + csi2_try_format(csi2, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == CSI2_PAD_SINK) { - format = __csi2_get_format(csi2, cfg, CSI2_PAD_SOURCE, + format = __csi2_get_format(csi2, sd_state, CSI2_PAD_SOURCE, fmt->which); *format = fmt->format; - csi2_try_format(csi2, cfg, CSI2_PAD_SOURCE, format, fmt->which); + csi2_try_format(csi2, sd_state, CSI2_PAD_SOURCE, format, + fmt->which); } return 0; @@ -1050,7 +1053,7 @@ static int csi2_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10; format.format.width = 4096; format.format.height = 4096; - csi2_set_format(sd, fh ? fh->pad : NULL, &format); + csi2_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/staging/media/omap4iss/iss_ipipe.c b/drivers/staging/media/omap4iss/iss_ipipe.c index 26be078b69f3..23f707cb336f 100644 --- a/drivers/staging/media/omap4iss/iss_ipipe.c +++ b/drivers/staging/media/omap4iss/iss_ipipe.c @@ -21,7 +21,7 @@ static struct v4l2_mbus_framefmt * __ipipe_get_format(struct iss_ipipe_device *ipipe, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which); @@ -175,12 +175,13 @@ static int ipipe_set_stream(struct v4l2_subdev *sd, int enable) static struct v4l2_mbus_framefmt * __ipipe_get_format(struct iss_ipipe_device *ipipe, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&ipipe->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&ipipe->subdev, sd_state, + pad); return &ipipe->formats[pad]; } @@ -194,7 +195,7 @@ __ipipe_get_format(struct iss_ipipe_device *ipipe, */ static void ipipe_try_format(struct iss_ipipe_device *ipipe, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) @@ -222,7 +223,8 @@ ipipe_try_format(struct iss_ipipe_device *ipipe, break; case IPIPE_PAD_SOURCE_VP: - format = __ipipe_get_format(ipipe, cfg, IPIPE_PAD_SINK, which); + format = __ipipe_get_format(ipipe, sd_state, IPIPE_PAD_SINK, + which); memcpy(fmt, format, sizeof(*fmt)); fmt->code = MEDIA_BUS_FMT_UYVY8_1X16; @@ -243,7 +245,7 @@ ipipe_try_format(struct iss_ipipe_device *ipipe, * return -EINVAL or zero on success */ static int ipipe_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { switch (code->pad) { @@ -270,7 +272,7 @@ static int ipipe_enum_mbus_code(struct v4l2_subdev *sd, } static int ipipe_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct iss_ipipe_device *ipipe = v4l2_get_subdevdata(sd); @@ -282,7 +284,7 @@ static int ipipe_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - ipipe_try_format(ipipe, cfg, fse->pad, &format, fse->which); + ipipe_try_format(ipipe, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -292,7 +294,7 @@ static int ipipe_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - ipipe_try_format(ipipe, cfg, fse->pad, &format, fse->which); + ipipe_try_format(ipipe, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -309,13 +311,13 @@ static int ipipe_enum_frame_size(struct v4l2_subdev *sd, * to the format type. */ static int ipipe_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_ipipe_device *ipipe = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ipipe_get_format(ipipe, cfg, fmt->pad, fmt->which); + format = __ipipe_get_format(ipipe, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; @@ -333,25 +335,26 @@ static int ipipe_get_format(struct v4l2_subdev *sd, * to the format type. */ static int ipipe_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_ipipe_device *ipipe = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ipipe_get_format(ipipe, cfg, fmt->pad, fmt->which); + format = __ipipe_get_format(ipipe, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; - ipipe_try_format(ipipe, cfg, fmt->pad, &fmt->format, fmt->which); + ipipe_try_format(ipipe, sd_state, fmt->pad, &fmt->format, fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == IPIPE_PAD_SINK) { - format = __ipipe_get_format(ipipe, cfg, IPIPE_PAD_SOURCE_VP, + format = __ipipe_get_format(ipipe, sd_state, + IPIPE_PAD_SOURCE_VP, fmt->which); *format = fmt->format; - ipipe_try_format(ipipe, cfg, IPIPE_PAD_SOURCE_VP, format, + ipipe_try_format(ipipe, sd_state, IPIPE_PAD_SOURCE_VP, format, fmt->which); } @@ -392,7 +395,7 @@ static int ipipe_init_formats(struct v4l2_subdev *sd, struct v4l2_subdev_fh *fh) format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10; format.format.width = 4096; format.format.height = 4096; - ipipe_set_format(sd, fh ? fh->pad : NULL, &format); + ipipe_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/staging/media/omap4iss/iss_ipipeif.c b/drivers/staging/media/omap4iss/iss_ipipeif.c index c2978d02e797..5e7f25cd53ac 100644 --- a/drivers/staging/media/omap4iss/iss_ipipeif.c +++ b/drivers/staging/media/omap4iss/iss_ipipeif.c @@ -357,11 +357,12 @@ static int ipipeif_set_stream(struct v4l2_subdev *sd, int enable) static struct v4l2_mbus_framefmt * __ipipeif_get_format(struct iss_ipipeif_device *ipipeif, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&ipipeif->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&ipipeif->subdev, sd_state, + pad); return &ipipeif->formats[pad]; } @@ -374,7 +375,7 @@ __ipipeif_get_format(struct iss_ipipeif_device *ipipeif, */ static void ipipeif_try_format(struct iss_ipipeif_device *ipipeif, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) { @@ -403,7 +404,8 @@ ipipeif_try_format(struct iss_ipipeif_device *ipipeif, break; case IPIPEIF_PAD_SOURCE_ISIF_SF: - format = __ipipeif_get_format(ipipeif, cfg, IPIPEIF_PAD_SINK, + format = __ipipeif_get_format(ipipeif, sd_state, + IPIPEIF_PAD_SINK, which); memcpy(fmt, format, sizeof(*fmt)); @@ -418,7 +420,8 @@ ipipeif_try_format(struct iss_ipipeif_device *ipipeif, break; case IPIPEIF_PAD_SOURCE_VP: - format = __ipipeif_get_format(ipipeif, cfg, IPIPEIF_PAD_SINK, + format = __ipipeif_get_format(ipipeif, sd_state, + IPIPEIF_PAD_SINK, which); memcpy(fmt, format, sizeof(*fmt)); @@ -442,7 +445,7 @@ ipipeif_try_format(struct iss_ipipeif_device *ipipeif, * return -EINVAL or zero on success */ static int ipipeif_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct iss_ipipeif_device *ipipeif = v4l2_get_subdevdata(sd); @@ -462,7 +465,8 @@ static int ipipeif_enum_mbus_code(struct v4l2_subdev *sd, if (code->index != 0) return -EINVAL; - format = __ipipeif_get_format(ipipeif, cfg, IPIPEIF_PAD_SINK, + format = __ipipeif_get_format(ipipeif, sd_state, + IPIPEIF_PAD_SINK, code->which); code->code = format->code; @@ -476,7 +480,7 @@ static int ipipeif_enum_mbus_code(struct v4l2_subdev *sd, } static int ipipeif_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct iss_ipipeif_device *ipipeif = v4l2_get_subdevdata(sd); @@ -488,7 +492,7 @@ static int ipipeif_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - ipipeif_try_format(ipipeif, cfg, fse->pad, &format, fse->which); + ipipeif_try_format(ipipeif, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -498,7 +502,7 @@ static int ipipeif_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - ipipeif_try_format(ipipeif, cfg, fse->pad, &format, fse->which); + ipipeif_try_format(ipipeif, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -515,13 +519,13 @@ static int ipipeif_enum_frame_size(struct v4l2_subdev *sd, * to the format type. */ static int ipipeif_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_ipipeif_device *ipipeif = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ipipeif_get_format(ipipeif, cfg, fmt->pad, fmt->which); + format = __ipipeif_get_format(ipipeif, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; @@ -539,33 +543,36 @@ static int ipipeif_get_format(struct v4l2_subdev *sd, * to the format type. */ static int ipipeif_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_ipipeif_device *ipipeif = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __ipipeif_get_format(ipipeif, cfg, fmt->pad, fmt->which); + format = __ipipeif_get_format(ipipeif, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; - ipipeif_try_format(ipipeif, cfg, fmt->pad, &fmt->format, fmt->which); + ipipeif_try_format(ipipeif, sd_state, fmt->pad, &fmt->format, + fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == IPIPEIF_PAD_SINK) { - format = __ipipeif_get_format(ipipeif, cfg, + format = __ipipeif_get_format(ipipeif, sd_state, IPIPEIF_PAD_SOURCE_ISIF_SF, fmt->which); *format = fmt->format; - ipipeif_try_format(ipipeif, cfg, IPIPEIF_PAD_SOURCE_ISIF_SF, + ipipeif_try_format(ipipeif, sd_state, + IPIPEIF_PAD_SOURCE_ISIF_SF, format, fmt->which); - format = __ipipeif_get_format(ipipeif, cfg, + format = __ipipeif_get_format(ipipeif, sd_state, IPIPEIF_PAD_SOURCE_VP, fmt->which); *format = fmt->format; - ipipeif_try_format(ipipeif, cfg, IPIPEIF_PAD_SOURCE_VP, format, + ipipeif_try_format(ipipeif, sd_state, IPIPEIF_PAD_SOURCE_VP, + format, fmt->which); } @@ -608,7 +615,7 @@ static int ipipeif_init_formats(struct v4l2_subdev *sd, format.format.code = MEDIA_BUS_FMT_SGRBG10_1X10; format.format.width = 4096; format.format.height = 4096; - ipipeif_set_format(sd, fh ? fh->pad : NULL, &format); + ipipeif_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/staging/media/omap4iss/iss_resizer.c b/drivers/staging/media/omap4iss/iss_resizer.c index 3b6875cbca9b..a5f8f9f1ab16 100644 --- a/drivers/staging/media/omap4iss/iss_resizer.c +++ b/drivers/staging/media/omap4iss/iss_resizer.c @@ -416,11 +416,12 @@ static int resizer_set_stream(struct v4l2_subdev *sd, int enable) static struct v4l2_mbus_framefmt * __resizer_get_format(struct iss_resizer_device *resizer, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, unsigned int pad, enum v4l2_subdev_format_whence which) { if (which == V4L2_SUBDEV_FORMAT_TRY) - return v4l2_subdev_get_try_format(&resizer->subdev, cfg, pad); + return v4l2_subdev_get_try_format(&resizer->subdev, sd_state, + pad); return &resizer->formats[pad]; } @@ -433,7 +434,7 @@ __resizer_get_format(struct iss_resizer_device *resizer, */ static void resizer_try_format(struct iss_resizer_device *resizer, - struct v4l2_subdev_pad_config *cfg, unsigned int pad, + struct v4l2_subdev_state *sd_state, unsigned int pad, struct v4l2_mbus_framefmt *fmt, enum v4l2_subdev_format_whence which) { @@ -461,7 +462,8 @@ resizer_try_format(struct iss_resizer_device *resizer, case RESIZER_PAD_SOURCE_MEM: pixelcode = fmt->code; - format = __resizer_get_format(resizer, cfg, RESIZER_PAD_SINK, + format = __resizer_get_format(resizer, sd_state, + RESIZER_PAD_SINK, which); memcpy(fmt, format, sizeof(*fmt)); @@ -492,7 +494,7 @@ resizer_try_format(struct iss_resizer_device *resizer, * return -EINVAL or zero on success */ static int resizer_enum_mbus_code(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { struct iss_resizer_device *resizer = v4l2_get_subdevdata(sd); @@ -507,7 +509,8 @@ static int resizer_enum_mbus_code(struct v4l2_subdev *sd, break; case RESIZER_PAD_SOURCE_MEM: - format = __resizer_get_format(resizer, cfg, RESIZER_PAD_SINK, + format = __resizer_get_format(resizer, sd_state, + RESIZER_PAD_SINK, code->which); if (code->index == 0) { @@ -537,7 +540,7 @@ static int resizer_enum_mbus_code(struct v4l2_subdev *sd, } static int resizer_enum_frame_size(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { struct iss_resizer_device *resizer = v4l2_get_subdevdata(sd); @@ -549,7 +552,7 @@ static int resizer_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = 1; format.height = 1; - resizer_try_format(resizer, cfg, fse->pad, &format, fse->which); + resizer_try_format(resizer, sd_state, fse->pad, &format, fse->which); fse->min_width = format.width; fse->min_height = format.height; @@ -559,7 +562,7 @@ static int resizer_enum_frame_size(struct v4l2_subdev *sd, format.code = fse->code; format.width = -1; format.height = -1; - resizer_try_format(resizer, cfg, fse->pad, &format, fse->which); + resizer_try_format(resizer, sd_state, fse->pad, &format, fse->which); fse->max_width = format.width; fse->max_height = format.height; @@ -576,13 +579,13 @@ static int resizer_enum_frame_size(struct v4l2_subdev *sd, * to the format type. */ static int resizer_get_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_resizer_device *resizer = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __resizer_get_format(resizer, cfg, fmt->pad, fmt->which); + format = __resizer_get_format(resizer, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; @@ -600,26 +603,28 @@ static int resizer_get_format(struct v4l2_subdev *sd, * to the format type. */ static int resizer_set_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct iss_resizer_device *resizer = v4l2_get_subdevdata(sd); struct v4l2_mbus_framefmt *format; - format = __resizer_get_format(resizer, cfg, fmt->pad, fmt->which); + format = __resizer_get_format(resizer, sd_state, fmt->pad, fmt->which); if (!format) return -EINVAL; - resizer_try_format(resizer, cfg, fmt->pad, &fmt->format, fmt->which); + resizer_try_format(resizer, sd_state, fmt->pad, &fmt->format, + fmt->which); *format = fmt->format; /* Propagate the format from sink to source */ if (fmt->pad == RESIZER_PAD_SINK) { - format = __resizer_get_format(resizer, cfg, + format = __resizer_get_format(resizer, sd_state, RESIZER_PAD_SOURCE_MEM, fmt->which); *format = fmt->format; - resizer_try_format(resizer, cfg, RESIZER_PAD_SOURCE_MEM, format, + resizer_try_format(resizer, sd_state, RESIZER_PAD_SOURCE_MEM, + format, fmt->which); } @@ -662,7 +667,7 @@ static int resizer_init_formats(struct v4l2_subdev *sd, format.format.code = MEDIA_BUS_FMT_UYVY8_1X16; format.format.width = 4096; format.format.height = 4096; - resizer_set_format(sd, fh ? fh->pad : NULL, &format); + resizer_set_format(sd, fh ? fh->state : NULL, &format); return 0; } diff --git a/drivers/staging/media/omap4iss/iss_video.c b/drivers/staging/media/omap4iss/iss_video.c index 930f638f51eb..d0da083deed5 100644 --- a/drivers/staging/media/omap4iss/iss_video.c +++ b/drivers/staging/media/omap4iss/iss_video.c @@ -399,7 +399,7 @@ static void iss_video_buf_queue(struct vb2_buffer *vb) if (start) omap4iss_pipeline_set_stream(pipe, - ISS_PIPELINE_STREAM_SINGLESHOT); + ISS_PIPELINE_STREAM_SINGLESHOT); } } @@ -960,7 +960,7 @@ iss_video_streamon(struct file *file, void *fh, enum v4l2_buf_type type) unsigned long flags; ret = omap4iss_pipeline_set_stream(pipe, - ISS_PIPELINE_STREAM_CONTINUOUS); + ISS_PIPELINE_STREAM_CONTINUOUS); if (ret < 0) goto err_omap4iss_set_stream; spin_lock_irqsave(&video->qlock, flags); diff --git a/drivers/staging/media/rkvdec/rkvdec.c b/drivers/staging/media/rkvdec/rkvdec.c index d821661d30f3..7131156c1f2c 100644 --- a/drivers/staging/media/rkvdec/rkvdec.c +++ b/drivers/staging/media/rkvdec/rkvdec.c @@ -481,7 +481,15 @@ static int rkvdec_buf_prepare(struct vb2_buffer *vb) if (vb2_plane_size(vb, i) < sizeimage) return -EINVAL; } - vb2_set_plane_payload(vb, 0, f->fmt.pix_mp.plane_fmt[0].sizeimage); + + /* + * Buffer's bytesused must be written by driver for CAPTURE buffers. + * (for OUTPUT buffers, if userspace passes 0 bytesused, v4l2-core sets + * it to buffer length). + */ + if (V4L2_TYPE_IS_CAPTURE(vq->type)) + vb2_set_plane_payload(vb, 0, f->fmt.pix_mp.plane_fmt[0].sizeimage); + return 0; } @@ -658,7 +666,7 @@ static void rkvdec_device_run(void *priv) if (WARN_ON(!desc)) return; - ret = pm_runtime_get_sync(rkvdec->dev); + ret = pm_runtime_resume_and_get(rkvdec->dev); if (ret < 0) { rkvdec_job_finish_no_pm(ctx, VB2_BUF_STATE_ERROR); return; diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c index 92812d1a39d4..c0d005dafc6c 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus.c @@ -31,13 +31,19 @@ static const struct cedrus_control cedrus_controls[] = { { .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS, + .id = V4L2_CID_STATELESS_MPEG2_SEQUENCE, }, .codec = CEDRUS_CODEC_MPEG2, }, { .cfg = { - .id = V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION, + .id = V4L2_CID_STATELESS_MPEG2_PICTURE, + }, + .codec = CEDRUS_CODEC_MPEG2, + }, + { + .cfg = { + .id = V4L2_CID_STATELESS_MPEG2_QUANTISATION, }, .codec = CEDRUS_CODEC_MPEG2, }, @@ -151,6 +157,12 @@ static const struct cedrus_control cedrus_controls[] = { }, .codec = CEDRUS_CODEC_VP8, }, + { + .cfg = { + .id = V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS, + }, + .codec = CEDRUS_CODEC_H265, + }, }; #define CEDRUS_CONTROLS_COUNT ARRAY_SIZE(cedrus_controls) diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.h b/drivers/staging/media/sunxi/cedrus/cedrus.h index 15f147dad4cb..88afba17b78b 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.h +++ b/drivers/staging/media/sunxi/cedrus/cedrus.h @@ -68,14 +68,16 @@ struct cedrus_h264_run { }; struct cedrus_mpeg2_run { - const struct v4l2_ctrl_mpeg2_slice_params *slice_params; - const struct v4l2_ctrl_mpeg2_quantization *quantization; + const struct v4l2_ctrl_mpeg2_sequence *sequence; + const struct v4l2_ctrl_mpeg2_picture *picture; + const struct v4l2_ctrl_mpeg2_quantisation *quantisation; }; struct cedrus_h265_run { const struct v4l2_ctrl_hevc_sps *sps; const struct v4l2_ctrl_hevc_pps *pps; const struct v4l2_ctrl_hevc_slice_params *slice_params; + const struct v4l2_ctrl_hevc_decode_params *decode_params; }; struct cedrus_vp8_run { diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c index d696b3ec70c0..40e8c4123f76 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c @@ -40,10 +40,12 @@ void cedrus_device_run(void *priv) switch (ctx->src_fmt.pixelformat) { case V4L2_PIX_FMT_MPEG2_SLICE: - run.mpeg2.slice_params = cedrus_find_control_data(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS); - run.mpeg2.quantization = cedrus_find_control_data(ctx, - V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION); + run.mpeg2.sequence = cedrus_find_control_data(ctx, + V4L2_CID_STATELESS_MPEG2_SEQUENCE); + run.mpeg2.picture = cedrus_find_control_data(ctx, + V4L2_CID_STATELESS_MPEG2_PICTURE); + run.mpeg2.quantisation = cedrus_find_control_data(ctx, + V4L2_CID_STATELESS_MPEG2_QUANTISATION); break; case V4L2_PIX_FMT_H264_SLICE: @@ -68,6 +70,8 @@ void cedrus_device_run(void *priv) V4L2_CID_MPEG_VIDEO_HEVC_PPS); run.h265.slice_params = cedrus_find_control_data(ctx, V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS); + run.h265.decode_params = cedrus_find_control_data(ctx, + V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS); break; case V4L2_PIX_FMT_VP8_FRAME: diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c index ce497d0197df..6821e3d05d34 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_h265.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_h265.c @@ -245,6 +245,7 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, const struct v4l2_ctrl_hevc_sps *sps; const struct v4l2_ctrl_hevc_pps *pps; const struct v4l2_ctrl_hevc_slice_params *slice_params; + const struct v4l2_ctrl_hevc_decode_params *decode_params; const struct v4l2_hevc_pred_weight_table *pred_weight_table; dma_addr_t src_buf_addr; dma_addr_t src_buf_end_addr; @@ -256,6 +257,7 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, sps = run->h265.sps; pps = run->h265.pps; slice_params = run->h265.slice_params; + decode_params = run->h265.decode_params; pred_weight_table = &slice_params->pred_weight_table; /* MV column buffer size and allocation. */ @@ -477,8 +479,8 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, slice_params->flags); reg |= VE_DEC_H265_FLAG(VE_DEC_H265_DEC_SLICE_HDR_INFO0_FLAG_DEPENDENT_SLICE_SEGMENT, - V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT, - pps->flags); + V4L2_HEVC_SLICE_PARAMS_FLAG_DEPENDENT_SLICE_SEGMENT, + slice_params->flags); /* FIXME: For multi-slice support. */ reg |= VE_DEC_H265_DEC_SLICE_HDR_INFO0_FLAG_FIRST_SLICE_SEGMENT_IN_PIC; @@ -487,7 +489,7 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, reg = VE_DEC_H265_DEC_SLICE_HDR_INFO1_SLICE_TC_OFFSET_DIV2(slice_params->slice_tc_offset_div2) | VE_DEC_H265_DEC_SLICE_HDR_INFO1_SLICE_BETA_OFFSET_DIV2(slice_params->slice_beta_offset_div2) | - VE_DEC_H265_DEC_SLICE_HDR_INFO1_SLICE_POC_BIGEST_IN_RPS_ST(slice_params->num_rps_poc_st_curr_after == 0) | + VE_DEC_H265_DEC_SLICE_HDR_INFO1_SLICE_POC_BIGEST_IN_RPS_ST(decode_params->num_poc_st_curr_after == 0) | VE_DEC_H265_DEC_SLICE_HDR_INFO1_SLICE_CR_QP_OFFSET(slice_params->slice_cr_qp_offset) | VE_DEC_H265_DEC_SLICE_HDR_INFO1_SLICE_CB_QP_OFFSET(slice_params->slice_cb_qp_offset) | VE_DEC_H265_DEC_SLICE_HDR_INFO1_SLICE_QP_DELTA(slice_params->slice_qp_delta); @@ -527,8 +529,8 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, cedrus_write(dev, VE_DEC_H265_NEIGHBOR_INFO_ADDR, reg); /* Write decoded picture buffer in pic list. */ - cedrus_h265_frame_info_write_dpb(ctx, slice_params->dpb, - slice_params->num_active_dpb_entries); + cedrus_h265_frame_info_write_dpb(ctx, decode_params->dpb, + decode_params->num_active_dpb_entries); /* Output frame. */ @@ -545,7 +547,7 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, /* Reference picture list 0 (for P/B frames). */ if (slice_params->slice_type != V4L2_HEVC_SLICE_TYPE_I) { - cedrus_h265_ref_pic_list_write(dev, slice_params->dpb, + cedrus_h265_ref_pic_list_write(dev, decode_params->dpb, slice_params->ref_idx_l0, slice_params->num_ref_idx_l0_active_minus1 + 1, VE_DEC_H265_SRAM_OFFSET_REF_PIC_LIST0); @@ -564,7 +566,7 @@ static void cedrus_h265_setup(struct cedrus_ctx *ctx, /* Reference picture list 1 (for B frames). */ if (slice_params->slice_type == V4L2_HEVC_SLICE_TYPE_B) { - cedrus_h265_ref_pic_list_write(dev, slice_params->dpb, + cedrus_h265_ref_pic_list_write(dev, decode_params->dpb, slice_params->ref_idx_l1, slice_params->num_ref_idx_l1_active_minus1 + 1, VE_DEC_H265_SRAM_OFFSET_REF_PIC_LIST1); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c b/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c index 8bcd6b8f9e2d..5dad2f296c6d 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_mpeg2.c @@ -13,30 +13,6 @@ #include "cedrus_hw.h" #include "cedrus_regs.h" -/* Default MPEG-2 quantization coefficients, from the specification. */ - -static const u8 intra_quantization_matrix_default[64] = { - 8, 16, 16, 19, 16, 19, 22, 22, - 22, 22, 22, 22, 26, 24, 26, 27, - 27, 27, 26, 26, 26, 26, 27, 27, - 27, 29, 29, 29, 34, 34, 34, 29, - 29, 29, 27, 27, 29, 29, 32, 32, - 34, 34, 37, 38, 37, 35, 35, 34, - 35, 38, 38, 40, 40, 40, 48, 48, - 46, 46, 56, 56, 58, 69, 69, 83 -}; - -static const u8 non_intra_quantization_matrix_default[64] = { - 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 16 -}; - static enum cedrus_irq_status cedrus_mpeg2_irq_status(struct cedrus_ctx *ctx) { struct cedrus_dev *dev = ctx->dev; @@ -74,10 +50,9 @@ static void cedrus_mpeg2_irq_disable(struct cedrus_ctx *ctx) static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) { - const struct v4l2_ctrl_mpeg2_slice_params *slice_params; - const struct v4l2_mpeg2_sequence *sequence; - const struct v4l2_mpeg2_picture *picture; - const struct v4l2_ctrl_mpeg2_quantization *quantization; + const struct v4l2_ctrl_mpeg2_sequence *seq; + const struct v4l2_ctrl_mpeg2_picture *pic; + const struct v4l2_ctrl_mpeg2_quantisation *quantisation; dma_addr_t src_buf_addr, dst_luma_addr, dst_chroma_addr; dma_addr_t fwd_luma_addr, fwd_chroma_addr; dma_addr_t bwd_luma_addr, bwd_chroma_addr; @@ -89,22 +64,16 @@ static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) unsigned int i; u32 reg; - slice_params = run->mpeg2.slice_params; - sequence = &slice_params->sequence; - picture = &slice_params->picture; + seq = run->mpeg2.sequence; + pic = run->mpeg2.picture; - quantization = run->mpeg2.quantization; + quantisation = run->mpeg2.quantisation; /* Activate MPEG engine. */ cedrus_engine_enable(ctx, CEDRUS_CODEC_MPEG2); - /* Set intra quantization matrix. */ - - if (quantization && quantization->load_intra_quantiser_matrix) - matrix = quantization->intra_quantiser_matrix; - else - matrix = intra_quantization_matrix_default; - + /* Set intra quantisation matrix. */ + matrix = quantisation->intra_quantiser_matrix; for (i = 0; i < 64; i++) { reg = VE_DEC_MPEG_IQMINPUT_WEIGHT(i, matrix[i]); reg |= VE_DEC_MPEG_IQMINPUT_FLAG_INTRA; @@ -112,13 +81,8 @@ static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) cedrus_write(dev, VE_DEC_MPEG_IQMINPUT, reg); } - /* Set non-intra quantization matrix. */ - - if (quantization && quantization->load_non_intra_quantiser_matrix) - matrix = quantization->non_intra_quantiser_matrix; - else - matrix = non_intra_quantization_matrix_default; - + /* Set non-intra quantisation matrix. */ + matrix = quantisation->non_intra_quantiser_matrix; for (i = 0; i < 64; i++) { reg = VE_DEC_MPEG_IQMINPUT_WEIGHT(i, matrix[i]); reg |= VE_DEC_MPEG_IQMINPUT_FLAG_NON_INTRA; @@ -128,19 +92,19 @@ static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) /* Set MPEG picture header. */ - reg = VE_DEC_MPEG_MP12HDR_SLICE_TYPE(picture->picture_coding_type); - reg |= VE_DEC_MPEG_MP12HDR_F_CODE(0, 0, picture->f_code[0][0]); - reg |= VE_DEC_MPEG_MP12HDR_F_CODE(0, 1, picture->f_code[0][1]); - reg |= VE_DEC_MPEG_MP12HDR_F_CODE(1, 0, picture->f_code[1][0]); - reg |= VE_DEC_MPEG_MP12HDR_F_CODE(1, 1, picture->f_code[1][1]); - reg |= VE_DEC_MPEG_MP12HDR_INTRA_DC_PRECISION(picture->intra_dc_precision); - reg |= VE_DEC_MPEG_MP12HDR_INTRA_PICTURE_STRUCTURE(picture->picture_structure); - reg |= VE_DEC_MPEG_MP12HDR_TOP_FIELD_FIRST(picture->top_field_first); - reg |= VE_DEC_MPEG_MP12HDR_FRAME_PRED_FRAME_DCT(picture->frame_pred_frame_dct); - reg |= VE_DEC_MPEG_MP12HDR_CONCEALMENT_MOTION_VECTORS(picture->concealment_motion_vectors); - reg |= VE_DEC_MPEG_MP12HDR_Q_SCALE_TYPE(picture->q_scale_type); - reg |= VE_DEC_MPEG_MP12HDR_INTRA_VLC_FORMAT(picture->intra_vlc_format); - reg |= VE_DEC_MPEG_MP12HDR_ALTERNATE_SCAN(picture->alternate_scan); + reg = VE_DEC_MPEG_MP12HDR_SLICE_TYPE(pic->picture_coding_type); + reg |= VE_DEC_MPEG_MP12HDR_F_CODE(0, 0, pic->f_code[0][0]); + reg |= VE_DEC_MPEG_MP12HDR_F_CODE(0, 1, pic->f_code[0][1]); + reg |= VE_DEC_MPEG_MP12HDR_F_CODE(1, 0, pic->f_code[1][0]); + reg |= VE_DEC_MPEG_MP12HDR_F_CODE(1, 1, pic->f_code[1][1]); + reg |= VE_DEC_MPEG_MP12HDR_INTRA_DC_PRECISION(pic->intra_dc_precision); + reg |= VE_DEC_MPEG_MP12HDR_INTRA_PICTURE_STRUCTURE(pic->picture_structure); + reg |= VE_DEC_MPEG_MP12HDR_TOP_FIELD_FIRST(pic->flags & V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST); + reg |= VE_DEC_MPEG_MP12HDR_FRAME_PRED_FRAME_DCT(pic->flags & V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT); + reg |= VE_DEC_MPEG_MP12HDR_CONCEALMENT_MOTION_VECTORS(pic->flags & V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV); + reg |= VE_DEC_MPEG_MP12HDR_Q_SCALE_TYPE(pic->flags & V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE); + reg |= VE_DEC_MPEG_MP12HDR_INTRA_VLC_FORMAT(pic->flags & V4L2_MPEG2_PIC_FLAG_INTRA_VLC); + reg |= VE_DEC_MPEG_MP12HDR_ALTERNATE_SCAN(pic->flags & V4L2_MPEG2_PIC_FLAG_ALT_SCAN); reg |= VE_DEC_MPEG_MP12HDR_FULL_PEL_FORWARD_VECTOR(0); reg |= VE_DEC_MPEG_MP12HDR_FULL_PEL_BACKWARD_VECTOR(0); @@ -148,8 +112,8 @@ static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) /* Set frame dimensions. */ - reg = VE_DEC_MPEG_PICCODEDSIZE_WIDTH(sequence->horizontal_size); - reg |= VE_DEC_MPEG_PICCODEDSIZE_HEIGHT(sequence->vertical_size); + reg = VE_DEC_MPEG_PICCODEDSIZE_WIDTH(seq->horizontal_size); + reg |= VE_DEC_MPEG_PICCODEDSIZE_HEIGHT(seq->vertical_size); cedrus_write(dev, VE_DEC_MPEG_PICCODEDSIZE, reg); @@ -162,14 +126,14 @@ static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) vq = v4l2_m2m_get_vq(ctx->fh.m2m_ctx, V4L2_BUF_TYPE_VIDEO_CAPTURE); - forward_idx = vb2_find_timestamp(vq, slice_params->forward_ref_ts, 0); + forward_idx = vb2_find_timestamp(vq, pic->forward_ref_ts, 0); fwd_luma_addr = cedrus_dst_buf_addr(ctx, forward_idx, 0); fwd_chroma_addr = cedrus_dst_buf_addr(ctx, forward_idx, 1); cedrus_write(dev, VE_DEC_MPEG_FWD_REF_LUMA_ADDR, fwd_luma_addr); cedrus_write(dev, VE_DEC_MPEG_FWD_REF_CHROMA_ADDR, fwd_chroma_addr); - backward_idx = vb2_find_timestamp(vq, slice_params->backward_ref_ts, 0); + backward_idx = vb2_find_timestamp(vq, pic->backward_ref_ts, 0); bwd_luma_addr = cedrus_dst_buf_addr(ctx, backward_idx, 0); bwd_chroma_addr = cedrus_dst_buf_addr(ctx, backward_idx, 1); @@ -186,10 +150,9 @@ static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) /* Source offset and length in bits. */ - cedrus_write(dev, VE_DEC_MPEG_VLD_OFFSET, - slice_params->data_bit_offset); + cedrus_write(dev, VE_DEC_MPEG_VLD_OFFSET, 0); - reg = slice_params->bit_size - slice_params->data_bit_offset; + reg = vb2_get_plane_payload(&run->src->vb2_buf, 0) * 8; cedrus_write(dev, VE_DEC_MPEG_VLD_LEN, reg); /* Source beginning and end addresses. */ @@ -203,7 +166,7 @@ static void cedrus_mpeg2_setup(struct cedrus_ctx *ctx, struct cedrus_run *run) cedrus_write(dev, VE_DEC_MPEG_VLD_ADDR, reg); - reg = src_buf_addr + DIV_ROUND_UP(slice_params->bit_size, 8); + reg = src_buf_addr + vb2_get_plane_payload(&run->src->vb2_buf, 0); cedrus_write(dev, VE_DEC_MPEG_VLD_END_ADDR, reg); /* Macroblock address: start at the beginning. */ diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_video.c b/drivers/staging/media/sunxi/cedrus/cedrus_video.c index b62eb8e84057..32c13ecb22d8 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_video.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_video.c @@ -457,7 +457,13 @@ static int cedrus_buf_prepare(struct vb2_buffer *vb) if (vb2_plane_size(vb, 0) < pix_fmt->sizeimage) return -EINVAL; - vb2_set_plane_payload(vb, 0, pix_fmt->sizeimage); + /* + * Buffer's bytesused must be written by driver for CAPTURE buffers. + * (for OUTPUT buffers, if userspace passes 0 bytesused, v4l2-core sets + * it to buffer length). + */ + if (V4L2_TYPE_IS_CAPTURE(vq->type)) + vb2_set_plane_payload(vb, 0, pix_fmt->sizeimage); return 0; } @@ -490,11 +496,9 @@ static int cedrus_start_streaming(struct vb2_queue *vq, unsigned int count) } if (V4L2_TYPE_IS_OUTPUT(vq->type)) { - ret = pm_runtime_get_sync(dev->dev); - if (ret < 0) { - pm_runtime_put_noidle(dev->dev); + ret = pm_runtime_resume_and_get(dev->dev); + if (ret < 0) goto err_cleanup; - } if (dev->dec_ops[ctx->current_codec]->start) { ret = dev->dec_ops[ctx->current_codec]->start(ctx); diff --git a/drivers/staging/media/tegra-vde/vde.c b/drivers/staging/media/tegra-vde/vde.c index 28845b5bafaf..ed4c1250b303 100644 --- a/drivers/staging/media/tegra-vde/vde.c +++ b/drivers/staging/media/tegra-vde/vde.c @@ -775,9 +775,9 @@ static int tegra_vde_ioctl_decode_h264(struct tegra_vde *vde, if (ret) goto release_dpb_frames; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) - goto put_runtime_pm; + goto unlock; /* * We rely on the VDE registers reset value, otherwise VDE @@ -843,6 +843,8 @@ static int tegra_vde_ioctl_decode_h264(struct tegra_vde *vde, put_runtime_pm: pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); + +unlock: mutex_unlock(&vde->lock); release_dpb_frames: @@ -1069,11 +1071,20 @@ static int tegra_vde_probe(struct platform_device *pdev) * power-cycle it in order to put hardware into a predictable lower * power state. */ - pm_runtime_get_sync(dev); + err = pm_runtime_resume_and_get(dev); + if (err) + goto err_pm_runtime; + pm_runtime_put(dev); return 0; +err_pm_runtime: + misc_deregister(&vde->miscdev); + + pm_runtime_dont_use_autosuspend(dev); + pm_runtime_disable(dev); + err_deinit_iommu: tegra_vde_iommu_deinit(vde); @@ -1089,7 +1100,12 @@ static int tegra_vde_remove(struct platform_device *pdev) struct tegra_vde *vde = platform_get_drvdata(pdev); struct device *dev = &pdev->dev; + /* + * As it increments RPM usage_count even on errors, we don't need to + * check the returned code here. + */ pm_runtime_get_sync(dev); + pm_runtime_dont_use_autosuspend(dev); pm_runtime_disable(dev); diff --git a/drivers/staging/media/tegra-video/csi.c b/drivers/staging/media/tegra-video/csi.c index 033a6935c26d..b26e44adb2be 100644 --- a/drivers/staging/media/tegra-video/csi.c +++ b/drivers/staging/media/tegra-video/csi.c @@ -64,7 +64,7 @@ static const struct v4l2_frmsize_discrete tegra_csi_tpg_sizes[] = { * V4L2 Subdevice Pad Operations */ static int csi_enum_bus_code(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_mbus_code_enum *code) { if (!IS_ENABLED(CONFIG_VIDEO_TEGRA_TPG)) @@ -79,7 +79,7 @@ static int csi_enum_bus_code(struct v4l2_subdev *subdev, } static int csi_get_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct tegra_csi_channel *csi_chan = to_csi_chan(subdev); @@ -127,7 +127,7 @@ static void csi_chan_update_blank_intervals(struct tegra_csi_channel *csi_chan, } static int csi_enum_framesizes(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_size_enum *fse) { unsigned int i; @@ -154,7 +154,7 @@ static int csi_enum_framesizes(struct v4l2_subdev *subdev, } static int csi_enum_frameintervals(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_frame_interval_enum *fie) { struct tegra_csi_channel *csi_chan = to_csi_chan(subdev); @@ -181,7 +181,7 @@ static int csi_enum_frameintervals(struct v4l2_subdev *subdev, } static int csi_set_format(struct v4l2_subdev *subdev, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *sd_state, struct v4l2_subdev_format *fmt) { struct tegra_csi_channel *csi_chan = to_csi_chan(subdev); @@ -298,10 +298,9 @@ static int tegra_csi_enable_stream(struct v4l2_subdev *subdev) struct tegra_csi *csi = csi_chan->csi; int ret, err; - ret = pm_runtime_get_sync(csi->dev); + ret = pm_runtime_resume_and_get(csi->dev); if (ret < 0) { dev_err(csi->dev, "failed to get runtime PM: %d\n", ret); - pm_runtime_put_noidle(csi->dev); return ret; } diff --git a/drivers/staging/media/tegra-video/vi.c b/drivers/staging/media/tegra-video/vi.c index df5ca3596470..89709cd06d4d 100644 --- a/drivers/staging/media/tegra-video/vi.c +++ b/drivers/staging/media/tegra-video/vi.c @@ -297,10 +297,9 @@ static int tegra_channel_start_streaming(struct vb2_queue *vq, u32 count) struct tegra_vi_channel *chan = vb2_get_drv_priv(vq); int ret; - ret = pm_runtime_get_sync(chan->vi->dev); + ret = pm_runtime_resume_and_get(chan->vi->dev); if (ret < 0) { dev_err(chan->vi->dev, "failed to get runtime PM: %d\n", ret); - pm_runtime_put_noidle(chan->vi->dev); return ret; } @@ -494,7 +493,7 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, const struct tegra_video_format *fmtinfo; struct v4l2_subdev *subdev; struct v4l2_subdev_format fmt; - struct v4l2_subdev_pad_config *pad_cfg; + struct v4l2_subdev_state *sd_state; struct v4l2_subdev_frame_size_enum fse = { .which = V4L2_SUBDEV_FORMAT_TRY, }; @@ -508,8 +507,8 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, if (!subdev) return -ENODEV; - pad_cfg = v4l2_subdev_alloc_pad_config(subdev); - if (!pad_cfg) + sd_state = v4l2_subdev_alloc_state(subdev); + if (!sd_state) return -ENOMEM; /* * Retrieve the format information and if requested format isn't @@ -533,33 +532,33 @@ static int __tegra_channel_try_format(struct tegra_vi_channel *chan, * If not available, try to get crop boundary from subdev. */ fse.code = fmtinfo->code; - ret = v4l2_subdev_call(subdev, pad, enum_frame_size, pad_cfg, &fse); + ret = v4l2_subdev_call(subdev, pad, enum_frame_size, sd_state, &fse); if (ret) { if (!v4l2_subdev_has_op(subdev, pad, get_selection)) { - pad_cfg->try_crop.width = 0; - pad_cfg->try_crop.height = 0; + sd_state->pads->try_crop.width = 0; + sd_state->pads->try_crop.height = 0; } else { ret = v4l2_subdev_call(subdev, pad, get_selection, NULL, &sdsel); if (ret) return -EINVAL; - pad_cfg->try_crop.width = sdsel.r.width; - pad_cfg->try_crop.height = sdsel.r.height; + sd_state->pads->try_crop.width = sdsel.r.width; + sd_state->pads->try_crop.height = sdsel.r.height; } } else { - pad_cfg->try_crop.width = fse.max_width; - pad_cfg->try_crop.height = fse.max_height; + sd_state->pads->try_crop.width = fse.max_width; + sd_state->pads->try_crop.height = fse.max_height; } - ret = v4l2_subdev_call(subdev, pad, set_fmt, pad_cfg, &fmt); + ret = v4l2_subdev_call(subdev, pad, set_fmt, sd_state, &fmt); if (ret < 0) return ret; v4l2_fill_pix_format(pix, &fmt.format); tegra_channel_fmt_align(chan, pix, fmtinfo->bpp); - v4l2_subdev_free_pad_config(pad_cfg); + v4l2_subdev_free_state(sd_state); return 0; } @@ -1812,8 +1811,8 @@ static int tegra_vi_graph_parse_one(struct tegra_vi_channel *chan, continue; } - tvge = v4l2_async_notifier_add_fwnode_subdev(&chan->notifier, - remote, struct tegra_vi_graph_entity); + tvge = v4l2_async_notifier_add_fwnode_subdev(&chan->notifier, remote, + struct tegra_vi_graph_entity); if (IS_ERR(tvge)) { ret = PTR_ERR(tvge); dev_err(vi->dev, diff --git a/drivers/staging/media/zoran/zoran.h b/drivers/staging/media/zoran/zoran.h index e7fe8da7732c..b1ad2a2b914c 100644 --- a/drivers/staging/media/zoran/zoran.h +++ b/drivers/staging/media/zoran/zoran.h @@ -158,7 +158,6 @@ struct zoran_jpg_settings { struct v4l2_jpegcompression jpg_comp; /* JPEG-specific capture settings */ }; - struct zoran; /* zoran_fh contains per-open() settings */ diff --git a/drivers/staging/media/zoran/zoran_card.c b/drivers/staging/media/zoran/zoran_card.c index dfc60e2e9dd7..f259585b0689 100644 --- a/drivers/staging/media/zoran/zoran_card.c +++ b/drivers/staging/media/zoran/zoran_card.c @@ -37,9 +37,10 @@ module_param_array(card, int, NULL, 0444); MODULE_PARM_DESC(card, "Card type"); /* - * The video mem address of the video card. The driver has a little database for some videocards - * to determine it from there. If your video card is not in there you have either to give it to - * the driver as a parameter or set in in a VIDIOCSFBUF ioctl + * The video mem address of the video card. The driver has a little database + * for some videocards to determine it from there. If your video card is not + * in there you have either to give it to the driver as a parameter or set + * in a VIDIOCSFBUF ioctl */ static unsigned long vidmem; /* default = 0 - Video memory base address */ diff --git a/drivers/staging/media/zoran/zoran_device.c b/drivers/staging/media/zoran/zoran_device.c index cf788d9cd1df..5b12a730a229 100644 --- a/drivers/staging/media/zoran/zoran_device.c +++ b/drivers/staging/media/zoran/zoran_device.c @@ -147,71 +147,6 @@ int post_office_read(struct zoran *zr, unsigned int guest, unsigned int reg) return btread(ZR36057_POR) & 0xFF; } -/* - * detect guests - */ - -static void dump_guests(struct zoran *zr) -{ - if (zr36067_debug > 2) { - int i, guest[8]; - - /* do not print random data */ - guest[0] = 0; - - for (i = 1; i < 8; i++) /* Don't read jpeg codec here */ - guest[i] = post_office_read(zr, i, 0); - - pci_info(zr->pci_dev, "Guests: %*ph\n", 8, guest); - } -} - -void detect_guest_activity(struct zoran *zr) -{ - int timeout, i, j, res, guest[8], guest0[8], change[8][3]; - ktime_t t0, t1; - - /* do not print random data */ - guest[0] = 0; - guest0[0] = 0; - - dump_guests(zr); - pci_info(zr->pci_dev, "Detecting guests activity, please wait...\n"); - for (i = 1; i < 8; i++) /* Don't read jpeg codec here */ - guest0[i] = guest[i] = post_office_read(zr, i, 0); - - timeout = 0; - j = 0; - t0 = ktime_get(); - while (timeout < 10000) { - udelay(10); - timeout++; - for (i = 1; (i < 8) && (j < 8); i++) { - res = post_office_read(zr, i, 0); - if (res != guest[i]) { - t1 = ktime_get(); - change[j][0] = ktime_to_us(ktime_sub(t1, t0)); - t0 = t1; - change[j][1] = i; - change[j][2] = res; - j++; - guest[i] = res; - } - } - if (j >= 8) - break; - } - - pci_info(zr->pci_dev, "Guests: %*ph\n", 8, guest0); - - if (j == 0) { - pci_info(zr->pci_dev, "No activity detected.\n"); - return; - } - for (i = 0; i < j; i++) - pci_info(zr->pci_dev, "%6d: %d => 0x%02x\n", change[i][0], change[i][1], change[i][2]); -} - /* * JPEG Codec access */ diff --git a/drivers/staging/media/zoran/zoran_device.h b/drivers/staging/media/zoran/zoran_device.h index 24be19a61b6d..6c5d70238228 100644 --- a/drivers/staging/media/zoran/zoran_device.h +++ b/drivers/staging/media/zoran/zoran_device.h @@ -20,8 +20,6 @@ extern int post_office_wait(struct zoran *zr); extern int post_office_write(struct zoran *zr, unsigned int guest, unsigned int reg, unsigned int value); extern int post_office_read(struct zoran *zr, unsigned int guest, unsigned int reg); -extern void detect_guest_activity(struct zoran *zr); - extern void jpeg_codec_sleep(struct zoran *zr, int sleep); extern int jpeg_codec_reset(struct zoran *zr); diff --git a/drivers/staging/media/zoran/zoran_driver.c b/drivers/staging/media/zoran/zoran_driver.c index e8902f824d6c..46382e43f1bf 100644 --- a/drivers/staging/media/zoran/zoran_driver.c +++ b/drivers/staging/media/zoran/zoran_driver.c @@ -678,12 +678,14 @@ static int zoran_g_selection(struct file *file, void *__fh, struct v4l2_selectio sel->r.height = zr->jpg_settings.img_height; break; case V4L2_SEL_TGT_CROP_DEFAULT: - sel->r.top = sel->r.left = 0; + sel->r.top = 0; + sel->r.left = 0; sel->r.width = BUZ_MIN_WIDTH; sel->r.height = BUZ_MIN_HEIGHT; break; case V4L2_SEL_TGT_CROP_BOUNDS: - sel->r.top = sel->r.left = 0; + sel->r.top = 0; + sel->r.left = 0; sel->r.width = BUZ_MAX_WIDTH; sel->r.height = BUZ_MAX_HEIGHT; break; diff --git a/drivers/staging/media/zoran/zr36016.c b/drivers/staging/media/zoran/zr36016.c index 2d7dc7abde79..9b350a885879 100644 --- a/drivers/staging/media/zoran/zr36016.c +++ b/drivers/staging/media/zoran/zr36016.c @@ -361,7 +361,8 @@ static int zr36016_setup(struct videocodec *codec) return -ENOSPC; } //mem structure init - codec->data = ptr = kzalloc(sizeof(struct zr36016), GFP_KERNEL); + ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); + codec->data = ptr; if (!ptr) return -ENOMEM; diff --git a/drivers/staging/media/zoran/zr36050.c b/drivers/staging/media/zoran/zr36050.c index 2826f4e5d37b..c62af27f2683 100644 --- a/drivers/staging/media/zoran/zr36050.c +++ b/drivers/staging/media/zoran/zr36050.c @@ -16,7 +16,7 @@ #include /* I/O commands, error codes */ -#include +#include /* headerfile of this module */ #include "zr36050.h" @@ -754,7 +754,8 @@ static int zr36050_setup(struct videocodec *codec) return -ENOSPC; } //mem structure init - codec->data = ptr = kzalloc(sizeof(struct zr36050), GFP_KERNEL); + ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); + codec->data = ptr; if (!ptr) return -ENOMEM; diff --git a/drivers/staging/media/zoran/zr36057.h b/drivers/staging/media/zoran/zr36057.h index 71b651add35a..a2a75fd9f535 100644 --- a/drivers/staging/media/zoran/zr36057.h +++ b/drivers/staging/media/zoran/zr36057.h @@ -30,13 +30,13 @@ #define ZR36057_VFESPFR_HOR_DCM 14 #define ZR36057_VFESPFR_VER_DCM 8 #define ZR36057_VFESPFR_DISP_MODE 6 -#define ZR36057_VFESPFR_YUV422 (0<<3) -#define ZR36057_VFESPFR_RGB888 (1<<3) -#define ZR36057_VFESPFR_RGB565 (2<<3) -#define ZR36057_VFESPFR_RGB555 (3<<3) -#define ZR36057_VFESPFR_ERR_DIF (1<<2) -#define ZR36057_VFESPFR_PACK24 (1<<1) -#define ZR36057_VFESPFR_LITTLE_ENDIAN (1<<0) +#define ZR36057_VFESPFR_YUV422 (0 << 3) +#define ZR36057_VFESPFR_RGB888 (1 << 3) +#define ZR36057_VFESPFR_RGB565 (2 << 3) +#define ZR36057_VFESPFR_RGB555 (3 << 3) +#define ZR36057_VFESPFR_ERR_DIF (1 << 2) +#define ZR36057_VFESPFR_PACK24 (1 << 1) +#define ZR36057_VFESPFR_LITTLE_ENDIAN (1 << 0) #define ZR36057_VDTR 0x00c /* Video Display "Top" Register */ diff --git a/drivers/staging/media/zoran/zr36060.c b/drivers/staging/media/zoran/zr36060.c index 4f9eb9ff2c42..1c3af11b5f24 100644 --- a/drivers/staging/media/zoran/zr36060.c +++ b/drivers/staging/media/zoran/zr36060.c @@ -790,7 +790,8 @@ static int zr36060_setup(struct videocodec *codec) return -ENOSPC; } //mem structure init - codec->data = ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); + ptr = kzalloc(sizeof(*ptr), GFP_KERNEL); + codec->data = ptr; if (!ptr) return -ENOMEM; diff --git a/drivers/tee/tee_core.c b/drivers/tee/tee_core.c index 480d294a23ab..2b37bc408fc3 100644 --- a/drivers/tee/tee_core.c +++ b/drivers/tee/tee_core.c @@ -452,6 +452,7 @@ static int params_to_user(struct tee_ioctl_param __user *uparams, case TEE_IOCTL_PARAM_ATTR_TYPE_MEMREF_INOUT: if (put_user((u64)p->u.memref.size, &up->b)) return -EFAULT; + break; default: break; } diff --git a/drivers/watchdog/machzwd.c b/drivers/watchdog/machzwd.c index 743377c5b173..73f2221f6222 100644 --- a/drivers/watchdog/machzwd.c +++ b/drivers/watchdog/machzwd.c @@ -174,6 +174,7 @@ static inline void zf_set_timer(unsigned short new, unsigned char n) fallthrough; case WD2: zf_writeb(COUNTER_2, new > 0xff ? 0xff : new); + fallthrough; default: return; } diff --git a/fs/btrfs/Kconfig b/fs/btrfs/Kconfig index 68b95ad82126..520a0f6a7d9e 100644 --- a/fs/btrfs/Kconfig +++ b/fs/btrfs/Kconfig @@ -18,6 +18,8 @@ config BTRFS_FS select RAID6_PQ select XOR_BLOCKS select SRCU + depends on !PPC_256K_PAGES # powerpc + depends on !PAGE_SIZE_256KB # hexagon help Btrfs is a general purpose copy-on-write filesystem with extents, diff --git a/fs/btrfs/backref.c b/fs/btrfs/backref.c index 117d423fdb93..7a8a2fc19533 100644 --- a/fs/btrfs/backref.c +++ b/fs/btrfs/backref.c @@ -2675,7 +2675,7 @@ static int handle_direct_tree_backref(struct btrfs_backref_cache *cache, * * @ref_key: The same as @ref_key in handle_direct_tree_backref() * @tree_key: The first key of this tree block. - * @path: A clean (released) path, to avoid allocating path everytime + * @path: A clean (released) path, to avoid allocating path every time * the function get called. */ static int handle_indirect_tree_backref(struct btrfs_backref_cache *cache, diff --git a/fs/btrfs/block-group.c b/fs/btrfs/block-group.c index 6d5c4e45cfef..38b127b9edfc 100644 --- a/fs/btrfs/block-group.c +++ b/fs/btrfs/block-group.c @@ -1399,7 +1399,6 @@ void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info) btrfs_space_info_update_bytes_pinned(fs_info, space_info, -block_group->pinned); space_info->bytes_readonly += block_group->pinned; - __btrfs_mod_total_bytes_pinned(space_info, -block_group->pinned); block_group->pinned = 0; spin_unlock(&block_group->lock); @@ -1491,7 +1490,7 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) container_of(work, struct btrfs_fs_info, reclaim_bgs_work); struct btrfs_block_group *bg; struct btrfs_space_info *space_info; - int ret; + LIST_HEAD(again_list); if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags)) return; @@ -1502,6 +1501,8 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) mutex_lock(&fs_info->reclaim_bgs_lock); spin_lock(&fs_info->unused_bgs_lock); while (!list_empty(&fs_info->reclaim_bgs)) { + int ret = 0; + bg = list_first_entry(&fs_info->reclaim_bgs, struct btrfs_block_group, bg_list); @@ -1547,9 +1548,13 @@ void btrfs_reclaim_bgs_work(struct work_struct *work) bg->start); next: - btrfs_put_block_group(bg); spin_lock(&fs_info->unused_bgs_lock); + if (ret == -EAGAIN && list_empty(&bg->bg_list)) + list_add_tail(&bg->bg_list, &again_list); + else + btrfs_put_block_group(bg); } + list_splice_tail(&again_list, &fs_info->reclaim_bgs); spin_unlock(&fs_info->unused_bgs_lock); mutex_unlock(&fs_info->reclaim_bgs_lock); btrfs_exclop_finish(fs_info); @@ -2505,7 +2510,7 @@ static int cache_save_setup(struct btrfs_block_group *block_group, struct extent_changeset *data_reserved = NULL; u64 alloc_hint = 0; int dcs = BTRFS_DC_ERROR; - u64 num_pages = 0; + u64 cache_size = 0; int retries = 0; int ret = 0; @@ -2617,20 +2622,20 @@ static int cache_save_setup(struct btrfs_block_group *block_group, * taking up quite a bit since it's not folded into the other space * cache. */ - num_pages = div_u64(block_group->length, SZ_256M); - if (!num_pages) - num_pages = 1; + cache_size = div_u64(block_group->length, SZ_256M); + if (!cache_size) + cache_size = 1; - num_pages *= 16; - num_pages *= PAGE_SIZE; + cache_size *= 16; + cache_size *= fs_info->sectorsize; ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved, 0, - num_pages); + cache_size); if (ret) goto out_put; - ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages, - num_pages, num_pages, + ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, cache_size, + cache_size, cache_size, &alloc_hint); /* * Our cache requires contiguous chunks so that we don't modify a bunch @@ -3062,8 +3067,6 @@ int btrfs_update_block_group(struct btrfs_trans_handle *trans, spin_unlock(&cache->lock); spin_unlock(&cache->space_info->lock); - __btrfs_mod_total_bytes_pinned(cache->space_info, - num_bytes); set_extent_dirty(&trans->transaction->pinned_extents, bytenr, bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL); diff --git a/fs/btrfs/compression.c b/fs/btrfs/compression.c index 1346d698463a..9a023ae0f98b 100644 --- a/fs/btrfs/compression.c +++ b/fs/btrfs/compression.c @@ -149,7 +149,7 @@ static int check_compressed_csum(struct btrfs_inode *inode, struct bio *bio, const u32 csum_size = fs_info->csum_size; const u32 sectorsize = fs_info->sectorsize; struct page *page; - unsigned long i; + unsigned int i; char *kaddr; u8 csum[BTRFS_CSUM_SIZE]; struct compressed_bio *cb = bio->bi_private; @@ -208,7 +208,7 @@ static void end_compressed_bio_read(struct bio *bio) struct compressed_bio *cb = bio->bi_private; struct inode *inode; struct page *page; - unsigned long index; + unsigned int index; unsigned int mirror = btrfs_io_bio(bio)->mirror_num; int ret = 0; @@ -334,7 +334,7 @@ static void end_compressed_bio_write(struct bio *bio) struct compressed_bio *cb = bio->bi_private; struct inode *inode; struct page *page; - unsigned long index; + unsigned int index; if (bio->bi_status) cb->errors = 1; @@ -349,12 +349,10 @@ static void end_compressed_bio_write(struct bio *bio) * call back into the FS and do all the end_io operations */ inode = cb->inode; - cb->compressed_pages[0]->mapping = cb->inode->i_mapping; btrfs_record_physical_zoned(inode, cb->start, bio); - btrfs_writepage_endio_finish_ordered(cb->compressed_pages[0], + btrfs_writepage_endio_finish_ordered(BTRFS_I(inode), NULL, cb->start, cb->start + cb->len - 1, bio->bi_status == BLK_STS_OK); - cb->compressed_pages[0]->mapping = NULL; end_compressed_writeback(inode, cb); /* note, our inode could be gone now */ @@ -387,10 +385,10 @@ static void end_compressed_bio_write(struct bio *bio) * the end io hooks. */ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, - unsigned long len, u64 disk_start, - unsigned long compressed_len, + unsigned int len, u64 disk_start, + unsigned int compressed_len, struct page **compressed_pages, - unsigned long nr_pages, + unsigned int nr_pages, unsigned int write_flags, struct cgroup_subsys_state *blkcg_css) { @@ -427,24 +425,16 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, bio->bi_end_io = end_compressed_bio_write; if (use_append) { - struct extent_map *em; - struct map_lookup *map; - struct block_device *bdev; + struct btrfs_device *device; - em = btrfs_get_chunk_map(fs_info, disk_start, PAGE_SIZE); - if (IS_ERR(em)) { + device = btrfs_zoned_get_device(fs_info, disk_start, PAGE_SIZE); + if (IS_ERR(device)) { kfree(cb); bio_put(bio); return BLK_STS_NOTSUPP; } - map = em->map_lookup; - /* We only support single profile for now */ - ASSERT(map->num_stripes == 1); - bdev = map->stripes[0].dev->bdev; - - bio_set_dev(bio, bdev); - free_extent_map(em); + bio_set_dev(bio, device->bdev); } if (blkcg_css) { @@ -515,7 +505,7 @@ blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, } if (bytes_left < PAGE_SIZE) { btrfs_info(fs_info, - "bytes left %lu compress len %lu nr %lu", + "bytes left %lu compress len %u nr %u", bytes_left, cb->compressed_len, cb->nr_pages); } bytes_left -= PAGE_SIZE; @@ -677,9 +667,9 @@ blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct extent_map_tree *em_tree; struct compressed_bio *cb; - unsigned long compressed_len; - unsigned long nr_pages; - unsigned long pg_index; + unsigned int compressed_len; + unsigned int nr_pages; + unsigned int pg_index; struct page *page; struct bio *comp_bio; u64 cur_disk_byte = bio->bi_iter.bi_sector << 9; @@ -1202,9 +1192,6 @@ static unsigned int btrfs_compress_set_level(int type, unsigned level) * * @total_out is an in/out parameter, must be set to the input length and will * be also used to return the total number of compressed bytes - * - * @max_out tells us the max number of bytes that we're allowed to - * stuff into pages */ int btrfs_compress_pages(unsigned int type_level, struct address_space *mapping, u64 start, struct page **pages, @@ -1225,20 +1212,6 @@ int btrfs_compress_pages(unsigned int type_level, struct address_space *mapping, return ret; } -/* - * pages_in is an array of pages with compressed data. - * - * disk_start is the starting logical offset of this array in the file - * - * orig_bio contains the pages from the file that we want to decompress into - * - * srclen is the number of bytes in pages_in - * - * The basic idea is that we have a bio that was created by readpages. - * The pages in the bio are for the uncompressed data, and they may not - * be contiguous. They all correspond to the range of bytes covered by - * the compressed extent. - */ static int btrfs_decompress_bio(struct compressed_bio *cb) { struct list_head *workspace; diff --git a/fs/btrfs/compression.h b/fs/btrfs/compression.h index 8001b700ea3a..c359f20920d0 100644 --- a/fs/btrfs/compression.h +++ b/fs/btrfs/compression.h @@ -31,6 +31,9 @@ struct compressed_bio { /* number of bios pending for this compressed extent */ refcount_t pending_bios; + /* Number of compressed pages in the array */ + unsigned int nr_pages; + /* the pages with the compressed data on them */ struct page **compressed_pages; @@ -40,20 +43,17 @@ struct compressed_bio { /* starting offset in the inode for our pages */ u64 start; - /* number of bytes in the inode we're working on */ - unsigned long len; + /* Number of bytes in the inode we're working on */ + unsigned int len; - /* number of bytes on disk */ - unsigned long compressed_len; + /* Number of bytes on disk */ + unsigned int compressed_len; - /* the compression algorithm for this bio */ - int compress_type; - - /* number of compressed pages in the array */ - unsigned long nr_pages; + /* The compression algorithm for this bio */ + u8 compress_type; /* IO errors */ - int errors; + u8 errors; int mirror_num; /* for reads, this is the bio we are copying the data into */ @@ -91,10 +91,10 @@ int btrfs_decompress_buf2page(const char *buf, unsigned long buf_start, struct bio *bio); blk_status_t btrfs_submit_compressed_write(struct btrfs_inode *inode, u64 start, - unsigned long len, u64 disk_start, - unsigned long compressed_len, + unsigned int len, u64 disk_start, + unsigned int compressed_len, struct page **compressed_pages, - unsigned long nr_pages, + unsigned int nr_pages, unsigned int write_flags, struct cgroup_subsys_state *blkcg_css); blk_status_t btrfs_submit_compressed_read(struct inode *inode, struct bio *bio, diff --git a/fs/btrfs/ctree.c b/fs/btrfs/ctree.c index a484fb72a01f..4bc3ca2cbd7d 100644 --- a/fs/btrfs/ctree.c +++ b/fs/btrfs/ctree.c @@ -596,7 +596,6 @@ noinline int btrfs_cow_block(struct btrfs_trans_handle *trans, trans->transid, fs_info->generation); if (!should_cow_block(trans, root, buf)) { - trans->dirty = true; *cow_ret = buf; return 0; } @@ -1788,10 +1787,8 @@ int btrfs_search_slot(struct btrfs_trans_handle *trans, struct btrfs_root *root, * then we don't want to set the path blocking, * so we test it here */ - if (!should_cow_block(trans, root, b)) { - trans->dirty = true; + if (!should_cow_block(trans, root, b)) goto cow_done; - } /* * must have write locks on this node and the diff --git a/fs/btrfs/ctree.h b/fs/btrfs/ctree.h index 9fb76829a281..e5e53e592d4f 100644 --- a/fs/btrfs/ctree.h +++ b/fs/btrfs/ctree.h @@ -561,10 +561,16 @@ enum { /* * Indicate that balance has been set up from the ioctl and is in the * main phase. The fs_info::balance_ctl is initialized. - * Set and cleared while holding fs_info::balance_mutex. */ BTRFS_FS_BALANCE_RUNNING, + /* + * Indicate that relocation of a chunk has started, it's set per chunk + * and is toggled between chunks. + * Set, tested and cleared while holding fs_info::send_reloc_lock. + */ + BTRFS_FS_RELOC_RUNNING, + /* Indicate that the cleaner thread is awake and doing something. */ BTRFS_FS_CLEANER_RUNNING, @@ -817,8 +823,6 @@ struct btrfs_fs_info { struct kobject *space_info_kobj; struct kobject *qgroups_kobj; - u64 total_pinned; - /* used to keep from writing metadata until there is a nice batch */ struct percpu_counter dirty_metadata_bytes; struct percpu_counter delalloc_bytes; @@ -871,6 +875,9 @@ struct btrfs_fs_info { struct btrfs_balance_control *balance_ctl; wait_queue_head_t balance_wait_q; + /* Cancellation requests for chunk relocation */ + atomic_t reloc_cancel_req; + u32 data_chunk_allocations; u32 metadata_ratio; @@ -986,14 +993,15 @@ struct btrfs_fs_info { struct crypto_shash *csum_shash; + spinlock_t send_reloc_lock; /* * Number of send operations in progress. - * Updated while holding fs_info::balance_mutex. + * Updated while holding fs_info::send_reloc_lock. */ int send_in_progress; - /* Type of exclusive operation running */ - unsigned long exclusive_operation; + /* Type of exclusive operation running, protected by super_lock */ + enum btrfs_exclusive_operation exclusive_operation; /* * Zone size > 0 when in ZONED mode, otherwise it's used for a check @@ -1375,38 +1383,39 @@ static inline u32 BTRFS_MAX_XATTR_SIZE(const struct btrfs_fs_info *info) * * Note: don't forget to add new options to btrfs_show_options() */ -#define BTRFS_MOUNT_NODATASUM (1 << 0) -#define BTRFS_MOUNT_NODATACOW (1 << 1) -#define BTRFS_MOUNT_NOBARRIER (1 << 2) -#define BTRFS_MOUNT_SSD (1 << 3) -#define BTRFS_MOUNT_DEGRADED (1 << 4) -#define BTRFS_MOUNT_COMPRESS (1 << 5) -#define BTRFS_MOUNT_NOTREELOG (1 << 6) -#define BTRFS_MOUNT_FLUSHONCOMMIT (1 << 7) -#define BTRFS_MOUNT_SSD_SPREAD (1 << 8) -#define BTRFS_MOUNT_NOSSD (1 << 9) -#define BTRFS_MOUNT_DISCARD_SYNC (1 << 10) -#define BTRFS_MOUNT_FORCE_COMPRESS (1 << 11) -#define BTRFS_MOUNT_SPACE_CACHE (1 << 12) -#define BTRFS_MOUNT_CLEAR_CACHE (1 << 13) -#define BTRFS_MOUNT_USER_SUBVOL_RM_ALLOWED (1 << 14) -#define BTRFS_MOUNT_ENOSPC_DEBUG (1 << 15) -#define BTRFS_MOUNT_AUTO_DEFRAG (1 << 16) -/* bit 17 is free */ -#define BTRFS_MOUNT_USEBACKUPROOT (1 << 18) -#define BTRFS_MOUNT_SKIP_BALANCE (1 << 19) -#define BTRFS_MOUNT_CHECK_INTEGRITY (1 << 20) -#define BTRFS_MOUNT_CHECK_INTEGRITY_INCLUDING_EXTENT_DATA (1 << 21) -#define BTRFS_MOUNT_PANIC_ON_FATAL_ERROR (1 << 22) -#define BTRFS_MOUNT_RESCAN_UUID_TREE (1 << 23) -#define BTRFS_MOUNT_FRAGMENT_DATA (1 << 24) -#define BTRFS_MOUNT_FRAGMENT_METADATA (1 << 25) -#define BTRFS_MOUNT_FREE_SPACE_TREE (1 << 26) -#define BTRFS_MOUNT_NOLOGREPLAY (1 << 27) -#define BTRFS_MOUNT_REF_VERIFY (1 << 28) -#define BTRFS_MOUNT_DISCARD_ASYNC (1 << 29) -#define BTRFS_MOUNT_IGNOREBADROOTS (1 << 30) -#define BTRFS_MOUNT_IGNOREDATACSUMS (1 << 31) +enum { + BTRFS_MOUNT_NODATASUM = (1UL << 0), + BTRFS_MOUNT_NODATACOW = (1UL << 1), + BTRFS_MOUNT_NOBARRIER = (1UL << 2), + BTRFS_MOUNT_SSD = (1UL << 3), + BTRFS_MOUNT_DEGRADED = (1UL << 4), + BTRFS_MOUNT_COMPRESS = (1UL << 5), + BTRFS_MOUNT_NOTREELOG = (1UL << 6), + BTRFS_MOUNT_FLUSHONCOMMIT = (1UL << 7), + BTRFS_MOUNT_SSD_SPREAD = (1UL << 8), + BTRFS_MOUNT_NOSSD = (1UL << 9), + BTRFS_MOUNT_DISCARD_SYNC = (1UL << 10), + BTRFS_MOUNT_FORCE_COMPRESS = (1UL << 11), + BTRFS_MOUNT_SPACE_CACHE = (1UL << 12), + BTRFS_MOUNT_CLEAR_CACHE = (1UL << 13), + BTRFS_MOUNT_USER_SUBVOL_RM_ALLOWED = (1UL << 14), + BTRFS_MOUNT_ENOSPC_DEBUG = (1UL << 15), + BTRFS_MOUNT_AUTO_DEFRAG = (1UL << 16), + BTRFS_MOUNT_USEBACKUPROOT = (1UL << 17), + BTRFS_MOUNT_SKIP_BALANCE = (1UL << 18), + BTRFS_MOUNT_CHECK_INTEGRITY = (1UL << 19), + BTRFS_MOUNT_CHECK_INTEGRITY_DATA = (1UL << 20), + BTRFS_MOUNT_PANIC_ON_FATAL_ERROR = (1UL << 21), + BTRFS_MOUNT_RESCAN_UUID_TREE = (1UL << 22), + BTRFS_MOUNT_FRAGMENT_DATA = (1UL << 23), + BTRFS_MOUNT_FRAGMENT_METADATA = (1UL << 24), + BTRFS_MOUNT_FREE_SPACE_TREE = (1UL << 25), + BTRFS_MOUNT_NOLOGREPLAY = (1UL << 26), + BTRFS_MOUNT_REF_VERIFY = (1UL << 27), + BTRFS_MOUNT_DISCARD_ASYNC = (1UL << 28), + BTRFS_MOUNT_IGNOREBADROOTS = (1UL << 29), + BTRFS_MOUNT_IGNOREDATACSUMS = (1UL << 30), +}; #define BTRFS_DEFAULT_COMMIT_INTERVAL (30) #define BTRFS_DEFAULT_MAX_INLINE (2048) @@ -2216,11 +2225,13 @@ BTRFS_SETGET_STACK_FUNCS(root_rtransid, struct btrfs_root_item, static inline bool btrfs_root_readonly(const struct btrfs_root *root) { + /* Byte-swap the constant at compile time, root_item::flags is LE */ return (root->root_item.flags & cpu_to_le64(BTRFS_ROOT_SUBVOL_RDONLY)) != 0; } static inline bool btrfs_root_dead(const struct btrfs_root *root) { + /* Byte-swap the constant at compile time, root_item::flags is LE */ return (root->root_item.flags & cpu_to_le64(BTRFS_ROOT_SUBVOL_DEAD)) != 0; } @@ -2746,9 +2757,9 @@ enum btrfs_reserve_flush_enum { /* * Flush space by above mentioned methods and by: * - Running delayed iputs - * - Commiting transaction + * - Committing transaction * - * Can be interruped by fatal signal. + * Can be interrupted by a fatal signal. */ BTRFS_RESERVE_FLUSH_DATA, BTRFS_RESERVE_FLUSH_FREE_SPACE_INODE, @@ -2758,7 +2769,7 @@ enum btrfs_reserve_flush_enum { * Pretty much the same as FLUSH_ALL, but can also steal space from * global rsv. * - * Can be interruped by fatal signal. + * Can be interrupted by a fatal signal. */ BTRFS_RESERVE_FLUSH_ALL_STEAL, }; @@ -2774,7 +2785,6 @@ enum btrfs_flush_state { ALLOC_CHUNK_FORCE = 8, RUN_DELAYED_IPUTS = 9, COMMIT_TRANS = 10, - FORCE_COMMIT_TRANS = 11, }; int btrfs_subvolume_reserve_metadata(struct btrfs_root *root, @@ -3100,8 +3110,8 @@ u64 btrfs_file_extent_end(const struct btrfs_path *path); /* inode.c */ blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio, int mirror_num, unsigned long bio_flags); -int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset, - struct page *page, u64 start, u64 end); +unsigned int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset, + struct page *page, u64 start, u64 end); struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode, u64 start, u64 len); noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len, @@ -3125,7 +3135,7 @@ int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len, int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_inode *inode, u64 new_size, - u32 min_type); + u32 min_type, u64 *extents_found); int btrfs_start_delalloc_snapshot(struct btrfs_root *root, bool in_reclaim_context); int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, long nr, @@ -3146,9 +3156,7 @@ void btrfs_split_delalloc_extent(struct inode *inode, struct extent_state *orig, u64 split); int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, unsigned long bio_flags); -bool btrfs_bio_fits_in_ordered_extent(struct page *page, struct bio *bio, - unsigned int size); -void btrfs_set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end); +void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end); vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf); int btrfs_readpage(struct file *file, struct page *page); void btrfs_evict_inode(struct inode *inode); @@ -3187,7 +3195,8 @@ int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct page *locked_page u64 start, u64 end, int *page_started, unsigned long *nr_written, struct writeback_control *wbc); int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end); -void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, +void btrfs_writepage_endio_finish_ordered(struct btrfs_inode *inode, + struct page *page, u64 start, u64 end, int uptodate); extern const struct dentry_operations btrfs_dentry_operations; extern const struct iomap_ops btrfs_dio_iomap_ops; @@ -3222,6 +3231,9 @@ void btrfs_update_ioctl_balance_args(struct btrfs_fs_info *fs_info, struct btrfs_ioctl_balance_args *bargs); bool btrfs_exclop_start(struct btrfs_fs_info *fs_info, enum btrfs_exclusive_operation type); +bool btrfs_exclop_start_try_lock(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type); +void btrfs_exclop_start_unlock(struct btrfs_fs_info *fs_info); void btrfs_exclop_finish(struct btrfs_fs_info *fs_info); /* file.c */ @@ -3786,4 +3798,14 @@ static inline bool btrfs_is_zoned(const struct btrfs_fs_info *fs_info) return fs_info->zoned != 0; } +/* + * We use page status Private2 to indicate there is an ordered extent with + * unfinished IO. + * + * Rename the Private2 accessors to Ordered, to improve readability. + */ +#define PageOrdered(page) PagePrivate2(page) +#define SetPageOrdered(page) SetPagePrivate2(page) +#define ClearPageOrdered(page) ClearPagePrivate2(page) + #endif diff --git a/fs/btrfs/delalloc-space.c b/fs/btrfs/delalloc-space.c index 56642ca7af10..2059d1504149 100644 --- a/fs/btrfs/delalloc-space.c +++ b/fs/btrfs/delalloc-space.c @@ -89,7 +89,7 @@ * ->outstanding_extents += 1 (current value is 1) * * -> set_delalloc - * ->outstanding_extents += 1 (currrent value is 2) + * ->outstanding_extents += 1 (current value is 2) * * -> btrfs_delalloc_release_extents() * ->outstanding_extents -= 1 (current value is 1) diff --git a/fs/btrfs/delayed-inode.c b/fs/btrfs/delayed-inode.c index 1a88f6214ebc..257c1e18abd4 100644 --- a/fs/btrfs/delayed-inode.c +++ b/fs/btrfs/delayed-inode.c @@ -681,7 +681,7 @@ static int btrfs_batch_insert_items(struct btrfs_root *root, { struct btrfs_delayed_item *curr, *next; int free_space; - int total_data_size = 0, total_size = 0; + int total_size = 0; struct extent_buffer *leaf; char *data_ptr; struct btrfs_key *keys; @@ -706,7 +706,6 @@ static int btrfs_batch_insert_items(struct btrfs_root *root, */ while (total_size + next->data_len + sizeof(struct btrfs_item) <= free_space) { - total_data_size += next->data_len; total_size += next->data_len + sizeof(struct btrfs_item); list_add_tail(&next->tree_list, &head); nitems++; @@ -974,14 +973,16 @@ static void btrfs_release_delayed_inode(struct btrfs_delayed_node *delayed_node) static void btrfs_release_delayed_iref(struct btrfs_delayed_node *delayed_node) { - struct btrfs_delayed_root *delayed_root; - ASSERT(delayed_node->root); - clear_bit(BTRFS_DELAYED_NODE_DEL_IREF, &delayed_node->flags); - delayed_node->count--; + if (test_and_clear_bit(BTRFS_DELAYED_NODE_DEL_IREF, &delayed_node->flags)) { + struct btrfs_delayed_root *delayed_root; - delayed_root = delayed_node->root->fs_info->delayed_root; - finish_one_item(delayed_root); + ASSERT(delayed_node->root); + delayed_node->count--; + + delayed_root = delayed_node->root->fs_info->delayed_root; + finish_one_item(delayed_root); + } } static int __btrfs_update_delayed_inode(struct btrfs_trans_handle *trans, @@ -1009,12 +1010,10 @@ static int __btrfs_update_delayed_inode(struct btrfs_trans_handle *trans, nofs_flag = memalloc_nofs_save(); ret = btrfs_lookup_inode(trans, root, path, &key, mod); memalloc_nofs_restore(nofs_flag); - if (ret > 0) { - btrfs_release_path(path); - return -ENOENT; - } else if (ret < 0) { - return ret; - } + if (ret > 0) + ret = -ENOENT; + if (ret < 0) + goto out; leaf = path->nodes[0]; inode_item = btrfs_item_ptr(leaf, path->slots[0], @@ -1024,7 +1023,7 @@ static int __btrfs_update_delayed_inode(struct btrfs_trans_handle *trans, btrfs_mark_buffer_dirty(leaf); if (!test_bit(BTRFS_DELAYED_NODE_DEL_IREF, &node->flags)) - goto no_iref; + goto out; path->slots[0]++; if (path->slots[0] >= btrfs_header_nritems(leaf)) @@ -1046,12 +1045,19 @@ static int __btrfs_update_delayed_inode(struct btrfs_trans_handle *trans, btrfs_del_item(trans, root, path); out: btrfs_release_delayed_iref(node); -no_iref: btrfs_release_path(path); err_out: btrfs_delayed_inode_release_metadata(fs_info, node, (ret < 0)); btrfs_release_delayed_inode(node); + /* + * If we fail to update the delayed inode we need to abort the + * transaction, because we could leave the inode with the improper + * counts behind. + */ + if (ret && ret != -ENOENT) + btrfs_abort_transaction(trans, ret); + return ret; search: @@ -1898,8 +1904,7 @@ static void __btrfs_kill_delayed_node(struct btrfs_delayed_node *delayed_node) btrfs_release_delayed_item(prev_item); } - if (test_bit(BTRFS_DELAYED_NODE_DEL_IREF, &delayed_node->flags)) - btrfs_release_delayed_iref(delayed_node); + btrfs_release_delayed_iref(delayed_node); if (test_bit(BTRFS_DELAYED_NODE_INODE_DIRTY, &delayed_node->flags)) { btrfs_delayed_inode_release_metadata(fs_info, delayed_node, false); diff --git a/fs/btrfs/delayed-ref.c b/fs/btrfs/delayed-ref.c index c92d9d4f5f46..06bc842ecdb3 100644 --- a/fs/btrfs/delayed-ref.c +++ b/fs/btrfs/delayed-ref.c @@ -641,7 +641,6 @@ static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans, struct btrfs_delayed_ref_root *delayed_refs = &trans->transaction->delayed_refs; struct btrfs_fs_info *fs_info = trans->fs_info; - u64 flags = btrfs_ref_head_to_space_flags(existing); int old_ref_mod; BUG_ON(existing->is_data != update->is_data); @@ -711,26 +710,6 @@ static noinline void update_existing_head_ref(struct btrfs_trans_handle *trans, } } - /* - * This handles the following conditions: - * - * 1. We had a ref mod of 0 or more and went negative, indicating that - * we may be freeing space, so add our space to the - * total_bytes_pinned counter. - * 2. We were negative and went to 0 or positive, so no longer can say - * that the space would be pinned, decrement our counter from the - * total_bytes_pinned counter. - * 3. We are now at 0 and have ->must_insert_reserved set, which means - * this was a new allocation and then we dropped it, and thus must - * add our space to the total_bytes_pinned counter. - */ - if (existing->total_ref_mod < 0 && old_ref_mod >= 0) - btrfs_mod_total_bytes_pinned(fs_info, flags, existing->num_bytes); - else if (existing->total_ref_mod >= 0 && old_ref_mod < 0) - btrfs_mod_total_bytes_pinned(fs_info, flags, -existing->num_bytes); - else if (existing->total_ref_mod == 0 && existing->must_insert_reserved) - btrfs_mod_total_bytes_pinned(fs_info, flags, existing->num_bytes); - spin_unlock(&existing->lock); } @@ -835,17 +814,12 @@ add_delayed_ref_head(struct btrfs_trans_handle *trans, kmem_cache_free(btrfs_delayed_ref_head_cachep, head_ref); head_ref = existing; } else { - u64 flags = btrfs_ref_head_to_space_flags(head_ref); - if (head_ref->is_data && head_ref->ref_mod < 0) { delayed_refs->pending_csums += head_ref->num_bytes; trans->delayed_ref_updates += btrfs_csum_bytes_to_leaves(trans->fs_info, head_ref->num_bytes); } - if (head_ref->ref_mod < 0) - btrfs_mod_total_bytes_pinned(trans->fs_info, flags, - head_ref->num_bytes); delayed_refs->num_heads++; delayed_refs->num_heads_ready++; atomic_inc(&delayed_refs->num_entries); diff --git a/fs/btrfs/dev-replace.c b/fs/btrfs/dev-replace.c index d05f73530af7..d029be40ea6f 100644 --- a/fs/btrfs/dev-replace.c +++ b/fs/btrfs/dev-replace.c @@ -37,7 +37,7 @@ * - Write duplication * * All new writes will be written to both target and source devices, so even - * if replace gets canceled, sources device still contans up-to-date data. + * if replace gets canceled, sources device still contains up-to-date data. * * Location: handle_ops_on_dev_replace() from __btrfs_map_block() * Start: btrfs_dev_replace_start() diff --git a/fs/btrfs/discard.c b/fs/btrfs/discard.c index 306ff20af70f..e1b7bd927d69 100644 --- a/fs/btrfs/discard.c +++ b/fs/btrfs/discard.c @@ -624,7 +624,7 @@ void btrfs_discard_update_discardable(struct btrfs_block_group *block_group) * @fs_info: fs_info of interest * * The unused_bgs list needs to be punted to the discard lists because the - * order of operations is changed. In the normal sychronous discard path, the + * order of operations is changed. In the normal synchronous discard path, the * block groups are trimmed via a single large trim in transaction commit. This * is ultimately what we are trying to avoid with asynchronous discard. Thus, * it must be done before going down the unused_bgs path. diff --git a/fs/btrfs/disk-io.c b/fs/btrfs/disk-io.c index 8d386a5587ee..b117dd3b8172 100644 --- a/fs/btrfs/disk-io.c +++ b/fs/btrfs/disk-io.c @@ -241,7 +241,6 @@ static int verify_parent_transid(struct extent_io_tree *io_tree, { struct extent_state *cached_state = NULL; int ret; - bool need_lock = (current->journal_info == BTRFS_SEND_TRANS_STUB); if (!parent_transid || btrfs_header_generation(eb) == parent_transid) return 0; @@ -249,9 +248,6 @@ static int verify_parent_transid(struct extent_io_tree *io_tree, if (atomic) return -EAGAIN; - if (need_lock) - btrfs_tree_read_lock(eb); - lock_extent_bits(io_tree, eb->start, eb->start + eb->len - 1, &cached_state); if (extent_buffer_uptodate(eb) && @@ -264,22 +260,10 @@ static int verify_parent_transid(struct extent_io_tree *io_tree, eb->start, parent_transid, btrfs_header_generation(eb)); ret = 1; - - /* - * Things reading via commit roots that don't have normal protection, - * like send, can have a really old block in cache that may point at a - * block that has been freed and re-allocated. So don't clear uptodate - * if we find an eb that is under IO (dirty/writeback) because we could - * end up reading in the stale data and then writing it back out and - * making everybody very sad. - */ - if (!extent_buffer_under_io(eb)) - clear_extent_buffer_uptodate(eb); + clear_extent_buffer_uptodate(eb); out: unlock_extent_cached(io_tree, eb->start, eb->start + eb->len - 1, &cached_state); - if (need_lock) - btrfs_tree_read_unlock(eb); return ret; } @@ -584,6 +568,7 @@ static int validate_extent_buffer(struct extent_buffer *eb) const u32 csum_size = fs_info->csum_size; u8 found_level; u8 result[BTRFS_CSUM_SIZE]; + const u8 *header_csum; int ret = 0; found_start = btrfs_header_bytenr(eb); @@ -608,15 +593,14 @@ static int validate_extent_buffer(struct extent_buffer *eb) } csum_tree_block(eb, result); + header_csum = page_address(eb->pages[0]) + + get_eb_offset_in_page(eb, offsetof(struct btrfs_header, csum)); - if (memcmp_extent_buffer(eb, result, 0, csum_size)) { - u8 val[BTRFS_CSUM_SIZE] = { 0 }; - - read_extent_buffer(eb, &val, 0, csum_size); + if (memcmp(result, header_csum, csum_size) != 0) { btrfs_warn_rl(fs_info, - "%s checksum verify failed on %llu wanted " CSUM_FMT " found " CSUM_FMT " level %d", - fs_info->sb->s_id, eb->start, - CSUM_FMT_VALUE(csum_size, val), + "checksum verify failed on %llu wanted " CSUM_FMT " found " CSUM_FMT " level %d", + eb->start, + CSUM_FMT_VALUE(csum_size, header_csum), CSUM_FMT_VALUE(csum_size, result), btrfs_header_level(eb)); ret = -EUCLEAN; @@ -917,23 +901,22 @@ static blk_status_t btree_submit_bio_start(struct inode *inode, struct bio *bio, return btree_csum_one_bio(bio); } -static int check_async_write(struct btrfs_fs_info *fs_info, +static bool should_async_write(struct btrfs_fs_info *fs_info, struct btrfs_inode *bi) { if (btrfs_is_zoned(fs_info)) - return 0; + return false; if (atomic_read(&bi->sync_writers)) - return 0; + return false; if (test_bit(BTRFS_FS_CSUM_IMPL_FAST, &fs_info->flags)) - return 0; - return 1; + return false; + return true; } blk_status_t btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, int mirror_num, unsigned long bio_flags) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); - int async = check_async_write(fs_info, BTRFS_I(inode)); blk_status_t ret; if (btrfs_op(bio) != BTRFS_MAP_WRITE) { @@ -946,7 +929,7 @@ blk_status_t btrfs_submit_metadata_bio(struct inode *inode, struct bio *bio, if (ret) goto out_w_error; ret = btrfs_map_bio(fs_info, bio, mirror_num); - } else if (!async) { + } else if (!should_async_write(fs_info, BTRFS_I(inode))) { ret = btree_csum_one_bio(bio); if (ret) goto out_w_error; @@ -2252,6 +2235,7 @@ static void btrfs_init_balance(struct btrfs_fs_info *fs_info) atomic_set(&fs_info->balance_cancel_req, 0); fs_info->balance_ctl = NULL; init_waitqueue_head(&fs_info->balance_wait_q); + atomic_set(&fs_info->reloc_cancel_req, 0); } static void btrfs_init_btree_inode(struct btrfs_fs_info *fs_info) @@ -2999,6 +2983,7 @@ void btrfs_init_fs_info(struct btrfs_fs_info *fs_info) spin_lock_init(&fs_info->swapfile_pins_lock); fs_info->swapfile_pins = RB_ROOT; + spin_lock_init(&fs_info->send_reloc_lock); fs_info->send_in_progress = 0; fs_info->bg_reclaim_threshold = BTRFS_DEFAULT_RECLAIM_THRESH; @@ -3471,7 +3456,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device * At this point we know all the devices that make this filesystem, * including the seed devices but we don't know yet if the replace * target is required. So free devices that are not part of this - * filesystem but skip the replace traget device which is checked + * filesystem but skip the replace target device which is checked * below in btrfs_init_dev_replace(). */ btrfs_free_extra_devids(fs_devices); @@ -3598,8 +3583,7 @@ int __cold open_ctree(struct super_block *sb, struct btrfs_fs_devices *fs_device if (btrfs_test_opt(fs_info, CHECK_INTEGRITY)) { ret = btrfsic_mount(fs_info, fs_devices, btrfs_test_opt(fs_info, - CHECK_INTEGRITY_INCLUDING_EXTENT_DATA) ? - 1 : 0, + CHECK_INTEGRITY_DATA) ? 1 : 0, fs_info->check_integrity_print_mask); if (ret) btrfs_warn(fs_info, @@ -4696,9 +4680,6 @@ static int btrfs_destroy_delayed_refs(struct btrfs_transaction *trans, cache->space_info->bytes_reserved -= head->num_bytes; spin_unlock(&cache->lock); spin_unlock(&cache->space_info->lock); - percpu_counter_add_batch( - &cache->space_info->total_bytes_pinned, - head->num_bytes, BTRFS_TOTAL_BYTES_PINNED_BATCH); btrfs_put_block_group(cache); diff --git a/fs/btrfs/extent-tree.c b/fs/btrfs/extent-tree.c index 3d5c35e4cb76..d296483d148f 100644 --- a/fs/btrfs/extent-tree.c +++ b/fs/btrfs/extent-tree.c @@ -1425,7 +1425,7 @@ int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans, * bytenr of the parent block. Since new extents are always * created with indirect references, this will only be the case * when relocating a shared extent. In that case, root_objectid - * will be BTRFS_TREE_RELOC_OBJECTID. Otheriwse, parent must + * will be BTRFS_TREE_RELOC_OBJECTID. Otherwise, parent must * be 0 * * @root_objectid: The id of the root where this modification has originated, @@ -1804,19 +1804,6 @@ void btrfs_cleanup_ref_head_accounting(struct btrfs_fs_info *fs_info, nr_items += btrfs_csum_bytes_to_leaves(fs_info, head->num_bytes); } - /* - * We were dropping refs, or had a new ref and dropped it, and thus must - * adjust down our total_bytes_pinned, the space may or may not have - * been pinned and so is accounted for properly in the pinned space by - * now. - */ - if (head->total_ref_mod < 0 || - (head->total_ref_mod == 0 && head->must_insert_reserved)) { - u64 flags = btrfs_ref_head_to_space_flags(head); - - btrfs_mod_total_bytes_pinned(fs_info, flags, -head->num_bytes); - } - btrfs_delayed_refs_rsv_release(fs_info, nr_items); } @@ -2551,7 +2538,6 @@ static int pin_down_extent(struct btrfs_trans_handle *trans, spin_unlock(&cache->lock); spin_unlock(&cache->space_info->lock); - __btrfs_mod_total_bytes_pinned(cache->space_info, num_bytes); set_extent_dirty(&trans->transaction->pinned_extents, bytenr, bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL); return 0; @@ -2762,7 +2748,6 @@ static int unpin_extent_range(struct btrfs_fs_info *fs_info, cache->pinned -= len; btrfs_space_info_update_bytes_pinned(fs_info, space_info, -len); space_info->max_extent_size = 0; - __btrfs_mod_total_bytes_pinned(space_info, -len); if (cache->ro) { space_info->bytes_readonly += len; readonly = true; @@ -4784,7 +4769,6 @@ btrfs_init_new_buffer(struct btrfs_trans_handle *trans, struct btrfs_root *root, set_extent_dirty(&trans->transaction->dirty_pages, buf->start, buf->start + buf->len - 1, GFP_NOFS); } - trans->dirty = true; /* this returns a buffer locked for blocking */ return buf; } diff --git a/fs/btrfs/extent_io.c b/fs/btrfs/extent_io.c index dee2dafbc872..9e81d25dea70 100644 --- a/fs/btrfs/extent_io.c +++ b/fs/btrfs/extent_io.c @@ -136,7 +136,7 @@ struct tree_entry { }; struct extent_page_data { - struct bio *bio; + struct btrfs_bio_ctrl bio_ctrl; /* tells writepage not to lock the state bits for this range * it still does the unlocking */ @@ -185,10 +185,12 @@ int __must_check submit_one_bio(struct bio *bio, int mirror_num, /* Cleanup unsubmitted bios */ static void end_write_bio(struct extent_page_data *epd, int ret) { - if (epd->bio) { - epd->bio->bi_status = errno_to_blk_status(ret); - bio_endio(epd->bio); - epd->bio = NULL; + struct bio *bio = epd->bio_ctrl.bio; + + if (bio) { + bio->bi_status = errno_to_blk_status(ret); + bio_endio(bio); + epd->bio_ctrl.bio = NULL; } } @@ -201,9 +203,10 @@ static void end_write_bio(struct extent_page_data *epd, int ret) static int __must_check flush_write_bio(struct extent_page_data *epd) { int ret = 0; + struct bio *bio = epd->bio_ctrl.bio; - if (epd->bio) { - ret = submit_one_bio(epd->bio, 0, 0); + if (bio) { + ret = submit_one_bio(bio, 0, 0); /* * Clean up of epd->bio is handled by its endio function. * And endio is either triggered by successful bio execution @@ -211,7 +214,7 @@ static int __must_check flush_write_bio(struct extent_page_data *epd) * So at this point, no matter what happened, we don't need * to clean up epd->bio. */ - epd->bio = NULL; + epd->bio_ctrl.bio = NULL; } return ret; } @@ -1805,10 +1808,130 @@ bool btrfs_find_delalloc_range(struct extent_io_tree *tree, u64 *start, return found; } +/* + * Process one page for __process_pages_contig(). + * + * Return >0 if we hit @page == @locked_page. + * Return 0 if we updated the page status. + * Return -EGAIN if the we need to try again. + * (For PAGE_LOCK case but got dirty page or page not belong to mapping) + */ +static int process_one_page(struct btrfs_fs_info *fs_info, + struct address_space *mapping, + struct page *page, struct page *locked_page, + unsigned long page_ops, u64 start, u64 end) +{ + u32 len; + + ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX); + len = end + 1 - start; + + if (page_ops & PAGE_SET_ORDERED) + btrfs_page_clamp_set_ordered(fs_info, page, start, len); + if (page_ops & PAGE_SET_ERROR) + btrfs_page_clamp_set_error(fs_info, page, start, len); + if (page_ops & PAGE_START_WRITEBACK) { + btrfs_page_clamp_clear_dirty(fs_info, page, start, len); + btrfs_page_clamp_set_writeback(fs_info, page, start, len); + } + if (page_ops & PAGE_END_WRITEBACK) + btrfs_page_clamp_clear_writeback(fs_info, page, start, len); + + if (page == locked_page) + return 1; + + if (page_ops & PAGE_LOCK) { + int ret; + + ret = btrfs_page_start_writer_lock(fs_info, page, start, len); + if (ret) + return ret; + if (!PageDirty(page) || page->mapping != mapping) { + btrfs_page_end_writer_lock(fs_info, page, start, len); + return -EAGAIN; + } + } + if (page_ops & PAGE_UNLOCK) + btrfs_page_end_writer_lock(fs_info, page, start, len); + return 0; +} + static int __process_pages_contig(struct address_space *mapping, struct page *locked_page, - pgoff_t start_index, pgoff_t end_index, - unsigned long page_ops, pgoff_t *index_ret); + u64 start, u64 end, unsigned long page_ops, + u64 *processed_end) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(mapping->host->i_sb); + pgoff_t start_index = start >> PAGE_SHIFT; + pgoff_t end_index = end >> PAGE_SHIFT; + pgoff_t index = start_index; + unsigned long nr_pages = end_index - start_index + 1; + unsigned long pages_processed = 0; + struct page *pages[16]; + int err = 0; + int i; + + if (page_ops & PAGE_LOCK) { + ASSERT(page_ops == PAGE_LOCK); + ASSERT(processed_end && *processed_end == start); + } + + if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0) + mapping_set_error(mapping, -EIO); + + while (nr_pages > 0) { + int found_pages; + + found_pages = find_get_pages_contig(mapping, index, + min_t(unsigned long, + nr_pages, ARRAY_SIZE(pages)), pages); + if (found_pages == 0) { + /* + * Only if we're going to lock these pages, we can find + * nothing at @index. + */ + ASSERT(page_ops & PAGE_LOCK); + err = -EAGAIN; + goto out; + } + + for (i = 0; i < found_pages; i++) { + int process_ret; + + process_ret = process_one_page(fs_info, mapping, + pages[i], locked_page, page_ops, + start, end); + if (process_ret < 0) { + for (; i < found_pages; i++) + put_page(pages[i]); + err = -EAGAIN; + goto out; + } + put_page(pages[i]); + pages_processed++; + } + nr_pages -= found_pages; + index += found_pages; + cond_resched(); + } +out: + if (err && processed_end) { + /* + * Update @processed_end. I know this is awful since it has + * two different return value patterns (inclusive vs exclusive). + * + * But the exclusive pattern is necessary if @start is 0, or we + * underflow and check against processed_end won't work as + * expected. + */ + if (pages_processed) + *processed_end = min(end, + ((u64)(start_index + pages_processed) << PAGE_SHIFT) - 1); + else + *processed_end = start; + } + return err; +} static noinline void __unlock_for_delalloc(struct inode *inode, struct page *locked_page, @@ -1821,7 +1944,7 @@ static noinline void __unlock_for_delalloc(struct inode *inode, if (index == locked_page->index && end_index == index) return; - __process_pages_contig(inode->i_mapping, locked_page, index, end_index, + __process_pages_contig(inode->i_mapping, locked_page, start, end, PAGE_UNLOCK, NULL); } @@ -1831,19 +1954,19 @@ static noinline int lock_delalloc_pages(struct inode *inode, u64 delalloc_end) { unsigned long index = delalloc_start >> PAGE_SHIFT; - unsigned long index_ret = index; unsigned long end_index = delalloc_end >> PAGE_SHIFT; + u64 processed_end = delalloc_start; int ret; ASSERT(locked_page); if (index == locked_page->index && index == end_index) return 0; - ret = __process_pages_contig(inode->i_mapping, locked_page, index, - end_index, PAGE_LOCK, &index_ret); - if (ret == -EAGAIN) + ret = __process_pages_contig(inode->i_mapping, locked_page, delalloc_start, + delalloc_end, PAGE_LOCK, &processed_end); + if (ret == -EAGAIN && processed_end > delalloc_start) __unlock_for_delalloc(inode, locked_page, delalloc_start, - (u64)index_ret << PAGE_SHIFT); + processed_end); return ret; } @@ -1936,84 +2059,6 @@ noinline_for_stack bool find_lock_delalloc_range(struct inode *inode, return found; } -static int __process_pages_contig(struct address_space *mapping, - struct page *locked_page, - pgoff_t start_index, pgoff_t end_index, - unsigned long page_ops, pgoff_t *index_ret) -{ - unsigned long nr_pages = end_index - start_index + 1; - unsigned long pages_processed = 0; - pgoff_t index = start_index; - struct page *pages[16]; - unsigned ret; - int err = 0; - int i; - - if (page_ops & PAGE_LOCK) { - ASSERT(page_ops == PAGE_LOCK); - ASSERT(index_ret && *index_ret == start_index); - } - - if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0) - mapping_set_error(mapping, -EIO); - - while (nr_pages > 0) { - ret = find_get_pages_contig(mapping, index, - min_t(unsigned long, - nr_pages, ARRAY_SIZE(pages)), pages); - if (ret == 0) { - /* - * Only if we're going to lock these pages, - * can we find nothing at @index. - */ - ASSERT(page_ops & PAGE_LOCK); - err = -EAGAIN; - goto out; - } - - for (i = 0; i < ret; i++) { - if (page_ops & PAGE_SET_PRIVATE2) - SetPagePrivate2(pages[i]); - - if (locked_page && pages[i] == locked_page) { - put_page(pages[i]); - pages_processed++; - continue; - } - if (page_ops & PAGE_START_WRITEBACK) { - clear_page_dirty_for_io(pages[i]); - set_page_writeback(pages[i]); - } - if (page_ops & PAGE_SET_ERROR) - SetPageError(pages[i]); - if (page_ops & PAGE_END_WRITEBACK) - end_page_writeback(pages[i]); - if (page_ops & PAGE_UNLOCK) - unlock_page(pages[i]); - if (page_ops & PAGE_LOCK) { - lock_page(pages[i]); - if (!PageDirty(pages[i]) || - pages[i]->mapping != mapping) { - unlock_page(pages[i]); - for (; i < ret; i++) - put_page(pages[i]); - err = -EAGAIN; - goto out; - } - } - put_page(pages[i]); - pages_processed++; - } - nr_pages -= ret; - index += ret; - cond_resched(); - } -out: - if (err && index_ret) - *index_ret = start_index + pages_processed - 1; - return err; -} - void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, struct page *locked_page, u32 clear_bits, unsigned long page_ops) @@ -2021,8 +2066,7 @@ void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end, clear_extent_bit(&inode->io_tree, start, end, clear_bits, 1, 0, NULL); __process_pages_contig(inode->vfs_inode.i_mapping, locked_page, - start >> PAGE_SHIFT, end >> PAGE_SHIFT, - page_ops, NULL); + start, end, page_ops, NULL); } /* @@ -2381,13 +2425,6 @@ int clean_io_failure(struct btrfs_fs_info *fs_info, BUG_ON(!failrec->this_mirror); - if (failrec->in_validation) { - /* there was no real error, just free the record */ - btrfs_debug(fs_info, - "clean_io_failure: freeing dummy error at %llu", - failrec->start); - goto out; - } if (sb_rdonly(fs_info->sb)) goto out; @@ -2449,7 +2486,7 @@ void btrfs_free_io_failure_record(struct btrfs_inode *inode, u64 start, u64 end) } static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode, - u64 start, u64 end) + u64 start) { struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct io_failure_record *failrec; @@ -2457,15 +2494,15 @@ static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree; struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree; + const u32 sectorsize = fs_info->sectorsize; int ret; u64 logical; failrec = get_state_failrec(failure_tree, start); if (!IS_ERR(failrec)) { btrfs_debug(fs_info, - "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d", - failrec->logical, failrec->start, failrec->len, - failrec->in_validation); + "Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu", + failrec->logical, failrec->start, failrec->len); /* * when data can be on disk more than twice, add to failrec here * (e.g. with a list for failed_mirror) to make @@ -2480,10 +2517,9 @@ static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode return ERR_PTR(-ENOMEM); failrec->start = start; - failrec->len = end - start + 1; + failrec->len = sectorsize; failrec->this_mirror = 0; failrec->bio_flags = 0; - failrec->in_validation = 0; read_lock(&em_tree->lock); em = lookup_extent_mapping(em_tree, start, failrec->len); @@ -2519,12 +2555,13 @@ static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode free_extent_map(em); /* Set the bits in the private failure tree */ - ret = set_extent_bits(failure_tree, start, end, + ret = set_extent_bits(failure_tree, start, start + sectorsize - 1, EXTENT_LOCKED | EXTENT_DIRTY); if (ret >= 0) { ret = set_state_failrec(failure_tree, start, failrec); /* Set the bits in the inode's tree */ - ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED); + ret = set_extent_bits(tree, start, start + sectorsize - 1, + EXTENT_DAMAGED); } else if (ret < 0) { kfree(failrec); return ERR_PTR(ret); @@ -2533,7 +2570,7 @@ static struct io_failure_record *btrfs_get_io_failure_record(struct inode *inode return failrec; } -static bool btrfs_check_repairable(struct inode *inode, bool needs_validation, +static bool btrfs_check_repairable(struct inode *inode, struct io_failure_record *failrec, int failed_mirror) { @@ -2553,39 +2590,22 @@ static bool btrfs_check_repairable(struct inode *inode, bool needs_validation, return false; } + /* The failure record should only contain one sector */ + ASSERT(failrec->len == fs_info->sectorsize); + /* - * there are two premises: - * a) deliver good data to the caller - * b) correct the bad sectors on disk + * There are two premises: + * a) deliver good data to the caller + * b) correct the bad sectors on disk + * + * Since we're only doing repair for one sector, we only need to get + * a good copy of the failed sector and if we succeed, we have setup + * everything for repair_io_failure to do the rest for us. */ - if (needs_validation) { - /* - * to fulfill b), we need to know the exact failing sectors, as - * we don't want to rewrite any more than the failed ones. thus, - * we need separate read requests for the failed bio - * - * if the following BUG_ON triggers, our validation request got - * merged. we need separate requests for our algorithm to work. - */ - BUG_ON(failrec->in_validation); - failrec->in_validation = 1; - failrec->this_mirror = failed_mirror; - } else { - /* - * we're ready to fulfill a) and b) alongside. get a good copy - * of the failed sector and if we succeed, we have setup - * everything for repair_io_failure to do the rest for us. - */ - if (failrec->in_validation) { - BUG_ON(failrec->this_mirror != failed_mirror); - failrec->in_validation = 0; - failrec->this_mirror = 0; - } - failrec->failed_mirror = failed_mirror; + failrec->failed_mirror = failed_mirror; + failrec->this_mirror++; + if (failrec->this_mirror == failed_mirror) failrec->this_mirror++; - if (failrec->this_mirror == failed_mirror) - failrec->this_mirror++; - } if (failrec->this_mirror > num_copies) { btrfs_debug(fs_info, @@ -2597,53 +2617,11 @@ static bool btrfs_check_repairable(struct inode *inode, bool needs_validation, return true; } -static bool btrfs_io_needs_validation(struct inode *inode, struct bio *bio) -{ - u64 len = 0; - const u32 blocksize = inode->i_sb->s_blocksize; - - /* - * If bi_status is BLK_STS_OK, then this was a checksum error, not an - * I/O error. In this case, we already know exactly which sector was - * bad, so we don't need to validate. - */ - if (bio->bi_status == BLK_STS_OK) - return false; - - /* - * We need to validate each sector individually if the failed I/O was - * for multiple sectors. - * - * There are a few possible bios that can end up here: - * 1. A buffered read bio, which is not cloned. - * 2. A direct I/O read bio, which is cloned. - * 3. A (buffered or direct) repair bio, which is not cloned. - * - * For cloned bios (case 2), we can get the size from - * btrfs_io_bio->iter; for non-cloned bios (cases 1 and 3), we can get - * it from the bvecs. - */ - if (bio_flagged(bio, BIO_CLONED)) { - if (btrfs_io_bio(bio)->iter.bi_size > blocksize) - return true; - } else { - struct bio_vec *bvec; - int i; - - bio_for_each_bvec_all(bvec, bio, i) { - len += bvec->bv_len; - if (len > blocksize) - return true; - } - } - return false; -} - -blk_status_t btrfs_submit_read_repair(struct inode *inode, - struct bio *failed_bio, u32 bio_offset, - struct page *page, unsigned int pgoff, - u64 start, u64 end, int failed_mirror, - submit_bio_hook_t *submit_bio_hook) +int btrfs_repair_one_sector(struct inode *inode, + struct bio *failed_bio, u32 bio_offset, + struct page *page, unsigned int pgoff, + u64 start, int failed_mirror, + submit_bio_hook_t *submit_bio_hook) { struct io_failure_record *failrec; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); @@ -2651,7 +2629,6 @@ blk_status_t btrfs_submit_read_repair(struct inode *inode, struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree; struct btrfs_io_bio *failed_io_bio = btrfs_io_bio(failed_bio); const int icsum = bio_offset >> fs_info->sectorsize_bits; - bool need_validation; struct bio *repair_bio; struct btrfs_io_bio *repair_io_bio; blk_status_t status; @@ -2661,23 +2638,19 @@ blk_status_t btrfs_submit_read_repair(struct inode *inode, BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); - failrec = btrfs_get_io_failure_record(inode, start, end); + failrec = btrfs_get_io_failure_record(inode, start); if (IS_ERR(failrec)) - return errno_to_blk_status(PTR_ERR(failrec)); + return PTR_ERR(failrec); - need_validation = btrfs_io_needs_validation(inode, failed_bio); - if (!btrfs_check_repairable(inode, need_validation, failrec, - failed_mirror)) { + if (!btrfs_check_repairable(inode, failrec, failed_mirror)) { free_io_failure(failure_tree, tree, failrec); - return BLK_STS_IOERR; + return -EIO; } repair_bio = btrfs_io_bio_alloc(1); repair_io_bio = btrfs_io_bio(repair_bio); repair_bio->bi_opf = REQ_OP_READ; - if (need_validation) - repair_bio->bi_opf |= REQ_FAILFAST_DEV; repair_bio->bi_end_io = failed_bio->bi_end_io; repair_bio->bi_iter.bi_sector = failrec->logical >> 9; repair_bio->bi_private = failed_bio->bi_private; @@ -2695,8 +2668,8 @@ blk_status_t btrfs_submit_read_repair(struct inode *inode, repair_io_bio->iter = repair_bio->bi_iter; btrfs_debug(btrfs_sb(inode->i_sb), -"repair read error: submitting new read to mirror %d, in_validation=%d", - failrec->this_mirror, failrec->in_validation); + "repair read error: submitting new read to mirror %d", + failrec->this_mirror); status = submit_bio_hook(inode, repair_bio, failrec->this_mirror, failrec->bio_flags); @@ -2704,17 +2677,114 @@ blk_status_t btrfs_submit_read_repair(struct inode *inode, free_io_failure(failure_tree, tree, failrec); bio_put(repair_bio); } - return status; + return blk_status_to_errno(status); +} + +static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); + + ASSERT(page_offset(page) <= start && + start + len <= page_offset(page) + PAGE_SIZE); + + if (uptodate) { + btrfs_page_set_uptodate(fs_info, page, start, len); + } else { + btrfs_page_clear_uptodate(fs_info, page, start, len); + btrfs_page_set_error(fs_info, page, start, len); + } + + if (fs_info->sectorsize == PAGE_SIZE) + unlock_page(page); + else + btrfs_subpage_end_reader(fs_info, page, start, len); +} + +static blk_status_t submit_read_repair(struct inode *inode, + struct bio *failed_bio, u32 bio_offset, + struct page *page, unsigned int pgoff, + u64 start, u64 end, int failed_mirror, + unsigned int error_bitmap, + submit_bio_hook_t *submit_bio_hook) +{ + struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + const u32 sectorsize = fs_info->sectorsize; + const int nr_bits = (end + 1 - start) >> fs_info->sectorsize_bits; + int error = 0; + int i; + + BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE); + + /* We're here because we had some read errors or csum mismatch */ + ASSERT(error_bitmap); + + /* + * We only get called on buffered IO, thus page must be mapped and bio + * must not be cloned. + */ + ASSERT(page->mapping && !bio_flagged(failed_bio, BIO_CLONED)); + + /* Iterate through all the sectors in the range */ + for (i = 0; i < nr_bits; i++) { + const unsigned int offset = i * sectorsize; + struct extent_state *cached = NULL; + bool uptodate = false; + int ret; + + if (!(error_bitmap & (1U << i))) { + /* + * This sector has no error, just end the page read + * and unlock the range. + */ + uptodate = true; + goto next; + } + + ret = btrfs_repair_one_sector(inode, failed_bio, + bio_offset + offset, + page, pgoff + offset, start + offset, + failed_mirror, submit_bio_hook); + if (!ret) { + /* + * We have submitted the read repair, the page release + * will be handled by the endio function of the + * submitted repair bio. + * Thus we don't need to do any thing here. + */ + continue; + } + /* + * Repair failed, just record the error but still continue. + * Or the remaining sectors will not be properly unlocked. + */ + if (!error) + error = ret; +next: + end_page_read(page, uptodate, start + offset, sectorsize); + if (uptodate) + set_extent_uptodate(&BTRFS_I(inode)->io_tree, + start + offset, + start + offset + sectorsize - 1, + &cached, GFP_ATOMIC); + unlock_extent_cached_atomic(&BTRFS_I(inode)->io_tree, + start + offset, + start + offset + sectorsize - 1, + &cached); + } + return errno_to_blk_status(error); } /* lots and lots of room for performance fixes in the end_bio funcs */ void end_extent_writepage(struct page *page, int err, u64 start, u64 end) { + struct btrfs_inode *inode; int uptodate = (err == 0); int ret = 0; - btrfs_writepage_endio_finish_ordered(page, start, end, uptodate); + ASSERT(page && page->mapping); + inode = BTRFS_I(page->mapping->host); + btrfs_writepage_endio_finish_ordered(inode, page, start, end, uptodate); if (!uptodate) { ClearPageUptodate(page); @@ -2747,25 +2817,20 @@ static void end_bio_extent_writepage(struct bio *bio) struct page *page = bvec->bv_page; struct inode *inode = page->mapping->host; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); + const u32 sectorsize = fs_info->sectorsize; - /* We always issue full-page reads, but if some block - * in a page fails to read, blk_update_request() will - * advance bv_offset and adjust bv_len to compensate. - * Print a warning for nonzero offsets, and an error - * if they don't add up to a full page. */ - if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) { - if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE) - btrfs_err(fs_info, - "partial page write in btrfs with offset %u and length %u", - bvec->bv_offset, bvec->bv_len); - else - btrfs_info(fs_info, - "incomplete page write in btrfs with offset %u and length %u", - bvec->bv_offset, bvec->bv_len); - } + /* Our read/write should always be sector aligned. */ + if (!IS_ALIGNED(bvec->bv_offset, sectorsize)) + btrfs_err(fs_info, + "partial page write in btrfs with offset %u and length %u", + bvec->bv_offset, bvec->bv_len); + else if (!IS_ALIGNED(bvec->bv_len, sectorsize)) + btrfs_info(fs_info, + "incomplete page write with offset %u and length %u", + bvec->bv_offset, bvec->bv_len); - start = page_offset(page); - end = start + bvec->bv_offset + bvec->bv_len - 1; + start = page_offset(page) + bvec->bv_offset; + end = start + bvec->bv_len - 1; if (first_bvec) { btrfs_record_physical_zoned(inode, start, bio); @@ -2773,7 +2838,8 @@ static void end_bio_extent_writepage(struct bio *bio) } end_extent_writepage(page, error, start, end); - end_page_writeback(page); + + btrfs_page_clear_writeback(fs_info, page, start, bvec->bv_len); } bio_put(bio); @@ -2862,30 +2928,6 @@ static void begin_page_read(struct btrfs_fs_info *fs_info, struct page *page) btrfs_subpage_start_reader(fs_info, page, page_offset(page), PAGE_SIZE); } -static void end_page_read(struct page *page, bool uptodate, u64 start, u32 len) -{ - struct btrfs_fs_info *fs_info = btrfs_sb(page->mapping->host->i_sb); - - ASSERT(page_offset(page) <= start && - start + len <= page_offset(page) + PAGE_SIZE); - - if (uptodate) { - btrfs_page_set_uptodate(fs_info, page, start, len); - } else { - btrfs_page_clear_uptodate(fs_info, page, start, len); - btrfs_page_set_error(fs_info, page, start, len); - } - - if (fs_info->sectorsize == PAGE_SIZE) - unlock_page(page); - else if (is_data_inode(page->mapping->host)) - /* - * For subpage data, unlock the page if we're the last reader. - * For subpage metadata, page lock is not utilized for read. - */ - btrfs_subpage_end_reader(fs_info, page, start, len); -} - /* * Find extent buffer for a givne bytenr. * @@ -2929,7 +2971,6 @@ static struct extent_buffer *find_extent_buffer_readpage( static void end_bio_extent_readpage(struct bio *bio) { struct bio_vec *bvec; - int uptodate = !bio->bi_status; struct btrfs_io_bio *io_bio = btrfs_io_bio(bio); struct extent_io_tree *tree, *failure_tree; struct processed_extent processed = { 0 }; @@ -2944,10 +2985,12 @@ static void end_bio_extent_readpage(struct bio *bio) ASSERT(!bio_flagged(bio, BIO_CLONED)); bio_for_each_segment_all(bvec, bio, iter_all) { + bool uptodate = !bio->bi_status; struct page *page = bvec->bv_page; struct inode *inode = page->mapping->host; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); const u32 sectorsize = fs_info->sectorsize; + unsigned int error_bitmap = (unsigned int)-1; u64 start; u64 end; u32 len; @@ -2982,14 +3025,16 @@ static void end_bio_extent_readpage(struct bio *bio) mirror = io_bio->mirror_num; if (likely(uptodate)) { - if (is_data_inode(inode)) - ret = btrfs_verify_data_csum(io_bio, + if (is_data_inode(inode)) { + error_bitmap = btrfs_verify_data_csum(io_bio, bio_offset, page, start, end); - else + ret = error_bitmap; + } else { ret = btrfs_validate_metadata_buffer(io_bio, page, start, end, mirror); + } if (ret) - uptodate = 0; + uptodate = false; else clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree, tree, start, @@ -3001,27 +3046,18 @@ static void end_bio_extent_readpage(struct bio *bio) goto readpage_ok; if (is_data_inode(inode)) { - /* - * The generic bio_readpage_error handles errors the - * following way: If possible, new read requests are - * created and submitted and will end up in - * end_bio_extent_readpage as well (if we're lucky, - * not in the !uptodate case). In that case it returns - * 0 and we just go on with the next page in our bio. - * If it can't handle the error it will return -EIO and - * we remain responsible for that page. + * btrfs_submit_read_repair() will handle all the good + * and bad sectors, we just continue to the next bvec. */ - if (!btrfs_submit_read_repair(inode, bio, bio_offset, - page, - start - page_offset(page), - start, end, mirror, - btrfs_submit_data_bio)) { - uptodate = !bio->bi_status; - ASSERT(bio_offset + len > bio_offset); - bio_offset += len; - continue; - } + submit_read_repair(inode, bio, bio_offset, page, + start - page_offset(page), start, + end, mirror, error_bitmap, + btrfs_submit_data_bio); + + ASSERT(bio_offset + len > bio_offset); + bio_offset += len; + continue; } else { struct extent_buffer *eb; @@ -3151,42 +3187,99 @@ struct bio *btrfs_bio_clone_partial(struct bio *orig, int offset, int size) * * Return true if successfully page added. Otherwise, return false. */ -static bool btrfs_bio_add_page(struct bio *bio, struct page *page, +static bool btrfs_bio_add_page(struct btrfs_bio_ctrl *bio_ctrl, + struct page *page, u64 disk_bytenr, unsigned int size, unsigned int pg_offset, - unsigned long prev_bio_flags, unsigned long bio_flags) { + struct bio *bio = bio_ctrl->bio; + u32 bio_size = bio->bi_iter.bi_size; const sector_t sector = disk_bytenr >> SECTOR_SHIFT; bool contig; int ret; - if (prev_bio_flags != bio_flags) + ASSERT(bio); + /* The limit should be calculated when bio_ctrl->bio is allocated */ + ASSERT(bio_ctrl->len_to_oe_boundary && bio_ctrl->len_to_stripe_boundary); + if (bio_ctrl->bio_flags != bio_flags) return false; - if (prev_bio_flags & EXTENT_BIO_COMPRESSED) + if (bio_ctrl->bio_flags & EXTENT_BIO_COMPRESSED) contig = bio->bi_iter.bi_sector == sector; else contig = bio_end_sector(bio) == sector; if (!contig) return false; - if (btrfs_bio_fits_in_stripe(page, size, bio, bio_flags)) + if (bio_size + size > bio_ctrl->len_to_oe_boundary || + bio_size + size > bio_ctrl->len_to_stripe_boundary) return false; - if (bio_op(bio) == REQ_OP_ZONE_APPEND) { - struct page *first_page = bio_first_bvec_all(bio)->bv_page; - - if (!btrfs_bio_fits_in_ordered_extent(first_page, bio, size)) - return false; + if (bio_op(bio) == REQ_OP_ZONE_APPEND) ret = bio_add_zone_append_page(bio, page, size, pg_offset); - } else { + else ret = bio_add_page(bio, page, size, pg_offset); - } return ret == size; } +static int calc_bio_boundaries(struct btrfs_bio_ctrl *bio_ctrl, + struct btrfs_inode *inode) +{ + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_io_geometry geom; + struct btrfs_ordered_extent *ordered; + struct extent_map *em; + u64 logical = (bio_ctrl->bio->bi_iter.bi_sector << SECTOR_SHIFT); + int ret; + + /* + * Pages for compressed extent are never submitted to disk directly, + * thus it has no real boundary, just set them to U32_MAX. + * + * The split happens for real compressed bio, which happens in + * btrfs_submit_compressed_read/write(). + */ + if (bio_ctrl->bio_flags & EXTENT_BIO_COMPRESSED) { + bio_ctrl->len_to_oe_boundary = U32_MAX; + bio_ctrl->len_to_stripe_boundary = U32_MAX; + return 0; + } + em = btrfs_get_chunk_map(fs_info, logical, fs_info->sectorsize); + if (IS_ERR(em)) + return PTR_ERR(em); + ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio_ctrl->bio), + logical, &geom); + free_extent_map(em); + if (ret < 0) { + return ret; + } + if (geom.len > U32_MAX) + bio_ctrl->len_to_stripe_boundary = U32_MAX; + else + bio_ctrl->len_to_stripe_boundary = (u32)geom.len; + + if (!btrfs_is_zoned(fs_info) || + bio_op(bio_ctrl->bio) != REQ_OP_ZONE_APPEND) { + bio_ctrl->len_to_oe_boundary = U32_MAX; + return 0; + } + + ASSERT(fs_info->max_zone_append_size > 0); + /* Ordered extent not yet created, so we're good */ + ordered = btrfs_lookup_ordered_extent(inode, logical); + if (!ordered) { + bio_ctrl->len_to_oe_boundary = U32_MAX; + return 0; + } + + bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX, + ordered->disk_bytenr + ordered->disk_num_bytes - logical); + btrfs_put_ordered_extent(ordered); + return 0; +} + /* * @opf: bio REQ_OP_* and REQ_* flags as one value * @wbc: optional writeback control for io accounting @@ -3203,12 +3296,11 @@ static bool btrfs_bio_add_page(struct bio *bio, struct page *page, */ static int submit_extent_page(unsigned int opf, struct writeback_control *wbc, + struct btrfs_bio_ctrl *bio_ctrl, struct page *page, u64 disk_bytenr, size_t size, unsigned long pg_offset, - struct bio **bio_ret, bio_end_io_t end_io_func, int mirror_num, - unsigned long prev_bio_flags, unsigned long bio_flags, bool force_bio_submit) { @@ -3219,19 +3311,19 @@ static int submit_extent_page(unsigned int opf, struct extent_io_tree *tree = &inode->io_tree; struct btrfs_fs_info *fs_info = inode->root->fs_info; - ASSERT(bio_ret); + ASSERT(bio_ctrl); - if (*bio_ret) { - bio = *bio_ret; + ASSERT(pg_offset < PAGE_SIZE && size <= PAGE_SIZE && + pg_offset + size <= PAGE_SIZE); + if (bio_ctrl->bio) { + bio = bio_ctrl->bio; if (force_bio_submit || - !btrfs_bio_add_page(bio, page, disk_bytenr, io_size, - pg_offset, prev_bio_flags, bio_flags)) { - ret = submit_one_bio(bio, mirror_num, prev_bio_flags); - if (ret < 0) { - *bio_ret = NULL; + !btrfs_bio_add_page(bio_ctrl, page, disk_bytenr, io_size, + pg_offset, bio_flags)) { + ret = submit_one_bio(bio, mirror_num, bio_ctrl->bio_flags); + bio_ctrl->bio = NULL; + if (ret < 0) return ret; - } - bio = NULL; } else { if (wbc) wbc_account_cgroup_owner(wbc, page, io_size); @@ -3254,22 +3346,18 @@ static int submit_extent_page(unsigned int opf, wbc_account_cgroup_owner(wbc, page, io_size); } if (btrfs_is_zoned(fs_info) && bio_op(bio) == REQ_OP_ZONE_APPEND) { - struct extent_map *em; - struct map_lookup *map; + struct btrfs_device *device; - em = btrfs_get_chunk_map(fs_info, disk_bytenr, io_size); - if (IS_ERR(em)) - return PTR_ERR(em); + device = btrfs_zoned_get_device(fs_info, disk_bytenr, io_size); + if (IS_ERR(device)) + return PTR_ERR(device); - map = em->map_lookup; - /* We only support single profile for now */ - ASSERT(map->num_stripes == 1); - btrfs_io_bio(bio)->device = map->stripes[0].dev; - - free_extent_map(em); + btrfs_io_bio(bio)->device = device; } - *bio_ret = bio; + bio_ctrl->bio = bio; + bio_ctrl->bio_flags = bio_flags; + ret = calc_bio_boundaries(bio_ctrl, inode); return ret; } @@ -3382,7 +3470,7 @@ __get_extent_map(struct inode *inode, struct page *page, size_t pg_offset, * return 0 on success, otherwise return error */ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, - struct bio **bio, unsigned long *bio_flags, + struct btrfs_bio_ctrl *bio_ctrl, unsigned int read_flags, u64 *prev_em_start) { struct inode *inode = page->mapping->host; @@ -3558,15 +3646,13 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, } ret = submit_extent_page(REQ_OP_READ | read_flags, NULL, - page, disk_bytenr, iosize, - pg_offset, bio, + bio_ctrl, page, disk_bytenr, iosize, + pg_offset, end_bio_extent_readpage, 0, - *bio_flags, this_bio_flag, force_bio_submit); if (!ret) { nr++; - *bio_flags = this_bio_flag; } else { unlock_extent(tree, cur, cur + iosize - 1); end_page_read(page, false, cur, iosize); @@ -3580,11 +3666,10 @@ int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, } static inline void contiguous_readpages(struct page *pages[], int nr_pages, - u64 start, u64 end, - struct extent_map **em_cached, - struct bio **bio, - unsigned long *bio_flags, - u64 *prev_em_start) + u64 start, u64 end, + struct extent_map **em_cached, + struct btrfs_bio_ctrl *bio_ctrl, + u64 *prev_em_start) { struct btrfs_inode *inode = BTRFS_I(pages[0]->mapping->host); int index; @@ -3592,7 +3677,7 @@ static inline void contiguous_readpages(struct page *pages[], int nr_pages, btrfs_lock_and_flush_ordered_range(inode, start, end, NULL); for (index = 0; index < nr_pages; index++) { - btrfs_do_readpage(pages[index], em_cached, bio, bio_flags, + btrfs_do_readpage(pages[index], em_cached, bio_ctrl, REQ_RAHEAD, prev_em_start); put_page(pages[index]); } @@ -3679,6 +3764,54 @@ static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode, return 0; } +/* + * Find the first byte we need to write. + * + * For subpage, one page can contain several sectors, and + * __extent_writepage_io() will just grab all extent maps in the page + * range and try to submit all non-inline/non-compressed extents. + * + * This is a big problem for subpage, we shouldn't re-submit already written + * data at all. + * This function will lookup subpage dirty bit to find which range we really + * need to submit. + * + * Return the next dirty range in [@start, @end). + * If no dirty range is found, @start will be page_offset(page) + PAGE_SIZE. + */ +static void find_next_dirty_byte(struct btrfs_fs_info *fs_info, + struct page *page, u64 *start, u64 *end) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + u64 orig_start = *start; + /* Declare as unsigned long so we can use bitmap ops */ + unsigned long dirty_bitmap; + unsigned long flags; + int nbits = (orig_start - page_offset(page)) >> fs_info->sectorsize_bits; + int range_start_bit = nbits; + int range_end_bit; + + /* + * For regular sector size == page size case, since one page only + * contains one sector, we return the page offset directly. + */ + if (fs_info->sectorsize == PAGE_SIZE) { + *start = page_offset(page); + *end = page_offset(page) + PAGE_SIZE; + return; + } + + /* We should have the page locked, but just in case */ + spin_lock_irqsave(&subpage->lock, flags); + dirty_bitmap = subpage->dirty_bitmap; + spin_unlock_irqrestore(&subpage->lock, flags); + + bitmap_next_set_region(&dirty_bitmap, &range_start_bit, &range_end_bit, + BTRFS_SUBPAGE_BITMAP_SIZE); + *start = page_offset(page) + range_start_bit * fs_info->sectorsize; + *end = page_offset(page) + range_end_bit * fs_info->sectorsize; +} + /* * helper for __extent_writepage. This calls the writepage start hooks, * and does the loop to map the page into extents and bios. @@ -3696,7 +3829,6 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, int *nr_ret) { struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct extent_io_tree *tree = &inode->io_tree; u64 start = page_offset(page); u64 end = start + PAGE_SIZE - 1; u64 cur = start; @@ -3727,15 +3859,26 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, while (cur <= end) { u64 disk_bytenr; u64 em_end; + u64 dirty_range_start = cur; + u64 dirty_range_end; u32 iosize; if (cur >= i_size) { - btrfs_writepage_endio_finish_ordered(page, cur, end, 1); + btrfs_writepage_endio_finish_ordered(inode, page, cur, + end, 1); break; } + + find_next_dirty_byte(fs_info, page, &dirty_range_start, + &dirty_range_end); + if (cur < dirty_range_start) { + cur = dirty_range_start; + continue; + } + em = btrfs_get_extent(inode, NULL, 0, cur, end - cur + 1); if (IS_ERR_OR_NULL(em)) { - SetPageError(page); + btrfs_page_set_error(fs_info, page, cur, end - cur + 1); ret = PTR_ERR_OR_ZERO(em); break; } @@ -3750,8 +3893,11 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags); disk_bytenr = em->block_start + extent_offset; - /* Note that em_end from extent_map_end() is exclusive */ - iosize = min(em_end, end + 1) - cur; + /* + * Note that em_end from extent_map_end() and dirty_range_end from + * find_next_dirty_byte() are all exclusive + */ + iosize = min(min(em_end, end + 1), dirty_range_end) - cur; if (btrfs_use_zone_append(inode, em->block_start)) opf = REQ_OP_ZONE_APPEND; @@ -3768,28 +3914,38 @@ static noinline_for_stack int __extent_writepage_io(struct btrfs_inode *inode, if (compressed) nr++; else - btrfs_writepage_endio_finish_ordered(page, cur, - cur + iosize - 1, 1); + btrfs_writepage_endio_finish_ordered(inode, + page, cur, cur + iosize - 1, 1); cur += iosize; continue; } - btrfs_set_range_writeback(tree, cur, cur + iosize - 1); + btrfs_set_range_writeback(inode, cur, cur + iosize - 1); if (!PageWriteback(page)) { btrfs_err(inode->root->fs_info, "page %lu not writeback, cur %llu end %llu", page->index, cur, end); } - ret = submit_extent_page(opf | write_flags, wbc, page, + /* + * Although the PageDirty bit is cleared before entering this + * function, subpage dirty bit is not cleared. + * So clear subpage dirty bit here so next time we won't submit + * page for range already written to disk. + */ + btrfs_page_clear_dirty(fs_info, page, cur, iosize); + + ret = submit_extent_page(opf | write_flags, wbc, + &epd->bio_ctrl, page, disk_bytenr, iosize, - cur - page_offset(page), &epd->bio, + cur - page_offset(page), end_bio_extent_writepage, - 0, 0, 0, false); + 0, 0, false); if (ret) { - SetPageError(page); + btrfs_page_set_error(fs_info, page, cur, iosize); if (PageWriteback(page)) - end_page_writeback(page); + btrfs_page_clear_writeback(fs_info, page, cur, + iosize); } cur += iosize; @@ -4098,12 +4254,15 @@ static struct extent_buffer *find_extent_buffer_nolock( * Unlike end_bio_extent_buffer_writepage(), we only call end_page_writeback() * after all extent buffers in the page has finished their writeback. */ -static void end_bio_subpage_eb_writepage(struct btrfs_fs_info *fs_info, - struct bio *bio) +static void end_bio_subpage_eb_writepage(struct bio *bio) { + struct btrfs_fs_info *fs_info; struct bio_vec *bvec; struct bvec_iter_all iter_all; + fs_info = btrfs_sb(bio_first_page_all(bio)->mapping->host->i_sb); + ASSERT(fs_info->sectorsize < PAGE_SIZE); + ASSERT(!bio_flagged(bio, BIO_CLONED)); bio_for_each_segment_all(bvec, bio, iter_all) { struct page *page = bvec->bv_page; @@ -4154,16 +4313,11 @@ static void end_bio_subpage_eb_writepage(struct btrfs_fs_info *fs_info, static void end_bio_extent_buffer_writepage(struct bio *bio) { - struct btrfs_fs_info *fs_info; struct bio_vec *bvec; struct extent_buffer *eb; int done; struct bvec_iter_all iter_all; - fs_info = btrfs_sb(bio_first_page_all(bio)->mapping->host->i_sb); - if (fs_info->sectorsize < PAGE_SIZE) - return end_bio_subpage_eb_writepage(fs_info, bio); - ASSERT(!bio_flagged(bio, BIO_CLONED)); bio_for_each_segment_all(bvec, bio, iter_all) { struct page *page = bvec->bv_page; @@ -4189,12 +4343,34 @@ static void end_bio_extent_buffer_writepage(struct bio *bio) bio_put(bio); } +static void prepare_eb_write(struct extent_buffer *eb) +{ + u32 nritems; + unsigned long start; + unsigned long end; + + clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags); + atomic_set(&eb->io_pages, num_extent_pages(eb)); + + /* Set btree blocks beyond nritems with 0 to avoid stale content */ + nritems = btrfs_header_nritems(eb); + if (btrfs_header_level(eb) > 0) { + end = btrfs_node_key_ptr_offset(nritems); + memzero_extent_buffer(eb, end, eb->len - end); + } else { + /* + * Leaf: + * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0 + */ + start = btrfs_item_nr_offset(nritems); + end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(eb); + memzero_extent_buffer(eb, start, end - start); + } +} + /* * Unlike the work in write_one_eb(), we rely completely on extent locking. * Page locking is only utilized at minimum to keep the VMM code happy. - * - * Caller should still call write_one_eb() other than this function directly. - * As write_one_eb() has extra preparation before submitting the extent buffer. */ static int write_one_subpage_eb(struct extent_buffer *eb, struct writeback_control *wbc, @@ -4206,6 +4382,8 @@ static int write_one_subpage_eb(struct extent_buffer *eb, bool no_dirty_ebs = false; int ret; + prepare_eb_write(eb); + /* clear_page_dirty_for_io() in subpage helper needs page locked */ lock_page(page); btrfs_subpage_set_writeback(fs_info, page, eb->start, eb->len); @@ -4216,10 +4394,10 @@ static int write_one_subpage_eb(struct extent_buffer *eb, if (no_dirty_ebs) clear_page_dirty_for_io(page); - ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, page, - eb->start, eb->len, eb->start - page_offset(page), - &epd->bio, end_bio_extent_buffer_writepage, 0, 0, 0, - false); + ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, + &epd->bio_ctrl, page, eb->start, eb->len, + eb->start - page_offset(page), + end_bio_subpage_eb_writepage, 0, 0, false); if (ret) { btrfs_subpage_clear_writeback(fs_info, page, eb->start, eb->len); set_btree_ioerr(page, eb); @@ -4244,45 +4422,23 @@ static noinline_for_stack int write_one_eb(struct extent_buffer *eb, struct extent_page_data *epd) { u64 disk_bytenr = eb->start; - u32 nritems; int i, num_pages; - unsigned long start, end; unsigned int write_flags = wbc_to_write_flags(wbc) | REQ_META; int ret = 0; - clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags); + prepare_eb_write(eb); + num_pages = num_extent_pages(eb); - atomic_set(&eb->io_pages, num_pages); - - /* set btree blocks beyond nritems with 0 to avoid stale content. */ - nritems = btrfs_header_nritems(eb); - if (btrfs_header_level(eb) > 0) { - end = btrfs_node_key_ptr_offset(nritems); - - memzero_extent_buffer(eb, end, eb->len - end); - } else { - /* - * leaf: - * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0 - */ - start = btrfs_item_nr_offset(nritems); - end = BTRFS_LEAF_DATA_OFFSET + leaf_data_end(eb); - memzero_extent_buffer(eb, start, end - start); - } - - if (eb->fs_info->sectorsize < PAGE_SIZE) - return write_one_subpage_eb(eb, wbc, epd); - for (i = 0; i < num_pages; i++) { struct page *p = eb->pages[i]; clear_page_dirty_for_io(p); set_page_writeback(p); ret = submit_extent_page(REQ_OP_WRITE | write_flags, wbc, - p, disk_bytenr, PAGE_SIZE, 0, - &epd->bio, + &epd->bio_ctrl, p, disk_bytenr, + PAGE_SIZE, 0, end_bio_extent_buffer_writepage, - 0, 0, 0, false); + 0, 0, false); if (ret) { set_btree_ioerr(p, eb); if (PageWriteback(p)) @@ -4386,7 +4542,7 @@ static int submit_eb_subpage(struct page *page, free_extent_buffer(eb); goto cleanup; } - ret = write_one_eb(eb, wbc, epd); + ret = write_one_subpage_eb(eb, wbc, epd); free_extent_buffer(eb); if (ret < 0) goto cleanup; @@ -4498,7 +4654,7 @@ int btree_write_cache_pages(struct address_space *mapping, { struct extent_buffer *eb_context = NULL; struct extent_page_data epd = { - .bio = NULL, + .bio_ctrl = { 0 }, .extent_locked = 0, .sync_io = wbc->sync_mode == WB_SYNC_ALL, }; @@ -4780,7 +4936,7 @@ int extent_write_full_page(struct page *page, struct writeback_control *wbc) { int ret; struct extent_page_data epd = { - .bio = NULL, + .bio_ctrl = { 0 }, .extent_locked = 0, .sync_io = wbc->sync_mode == WB_SYNC_ALL, }; @@ -4807,7 +4963,7 @@ int extent_write_locked_range(struct inode *inode, u64 start, u64 end, PAGE_SHIFT; struct extent_page_data epd = { - .bio = NULL, + .bio_ctrl = { 0 }, .extent_locked = 1, .sync_io = mode == WB_SYNC_ALL, }; @@ -4827,8 +4983,8 @@ int extent_write_locked_range(struct inode *inode, u64 start, u64 end, if (clear_page_dirty_for_io(page)) ret = __extent_writepage(page, &wbc_writepages, &epd); else { - btrfs_writepage_endio_finish_ordered(page, start, - start + PAGE_SIZE - 1, 1); + btrfs_writepage_endio_finish_ordered(BTRFS_I(inode), + page, start, start + PAGE_SIZE - 1, 1); unlock_page(page); } put_page(page); @@ -4850,7 +5006,7 @@ int extent_writepages(struct address_space *mapping, { int ret = 0; struct extent_page_data epd = { - .bio = NULL, + .bio_ctrl = { 0 }, .extent_locked = 0, .sync_io = wbc->sync_mode == WB_SYNC_ALL, }; @@ -4867,8 +5023,7 @@ int extent_writepages(struct address_space *mapping, void extent_readahead(struct readahead_control *rac) { - struct bio *bio = NULL; - unsigned long bio_flags = 0; + struct btrfs_bio_ctrl bio_ctrl = { 0 }; struct page *pagepool[16]; struct extent_map *em_cached = NULL; u64 prev_em_start = (u64)-1; @@ -4879,14 +5034,14 @@ void extent_readahead(struct readahead_control *rac) u64 contig_end = contig_start + readahead_batch_length(rac) - 1; contiguous_readpages(pagepool, nr, contig_start, contig_end, - &em_cached, &bio, &bio_flags, &prev_em_start); + &em_cached, &bio_ctrl, &prev_em_start); } if (em_cached) free_extent_map(em_cached); - if (bio) { - if (submit_one_bio(bio, 0, bio_flags)) + if (bio_ctrl.bio) { + if (submit_one_bio(bio_ctrl.bio, 0, bio_ctrl.bio_flags)) return; } } @@ -5429,6 +5584,12 @@ static bool page_range_has_eb(struct btrfs_fs_info *fs_info, struct page *page) subpage = (struct btrfs_subpage *)page->private; if (atomic_read(&subpage->eb_refs)) return true; + /* + * Even there is no eb refs here, we may still have + * end_page_read() call relying on page::private. + */ + if (atomic_read(&subpage->readers)) + return true; } return false; } @@ -5489,7 +5650,7 @@ static void detach_extent_buffer_page(struct extent_buffer *eb, struct page *pag /* * We can only detach the page private if there are no other ebs in the - * page range. + * page range and no unfinished IO. */ if (!page_range_has_eb(fs_info, page)) btrfs_detach_subpage(fs_info, page); @@ -6176,7 +6337,7 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, struct btrfs_fs_info *fs_info = eb->fs_info; struct extent_io_tree *io_tree; struct page *page = eb->pages[0]; - struct bio *bio = NULL; + struct btrfs_bio_ctrl bio_ctrl = { 0 }; int ret = 0; ASSERT(!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags)); @@ -6184,10 +6345,8 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, io_tree = &BTRFS_I(fs_info->btree_inode)->io_tree; if (wait == WAIT_NONE) { - ret = try_lock_extent(io_tree, eb->start, - eb->start + eb->len - 1); - if (ret <= 0) - return ret; + if (!try_lock_extent(io_tree, eb->start, eb->start + eb->len - 1)) + return -EAGAIN; } else { ret = lock_extent(io_tree, eb->start, eb->start + eb->len - 1); if (ret < 0) @@ -6209,9 +6368,11 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, check_buffer_tree_ref(eb); btrfs_subpage_clear_error(fs_info, page, eb->start, eb->len); - ret = submit_extent_page(REQ_OP_READ | REQ_META, NULL, page, eb->start, - eb->len, eb->start - page_offset(page), &bio, - end_bio_extent_readpage, mirror_num, 0, 0, + btrfs_subpage_start_reader(fs_info, page, eb->start, eb->len); + ret = submit_extent_page(REQ_OP_READ | REQ_META, NULL, &bio_ctrl, + page, eb->start, eb->len, + eb->start - page_offset(page), + end_bio_extent_readpage, mirror_num, 0, true); if (ret) { /* @@ -6221,10 +6382,11 @@ static int read_extent_buffer_subpage(struct extent_buffer *eb, int wait, */ atomic_dec(&eb->io_pages); } - if (bio) { + if (bio_ctrl.bio) { int tmp; - tmp = submit_one_bio(bio, mirror_num, 0); + tmp = submit_one_bio(bio_ctrl.bio, mirror_num, 0); + bio_ctrl.bio = NULL; if (tmp < 0) return tmp; } @@ -6247,8 +6409,7 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) int all_uptodate = 1; int num_pages; unsigned long num_reads = 0; - struct bio *bio = NULL; - unsigned long bio_flags = 0; + struct btrfs_bio_ctrl bio_ctrl = { 0 }; if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags)) return 0; @@ -6312,9 +6473,9 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) ClearPageError(page); err = submit_extent_page(REQ_OP_READ | REQ_META, NULL, - page, page_offset(page), PAGE_SIZE, 0, - &bio, end_bio_extent_readpage, - mirror_num, 0, 0, false); + &bio_ctrl, page, page_offset(page), + PAGE_SIZE, 0, end_bio_extent_readpage, + mirror_num, 0, false); if (err) { /* * We failed to submit the bio so it's the @@ -6331,8 +6492,9 @@ int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num) } } - if (bio) { - err = submit_one_bio(bio, mirror_num, bio_flags); + if (bio_ctrl.bio) { + err = submit_one_bio(bio_ctrl.bio, mirror_num, bio_ctrl.bio_flags); + bio_ctrl.bio = NULL; if (err) return err; } @@ -6515,9 +6677,10 @@ void write_extent_buffer_chunk_tree_uuid(const struct extent_buffer *eb, char *kaddr; assert_eb_page_uptodate(eb, eb->pages[0]); - kaddr = page_address(eb->pages[0]) + get_eb_offset_in_page(eb, 0); - memcpy(kaddr + offsetof(struct btrfs_header, chunk_tree_uuid), srcv, - BTRFS_FSID_SIZE); + kaddr = page_address(eb->pages[0]) + + get_eb_offset_in_page(eb, offsetof(struct btrfs_header, + chunk_tree_uuid)); + memcpy(kaddr, srcv, BTRFS_FSID_SIZE); } void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv) @@ -6525,9 +6688,9 @@ void write_extent_buffer_fsid(const struct extent_buffer *eb, const void *srcv) char *kaddr; assert_eb_page_uptodate(eb, eb->pages[0]); - kaddr = page_address(eb->pages[0]) + get_eb_offset_in_page(eb, 0); - memcpy(kaddr + offsetof(struct btrfs_header, fsid), srcv, - BTRFS_FSID_SIZE); + kaddr = page_address(eb->pages[0]) + + get_eb_offset_in_page(eb, offsetof(struct btrfs_header, fsid)); + memcpy(kaddr, srcv, BTRFS_FSID_SIZE); } void write_extent_buffer(const struct extent_buffer *eb, const void *srcv, diff --git a/fs/btrfs/extent_io.h b/fs/btrfs/extent_io.h index 227215a5722c..62027f551b44 100644 --- a/fs/btrfs/extent_io.h +++ b/fs/btrfs/extent_io.h @@ -39,7 +39,7 @@ enum { /* Page starts writeback, clear dirty bit and set writeback bit */ #define PAGE_START_WRITEBACK (1 << 1) #define PAGE_END_WRITEBACK (1 << 2) -#define PAGE_SET_PRIVATE2 (1 << 3) +#define PAGE_SET_ORDERED (1 << 3) #define PAGE_SET_ERROR (1 << 4) #define PAGE_LOCK (1 << 5) @@ -101,6 +101,17 @@ struct extent_buffer { #endif }; +/* + * Structure to record info about the bio being assembled, and other info like + * how many bytes are there before stripe/ordered extent boundary. + */ +struct btrfs_bio_ctrl { + struct bio *bio; + unsigned long bio_flags; + u32 len_to_stripe_boundary; + u32 len_to_oe_boundary; +}; + /* * Structure to record how many bytes and which ranges are set/cleared */ @@ -169,7 +180,7 @@ int try_release_extent_buffer(struct page *page); int __must_check submit_one_bio(struct bio *bio, int mirror_num, unsigned long bio_flags); int btrfs_do_readpage(struct page *page, struct extent_map **em_cached, - struct bio **bio, unsigned long *bio_flags, + struct btrfs_bio_ctrl *bio_ctrl, unsigned int read_flags, u64 *prev_em_start); int extent_write_full_page(struct page *page, struct writeback_control *wbc); int extent_write_locked_range(struct inode *inode, u64 start, u64 end, @@ -281,7 +292,7 @@ int btrfs_repair_eb_io_failure(const struct extent_buffer *eb, int mirror_num); * When IO fails, either with EIO or csum verification fails, we * try other mirrors that might have a good copy of the data. This * io_failure_record is used to record state as we go through all the - * mirrors. If another mirror has good data, the page is set up to date + * mirrors. If another mirror has good data, the sector is set up to date * and things continue. If a good mirror can't be found, the original * bio end_io callback is called to indicate things have failed. */ @@ -293,15 +304,13 @@ struct io_failure_record { unsigned long bio_flags; int this_mirror; int failed_mirror; - int in_validation; }; - -blk_status_t btrfs_submit_read_repair(struct inode *inode, - struct bio *failed_bio, u32 bio_offset, - struct page *page, unsigned int pgoff, - u64 start, u64 end, int failed_mirror, - submit_bio_hook_t *submit_bio_hook); +int btrfs_repair_one_sector(struct inode *inode, + struct bio *failed_bio, u32 bio_offset, + struct page *page, unsigned int pgoff, + u64 start, int failed_mirror, + submit_bio_hook_t *submit_bio_hook); #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS bool find_lock_delalloc_range(struct inode *inode, diff --git a/fs/btrfs/file-item.c b/fs/btrfs/file-item.c index 441cee7fbb62..df6631eefc65 100644 --- a/fs/btrfs/file-item.c +++ b/fs/btrfs/file-item.c @@ -618,7 +618,7 @@ int btrfs_lookup_csums_range(struct btrfs_root *root, u64 start, u64 end, * @file_start: offset in file this bio begins to describe * @contig: Boolean. If true/1 means all bio vecs in this bio are * contiguous and they begin at @file_start in the file. False/0 - * means this bio can contains potentially discontigous bio vecs + * means this bio can contain potentially discontiguous bio vecs * so the logical offset of each should be calculated separately. */ blk_status_t btrfs_csum_one_bio(struct btrfs_inode *inode, struct bio *bio, diff --git a/fs/btrfs/file.c b/fs/btrfs/file.c index 55f68422061d..28a05ba47060 100644 --- a/fs/btrfs/file.c +++ b/fs/btrfs/file.c @@ -28,6 +28,7 @@ #include "compression.h" #include "delalloc-space.h" #include "reflink.h" +#include "subpage.h" static struct kmem_cache *btrfs_inode_defrag_cachep; /* @@ -482,6 +483,7 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, start_pos = round_down(pos, fs_info->sectorsize); num_bytes = round_up(write_bytes + pos - start_pos, fs_info->sectorsize); + ASSERT(num_bytes <= U32_MAX); end_of_last_block = start_pos + num_bytes - 1; @@ -500,9 +502,10 @@ int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages, for (i = 0; i < num_pages; i++) { struct page *p = pages[i]; - SetPageUptodate(p); + + btrfs_page_clamp_set_uptodate(fs_info, p, start_pos, num_bytes); ClearPageChecked(p); - set_page_dirty(p); + btrfs_page_clamp_set_dirty(fs_info, p, start_pos, num_bytes); } /* @@ -2483,6 +2486,17 @@ static int btrfs_punch_hole_lock_range(struct inode *inode, const u64 lockend, struct extent_state **cached_state) { + /* + * For subpage case, if the range is not at page boundary, we could + * have pages at the leading/tailing part of the range. + * This could lead to dead loop since filemap_range_has_page() + * will always return true. + * So here we need to do extra page alignment for + * filemap_range_has_page(). + */ + const u64 page_lockstart = round_up(lockstart, PAGE_SIZE); + const u64 page_lockend = round_down(lockend + 1, PAGE_SIZE) - 1; + while (1) { struct btrfs_ordered_extent *ordered; int ret; @@ -2503,7 +2517,7 @@ static int btrfs_punch_hole_lock_range(struct inode *inode, (ordered->file_offset + ordered->num_bytes <= lockstart || ordered->file_offset > lockend)) && !filemap_range_has_page(inode->i_mapping, - lockstart, lockend)) { + page_lockstart, page_lockend)) { if (ordered) btrfs_put_ordered_extent(ordered); break; @@ -3034,22 +3048,20 @@ struct falloc_range { */ static int add_falloc_range(struct list_head *head, u64 start, u64 len) { - struct falloc_range *prev = NULL; struct falloc_range *range = NULL; - if (list_empty(head)) - goto insert; - - /* - * As fallocate iterate by bytenr order, we only need to check - * the last range. - */ - prev = list_entry(head->prev, struct falloc_range, list); - if (prev->start + prev->len == start) { - prev->len += len; - return 0; + if (!list_empty(head)) { + /* + * As fallocate iterates by bytenr order, we only need to check + * the last range. + */ + range = list_last_entry(head, struct falloc_range, list); + if (range->start + range->len == start) { + range->len += len; + return 0; + } } -insert: + range = kmalloc(sizeof(*range), GFP_KERNEL); if (!range) return -ENOMEM; diff --git a/fs/btrfs/free-space-cache.c b/fs/btrfs/free-space-cache.c index 4806295116d8..2131ae5b9ed7 100644 --- a/fs/btrfs/free-space-cache.c +++ b/fs/btrfs/free-space-cache.c @@ -327,7 +327,7 @@ int btrfs_truncate_free_space_cache(struct btrfs_trans_handle *trans, * need to check for -EAGAIN. */ ret = btrfs_truncate_inode_items(trans, root, BTRFS_I(inode), - 0, BTRFS_EXTENT_DATA_KEY); + 0, BTRFS_EXTENT_DATA_KEY, NULL); if (ret) goto fail; diff --git a/fs/btrfs/inode.c b/fs/btrfs/inode.c index 46f392943f4d..e6eb20987351 100644 --- a/fs/btrfs/inode.c +++ b/fs/btrfs/inode.c @@ -51,6 +51,7 @@ #include "block-group.h" #include "space-info.h" #include "zoned.h" +#include "subpage.h" struct btrfs_iget_args { u64 ino; @@ -166,22 +167,47 @@ static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode, struct page *page; while (index <= end_index) { + /* + * For locked page, we will call end_extent_writepage() on it + * in run_delalloc_range() for the error handling. That + * end_extent_writepage() function will call + * btrfs_mark_ordered_io_finished() to clear page Ordered and + * run the ordered extent accounting. + * + * Here we can't just clear the Ordered bit, or + * btrfs_mark_ordered_io_finished() would skip the accounting + * for the page range, and the ordered extent will never finish. + */ + if (index == (page_offset(locked_page) >> PAGE_SHIFT)) { + index++; + continue; + } page = find_get_page(inode->vfs_inode.i_mapping, index); index++; if (!page) continue; - ClearPagePrivate2(page); + + /* + * Here we just clear all Ordered bits for every page in the + * range, then __endio_write_update_ordered() will handle + * the ordered extent accounting for the range. + */ + btrfs_page_clamp_clear_ordered(inode->root->fs_info, page, + offset, bytes); put_page(page); } + /* The locked page covers the full range, nothing needs to be done */ + if (bytes + offset <= page_offset(locked_page) + PAGE_SIZE) + return; /* * In case this page belongs to the delalloc range being instantiated * then skip it, since the first page of a range is going to be * properly cleaned up by the caller of run_delalloc_range */ if (page_start >= offset && page_end <= (offset + bytes - 1)) { - offset += PAGE_SIZE; - bytes -= PAGE_SIZE; + bytes = offset + bytes - page_offset(locked_page) - PAGE_SIZE; + offset = page_offset(locked_page) + PAGE_SIZE; } return __endio_write_update_ordered(inode, offset, bytes, false); @@ -603,7 +629,7 @@ static noinline int compress_file_range(struct async_chunk *async_chunk) * inode has not been flagged as nocompress. This flag can * change at any time if we discover bad compression ratios. */ - if (inode_need_compress(BTRFS_I(inode), start, end)) { + if (nr_pages > 1 && inode_need_compress(BTRFS_I(inode), start, end)) { WARN_ON(pages); pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS); if (!pages) { @@ -946,7 +972,8 @@ static noinline void submit_compressed_extents(struct async_chunk *async_chunk) const u64 end = start + async_extent->ram_size - 1; p->mapping = inode->vfs_inode.i_mapping; - btrfs_writepage_endio_finish_ordered(p, start, end, 0); + btrfs_writepage_endio_finish_ordered(inode, p, start, + end, 0); p->mapping = NULL; extent_clear_unlock_delalloc(inode, start, end, NULL, 0, @@ -1064,7 +1091,8 @@ static noinline int cow_file_range(struct btrfs_inode *inode, * our outstanding extent for clearing delalloc for this * range. */ - extent_clear_unlock_delalloc(inode, start, end, NULL, + extent_clear_unlock_delalloc(inode, start, end, + locked_page, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW | EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING, PAGE_UNLOCK | @@ -1072,6 +1100,19 @@ static noinline int cow_file_range(struct btrfs_inode *inode, *nr_written = *nr_written + (end - start + PAGE_SIZE) / PAGE_SIZE; *page_started = 1; + /* + * locked_page is locked by the caller of + * writepage_delalloc(), not locked by + * __process_pages_contig(). + * + * We can't let __process_pages_contig() to unlock it, + * as it doesn't have any subpage::writers recorded. + * + * Here we manually unlock the page, since the caller + * can't use page_started to determine if it's an + * inline extent or a compressed extent. + */ + unlock_page(locked_page); goto out; } else if (ret < 0) { goto out_unlock; @@ -1150,15 +1191,16 @@ static noinline int cow_file_range(struct btrfs_inode *inode, btrfs_dec_block_group_reservations(fs_info, ins.objectid); - /* we're not doing compressed IO, don't unlock the first - * page (which the caller expects to stay locked), don't - * clear any dirty bits and don't set any writeback bits + /* + * We're not doing compressed IO, don't unlock the first page + * (which the caller expects to stay locked), don't clear any + * dirty bits and don't set any writeback bits * - * Do set the Private2 bit so we know this page was properly - * setup for writepage + * Do set the Ordered (Private2) bit so we know this page was + * properly setup for writepage. */ page_ops = unlock ? PAGE_UNLOCK : 0; - page_ops |= PAGE_SET_PRIVATE2; + page_ops |= PAGE_SET_ORDERED; extent_clear_unlock_delalloc(inode, start, start + ram_size - 1, locked_page, @@ -1822,7 +1864,7 @@ static noinline int run_delalloc_nocow(struct btrfs_inode *inode, locked_page, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_CLEAR_DATA_RESV, - PAGE_UNLOCK | PAGE_SET_PRIVATE2); + PAGE_UNLOCK | PAGE_SET_ORDERED); cur_offset = extent_end; @@ -2193,26 +2235,22 @@ int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio, struct inode *inode = page->mapping->host; struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); u64 logical = bio->bi_iter.bi_sector << 9; + u32 bio_len = bio->bi_iter.bi_size; struct extent_map *em; - u64 length = 0; - u64 map_length; int ret = 0; struct btrfs_io_geometry geom; if (bio_flags & EXTENT_BIO_COMPRESSED) return 0; - length = bio->bi_iter.bi_size; - map_length = length; - em = btrfs_get_chunk_map(fs_info, logical, map_length); + em = btrfs_get_chunk_map(fs_info, logical, fs_info->sectorsize); if (IS_ERR(em)) return PTR_ERR(em); - ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio), logical, - map_length, &geom); + ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(bio), logical, &geom); if (ret < 0) goto out; - if (geom.len < length + size) + if (geom.len < bio_len + size) ret = 1; out: free_extent_map(em); @@ -2233,33 +2271,6 @@ static blk_status_t btrfs_submit_bio_start(struct inode *inode, struct bio *bio, return btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0); } -bool btrfs_bio_fits_in_ordered_extent(struct page *page, struct bio *bio, - unsigned int size) -{ - struct btrfs_inode *inode = BTRFS_I(page->mapping->host); - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct btrfs_ordered_extent *ordered; - u64 len = bio->bi_iter.bi_size + size; - bool ret = true; - - ASSERT(btrfs_is_zoned(fs_info)); - ASSERT(fs_info->max_zone_append_size > 0); - ASSERT(bio_op(bio) == REQ_OP_ZONE_APPEND); - - /* Ordered extent not yet created, so we're good */ - ordered = btrfs_lookup_ordered_extent(inode, page_offset(page)); - if (!ordered) - return ret; - - if ((bio->bi_iter.bi_sector << SECTOR_SHIFT) + len > - ordered->disk_bytenr + ordered->disk_num_bytes) - ret = false; - - btrfs_put_ordered_extent(ordered); - - return ret; -} - static blk_status_t extract_ordered_extent(struct btrfs_inode *inode, struct bio *bio, loff_t file_offset) { @@ -2601,7 +2612,7 @@ static void btrfs_writepage_fixup_worker(struct btrfs_work *work) lock_extent_bits(&inode->io_tree, page_start, page_end, &cached_state); /* already ordered? We're done */ - if (PagePrivate2(page)) + if (PageOrdered(page)) goto out_reserved; ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE); @@ -2676,8 +2687,8 @@ int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end) struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_writepage_fixup *fixup; - /* this page is properly in the ordered list */ - if (TestClearPagePrivate2(page)) + /* This page has ordered extent covering it already */ + if (PageOrdered(page)) return 0; /* @@ -2773,7 +2784,7 @@ static int insert_reserved_file_extent(struct btrfs_trans_handle *trans, /* * If we dropped an inline extent here, we know the range where it is * was not marked with the EXTENT_DELALLOC_NEW bit, so we update the - * number of bytes only for that range contaning the inline extent. + * number of bytes only for that range containing the inline extent. * The remaining of the range will be processed when clearning the * EXTENT_DELALLOC_BIT bit through the ordered extent completion. */ @@ -3069,28 +3080,14 @@ static void finish_ordered_fn(struct btrfs_work *work) btrfs_finish_ordered_io(ordered_extent); } -void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start, +void btrfs_writepage_endio_finish_ordered(struct btrfs_inode *inode, + struct page *page, u64 start, u64 end, int uptodate) { - struct btrfs_inode *inode = BTRFS_I(page->mapping->host); - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct btrfs_ordered_extent *ordered_extent = NULL; - struct btrfs_workqueue *wq; + trace_btrfs_writepage_end_io_hook(inode, start, end, uptodate); - trace_btrfs_writepage_end_io_hook(page, start, end, uptodate); - - ClearPagePrivate2(page); - if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start, - end - start + 1, uptodate)) - return; - - if (btrfs_is_free_space_inode(inode)) - wq = fs_info->endio_freespace_worker; - else - wq = fs_info->endio_write_workers; - - btrfs_init_work(&ordered_extent->work, finish_ordered_fn, NULL, NULL); - btrfs_queue_work(wq, &ordered_extent->work); + btrfs_mark_ordered_io_finished(inode, page, start, end + 1 - start, + finish_ordered_fn, uptodate); } /* @@ -3152,15 +3149,19 @@ static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio, * @bio_offset: offset to the beginning of the bio (in bytes) * @start: file offset of the range start * @end: file offset of the range end (inclusive) + * + * Return a bitmap where bit set means a csum mismatch, and bit not set means + * csum match. */ -int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset, - struct page *page, u64 start, u64 end) +unsigned int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset, + struct page *page, u64 start, u64 end) { struct inode *inode = page->mapping->host; struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; struct btrfs_root *root = BTRFS_I(inode)->root; const u32 sectorsize = root->fs_info->sectorsize; u32 pg_off; + unsigned int result = 0; if (PageChecked(page)) { ClearPageChecked(page); @@ -3188,10 +3189,14 @@ int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u32 bio_offset, ret = check_data_csum(inode, io_bio, bio_offset, page, pg_off, page_offset(page) + pg_off); - if (ret < 0) - return -EIO; + if (ret < 0) { + const int nr_bit = (pg_off - offset_in_page(start)) >> + root->fs_info->sectorsize_bits; + + result |= (1U << nr_bit); + } } - return 0; + return result; } /* @@ -4109,7 +4114,7 @@ static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans, * This is a placeholder inode for a subvolume we didn't have a * reference to at the time of the snapshot creation. In the meantime * we could have renamed the real subvol link into our snapshot, so - * depending on btrfs_del_root_ref to return -ENOENT here is incorret. + * depending on btrfs_del_root_ref to return -ENOENT here is incorrect. * Instead simply lookup the dir_index_item for this entry so we can * remove it. Otherwise we know we have a ref to the root and we can * call btrfs_del_root_ref, and it _shouldn't_ fail. @@ -4464,20 +4469,36 @@ static int btrfs_rmdir(struct inode *dir, struct dentry *dentry) #define NEED_TRUNCATE_BLOCK 1 /* - * this can truncate away extent items, csum items and directory items. - * It starts at a high offset and removes keys until it can't find - * any higher than new_size + * Remove inode items from a given root. * - * csum items that cross the new i_size are truncated to the new size - * as well. + * @trans: A transaction handle. + * @root: The root from which to remove items. + * @inode: The inode whose items we want to remove. + * @new_size: The new i_size for the inode. This is only applicable when + * @min_type is BTRFS_EXTENT_DATA_KEY, must be 0 otherwise. + * @min_type: The minimum key type to remove. All keys with a type + * greater than this value are removed and all keys with + * this type are removed only if their offset is >= @new_size. + * @extents_found: Output parameter that will contain the number of file + * extent items that were removed or adjusted to the new + * inode i_size. The caller is responsible for initializing + * the counter. Also, it can be NULL if the caller does not + * need this counter. * - * min_type is the minimum key type to truncate down to. If set to 0, this - * will kill all the items on this inode, including the INODE_ITEM_KEY. + * Remove all keys associated with the inode from the given root that have a key + * with a type greater than or equals to @min_type. When @min_type has a value of + * BTRFS_EXTENT_DATA_KEY, only remove file extent items that have an offset value + * greater than or equals to @new_size. If a file extent item that starts before + * @new_size and ends after it is found, its length is adjusted. + * + * Returns: 0 on success, < 0 on error and NEED_TRUNCATE_BLOCK when @min_type is + * BTRFS_EXTENT_DATA_KEY and the caller must truncate the last block. */ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_inode *inode, - u64 new_size, u32 min_type) + u64 new_size, u32 min_type, + u64 *extents_found) { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_path *path; @@ -4623,6 +4644,9 @@ int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans, if (found_type != BTRFS_EXTENT_DATA_KEY) goto delete; + if (extents_found != NULL) + (*extents_found)++; + if (extent_type != BTRFS_FILE_EXTENT_INLINE) { u64 num_dec; @@ -4941,7 +4965,7 @@ int btrfs_truncate_block(struct btrfs_inode *inode, loff_t from, loff_t len, flush_dcache_page(page); } ClearPageChecked(page); - set_page_dirty(page); + btrfs_page_set_dirty(fs_info, page, block_start, block_end + 1 - block_start); unlock_extent_cached(io_tree, block_start, block_end, &cached_state); if (only_release_metadata) @@ -5455,7 +5479,7 @@ void btrfs_evict_inode(struct inode *inode) trans->block_rsv = rsv; ret = btrfs_truncate_inode_items(trans, root, BTRFS_I(inode), - 0, 0); + 0, 0, NULL); trans->block_rsv = &fs_info->trans_block_rsv; btrfs_end_transaction(trans); btrfs_btree_balance_dirty(fs_info); @@ -7937,19 +7961,17 @@ static blk_status_t btrfs_check_read_dio_bio(struct inode *inode, btrfs_ino(BTRFS_I(inode)), pgoff); } else { - blk_status_t status; + int ret; ASSERT((start - io_bio->logical) < UINT_MAX); - status = btrfs_submit_read_repair(inode, - &io_bio->bio, - start - io_bio->logical, - bvec.bv_page, pgoff, - start, - start + sectorsize - 1, - io_bio->mirror_num, - submit_dio_repair_bio); - if (status) - err = status; + ret = btrfs_repair_one_sector(inode, + &io_bio->bio, + start - io_bio->logical, + bvec.bv_page, pgoff, + start, io_bio->mirror_num, + submit_dio_repair_bio); + if (ret) + err = errno_to_blk_status(ret); } start += sectorsize; ASSERT(bio_offset + sectorsize > bio_offset); @@ -7964,41 +7986,8 @@ static void __endio_write_update_ordered(struct btrfs_inode *inode, const u64 offset, const u64 bytes, const bool uptodate) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; - struct btrfs_ordered_extent *ordered = NULL; - struct btrfs_workqueue *wq; - u64 ordered_offset = offset; - u64 ordered_bytes = bytes; - u64 last_offset; - - if (btrfs_is_free_space_inode(inode)) - wq = fs_info->endio_freespace_worker; - else - wq = fs_info->endio_write_workers; - - while (ordered_offset < offset + bytes) { - last_offset = ordered_offset; - if (btrfs_dec_test_first_ordered_pending(inode, &ordered, - &ordered_offset, - ordered_bytes, - uptodate)) { - btrfs_init_work(&ordered->work, finish_ordered_fn, NULL, - NULL); - btrfs_queue_work(wq, &ordered->work); - } - - /* No ordered extent found in the range, exit */ - if (ordered_offset == last_offset) - return; - /* - * Our bio might span multiple ordered extents. In this case - * we keep going until we have accounted the whole dio. - */ - if (ordered_offset < offset + bytes) { - ordered_bytes = offset + bytes - ordered_offset; - ordered = NULL; - } - } + btrfs_mark_ordered_io_finished(inode, NULL, offset, bytes, + finish_ordered_fn, uptodate); } static blk_status_t btrfs_submit_bio_start_direct_io(struct inode *inode, @@ -8172,7 +8161,7 @@ static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap, goto out_err_em; } ret = btrfs_get_io_geometry(fs_info, em, btrfs_op(dio_bio), - logical, submit_len, &geom); + logical, &geom); if (ret) { status = errno_to_blk_status(ret); goto out_err_em; @@ -8276,15 +8265,14 @@ int btrfs_readpage(struct file *file, struct page *page) struct btrfs_inode *inode = BTRFS_I(page->mapping->host); u64 start = page_offset(page); u64 end = start + PAGE_SIZE - 1; - unsigned long bio_flags = 0; - struct bio *bio = NULL; + struct btrfs_bio_ctrl bio_ctrl = { 0 }; int ret; btrfs_lock_and_flush_ordered_range(inode, start, end, NULL); - ret = btrfs_do_readpage(page, NULL, &bio, &bio_flags, 0, NULL); - if (bio) - ret = submit_one_bio(bio, 0, bio_flags); + ret = btrfs_do_readpage(page, NULL, &bio_ctrl, 0, NULL); + if (bio_ctrl.bio) + ret = submit_one_bio(bio_ctrl.bio, 0, bio_ctrl.bio_flags); return ret; } @@ -8353,9 +8341,9 @@ static int btrfs_migratepage(struct address_space *mapping, if (page_has_private(page)) attach_page_private(newpage, detach_page_private(page)); - if (PagePrivate2(page)) { - ClearPagePrivate2(page); - SetPagePrivate2(newpage); + if (PageOrdered(page)) { + ClearPageOrdered(page); + SetPageOrdered(newpage); } if (mode != MIGRATE_SYNC_NO_COPY) @@ -8370,27 +8358,42 @@ static void btrfs_invalidatepage(struct page *page, unsigned int offset, unsigned int length) { struct btrfs_inode *inode = BTRFS_I(page->mapping->host); + struct btrfs_fs_info *fs_info = inode->root->fs_info; struct extent_io_tree *tree = &inode->io_tree; - struct btrfs_ordered_extent *ordered; struct extent_state *cached_state = NULL; u64 page_start = page_offset(page); u64 page_end = page_start + PAGE_SIZE - 1; - u64 start; - u64 end; + u64 cur; int inode_evicting = inode->vfs_inode.i_state & I_FREEING; - bool found_ordered = false; - bool completed_ordered = false; /* - * we have the page locked, so new writeback can't start, - * and the dirty bit won't be cleared while we are here. + * We have page locked so no new ordered extent can be created on this + * page, nor bio can be submitted for this page. * - * Wait for IO on this page so that we can safely clear - * the PagePrivate2 bit and do ordered accounting + * But already submitted bio can still be finished on this page. + * Furthermore, endio function won't skip page which has Ordered + * (Private2) already cleared, so it's possible for endio and + * invalidatepage to do the same ordered extent accounting twice + * on one page. + * + * So here we wait for any submitted bios to finish, so that we won't + * do double ordered extent accounting on the same page. */ wait_on_page_writeback(page); - if (offset) { + /* + * For subpage case, we have call sites like + * btrfs_punch_hole_lock_range() which passes range not aligned to + * sectorsize. + * If the range doesn't cover the full page, we don't need to and + * shouldn't clear page extent mapped, as page->private can still + * record subpage dirty bits for other part of the range. + * + * For cases that can invalidate the full even the range doesn't + * cover the full page, like invalidating the last page, we're + * still safe to wait for ordered extent to finish. + */ + if (!(offset == 0 && length == PAGE_SIZE)) { btrfs_releasepage(page, GFP_NOFS); return; } @@ -8398,89 +8401,123 @@ static void btrfs_invalidatepage(struct page *page, unsigned int offset, if (!inode_evicting) lock_extent_bits(tree, page_start, page_end, &cached_state); - start = page_start; -again: - ordered = btrfs_lookup_ordered_range(inode, start, page_end - start + 1); - if (ordered) { - found_ordered = true; - end = min(page_end, - ordered->file_offset + ordered->num_bytes - 1); + cur = page_start; + while (cur < page_end) { + struct btrfs_ordered_extent *ordered; + bool delete_states; + u64 range_end; + u32 range_len; + + ordered = btrfs_lookup_first_ordered_range(inode, cur, + page_end + 1 - cur); + if (!ordered) { + range_end = page_end; + /* + * No ordered extent covering this range, we are safe + * to delete all extent states in the range. + */ + delete_states = true; + goto next; + } + if (ordered->file_offset > cur) { + /* + * There is a range between [cur, oe->file_offset) not + * covered by any ordered extent. + * We are safe to delete all extent states, and handle + * the ordered extent in the next iteration. + */ + range_end = ordered->file_offset - 1; + delete_states = true; + goto next; + } + + range_end = min(ordered->file_offset + ordered->num_bytes - 1, + page_end); + ASSERT(range_end + 1 - cur < U32_MAX); + range_len = range_end + 1 - cur; + if (!btrfs_page_test_ordered(fs_info, page, cur, range_len)) { + /* + * If Ordered (Private2) is cleared, it means endio has + * already been executed for the range. + * We can't delete the extent states as + * btrfs_finish_ordered_io() may still use some of them. + */ + delete_states = false; + goto next; + } + btrfs_page_clear_ordered(fs_info, page, cur, range_len); + /* * IO on this page will never be started, so we need to account * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW * here, must leave that up for the ordered extent completion. + * + * This will also unlock the range for incoming + * btrfs_finish_ordered_io(). */ if (!inode_evicting) - clear_extent_bit(tree, start, end, + clear_extent_bit(tree, cur, range_end, EXTENT_DELALLOC | EXTENT_LOCKED | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 0, &cached_state); - /* - * whoever cleared the private bit is responsible - * for the finish_ordered_io - */ - if (TestClearPagePrivate2(page)) { - spin_lock_irq(&inode->ordered_tree.lock); - set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); - ordered->truncated_len = min(ordered->truncated_len, - start - ordered->file_offset); - spin_unlock_irq(&inode->ordered_tree.lock); - if (btrfs_dec_test_ordered_pending(inode, &ordered, - start, - end - start + 1, 1)) { - btrfs_finish_ordered_io(ordered); - completed_ordered = true; - } + spin_lock_irq(&inode->ordered_tree.lock); + set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags); + ordered->truncated_len = min(ordered->truncated_len, + cur - ordered->file_offset); + spin_unlock_irq(&inode->ordered_tree.lock); + + if (btrfs_dec_test_ordered_pending(inode, &ordered, + cur, range_end + 1 - cur, 1)) { + btrfs_finish_ordered_io(ordered); + /* + * The ordered extent has finished, now we're again + * safe to delete all extent states of the range. + */ + delete_states = true; + } else { + /* + * btrfs_finish_ordered_io() will get executed by endio + * of other pages, thus we can't delete extent states + * anymore + */ + delete_states = false; } - btrfs_put_ordered_extent(ordered); +next: + if (ordered) + btrfs_put_ordered_extent(ordered); + /* + * Qgroup reserved space handler + * Sector(s) here will be either: + * + * 1) Already written to disk or bio already finished + * Then its QGROUP_RESERVED bit in io_tree is already cleared. + * Qgroup will be handled by its qgroup_record then. + * btrfs_qgroup_free_data() call will do nothing here. + * + * 2) Not written to disk yet + * Then btrfs_qgroup_free_data() call will clear the + * QGROUP_RESERVED bit of its io_tree, and free the qgroup + * reserved data space. + * Since the IO will never happen for this page. + */ + btrfs_qgroup_free_data(inode, NULL, cur, range_end + 1 - cur); if (!inode_evicting) { - cached_state = NULL; - lock_extent_bits(tree, start, end, - &cached_state); - } - - start = end + 1; - if (start < page_end) - goto again; - } - - /* - * Qgroup reserved space handler - * Page here will be either - * 1) Already written to disk or ordered extent already submitted - * Then its QGROUP_RESERVED bit in io_tree is already cleaned. - * Qgroup will be handled by its qgroup_record then. - * btrfs_qgroup_free_data() call will do nothing here. - * - * 2) Not written to disk yet - * Then btrfs_qgroup_free_data() call will clear the QGROUP_RESERVED - * bit of its io_tree, and free the qgroup reserved data space. - * Since the IO will never happen for this page. - */ - btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE); - if (!inode_evicting) { - bool delete = true; - - /* - * If there's an ordered extent for this range and we have not - * finished it ourselves, we must leave EXTENT_DELALLOC_NEW set - * in the range for the ordered extent completion. We must also - * not delete the range, otherwise we would lose that bit (and - * any other bits set in the range). Make sure EXTENT_UPTODATE - * is cleared if we don't delete, otherwise it can lead to - * corruptions if the i_size is extented later. - */ - if (found_ordered && !completed_ordered) - delete = false; - clear_extent_bit(tree, page_start, page_end, EXTENT_LOCKED | + clear_extent_bit(tree, cur, range_end, EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_UPTODATE | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, - delete, &cached_state); - - __btrfs_releasepage(page, GFP_NOFS); + delete_states, &cached_state); + } + cur = range_end + 1; } - + /* + * We have iterated through all ordered extents of the page, the page + * should not have Ordered (Private2) anymore, or the above iteration + * did something wrong. + */ + ASSERT(!PageOrdered(page)); + if (!inode_evicting) + __btrfs_releasepage(page, GFP_NOFS); ClearPageChecked(page); clear_page_extent_mapped(page); } @@ -8626,8 +8663,8 @@ vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf) flush_dcache_page(page); } ClearPageChecked(page); - set_page_dirty(page); - SetPageUptodate(page); + btrfs_page_set_dirty(fs_info, page, page_start, end + 1 - page_start); + btrfs_page_set_uptodate(fs_info, page, page_start, end + 1 - page_start); btrfs_set_inode_last_sub_trans(BTRFS_I(inode)); @@ -8661,6 +8698,7 @@ static int btrfs_truncate(struct inode *inode, bool skip_writeback) struct btrfs_trans_handle *trans; u64 mask = fs_info->sectorsize - 1; u64 min_size = btrfs_calc_metadata_size(fs_info, 1); + u64 extents_found = 0; if (!skip_writeback) { ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask), @@ -8718,20 +8756,13 @@ static int btrfs_truncate(struct inode *inode, bool skip_writeback) min_size, false); BUG_ON(ret); - /* - * So if we truncate and then write and fsync we normally would just - * write the extents that changed, which is a problem if we need to - * first truncate that entire inode. So set this flag so we write out - * all of the extents in the inode to the sync log so we're completely - * safe. - */ - set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); trans->block_rsv = rsv; while (1) { ret = btrfs_truncate_inode_items(trans, root, BTRFS_I(inode), inode->i_size, - BTRFS_EXTENT_DATA_KEY); + BTRFS_EXTENT_DATA_KEY, + &extents_found); trans->block_rsv = &fs_info->trans_block_rsv; if (ret != -ENOSPC && ret != -EAGAIN) break; @@ -8793,6 +8824,22 @@ static int btrfs_truncate(struct inode *inode, bool skip_writeback) } out: btrfs_free_block_rsv(fs_info, rsv); + /* + * So if we truncate and then write and fsync we normally would just + * write the extents that changed, which is a problem if we need to + * first truncate that entire inode. So set this flag so we write out + * all of the extents in the inode to the sync log so we're completely + * safe. + * + * If no extents were dropped or trimmed we don't need to force the next + * fsync to truncate all the inode's items from the log and re-log them + * all. This means the truncate operation did not change the file size, + * or changed it to a smaller size but there was only an implicit hole + * between the old i_size and the new i_size, and there were no prealloc + * extents beyond i_size to drop. + */ + if (extents_found > 0) + set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags); return ret; } @@ -10199,17 +10246,21 @@ static int btrfs_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, return ret; } -void btrfs_set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end) +void btrfs_set_range_writeback(struct btrfs_inode *inode, u64 start, u64 end) { - struct inode *inode = tree->private_data; + struct btrfs_fs_info *fs_info = inode->root->fs_info; unsigned long index = start >> PAGE_SHIFT; unsigned long end_index = end >> PAGE_SHIFT; struct page *page; + u32 len; + ASSERT(end + 1 - start <= U32_MAX); + len = end + 1 - start; while (index <= end_index) { - page = find_get_page(inode->i_mapping, index); + page = find_get_page(inode->vfs_inode.i_mapping, index); ASSERT(page); /* Pages should be in the extent_io_tree */ - set_page_writeback(page); + + btrfs_page_set_writeback(fs_info, page, start, len); put_page(page); index++; } diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c index 5dc2fd843ae3..0ba98e08a029 100644 --- a/fs/btrfs/ioctl.c +++ b/fs/btrfs/ioctl.c @@ -353,15 +353,55 @@ int btrfs_fileattr_set(struct user_namespace *mnt_userns, return ret; } +/* + * Start exclusive operation @type, return true on success + */ bool btrfs_exclop_start(struct btrfs_fs_info *fs_info, enum btrfs_exclusive_operation type) { - return !cmpxchg(&fs_info->exclusive_operation, BTRFS_EXCLOP_NONE, type); + bool ret = false; + + spin_lock(&fs_info->super_lock); + if (fs_info->exclusive_operation == BTRFS_EXCLOP_NONE) { + fs_info->exclusive_operation = type; + ret = true; + } + spin_unlock(&fs_info->super_lock); + + return ret; +} + +/* + * Conditionally allow to enter the exclusive operation in case it's compatible + * with the running one. This must be paired with btrfs_exclop_start_unlock and + * btrfs_exclop_finish. + * + * Compatibility: + * - the same type is already running + * - not BTRFS_EXCLOP_NONE - this is intentionally incompatible and the caller + * must check the condition first that would allow none -> @type + */ +bool btrfs_exclop_start_try_lock(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type) +{ + spin_lock(&fs_info->super_lock); + if (fs_info->exclusive_operation == type) + return true; + + spin_unlock(&fs_info->super_lock); + return false; +} + +void btrfs_exclop_start_unlock(struct btrfs_fs_info *fs_info) +{ + spin_unlock(&fs_info->super_lock); } void btrfs_exclop_finish(struct btrfs_fs_info *fs_info) { + spin_lock(&fs_info->super_lock); WRITE_ONCE(fs_info->exclusive_operation, BTRFS_EXCLOP_NONE); + spin_unlock(&fs_info->super_lock); sysfs_notify(&fs_info->fs_devices->fsid_kobj, NULL, "exclusive_operation"); } @@ -1455,7 +1495,7 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, if (btrfs_defrag_cancelled(fs_info)) { btrfs_debug(fs_info, "defrag_file cancelled"); ret = -EAGAIN; - break; + goto error; } if (!should_defrag_range(inode, (u64)i << PAGE_SHIFT, @@ -1533,6 +1573,8 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, } } + ret = defrag_count; +error: if ((range->flags & BTRFS_DEFRAG_RANGE_START_IO)) { filemap_flush(inode->i_mapping); if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, @@ -1546,8 +1588,6 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, btrfs_set_fs_incompat(fs_info, COMPRESS_ZSTD); } - ret = defrag_count; - out_ra: if (do_compress) { btrfs_inode_lock(inode, 0); @@ -1560,6 +1600,48 @@ int btrfs_defrag_file(struct inode *inode, struct file *file, return ret; } +/* + * Try to start exclusive operation @type or cancel it if it's running. + * + * Return: + * 0 - normal mode, newly claimed op started + * >0 - normal mode, something else is running, + * return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS to user space + * ECANCELED - cancel mode, successful cancel + * ENOTCONN - cancel mode, operation not running anymore + */ +static int exclop_start_or_cancel_reloc(struct btrfs_fs_info *fs_info, + enum btrfs_exclusive_operation type, bool cancel) +{ + if (!cancel) { + /* Start normal op */ + if (!btrfs_exclop_start(fs_info, type)) + return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; + /* Exclusive operation is now claimed */ + return 0; + } + + /* Cancel running op */ + if (btrfs_exclop_start_try_lock(fs_info, type)) { + /* + * This blocks any exclop finish from setting it to NONE, so we + * request cancellation. Either it runs and we will wait for it, + * or it has finished and no waiting will happen. + */ + atomic_inc(&fs_info->reloc_cancel_req); + btrfs_exclop_start_unlock(fs_info); + + if (test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) + wait_on_bit(&fs_info->flags, BTRFS_FS_RELOC_RUNNING, + TASK_INTERRUPTIBLE); + + return -ECANCELED; + } + + /* Something else is running or none */ + return -ENOTCONN; +} + static noinline int btrfs_ioctl_resize(struct file *file, void __user *arg) { @@ -1577,6 +1659,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, char *devstr = NULL; int ret = 0; int mod = 0; + bool cancel; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -1585,20 +1668,23 @@ static noinline int btrfs_ioctl_resize(struct file *file, if (ret) return ret; - if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_RESIZE)) { - mnt_drop_write_file(file); - return BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; - } - + /* + * Read the arguments before checking exclusivity to be able to + * distinguish regular resize and cancel + */ vol_args = memdup_user(arg, sizeof(*vol_args)); if (IS_ERR(vol_args)) { ret = PTR_ERR(vol_args); - goto out; + goto out_drop; } - vol_args->name[BTRFS_PATH_NAME_MAX] = '\0'; - sizestr = vol_args->name; + cancel = (strcmp("cancel", sizestr) == 0); + ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_RESIZE, cancel); + if (ret) + goto out_free; + /* Exclusive operation is now claimed */ + devstr = strchr(sizestr, ':'); if (devstr) { sizestr = devstr + 1; @@ -1606,10 +1692,10 @@ static noinline int btrfs_ioctl_resize(struct file *file, devstr = vol_args->name; ret = kstrtoull(devstr, 10, &devid); if (ret) - goto out_free; + goto out_finish; if (!devid) { ret = -EINVAL; - goto out_free; + goto out_finish; } btrfs_info(fs_info, "resizing devid %llu", devid); } @@ -1619,7 +1705,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, btrfs_info(fs_info, "resizer unable to find device %llu", devid); ret = -ENODEV; - goto out_free; + goto out_finish; } if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) { @@ -1627,7 +1713,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, "resizer unable to apply on readonly device %llu", devid); ret = -EPERM; - goto out_free; + goto out_finish; } if (!strcmp(sizestr, "max")) @@ -1643,13 +1729,13 @@ static noinline int btrfs_ioctl_resize(struct file *file, new_size = memparse(sizestr, &retptr); if (*retptr != '\0' || new_size == 0) { ret = -EINVAL; - goto out_free; + goto out_finish; } } if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) { ret = -EPERM; - goto out_free; + goto out_finish; } old_size = btrfs_device_get_total_bytes(device); @@ -1657,24 +1743,24 @@ static noinline int btrfs_ioctl_resize(struct file *file, if (mod < 0) { if (new_size > old_size) { ret = -EINVAL; - goto out_free; + goto out_finish; } new_size = old_size - new_size; } else if (mod > 0) { if (new_size > ULLONG_MAX - old_size) { ret = -ERANGE; - goto out_free; + goto out_finish; } new_size = old_size + new_size; } if (new_size < SZ_256M) { ret = -EINVAL; - goto out_free; + goto out_finish; } if (new_size > device->bdev->bd_inode->i_size) { ret = -EFBIG; - goto out_free; + goto out_finish; } new_size = round_down(new_size, fs_info->sectorsize); @@ -1683,7 +1769,7 @@ static noinline int btrfs_ioctl_resize(struct file *file, trans = btrfs_start_transaction(root, 0); if (IS_ERR(trans)) { ret = PTR_ERR(trans); - goto out_free; + goto out_finish; } ret = btrfs_grow_device(trans, device, new_size); btrfs_commit_transaction(trans); @@ -1696,10 +1782,11 @@ static noinline int btrfs_ioctl_resize(struct file *file, "resize device %s (devid %llu) from %llu to %llu", rcu_str_deref(device->name), device->devid, old_size, new_size); +out_finish: + btrfs_exclop_finish(fs_info); out_free: kfree(vol_args); -out: - btrfs_exclop_finish(fs_info); +out_drop: mnt_drop_write_file(file); return ret; } @@ -2897,7 +2984,7 @@ static noinline int btrfs_ioctl_snap_destroy(struct file *file, err = PTR_ERR(subvol_name_ptr); goto free_parent; } - /* subvol_name_ptr is already NULL termined */ + /* subvol_name_ptr is already nul terminated */ subvol_name = (char *)kbasename(subvol_name_ptr); } } else { @@ -3119,6 +3206,7 @@ static long btrfs_ioctl_rm_dev_v2(struct file *file, void __user *arg) struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_ioctl_vol_args_v2 *vol_args; int ret; + bool cancel = false; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -3137,18 +3225,22 @@ static long btrfs_ioctl_rm_dev_v2(struct file *file, void __user *arg) ret = -EOPNOTSUPP; goto out; } + vol_args->name[BTRFS_SUBVOL_NAME_MAX] = '\0'; + if (!(vol_args->flags & BTRFS_DEVICE_SPEC_BY_ID) && + strcmp("cancel", vol_args->name) == 0) + cancel = true; - if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REMOVE)) { - ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; + ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_DEV_REMOVE, + cancel); + if (ret) goto out; - } + /* Exclusive operation is now claimed */ - if (vol_args->flags & BTRFS_DEVICE_SPEC_BY_ID) { + if (vol_args->flags & BTRFS_DEVICE_SPEC_BY_ID) ret = btrfs_rm_device(fs_info, NULL, vol_args->devid); - } else { - vol_args->name[BTRFS_SUBVOL_NAME_MAX] = '\0'; + else ret = btrfs_rm_device(fs_info, vol_args->name, 0); - } + btrfs_exclop_finish(fs_info); if (!ret) { @@ -3172,6 +3264,7 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); struct btrfs_ioctl_vol_args *vol_args; int ret; + bool cancel; if (!capable(CAP_SYS_ADMIN)) return -EPERM; @@ -3180,25 +3273,24 @@ static long btrfs_ioctl_rm_dev(struct file *file, void __user *arg) if (ret) return ret; - if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_DEV_REMOVE)) { - ret = BTRFS_ERROR_DEV_EXCL_RUN_IN_PROGRESS; - goto out_drop_write; - } - vol_args = memdup_user(arg, sizeof(*vol_args)); if (IS_ERR(vol_args)) { ret = PTR_ERR(vol_args); - goto out; + goto out_drop_write; + } + vol_args->name[BTRFS_PATH_NAME_MAX] = '\0'; + cancel = (strcmp("cancel", vol_args->name) == 0); + + ret = exclop_start_or_cancel_reloc(fs_info, BTRFS_EXCLOP_DEV_REMOVE, + cancel); + if (ret == 0) { + ret = btrfs_rm_device(fs_info, vol_args->name, 0); + if (!ret) + btrfs_info(fs_info, "disk deleted %s", vol_args->name); + btrfs_exclop_finish(fs_info); } - vol_args->name[BTRFS_PATH_NAME_MAX] = '\0'; - ret = btrfs_rm_device(fs_info, vol_args->name, 0); - - if (!ret) - btrfs_info(fs_info, "disk deleted %s", vol_args->name); kfree(vol_args); -out: - btrfs_exclop_finish(fs_info); out_drop_write: mnt_drop_write_file(file); @@ -3551,7 +3643,7 @@ static noinline long btrfs_ioctl_start_sync(struct btrfs_root *root, goto out; } transid = trans->transid; - ret = btrfs_commit_transaction_async(trans, 0); + ret = btrfs_commit_transaction_async(trans); if (ret) { btrfs_end_transaction(trans); return ret; diff --git a/fs/btrfs/locking.c b/fs/btrfs/locking.c index 5fafc5e89bb7..313d9d685adb 100644 --- a/fs/btrfs/locking.c +++ b/fs/btrfs/locking.c @@ -57,7 +57,7 @@ void btrfs_tree_read_lock(struct extent_buffer *eb) /* * Try-lock for read. * - * Retrun 1 if the rwlock has been taken, 0 otherwise + * Return 1 if the rwlock has been taken, 0 otherwise */ int btrfs_try_tree_read_lock(struct extent_buffer *eb) { @@ -72,7 +72,7 @@ int btrfs_try_tree_read_lock(struct extent_buffer *eb) /* * Try-lock for write. * - * Retrun 1 if the rwlock has been taken, 0 otherwise + * Return 1 if the rwlock has been taken, 0 otherwise */ int btrfs_try_tree_write_lock(struct extent_buffer *eb) { diff --git a/fs/btrfs/ordered-data.c b/fs/btrfs/ordered-data.c index 6c413bb451a3..6eb41b7c0c84 100644 --- a/fs/btrfs/ordered-data.c +++ b/fs/btrfs/ordered-data.c @@ -16,6 +16,7 @@ #include "compression.h" #include "delalloc-space.h" #include "qgroup.h" +#include "subpage.h" static struct kmem_cache *btrfs_ordered_extent_cache; @@ -300,81 +301,142 @@ void btrfs_add_ordered_sum(struct btrfs_ordered_extent *entry, } /* - * Finish IO for one ordered extent across a given range. The range can - * contain several ordered extents. + * Mark all ordered extents io inside the specified range finished. * - * @found_ret: Return the finished ordered extent - * @file_offset: File offset for the finished IO - * Will also be updated to one byte past the range that is - * recordered as finished. This allows caller to walk forward. - * @io_size: Length of the finish IO range - * @uptodate: If the IO finished without problem + * @page: The invovled page for the opeartion. + * For uncompressed buffered IO, the page status also needs to be + * updated to indicate whether the pending ordered io is finished. + * Can be NULL for direct IO and compressed write. + * For these cases, callers are ensured they won't execute the + * endio function twice. + * @finish_func: The function to be executed when all the IO of an ordered + * extent are finished. * - * Return true if any ordered extent is finished in the range, and update - * @found_ret and @file_offset. - * Return false otherwise. - * - * NOTE: Although The range can cross multiple ordered extents, only one - * ordered extent will be updated during one call. The caller is responsible to - * iterate all ordered extents in the range. + * This function is called for endio, thus the range must have ordered + * extent(s) coveri it. */ -bool btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, - struct btrfs_ordered_extent **finished_ret, - u64 *file_offset, u64 io_size, int uptodate) +void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode, + struct page *page, u64 file_offset, + u64 num_bytes, btrfs_func_t finish_func, + bool uptodate) { - struct btrfs_fs_info *fs_info = inode->root->fs_info; struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree; + struct btrfs_fs_info *fs_info = inode->root->fs_info; + struct btrfs_workqueue *wq; struct rb_node *node; struct btrfs_ordered_extent *entry = NULL; - bool finished = false; unsigned long flags; - u64 dec_end; - u64 dec_start; - u64 to_dec; + u64 cur = file_offset; + + if (btrfs_is_free_space_inode(inode)) + wq = fs_info->endio_freespace_worker; + else + wq = fs_info->endio_write_workers; + + if (page) + ASSERT(page->mapping && page_offset(page) <= file_offset && + file_offset + num_bytes <= page_offset(page) + PAGE_SIZE); spin_lock_irqsave(&tree->lock, flags); - node = tree_search(tree, *file_offset); - if (!node) - goto out; + while (cur < file_offset + num_bytes) { + u64 entry_end; + u64 end; + u32 len; - entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); - if (!in_range(*file_offset, entry->file_offset, entry->num_bytes)) - goto out; + node = tree_search(tree, cur); + /* No ordered extents at all */ + if (!node) + break; - dec_start = max(*file_offset, entry->file_offset); - dec_end = min(*file_offset + io_size, - entry->file_offset + entry->num_bytes); - *file_offset = dec_end; - if (dec_start > dec_end) { - btrfs_crit(fs_info, "bad ordering dec_start %llu end %llu", - dec_start, dec_end); - } - to_dec = dec_end - dec_start; - if (to_dec > entry->bytes_left) { - btrfs_crit(fs_info, - "bad ordered accounting left %llu size %llu", - entry->bytes_left, to_dec); - } - entry->bytes_left -= to_dec; - if (!uptodate) - set_bit(BTRFS_ORDERED_IOERR, &entry->flags); - - if (entry->bytes_left == 0) { + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); + entry_end = entry->file_offset + entry->num_bytes; /* - * Ensure only one caller can set the flag and finished_ret - * accordingly + * |<-- OE --->| | + * cur + * Go to next OE. */ - finished = !test_and_set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags); - /* test_and_set_bit implies a barrier */ - cond_wake_up_nomb(&entry->wait); - } -out: - if (finished && finished_ret && entry) { - *finished_ret = entry; - refcount_inc(&entry->refs); + if (cur >= entry_end) { + node = rb_next(node); + /* No more ordered extents, exit */ + if (!node) + break; + entry = rb_entry(node, struct btrfs_ordered_extent, + rb_node); + + /* Go to next ordered extent and continue */ + cur = entry->file_offset; + continue; + } + /* + * | |<--- OE --->| + * cur + * Go to the start of OE. + */ + if (cur < entry->file_offset) { + cur = entry->file_offset; + continue; + } + + /* + * Now we are definitely inside one ordered extent. + * + * |<--- OE --->| + * | + * cur + */ + end = min(entry->file_offset + entry->num_bytes, + file_offset + num_bytes) - 1; + ASSERT(end + 1 - cur < U32_MAX); + len = end + 1 - cur; + + if (page) { + /* + * Ordered (Private2) bit indicates whether we still + * have pending io unfinished for the ordered extent. + * + * If there's no such bit, we need to skip to next range. + */ + if (!btrfs_page_test_ordered(fs_info, page, cur, len)) { + cur += len; + continue; + } + btrfs_page_clear_ordered(fs_info, page, cur, len); + } + + /* Now we're fine to update the accounting */ + if (unlikely(len > entry->bytes_left)) { + WARN_ON(1); + btrfs_crit(fs_info, +"bad ordered extent accounting, root=%llu ino=%llu OE offset=%llu OE len=%llu to_dec=%u left=%llu", + inode->root->root_key.objectid, + btrfs_ino(inode), + entry->file_offset, + entry->num_bytes, + len, entry->bytes_left); + entry->bytes_left = 0; + } else { + entry->bytes_left -= len; + } + + if (!uptodate) + set_bit(BTRFS_ORDERED_IOERR, &entry->flags); + + /* + * All the IO of the ordered extent is finished, we need to queue + * the finish_func to be executed. + */ + if (entry->bytes_left == 0) { + set_bit(BTRFS_ORDERED_IO_DONE, &entry->flags); + cond_wake_up(&entry->wait); + refcount_inc(&entry->refs); + spin_unlock_irqrestore(&tree->lock, flags); + btrfs_init_work(&entry->work, finish_func, NULL, NULL); + btrfs_queue_work(wq, &entry->work); + spin_lock_irqsave(&tree->lock, flags); + } + cur += len; } spin_unlock_irqrestore(&tree->lock, flags); - return finished; } /* @@ -869,6 +931,81 @@ btrfs_lookup_first_ordered_extent(struct btrfs_inode *inode, u64 file_offset) return entry; } +/* + * Lookup the first ordered extent that overlaps the range + * [@file_offset, @file_offset + @len). + * + * The difference between this and btrfs_lookup_first_ordered_extent() is + * that this one won't return any ordered extent that does not overlap the range. + * And the difference against btrfs_lookup_ordered_extent() is, this function + * ensures the first ordered extent gets returned. + */ +struct btrfs_ordered_extent *btrfs_lookup_first_ordered_range( + struct btrfs_inode *inode, u64 file_offset, u64 len) +{ + struct btrfs_ordered_inode_tree *tree = &inode->ordered_tree; + struct rb_node *node; + struct rb_node *cur; + struct rb_node *prev; + struct rb_node *next; + struct btrfs_ordered_extent *entry = NULL; + + spin_lock_irq(&tree->lock); + node = tree->tree.rb_node; + /* + * Here we don't want to use tree_search() which will use tree->last + * and screw up the search order. + * And __tree_search() can't return the adjacent ordered extents + * either, thus here we do our own search. + */ + while (node) { + entry = rb_entry(node, struct btrfs_ordered_extent, rb_node); + + if (file_offset < entry->file_offset) { + node = node->rb_left; + } else if (file_offset >= entry_end(entry)) { + node = node->rb_right; + } else { + /* + * Direct hit, got an ordered extent that starts at + * @file_offset + */ + goto out; + } + } + if (!entry) { + /* Empty tree */ + goto out; + } + + cur = &entry->rb_node; + /* We got an entry around @file_offset, check adjacent entries */ + if (entry->file_offset < file_offset) { + prev = cur; + next = rb_next(cur); + } else { + prev = rb_prev(cur); + next = cur; + } + if (prev) { + entry = rb_entry(prev, struct btrfs_ordered_extent, rb_node); + if (range_overlaps(entry, file_offset, len)) + goto out; + } + if (next) { + entry = rb_entry(next, struct btrfs_ordered_extent, rb_node); + if (range_overlaps(entry, file_offset, len)) + goto out; + } + /* No ordered extent in the range */ + entry = NULL; +out: + if (entry) + refcount_inc(&entry->refs); + spin_unlock_irq(&tree->lock); + return entry; +} + /* * btrfs_flush_ordered_range - Lock the passed range and ensures all pending * ordered extents in it are run to completion. diff --git a/fs/btrfs/ordered-data.h b/fs/btrfs/ordered-data.h index e60c07f36427..566472004edd 100644 --- a/fs/btrfs/ordered-data.h +++ b/fs/btrfs/ordered-data.h @@ -172,13 +172,13 @@ btrfs_ordered_inode_tree_init(struct btrfs_ordered_inode_tree *t) void btrfs_put_ordered_extent(struct btrfs_ordered_extent *entry); void btrfs_remove_ordered_extent(struct btrfs_inode *btrfs_inode, struct btrfs_ordered_extent *entry); +void btrfs_mark_ordered_io_finished(struct btrfs_inode *inode, + struct page *page, u64 file_offset, + u64 num_bytes, btrfs_func_t finish_func, + bool uptodate); bool btrfs_dec_test_ordered_pending(struct btrfs_inode *inode, struct btrfs_ordered_extent **cached, u64 file_offset, u64 io_size, int uptodate); -bool btrfs_dec_test_first_ordered_pending(struct btrfs_inode *inode, - struct btrfs_ordered_extent **finished_ret, - u64 *file_offset, u64 io_size, - int uptodate); int btrfs_add_ordered_extent(struct btrfs_inode *inode, u64 file_offset, u64 disk_bytenr, u64 num_bytes, u64 disk_num_bytes, int type); @@ -196,6 +196,8 @@ void btrfs_start_ordered_extent(struct btrfs_ordered_extent *entry, int wait); int btrfs_wait_ordered_range(struct inode *inode, u64 start, u64 len); struct btrfs_ordered_extent * btrfs_lookup_first_ordered_extent(struct btrfs_inode *inode, u64 file_offset); +struct btrfs_ordered_extent *btrfs_lookup_first_ordered_range( + struct btrfs_inode *inode, u64 file_offset, u64 len); struct btrfs_ordered_extent *btrfs_lookup_ordered_range( struct btrfs_inode *inode, u64 file_offset, diff --git a/fs/btrfs/props.c b/fs/btrfs/props.c index 2dcb1cb21634..b1cb5a8c2999 100644 --- a/fs/btrfs/props.c +++ b/fs/btrfs/props.c @@ -260,6 +260,10 @@ static int prop_compression_validate(const char *value, size_t len) if (btrfs_compress_is_valid_type(value, len)) return 0; + if ((len == 2 && strncmp("no", value, 2) == 0) || + (len == 4 && strncmp("none", value, 4) == 0)) + return 0; + return -EINVAL; } @@ -269,7 +273,17 @@ static int prop_compression_apply(struct inode *inode, const char *value, struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb); int type; + /* Reset to defaults */ if (len == 0) { + BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS; + BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS; + BTRFS_I(inode)->prop_compress = BTRFS_COMPRESS_NONE; + return 0; + } + + /* Set NOCOMPRESS flag */ + if ((len == 2 && strncmp("no", value, 2) == 0) || + (len == 4 && strncmp("none", value, 4) == 0)) { BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS; BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS; BTRFS_I(inode)->prop_compress = BTRFS_COMPRESS_NONE; @@ -348,7 +362,7 @@ static int inherit_props(struct btrfs_trans_handle *trans, /* * This is not strictly necessary as the property should be - * valid, but in case it isn't, don't propagate it futher. + * valid, but in case it isn't, don't propagate it further. */ ret = h->validate(value, strlen(value)); if (ret) diff --git a/fs/btrfs/qgroup.c b/fs/btrfs/qgroup.c index 3ded812f522c..07ec06d4e972 100644 --- a/fs/btrfs/qgroup.c +++ b/fs/btrfs/qgroup.c @@ -2521,7 +2521,7 @@ int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr, int ret = 0; /* - * If quotas get disabled meanwhile, the resouces need to be freed and + * If quotas get disabled meanwhile, the resources need to be freed and * we can't just exit here. */ if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags)) @@ -3545,13 +3545,7 @@ static int try_flush_qgroup(struct btrfs_root *root) struct btrfs_trans_handle *trans; int ret; - /* - * Can't hold an open transaction or we run the risk of deadlocking, - * and can't either be under the context of a send operation (where - * current->journal_info is set to BTRFS_SEND_TRANS_STUB), as that - * would result in a crash when starting a transaction and does not - * make sense either (send is a read-only operation). - */ + /* Can't hold an open transaction or we run the risk of deadlocking. */ ASSERT(current->journal_info == NULL); if (WARN_ON(current->journal_info)) return 0; diff --git a/fs/btrfs/reflink.c b/fs/btrfs/reflink.c index 9178da07cc9c..9b0814318e72 100644 --- a/fs/btrfs/reflink.c +++ b/fs/btrfs/reflink.c @@ -7,6 +7,7 @@ #include "delalloc-space.h" #include "reflink.h" #include "transaction.h" +#include "subpage.h" #define BTRFS_MAX_DEDUPE_LEN SZ_16M @@ -52,7 +53,8 @@ static int copy_inline_to_page(struct btrfs_inode *inode, const u64 datal, const u8 comp_type) { - const u64 block_size = btrfs_inode_sectorsize(inode); + struct btrfs_fs_info *fs_info = inode->root->fs_info; + const u32 block_size = fs_info->sectorsize; const u64 range_end = file_offset + block_size - 1; const size_t inline_size = size - btrfs_file_extent_calc_inline_size(0); char *data_start = inline_data + btrfs_file_extent_calc_inline_size(0); @@ -106,10 +108,12 @@ static int copy_inline_to_page(struct btrfs_inode *inode, set_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &inode->runtime_flags); if (comp_type == BTRFS_COMPRESS_NONE) { - memcpy_to_page(page, 0, data_start, datal); + memcpy_to_page(page, offset_in_page(file_offset), data_start, + datal); flush_dcache_page(page); } else { - ret = btrfs_decompress(comp_type, data_start, page, 0, + ret = btrfs_decompress(comp_type, data_start, page, + offset_in_page(file_offset), inline_size, datal); if (ret) goto out_unlock; @@ -133,9 +137,9 @@ static int copy_inline_to_page(struct btrfs_inode *inode, flush_dcache_page(page); } - SetPageUptodate(page); + btrfs_page_set_uptodate(fs_info, page, file_offset, block_size); ClearPageChecked(page); - set_page_dirty(page); + btrfs_page_set_dirty(fs_info, page, file_offset, block_size); out_unlock: if (page) { unlock_page(page); diff --git a/fs/btrfs/relocation.c b/fs/btrfs/relocation.c index b70be2ac2e9e..fc831597cb22 100644 --- a/fs/btrfs/relocation.c +++ b/fs/btrfs/relocation.c @@ -2876,11 +2876,12 @@ int setup_extent_mapping(struct inode *inode, u64 start, u64 end, } /* - * Allow error injection to test balance cancellation + * Allow error injection to test balance/relocation cancellation */ noinline int btrfs_should_cancel_balance(struct btrfs_fs_info *fs_info) { return atomic_read(&fs_info->balance_cancel_req) || + atomic_read(&fs_info->reloc_cancel_req) || fatal_signal_pending(current); } ALLOW_ERROR_INJECTION(btrfs_should_cancel_balance, TRUE); @@ -3780,6 +3781,60 @@ struct inode *create_reloc_inode(struct btrfs_fs_info *fs_info, return inode; } +/* + * Mark start of chunk relocation that is cancellable. Check if the cancellation + * has been requested meanwhile and don't start in that case. + * + * Return: + * 0 success + * -EINPROGRESS operation is already in progress, that's probably a bug + * -ECANCELED cancellation request was set before the operation started + * -EAGAIN can not start because there are ongoing send operations + */ +static int reloc_chunk_start(struct btrfs_fs_info *fs_info) +{ + spin_lock(&fs_info->send_reloc_lock); + if (fs_info->send_in_progress) { + btrfs_warn_rl(fs_info, +"cannot run relocation while send operations are in progress (%d in progress)", + fs_info->send_in_progress); + spin_unlock(&fs_info->send_reloc_lock); + return -EAGAIN; + } + if (test_and_set_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) { + /* This should not happen */ + spin_unlock(&fs_info->send_reloc_lock); + btrfs_err(fs_info, "reloc already running, cannot start"); + return -EINPROGRESS; + } + spin_unlock(&fs_info->send_reloc_lock); + + if (atomic_read(&fs_info->reloc_cancel_req) > 0) { + btrfs_info(fs_info, "chunk relocation canceled on start"); + /* + * On cancel, clear all requests but let the caller mark + * the end after cleanup operations. + */ + atomic_set(&fs_info->reloc_cancel_req, 0); + return -ECANCELED; + } + return 0; +} + +/* + * Mark end of chunk relocation that is cancellable and wake any waiters. + */ +static void reloc_chunk_end(struct btrfs_fs_info *fs_info) +{ + /* Requested after start, clear bit first so any waiters can continue */ + if (atomic_read(&fs_info->reloc_cancel_req) > 0) + btrfs_info(fs_info, "chunk relocation canceled during operation"); + spin_lock(&fs_info->send_reloc_lock); + clear_and_wake_up_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags); + spin_unlock(&fs_info->send_reloc_lock); + atomic_set(&fs_info->reloc_cancel_req, 0); +} + static struct reloc_control *alloc_reloc_control(struct btrfs_fs_info *fs_info) { struct reloc_control *rc; @@ -3862,6 +3917,12 @@ int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start) return -ENOMEM; } + ret = reloc_chunk_start(fs_info); + if (ret < 0) { + err = ret; + goto out_put_bg; + } + rc->extent_root = extent_root; rc->block_group = bg; @@ -3952,7 +4013,9 @@ int btrfs_relocate_block_group(struct btrfs_fs_info *fs_info, u64 group_start) if (err && rw) btrfs_dec_block_group_ro(rc->block_group); iput(rc->data_inode); - btrfs_put_block_group(rc->block_group); +out_put_bg: + btrfs_put_block_group(bg); + reloc_chunk_end(fs_info); free_reloc_control(rc); return err; } @@ -4073,6 +4136,12 @@ int btrfs_recover_relocation(struct btrfs_root *root) goto out; } + ret = reloc_chunk_start(fs_info); + if (ret < 0) { + err = ret; + goto out_end; + } + rc->extent_root = fs_info->extent_root; set_reloc_control(rc); @@ -4137,6 +4206,8 @@ int btrfs_recover_relocation(struct btrfs_root *root) err = ret; out_unset: unset_reloc_control(rc); +out_end: + reloc_chunk_end(fs_info); free_reloc_control(rc); out: free_reloc_roots(&reloc_roots); diff --git a/fs/btrfs/scrub.c b/fs/btrfs/scrub.c index 485cda3eb8d7..088641ba7a8e 100644 --- a/fs/btrfs/scrub.c +++ b/fs/btrfs/scrub.c @@ -165,6 +165,10 @@ struct scrub_ctx { int readonly; int pages_per_rd_bio; + /* State of IO submission throttling affecting the associated device */ + ktime_t throttle_deadline; + u64 throttle_sent; + int is_dev_replace; u64 write_pointer; @@ -605,6 +609,7 @@ static noinline_for_stack struct scrub_ctx *scrub_setup_ctx( spin_lock_init(&sctx->list_lock); spin_lock_init(&sctx->stat_lock); init_waitqueue_head(&sctx->list_wait); + sctx->throttle_deadline = 0; WARN_ON(sctx->wr_curr_bio != NULL); mutex_init(&sctx->wr_lock); @@ -626,7 +631,6 @@ static noinline_for_stack struct scrub_ctx *scrub_setup_ctx( static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, void *warn_ctx) { - u64 isize; u32 nlink; int ret; int i; @@ -662,7 +666,6 @@ static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, eb = swarn->path->nodes[0]; inode_item = btrfs_item_ptr(eb, swarn->path->slots[0], struct btrfs_inode_item); - isize = btrfs_inode_size(eb, inode_item); nlink = btrfs_inode_nlink(eb, inode_item); btrfs_release_path(swarn->path); @@ -691,12 +694,12 @@ static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root, */ for (i = 0; i < ipath->fspath->elem_cnt; ++i) btrfs_warn_in_rcu(fs_info, -"%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)", +"%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %u, links %u (path: %s)", swarn->errstr, swarn->logical, rcu_str_deref(swarn->dev->name), swarn->physical, root, inum, offset, - min(isize - offset, (u64)PAGE_SIZE), nlink, + fs_info->sectorsize, nlink, (char *)(unsigned long)ipath->fspath->val[i]); btrfs_put_root(local_root); @@ -885,25 +888,25 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) * read all mirrors one after the other. This includes to * re-read the extent or metadata block that failed (that was * the cause that this fixup code is called) another time, - * page by page this time in order to know which pages + * sector by sector this time in order to know which sectors * caused I/O errors and which ones are good (for all mirrors). * It is the goal to handle the situation when more than one * mirror contains I/O errors, but the errors do not * overlap, i.e. the data can be repaired by selecting the - * pages from those mirrors without I/O error on the - * particular pages. One example (with blocks >= 2 * PAGE_SIZE) - * would be that mirror #1 has an I/O error on the first page, - * the second page is good, and mirror #2 has an I/O error on - * the second page, but the first page is good. - * Then the first page of the first mirror can be repaired by - * taking the first page of the second mirror, and the - * second page of the second mirror can be repaired by - * copying the contents of the 2nd page of the 1st mirror. - * One more note: if the pages of one mirror contain I/O + * sectors from those mirrors without I/O error on the + * particular sectors. One example (with blocks >= 2 * sectorsize) + * would be that mirror #1 has an I/O error on the first sector, + * the second sector is good, and mirror #2 has an I/O error on + * the second sector, but the first sector is good. + * Then the first sector of the first mirror can be repaired by + * taking the first sector of the second mirror, and the + * second sector of the second mirror can be repaired by + * copying the contents of the 2nd sector of the 1st mirror. + * One more note: if the sectors of one mirror contain I/O * errors, the checksum cannot be verified. In order to get * the best data for repairing, the first attempt is to find * a mirror without I/O errors and with a validated checksum. - * Only if this is not possible, the pages are picked from + * Only if this is not possible, the sectors are picked from * mirrors with I/O errors without considering the checksum. * If the latter is the case, at the end, the checksum of the * repaired area is verified in order to correctly maintain @@ -1060,26 +1063,26 @@ static int scrub_handle_errored_block(struct scrub_block *sblock_to_check) /* * In case of I/O errors in the area that is supposed to be - * repaired, continue by picking good copies of those pages. - * Select the good pages from mirrors to rewrite bad pages from + * repaired, continue by picking good copies of those sectors. + * Select the good sectors from mirrors to rewrite bad sectors from * the area to fix. Afterwards verify the checksum of the block * that is supposed to be repaired. This verification step is * only done for the purpose of statistic counting and for the * final scrub report, whether errors remain. * A perfect algorithm could make use of the checksum and try - * all possible combinations of pages from the different mirrors + * all possible combinations of sectors from the different mirrors * until the checksum verification succeeds. For example, when - * the 2nd page of mirror #1 faces I/O errors, and the 2nd page + * the 2nd sector of mirror #1 faces I/O errors, and the 2nd sector * of mirror #2 is readable but the final checksum test fails, - * then the 2nd page of mirror #3 could be tried, whether now + * then the 2nd sector of mirror #3 could be tried, whether now * the final checksum succeeds. But this would be a rare * exception and is therefore not implemented. At least it is * avoided that the good copy is overwritten. * A more useful improvement would be to pick the sectors * without I/O error based on sector sizes (512 bytes on legacy - * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one + * disks) instead of on sectorsize. Then maybe 512 byte of one * mirror could be repaired by taking 512 byte of a different - * mirror, even if other 512 byte sectors in the same PAGE_SIZE + * mirror, even if other 512 byte sectors in the same sectorsize * area are unreadable. */ success = 1; @@ -1260,7 +1263,7 @@ static int scrub_setup_recheck_block(struct scrub_block *original_sblock, { struct scrub_ctx *sctx = original_sblock->sctx; struct btrfs_fs_info *fs_info = sctx->fs_info; - u64 length = original_sblock->page_count * PAGE_SIZE; + u64 length = original_sblock->page_count * fs_info->sectorsize; u64 logical = original_sblock->pagev[0]->logical; u64 generation = original_sblock->pagev[0]->generation; u64 flags = original_sblock->pagev[0]->flags; @@ -1283,13 +1286,13 @@ static int scrub_setup_recheck_block(struct scrub_block *original_sblock, */ while (length > 0) { - sublen = min_t(u64, length, PAGE_SIZE); + sublen = min_t(u64, length, fs_info->sectorsize); mapped_length = sublen; bbio = NULL; /* - * with a length of PAGE_SIZE, each returned stripe - * represents one mirror + * With a length of sectorsize, each returned stripe represents + * one mirror */ btrfs_bio_counter_inc_blocked(fs_info); ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, @@ -1480,7 +1483,7 @@ static void scrub_recheck_block(struct btrfs_fs_info *fs_info, bio = btrfs_io_bio_alloc(1); bio_set_dev(bio, spage->dev->bdev); - bio_add_page(bio, spage->page, PAGE_SIZE, 0); + bio_add_page(bio, spage->page, fs_info->sectorsize, 0); bio->bi_iter.bi_sector = spage->physical >> 9; bio->bi_opf = REQ_OP_READ; @@ -1544,6 +1547,7 @@ static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad, struct scrub_page *spage_bad = sblock_bad->pagev[page_num]; struct scrub_page *spage_good = sblock_good->pagev[page_num]; struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info; + const u32 sectorsize = fs_info->sectorsize; BUG_ON(spage_bad->page == NULL); BUG_ON(spage_good->page == NULL); @@ -1563,8 +1567,8 @@ static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad, bio->bi_iter.bi_sector = spage_bad->physical >> 9; bio->bi_opf = REQ_OP_WRITE; - ret = bio_add_page(bio, spage_good->page, PAGE_SIZE, 0); - if (PAGE_SIZE != ret) { + ret = bio_add_page(bio, spage_good->page, sectorsize, 0); + if (ret != sectorsize) { bio_put(bio); return -EIO; } @@ -1642,6 +1646,7 @@ static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, { struct scrub_bio *sbio; int ret; + const u32 sectorsize = sctx->fs_info->sectorsize; mutex_lock(&sctx->wr_lock); again: @@ -1681,16 +1686,16 @@ static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx, bio->bi_iter.bi_sector = sbio->physical >> 9; bio->bi_opf = REQ_OP_WRITE; sbio->status = 0; - } else if (sbio->physical + sbio->page_count * PAGE_SIZE != + } else if (sbio->physical + sbio->page_count * sectorsize != spage->physical_for_dev_replace || - sbio->logical + sbio->page_count * PAGE_SIZE != + sbio->logical + sbio->page_count * sectorsize != spage->logical) { scrub_wr_submit(sctx); goto again; } - ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); - if (ret != PAGE_SIZE) { + ret = bio_add_page(sbio->bio, spage->page, sectorsize, 0); + if (ret != sectorsize) { if (sbio->page_count < 1) { bio_put(sbio->bio); sbio->bio = NULL; @@ -1729,7 +1734,8 @@ static void scrub_wr_submit(struct scrub_ctx *sctx) btrfsic_submit_bio(sbio->bio); if (btrfs_is_zoned(sctx->fs_info)) - sctx->write_pointer = sbio->physical + sbio->page_count * PAGE_SIZE; + sctx->write_pointer = sbio->physical + sbio->page_count * + sctx->fs_info->sectorsize; } static void scrub_wr_bio_end_io(struct bio *bio) @@ -1988,6 +1994,65 @@ static void scrub_page_put(struct scrub_page *spage) } } +/* + * Throttling of IO submission, bandwidth-limit based, the timeslice is 1 + * second. Limit can be set via /sys/fs/UUID/devinfo/devid/scrub_speed_max. + */ +static void scrub_throttle(struct scrub_ctx *sctx) +{ + const int time_slice = 1000; + struct scrub_bio *sbio; + struct btrfs_device *device; + s64 delta; + ktime_t now; + u32 div; + u64 bwlimit; + + sbio = sctx->bios[sctx->curr]; + device = sbio->dev; + bwlimit = READ_ONCE(device->scrub_speed_max); + if (bwlimit == 0) + return; + + /* + * Slice is divided into intervals when the IO is submitted, adjust by + * bwlimit and maximum of 64 intervals. + */ + div = max_t(u32, 1, (u32)(bwlimit / (16 * 1024 * 1024))); + div = min_t(u32, 64, div); + + /* Start new epoch, set deadline */ + now = ktime_get(); + if (sctx->throttle_deadline == 0) { + sctx->throttle_deadline = ktime_add_ms(now, time_slice / div); + sctx->throttle_sent = 0; + } + + /* Still in the time to send? */ + if (ktime_before(now, sctx->throttle_deadline)) { + /* If current bio is within the limit, send it */ + sctx->throttle_sent += sbio->bio->bi_iter.bi_size; + if (sctx->throttle_sent <= div_u64(bwlimit, div)) + return; + + /* We're over the limit, sleep until the rest of the slice */ + delta = ktime_ms_delta(sctx->throttle_deadline, now); + } else { + /* New request after deadline, start new epoch */ + delta = 0; + } + + if (delta) { + long timeout; + + timeout = div_u64(delta * HZ, 1000); + schedule_timeout_interruptible(timeout); + } + + /* Next call will start the deadline period */ + sctx->throttle_deadline = 0; +} + static void scrub_submit(struct scrub_ctx *sctx) { struct scrub_bio *sbio; @@ -1995,6 +2060,8 @@ static void scrub_submit(struct scrub_ctx *sctx) if (sctx->curr == -1) return; + scrub_throttle(sctx); + sbio = sctx->bios[sctx->curr]; sctx->curr = -1; scrub_pending_bio_inc(sctx); @@ -2006,6 +2073,7 @@ static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, { struct scrub_block *sblock = spage->sblock; struct scrub_bio *sbio; + const u32 sectorsize = sctx->fs_info->sectorsize; int ret; again: @@ -2044,9 +2112,9 @@ static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, bio->bi_iter.bi_sector = sbio->physical >> 9; bio->bi_opf = REQ_OP_READ; sbio->status = 0; - } else if (sbio->physical + sbio->page_count * PAGE_SIZE != + } else if (sbio->physical + sbio->page_count * sectorsize != spage->physical || - sbio->logical + sbio->page_count * PAGE_SIZE != + sbio->logical + sbio->page_count * sectorsize != spage->logical || sbio->dev != spage->dev) { scrub_submit(sctx); @@ -2054,8 +2122,8 @@ static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx, } sbio->pagev[sbio->page_count] = spage; - ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0); - if (ret != PAGE_SIZE) { + ret = bio_add_page(sbio->bio, spage->page, sectorsize, 0); + if (ret != sectorsize) { if (sbio->page_count < 1) { bio_put(sbio->bio); sbio->bio = NULL; @@ -2398,7 +2466,7 @@ static void scrub_block_complete(struct scrub_block *sblock) if (sblock->sparity && corrupted && !sblock->data_corrected) { u64 start = sblock->pagev[0]->logical; u64 end = sblock->pagev[sblock->page_count - 1]->logical + - PAGE_SIZE; + sblock->sctx->fs_info->sectorsize; ASSERT(end - start <= U32_MAX); scrub_parity_mark_sectors_error(sblock->sparity, @@ -2418,7 +2486,7 @@ static void drop_csum_range(struct scrub_ctx *sctx, struct btrfs_ordered_sum *su * the csum into @csum. * * The search source is sctx->csum_list, which is a pre-populated list - * storing bytenr ordered csum ranges. We're reponsible to cleanup any range + * storing bytenr ordered csum ranges. We're responsible to cleanup any range * that is before @logical. * * Return 0 if there is no csum for the range. @@ -3138,28 +3206,23 @@ static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx, physical = map->stripes[num].physical; offset = 0; nstripes = div64_u64(length, map->stripe_len); + mirror_num = 1; + increment = map->stripe_len; if (map->type & BTRFS_BLOCK_GROUP_RAID0) { offset = map->stripe_len * num; increment = map->stripe_len * map->num_stripes; - mirror_num = 1; } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) { int factor = map->num_stripes / map->sub_stripes; offset = map->stripe_len * (num / map->sub_stripes); increment = map->stripe_len * factor; mirror_num = num % map->sub_stripes + 1; } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) { - increment = map->stripe_len; mirror_num = num % map->num_stripes + 1; } else if (map->type & BTRFS_BLOCK_GROUP_DUP) { - increment = map->stripe_len; mirror_num = num % map->num_stripes + 1; } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) { get_raid56_logic_offset(physical, num, map, &offset, NULL); increment = map->stripe_len * nr_data_stripes(map); - mirror_num = 1; - } else { - increment = map->stripe_len; - mirror_num = 1; } path = btrfs_alloc_path(); diff --git a/fs/btrfs/send.c b/fs/btrfs/send.c index bd69db72acc5..6ac37ae6c811 100644 --- a/fs/btrfs/send.c +++ b/fs/btrfs/send.c @@ -2077,16 +2077,6 @@ static struct name_cache_entry *name_cache_search(struct send_ctx *sctx, return NULL; } -/* - * Removes the entry from the list and adds it back to the end. This marks the - * entry as recently used so that name_cache_clean_unused does not remove it. - */ -static void name_cache_used(struct send_ctx *sctx, struct name_cache_entry *nce) -{ - list_del(&nce->list); - list_add_tail(&nce->list, &sctx->name_cache_list); -} - /* * Remove some entries from the beginning of name_cache_list. */ @@ -2147,7 +2137,13 @@ static int __get_cur_name_and_parent(struct send_ctx *sctx, kfree(nce); nce = NULL; } else { - name_cache_used(sctx, nce); + /* + * Removes the entry from the list and adds it back to + * the end. This marks the entry as recently used so + * that name_cache_clean_unused does not remove it. + */ + list_move_tail(&nce->list, &sctx->name_cache_list); + *parent_ino = nce->parent_ino; *parent_gen = nce->parent_gen; ret = fs_path_add(dest, nce->name, nce->name_len); @@ -4064,6 +4060,17 @@ static int process_recorded_refs(struct send_ctx *sctx, int *pending_move) if (ret < 0) goto out; } else { + /* + * If we previously orphanized a directory that + * collided with a new reference that we already + * processed, recompute the current path because + * that directory may be part of the path. + */ + if (orphanized_dir) { + ret = refresh_ref_path(sctx, cur); + if (ret < 0) + goto out; + } ret = send_unlink(sctx, cur->full_path); if (ret < 0) goto out; @@ -6507,7 +6514,7 @@ static int changed_extent(struct send_ctx *sctx, * updates the inode item, but it only changes the iversion (sequence * field in the inode item) of the inode, so if a file is deduplicated * the same amount of times in both the parent and send snapshots, its - * iversion becames the same in both snapshots, whence the inode item is + * iversion becomes the same in both snapshots, whence the inode item is * the same on both snapshots. */ if (sctx->cur_ino != sctx->cmp_key->objectid) @@ -7409,23 +7416,21 @@ long btrfs_ioctl_send(struct file *mnt_file, struct btrfs_ioctl_send_args *arg) if (ret) goto out; - mutex_lock(&fs_info->balance_mutex); - if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) { - mutex_unlock(&fs_info->balance_mutex); + spin_lock(&fs_info->send_reloc_lock); + if (test_bit(BTRFS_FS_RELOC_RUNNING, &fs_info->flags)) { + spin_unlock(&fs_info->send_reloc_lock); btrfs_warn_rl(fs_info, - "cannot run send because a balance operation is in progress"); + "cannot run send because a relocation operation is in progress"); ret = -EAGAIN; goto out; } fs_info->send_in_progress++; - mutex_unlock(&fs_info->balance_mutex); + spin_unlock(&fs_info->send_reloc_lock); - current->journal_info = BTRFS_SEND_TRANS_STUB; ret = send_subvol(sctx); - current->journal_info = NULL; - mutex_lock(&fs_info->balance_mutex); + spin_lock(&fs_info->send_reloc_lock); fs_info->send_in_progress--; - mutex_unlock(&fs_info->balance_mutex); + spin_unlock(&fs_info->send_reloc_lock); if (ret < 0) goto out; diff --git a/fs/btrfs/space-info.c b/fs/btrfs/space-info.c index 2dc674b7c3b1..f79bf85f2439 100644 --- a/fs/btrfs/space-info.c +++ b/fs/btrfs/space-info.c @@ -133,18 +133,13 @@ * operations, however they won't be usable until the transaction commits. * * COMMIT_TRANS - * may_commit_transaction() is the ultimate arbiter on whether we commit the - * transaction or not. In order to avoid constantly churning we do all the - * above flushing first and then commit the transaction as the last resort. - * However we need to take into account things like pinned space that would - * be freed, plus any delayed work we may not have gotten rid of in the case - * of metadata. - * - * FORCE_COMMIT_TRANS - * For use by the preemptive flusher. We use this to bypass the ticketing - * checks in may_commit_transaction, as we have more information about the - * overall state of the system and may want to commit the transaction ahead - * of actual ENOSPC conditions. + * This will commit the transaction. Historically we had a lot of logic + * surrounding whether or not we'd commit the transaction, but this waits born + * out of a pre-tickets era where we could end up committing the transaction + * thousands of times in a row without making progress. Now thanks to our + * ticketing system we know if we're not making progress and can error + * everybody out after a few commits rather than burning the disk hoping for + * a different answer. * * OVERCOMMIT * @@ -197,13 +192,6 @@ static int create_space_info(struct btrfs_fs_info *info, u64 flags) if (!space_info) return -ENOMEM; - ret = percpu_counter_init(&space_info->total_bytes_pinned, 0, - GFP_KERNEL); - if (ret) { - kfree(space_info); - return ret; - } - for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) INIT_LIST_HEAD(&space_info->block_groups[i]); init_rwsem(&space_info->groups_sem); @@ -389,7 +377,7 @@ void btrfs_try_granting_tickets(struct btrfs_fs_info *fs_info, ticket = list_first_entry(head, struct reserve_ticket, list); - /* Check and see if our ticket can be satisified now. */ + /* Check and see if our ticket can be satisfied now. */ if ((used + ticket->bytes <= space_info->total_bytes) || btrfs_can_overcommit(fs_info, space_info, ticket->bytes, flush)) { @@ -495,7 +483,8 @@ static inline u64 calc_reclaim_items_nr(struct btrfs_fs_info *fs_info, */ static void shrink_delalloc(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info, - u64 to_reclaim, bool wait_ordered) + u64 to_reclaim, bool wait_ordered, + bool for_preempt) { struct btrfs_trans_handle *trans; u64 delalloc_bytes; @@ -532,7 +521,7 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, * ordered extents, otherwise we'll waste time trying to flush delalloc * that likely won't give us the space back we need. */ - if (ordered_bytes > delalloc_bytes) + if (ordered_bytes > delalloc_bytes && !for_preempt) wait_ordered = true; loops = 0; @@ -551,6 +540,14 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, break; } + /* + * If we are for preemption we just want a one-shot of delalloc + * flushing so we can stop flushing if we decide we don't need + * to anymore. + */ + if (for_preempt) + break; + spin_lock(&space_info->lock); if (list_empty(&space_info->tickets) && list_empty(&space_info->priority_tickets)) { @@ -566,109 +563,6 @@ static void shrink_delalloc(struct btrfs_fs_info *fs_info, } } -/** - * Possibly commit the transaction if its ok to - * - * @fs_info: the filesystem - * @space_info: space_info we are checking for commit, either data or metadata - * - * This will check to make sure that committing the transaction will actually - * get us somewhere and then commit the transaction if it does. Otherwise it - * will return -ENOSPC. - */ -static int may_commit_transaction(struct btrfs_fs_info *fs_info, - struct btrfs_space_info *space_info) -{ - struct reserve_ticket *ticket = NULL; - struct btrfs_block_rsv *delayed_rsv = &fs_info->delayed_block_rsv; - struct btrfs_block_rsv *delayed_refs_rsv = &fs_info->delayed_refs_rsv; - struct btrfs_block_rsv *trans_rsv = &fs_info->trans_block_rsv; - struct btrfs_trans_handle *trans; - u64 reclaim_bytes = 0; - u64 bytes_needed = 0; - u64 cur_free_bytes = 0; - - trans = (struct btrfs_trans_handle *)current->journal_info; - if (trans) - return -EAGAIN; - - spin_lock(&space_info->lock); - cur_free_bytes = btrfs_space_info_used(space_info, true); - if (cur_free_bytes < space_info->total_bytes) - cur_free_bytes = space_info->total_bytes - cur_free_bytes; - else - cur_free_bytes = 0; - - if (!list_empty(&space_info->priority_tickets)) - ticket = list_first_entry(&space_info->priority_tickets, - struct reserve_ticket, list); - else if (!list_empty(&space_info->tickets)) - ticket = list_first_entry(&space_info->tickets, - struct reserve_ticket, list); - if (ticket) - bytes_needed = ticket->bytes; - - if (bytes_needed > cur_free_bytes) - bytes_needed -= cur_free_bytes; - else - bytes_needed = 0; - spin_unlock(&space_info->lock); - - if (!bytes_needed) - return 0; - - trans = btrfs_join_transaction(fs_info->extent_root); - if (IS_ERR(trans)) - return PTR_ERR(trans); - - /* - * See if there is enough pinned space to make this reservation, or if - * we have block groups that are going to be freed, allowing us to - * possibly do a chunk allocation the next loop through. - */ - if (test_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags) || - __percpu_counter_compare(&space_info->total_bytes_pinned, - bytes_needed, - BTRFS_TOTAL_BYTES_PINNED_BATCH) >= 0) - goto commit; - - /* - * See if there is some space in the delayed insertion reserve for this - * reservation. If the space_info's don't match (like for DATA or - * SYSTEM) then just go enospc, reclaiming this space won't recover any - * space to satisfy those reservations. - */ - if (space_info != delayed_rsv->space_info) - goto enospc; - - spin_lock(&delayed_rsv->lock); - reclaim_bytes += delayed_rsv->reserved; - spin_unlock(&delayed_rsv->lock); - - spin_lock(&delayed_refs_rsv->lock); - reclaim_bytes += delayed_refs_rsv->reserved; - spin_unlock(&delayed_refs_rsv->lock); - - spin_lock(&trans_rsv->lock); - reclaim_bytes += trans_rsv->reserved; - spin_unlock(&trans_rsv->lock); - - if (reclaim_bytes >= bytes_needed) - goto commit; - bytes_needed -= reclaim_bytes; - - if (__percpu_counter_compare(&space_info->total_bytes_pinned, - bytes_needed, - BTRFS_TOTAL_BYTES_PINNED_BATCH) < 0) - goto enospc; - -commit: - return btrfs_commit_transaction(trans); -enospc: - btrfs_end_transaction(trans); - return -ENOSPC; -} - /* * Try to flush some data based on policy set by @state. This is only advisory * and may fail for various reasons. The caller is supposed to examine the @@ -702,7 +596,7 @@ static void flush_space(struct btrfs_fs_info *fs_info, case FLUSH_DELALLOC: case FLUSH_DELALLOC_WAIT: shrink_delalloc(fs_info, space_info, num_bytes, - state == FLUSH_DELALLOC_WAIT); + state == FLUSH_DELALLOC_WAIT, for_preempt); break; case FLUSH_DELAYED_REFS_NR: case FLUSH_DELAYED_REFS: @@ -743,9 +637,7 @@ static void flush_space(struct btrfs_fs_info *fs_info, btrfs_wait_on_delayed_iputs(fs_info); break; case COMMIT_TRANS: - ret = may_commit_transaction(fs_info, space_info); - break; - case FORCE_COMMIT_TRANS: + ASSERT(current->journal_info == NULL); trans = btrfs_join_transaction(root); if (IS_ERR(trans)) { ret = PTR_ERR(trans); @@ -792,12 +684,14 @@ btrfs_calc_reclaim_metadata_size(struct btrfs_fs_info *fs_info, static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, struct btrfs_space_info *space_info) { + u64 global_rsv_size = fs_info->global_block_rsv.reserved; u64 ordered, delalloc; u64 thresh = div_factor_fine(space_info->total_bytes, 98); u64 used; /* If we're just plain full then async reclaim just slows us down. */ - if ((space_info->bytes_used + space_info->bytes_reserved) >= thresh) + if ((space_info->bytes_used + space_info->bytes_reserved + + global_rsv_size) >= thresh) return false; /* @@ -838,8 +732,10 @@ static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, thresh = calc_available_free_space(fs_info, space_info, BTRFS_RESERVE_FLUSH_ALL); - thresh += (space_info->total_bytes - space_info->bytes_used - - space_info->bytes_reserved - space_info->bytes_readonly); + used = space_info->bytes_used + space_info->bytes_reserved + + space_info->bytes_readonly + global_rsv_size; + if (used < space_info->total_bytes) + thresh += space_info->total_bytes - used; thresh >>= space_info->clamp; used = space_info->bytes_pinned; @@ -860,14 +756,20 @@ static bool need_preemptive_reclaim(struct btrfs_fs_info *fs_info, * clearly be heavy enough to warrant preemptive flushing. In the case * of heavy DIO or ordered reservations, preemptive flushing will just * waste time and cause us to slow down. + * + * We want to make sure we truly are maxed out on ordered however, so + * cut ordered in half, and if it's still higher than delalloc then we + * can keep flushing. This is to avoid the case where we start + * flushing, and now delalloc == ordered and we stop preemptively + * flushing when we could still have several gigs of delalloc to flush. */ - ordered = percpu_counter_read_positive(&fs_info->ordered_bytes); + ordered = percpu_counter_read_positive(&fs_info->ordered_bytes) >> 1; delalloc = percpu_counter_read_positive(&fs_info->delalloc_bytes); if (ordered >= delalloc) used += fs_info->delayed_refs_rsv.reserved + fs_info->delayed_block_rsv.reserved; else - used += space_info->bytes_may_use; + used += space_info->bytes_may_use - global_rsv_size; return (used >= thresh && !btrfs_fs_closing(fs_info) && !test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state)); @@ -921,7 +823,6 @@ static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info, { struct reserve_ticket *ticket; u64 tickets_id = space_info->tickets_id; - u64 first_ticket_bytes = 0; if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) { btrfs_info(fs_info, "cannot satisfy tickets, dumping space info"); @@ -937,21 +838,6 @@ static bool maybe_fail_all_tickets(struct btrfs_fs_info *fs_info, steal_from_global_rsv(fs_info, space_info, ticket)) return true; - /* - * may_commit_transaction will avoid committing the transaction - * if it doesn't feel like the space reclaimed by the commit - * would result in the ticket succeeding. However if we have a - * smaller ticket in the queue it may be small enough to be - * satisified by committing the transaction, so if any - * subsequent ticket is smaller than the first ticket go ahead - * and send us back for another loop through the enospc flushing - * code. - */ - if (first_ticket_bytes == 0) - first_ticket_bytes = ticket->bytes; - else if (first_ticket_bytes > ticket->bytes) - return true; - if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) btrfs_info(fs_info, "failing ticket with %llu bytes", ticket->bytes); @@ -1117,7 +1003,7 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) (delayed_block_rsv->reserved + delayed_refs_rsv->reserved)) { to_reclaim = space_info->bytes_pinned; - flush = FORCE_COMMIT_TRANS; + flush = COMMIT_TRANS; } else if (delayed_block_rsv->reserved > delayed_refs_rsv->reserved) { to_reclaim = delayed_block_rsv->reserved; @@ -1171,28 +1057,9 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) * immediately re-usable, it comes in the form of a delayed ref, which must be * run and then the transaction must be committed. * - * FLUSH_DELAYED_REFS - * The above two cases generate delayed refs that will affect - * ->total_bytes_pinned. However this counter can be inconsistent with - * reality if there are outstanding delayed refs. This is because we adjust - * the counter based solely on the current set of delayed refs and disregard - * any on-disk state which might include more refs. So for example, if we - * have an extent with 2 references, but we only drop 1, we'll see that there - * is a negative delayed ref count for the extent and assume that the space - * will be freed, and thus increase ->total_bytes_pinned. - * - * Running the delayed refs gives us the actual real view of what will be - * freed at the transaction commit time. This stage will not actually free - * space for us, it just makes sure that may_commit_transaction() has all of - * the information it needs to make the right decision. - * * COMMIT_TRANS - * This is where we reclaim all of the pinned space generated by the previous - * two stages. We will not commit the transaction if we don't think we're - * likely to satisfy our request, which means if our current free space + - * total_bytes_pinned < reservation we will not commit. This is why the - * previous states are actually important, to make sure we know for sure - * whether committing the transaction will allow us to make progress. + * This is where we reclaim all of the pinned space generated by running the + * iputs * * ALLOC_CHUNK_FORCE * For data we start with alloc chunk force, however we could have been full @@ -1202,7 +1069,6 @@ static void btrfs_preempt_reclaim_metadata_space(struct work_struct *work) static const enum btrfs_flush_state data_flush_states[] = { FLUSH_DELALLOC_WAIT, RUN_DELAYED_IPUTS, - FLUSH_DELAYED_REFS, COMMIT_TRANS, ALLOC_CHUNK_FORCE, }; @@ -1561,6 +1427,15 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, flush == BTRFS_RESERVE_FLUSH_DATA) { list_add_tail(&ticket.list, &space_info->tickets); if (!space_info->flush) { + /* + * We were forced to add a reserve ticket, so + * our preemptive flushing is unable to keep + * up. Clamp down on the threshold for the + * preemptive flushing in order to keep up with + * the workload. + */ + maybe_clamp_preempt(fs_info, space_info); + space_info->flush = 1; trace_btrfs_trigger_flush(fs_info, space_info->flags, @@ -1572,14 +1447,6 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, list_add_tail(&ticket.list, &space_info->priority_tickets); } - - /* - * We were forced to add a reserve ticket, so our preemptive - * flushing is unable to keep up. Clamp down on the threshold - * for the preemptive flushing in order to keep up with the - * workload. - */ - maybe_clamp_preempt(fs_info, space_info); } else if (!ret && space_info->flags & BTRFS_BLOCK_GROUP_METADATA) { used += orig_bytes; /* @@ -1588,8 +1455,8 @@ static int __reserve_bytes(struct btrfs_fs_info *fs_info, * the async reclaim as we will panic. */ if (!test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags) && - need_preemptive_reclaim(fs_info, space_info) && - !work_busy(&fs_info->preempt_reclaim_work)) { + !work_busy(&fs_info->preempt_reclaim_work) && + need_preemptive_reclaim(fs_info, space_info)) { trace_btrfs_trigger_flush(fs_info, space_info->flags, orig_bytes, flush, "preempt"); queue_work(system_unbound_wq, diff --git a/fs/btrfs/space-info.h b/fs/btrfs/space-info.h index b1a8ffb03b3e..cb5056472e79 100644 --- a/fs/btrfs/space-info.h +++ b/fs/btrfs/space-info.h @@ -43,18 +43,6 @@ struct btrfs_space_info { u64 flags; - /* - * bytes_pinned is kept in line with what is actually pinned, as in - * we've called update_block_group and dropped the bytes_used counter - * and increased the bytes_pinned counter. However this means that - * bytes_pinned does not reflect the bytes that will be pinned once the - * delayed refs are flushed, so this counter is inc'ed every time we - * call btrfs_free_extent so it is a realtime count of what will be - * freed once the transaction is committed. It will be zeroed every - * time the transaction commits. - */ - struct percpu_counter total_bytes_pinned; - struct list_head list; /* Protected by the spinlock 'lock'. */ struct list_head ro_bgs; @@ -157,22 +145,4 @@ static inline void btrfs_space_info_free_bytes_may_use( } int btrfs_reserve_data_bytes(struct btrfs_fs_info *fs_info, u64 bytes, enum btrfs_reserve_flush_enum flush); - -static inline void __btrfs_mod_total_bytes_pinned( - struct btrfs_space_info *space_info, - s64 mod) -{ - percpu_counter_add_batch(&space_info->total_bytes_pinned, mod, - BTRFS_TOTAL_BYTES_PINNED_BATCH); -} - -static inline void btrfs_mod_total_bytes_pinned(struct btrfs_fs_info *fs_info, - u64 flags, s64 mod) -{ - struct btrfs_space_info *space_info = btrfs_find_space_info(fs_info, flags); - - ASSERT(space_info); - __btrfs_mod_total_bytes_pinned(space_info, mod); -} - #endif /* BTRFS_SPACE_INFO_H */ diff --git a/fs/btrfs/subpage.c b/fs/btrfs/subpage.c index 2d19089ab625..640bcd21bf28 100644 --- a/fs/btrfs/subpage.c +++ b/fs/btrfs/subpage.c @@ -3,6 +3,7 @@ #include #include "ctree.h" #include "subpage.h" +#include "btrfs_inode.h" /* * Subpage (sectorsize < PAGE_SIZE) support overview: @@ -110,10 +111,12 @@ int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info, if (!*ret) return -ENOMEM; spin_lock_init(&(*ret)->lock); - if (type == BTRFS_SUBPAGE_METADATA) + if (type == BTRFS_SUBPAGE_METADATA) { atomic_set(&(*ret)->eb_refs, 0); - else + } else { atomic_set(&(*ret)->readers, 0); + atomic_set(&(*ret)->writers, 0); + } return 0; } @@ -183,12 +186,10 @@ void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, { struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; const int nbits = len >> fs_info->sectorsize_bits; - int ret; btrfs_subpage_assert(fs_info, page, start, len); - ret = atomic_add_return(nbits, &subpage->readers); - ASSERT(ret == nbits); + atomic_add(nbits, &subpage->readers); } void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, @@ -196,10 +197,95 @@ void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, { struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; const int nbits = len >> fs_info->sectorsize_bits; + bool is_data; + bool last; btrfs_subpage_assert(fs_info, page, start, len); + is_data = is_data_inode(page->mapping->host); ASSERT(atomic_read(&subpage->readers) >= nbits); - if (atomic_sub_and_test(nbits, &subpage->readers)) + last = atomic_sub_and_test(nbits, &subpage->readers); + + /* + * For data we need to unlock the page if the last read has finished. + * + * And please don't replace @last with atomic_sub_and_test() call + * inside if () condition. + * As we want the atomic_sub_and_test() to be always executed. + */ + if (is_data && last) + unlock_page(page); +} + +static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len) +{ + u64 orig_start = *start; + u32 orig_len = *len; + + *start = max_t(u64, page_offset(page), orig_start); + *len = min_t(u64, page_offset(page) + PAGE_SIZE, + orig_start + orig_len) - *start; +} + +void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const int nbits = (len >> fs_info->sectorsize_bits); + int ret; + + btrfs_subpage_assert(fs_info, page, start, len); + + ASSERT(atomic_read(&subpage->readers) == 0); + ret = atomic_add_return(nbits, &subpage->writers); + ASSERT(ret == nbits); +} + +bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const int nbits = (len >> fs_info->sectorsize_bits); + + btrfs_subpage_assert(fs_info, page, start, len); + + ASSERT(atomic_read(&subpage->writers) >= nbits); + return atomic_sub_and_test(nbits, &subpage->writers); +} + +/* + * Lock a page for delalloc page writeback. + * + * Return -EAGAIN if the page is not properly initialized. + * Return 0 with the page locked, and writer counter updated. + * + * Even with 0 returned, the page still need extra check to make sure + * it's really the correct page, as the caller is using + * find_get_pages_contig(), which can race with page invalidating. + */ +int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { + lock_page(page); + return 0; + } + lock_page(page); + if (!PagePrivate(page) || !page->private) { + unlock_page(page); + return -EAGAIN; + } + btrfs_subpage_clamp_range(page, &start, &len); + btrfs_subpage_start_writer(fs_info, page, start, len); + return 0; +} + +void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) + return unlock_page(page); + btrfs_subpage_clamp_range(page, &start, &len); + if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len)) unlock_page(page); } @@ -354,6 +440,32 @@ void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info, spin_unlock_irqrestore(&subpage->lock, flags); } +void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); + unsigned long flags; + + spin_lock_irqsave(&subpage->lock, flags); + subpage->ordered_bitmap |= tmp; + SetPageOrdered(page); + spin_unlock_irqrestore(&subpage->lock, flags); +} + +void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len) +{ + struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; + const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); + unsigned long flags; + + spin_lock_irqsave(&subpage->lock, flags); + subpage->ordered_bitmap &= ~tmp; + if (subpage->ordered_bitmap == 0) + ClearPageOrdered(page); + spin_unlock_irqrestore(&subpage->lock, flags); +} /* * Unlike set/clear which is dependent on each page status, for test all bits * are tested in the same way. @@ -376,6 +488,7 @@ IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate); IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error); IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty); IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback); +IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered); /* * Note that, in selftests (extent-io-tests), we can have empty fs_info passed @@ -408,6 +521,34 @@ bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ return test_page_func(page); \ return btrfs_subpage_test_##name(fs_info, page, start, len); \ +} \ +void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ + set_page_func(page); \ + return; \ + } \ + btrfs_subpage_clamp_range(page, &start, &len); \ + btrfs_subpage_set_##name(fs_info, page, start, len); \ +} \ +void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \ + clear_page_func(page); \ + return; \ + } \ + btrfs_subpage_clamp_range(page, &start, &len); \ + btrfs_subpage_clear_##name(fs_info, page, start, len); \ +} \ +bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len) \ +{ \ + if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \ + return test_page_func(page); \ + btrfs_subpage_clamp_range(page, &start, &len); \ + return btrfs_subpage_test_##name(fs_info, page, start, len); \ } IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate, PageUptodate); @@ -416,3 +557,5 @@ IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io, PageDirty); IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback, PageWriteback); +IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered, + PageOrdered); diff --git a/fs/btrfs/subpage.h b/fs/btrfs/subpage.h index bfd626e955be..4d7aca85d915 100644 --- a/fs/btrfs/subpage.h +++ b/fs/btrfs/subpage.h @@ -22,6 +22,14 @@ struct btrfs_subpage { u16 error_bitmap; u16 dirty_bitmap; u16 writeback_bitmap; + /* + * Both data and metadata needs to track how many readers are for the + * page. + * Data relies on @readers to unlock the page when last reader finished. + * While metadata doesn't need page unlock, it needs to prevent + * page::private get cleared before the last end_page_read(). + */ + atomic_t readers; union { /* * Structures only used by metadata @@ -32,7 +40,10 @@ struct btrfs_subpage { atomic_t eb_refs; /* Structures only used by data */ struct { - atomic_t readers; + atomic_t writers; + + /* Tracke pending ordered extent in this sector */ + u16 ordered_bitmap; }; }; }; @@ -63,6 +74,15 @@ void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info, void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, struct page *page, u64 start, u32 len); +void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len); +bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len); +int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len); +void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info, + struct page *page, u64 start, u32 len); + /* * Template for subpage related operations. * @@ -72,6 +92,10 @@ void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info, * btrfs_page_*() are for call sites where the page can either be subpage * specific or regular page. The function will handle both cases. * But the range still needs to be inside the page. + * + * btrfs_page_clamp_*() are similar to btrfs_page_*(), except the range doesn't + * need to be inside the page. Those functions will truncate the range + * automatically. */ #define DECLARE_BTRFS_SUBPAGE_OPS(name) \ void btrfs_subpage_set_##name(const struct btrfs_fs_info *fs_info, \ @@ -85,12 +109,19 @@ void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \ void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \ struct page *page, u64 start, u32 len); \ bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \ + struct page *page, u64 start, u32 len); \ +bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \ struct page *page, u64 start, u32 len); DECLARE_BTRFS_SUBPAGE_OPS(uptodate); DECLARE_BTRFS_SUBPAGE_OPS(error); DECLARE_BTRFS_SUBPAGE_OPS(dirty); DECLARE_BTRFS_SUBPAGE_OPS(writeback); +DECLARE_BTRFS_SUBPAGE_OPS(ordered); bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info, struct page *page, u64 start, u32 len); diff --git a/fs/btrfs/super.c b/fs/btrfs/super.c index 4a396c1147f1..d07b18b2b250 100644 --- a/fs/btrfs/super.c +++ b/fs/btrfs/super.c @@ -299,17 +299,6 @@ void __btrfs_abort_transaction(struct btrfs_trans_handle *trans, struct btrfs_fs_info *fs_info = trans->fs_info; WRITE_ONCE(trans->aborted, errno); - /* Nothing used. The other threads that have joined this - * transaction may be able to continue. */ - if (!trans->dirty && list_empty(&trans->new_bgs)) { - const char *errstr; - - errstr = btrfs_decode_error(errno); - btrfs_warn(fs_info, - "%s:%d: Aborting unused transaction(%s).", - function, line, errstr); - return; - } WRITE_ONCE(trans->transaction->aborted, errno); /* Wake up anybody who may be waiting on this transaction */ wake_up(&fs_info->transaction_wait); @@ -945,8 +934,7 @@ int btrfs_parse_options(struct btrfs_fs_info *info, char *options, case Opt_check_integrity_including_extent_data: btrfs_info(info, "enabling check integrity including extent data"); - btrfs_set_opt(info->mount_opt, - CHECK_INTEGRITY_INCLUDING_EXTENT_DATA); + btrfs_set_opt(info->mount_opt, CHECK_INTEGRITY_DATA); btrfs_set_opt(info->mount_opt, CHECK_INTEGRITY); break; case Opt_check_integrity: @@ -1527,7 +1515,7 @@ static int btrfs_show_options(struct seq_file *seq, struct dentry *dentry) if (btrfs_test_opt(info, SKIP_BALANCE)) seq_puts(seq, ",skip_balance"); #ifdef CONFIG_BTRFS_FS_CHECK_INTEGRITY - if (btrfs_test_opt(info, CHECK_INTEGRITY_INCLUDING_EXTENT_DATA)) + if (btrfs_test_opt(info, CHECK_INTEGRITY_DATA)) seq_puts(seq, ",check_int_data"); else if (btrfs_test_opt(info, CHECK_INTEGRITY)) seq_puts(seq, ",check_int"); diff --git a/fs/btrfs/sysfs.c b/fs/btrfs/sysfs.c index 436ac7b4b334..9d1d140118ff 100644 --- a/fs/btrfs/sysfs.c +++ b/fs/btrfs/sysfs.c @@ -429,7 +429,7 @@ static ssize_t btrfs_discard_bitmap_bytes_show(struct kobject *kobj, { struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); - return scnprintf(buf, PAGE_SIZE, "%lld\n", + return scnprintf(buf, PAGE_SIZE, "%llu\n", fs_info->discard_ctl.discard_bitmap_bytes); } BTRFS_ATTR(discard, discard_bitmap_bytes, btrfs_discard_bitmap_bytes_show); @@ -451,7 +451,7 @@ static ssize_t btrfs_discard_extent_bytes_show(struct kobject *kobj, { struct btrfs_fs_info *fs_info = discard_to_fs_info(kobj); - return scnprintf(buf, PAGE_SIZE, "%lld\n", + return scnprintf(buf, PAGE_SIZE, "%llu\n", fs_info->discard_ctl.discard_extent_bytes); } BTRFS_ATTR(discard, discard_extent_bytes, btrfs_discard_extent_bytes_show); @@ -665,15 +665,6 @@ static ssize_t btrfs_space_info_show_##field(struct kobject *kobj, \ } \ BTRFS_ATTR(space_info, field, btrfs_space_info_show_##field) -static ssize_t btrfs_space_info_show_total_bytes_pinned(struct kobject *kobj, - struct kobj_attribute *a, - char *buf) -{ - struct btrfs_space_info *sinfo = to_space_info(kobj); - s64 val = percpu_counter_sum(&sinfo->total_bytes_pinned); - return scnprintf(buf, PAGE_SIZE, "%lld\n", val); -} - SPACE_INFO_ATTR(flags); SPACE_INFO_ATTR(total_bytes); SPACE_INFO_ATTR(bytes_used); @@ -684,8 +675,6 @@ SPACE_INFO_ATTR(bytes_readonly); SPACE_INFO_ATTR(bytes_zone_unusable); SPACE_INFO_ATTR(disk_used); SPACE_INFO_ATTR(disk_total); -BTRFS_ATTR(space_info, total_bytes_pinned, - btrfs_space_info_show_total_bytes_pinned); static struct attribute *space_info_attrs[] = { BTRFS_ATTR_PTR(space_info, flags), @@ -698,7 +687,6 @@ static struct attribute *space_info_attrs[] = { BTRFS_ATTR_PTR(space_info, bytes_zone_unusable), BTRFS_ATTR_PTR(space_info, disk_used), BTRFS_ATTR_PTR(space_info, disk_total), - BTRFS_ATTR_PTR(space_info, total_bytes_pinned), NULL, }; ATTRIBUTE_GROUPS(space_info); @@ -706,7 +694,6 @@ ATTRIBUTE_GROUPS(space_info); static void space_info_release(struct kobject *kobj) { struct btrfs_space_info *sinfo = to_space_info(kobj); - percpu_counter_destroy(&sinfo->total_bytes_pinned); kfree(sinfo); } @@ -1455,6 +1442,33 @@ static ssize_t btrfs_devinfo_replace_target_show(struct kobject *kobj, } BTRFS_ATTR(devid, replace_target, btrfs_devinfo_replace_target_show); +static ssize_t btrfs_devinfo_scrub_speed_max_show(struct kobject *kobj, + struct kobj_attribute *a, + char *buf) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + return scnprintf(buf, PAGE_SIZE, "%llu\n", + READ_ONCE(device->scrub_speed_max)); +} + +static ssize_t btrfs_devinfo_scrub_speed_max_store(struct kobject *kobj, + struct kobj_attribute *a, + const char *buf, size_t len) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + char *endptr; + unsigned long long limit; + + limit = memparse(buf, &endptr); + WRITE_ONCE(device->scrub_speed_max, limit); + return len; +} +BTRFS_ATTR_RW(devid, scrub_speed_max, btrfs_devinfo_scrub_speed_max_show, + btrfs_devinfo_scrub_speed_max_store); + static ssize_t btrfs_devinfo_writeable_show(struct kobject *kobj, struct kobj_attribute *a, char *buf) { @@ -1468,10 +1482,40 @@ static ssize_t btrfs_devinfo_writeable_show(struct kobject *kobj, } BTRFS_ATTR(devid, writeable, btrfs_devinfo_writeable_show); +static ssize_t btrfs_devinfo_error_stats_show(struct kobject *kobj, + struct kobj_attribute *a, char *buf) +{ + struct btrfs_device *device = container_of(kobj, struct btrfs_device, + devid_kobj); + + if (!device->dev_stats_valid) + return scnprintf(buf, PAGE_SIZE, "invalid\n"); + + /* + * Print all at once so we get a snapshot of all values from the same + * time. Keep them in sync and in order of definition of + * btrfs_dev_stat_values. + */ + return scnprintf(buf, PAGE_SIZE, + "write_errs %d\n" + "read_errs %d\n" + "flush_errs %d\n" + "corruption_errs %d\n" + "generation_errs %d\n", + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_WRITE_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_READ_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_FLUSH_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_CORRUPTION_ERRS), + btrfs_dev_stat_read(device, BTRFS_DEV_STAT_GENERATION_ERRS)); +} +BTRFS_ATTR(devid, error_stats, btrfs_devinfo_error_stats_show); + static struct attribute *devid_attrs[] = { + BTRFS_ATTR_PTR(devid, error_stats), BTRFS_ATTR_PTR(devid, in_fs_metadata), BTRFS_ATTR_PTR(devid, missing), BTRFS_ATTR_PTR(devid, replace_target), + BTRFS_ATTR_PTR(devid, scrub_speed_max), BTRFS_ATTR_PTR(devid, writeable), NULL }; diff --git a/fs/btrfs/tests/extent-map-tests.c b/fs/btrfs/tests/extent-map-tests.c index c0aefe6dee0b..319fed82d741 100644 --- a/fs/btrfs/tests/extent-map-tests.c +++ b/fs/btrfs/tests/extent-map-tests.c @@ -557,7 +557,7 @@ int btrfs_test_extent_map(void) { /* * Test a chunk with 2 data stripes one of which - * interesects the physical address of the super block + * intersects the physical address of the super block * is correctly recognised. */ .raid_type = BTRFS_BLOCK_GROUP_RAID1, diff --git a/fs/btrfs/transaction.c b/fs/btrfs/transaction.c index f75de9f6c0ad..50318231c1a8 100644 --- a/fs/btrfs/transaction.c +++ b/fs/btrfs/transaction.c @@ -583,9 +583,6 @@ start_transaction(struct btrfs_root *root, unsigned int num_items, bool do_chunk_alloc = false; int ret; - /* Send isn't supposed to start transactions. */ - ASSERT(current->journal_info != BTRFS_SEND_TRANS_STUB); - if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state)) return ERR_PTR(-EROFS); @@ -1406,8 +1403,10 @@ int btrfs_defrag_root(struct btrfs_root *root) while (1) { trans = btrfs_start_transaction(root, 0); - if (IS_ERR(trans)) - return PTR_ERR(trans); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + break; + } ret = btrfs_defrag_leaves(trans, root); @@ -1476,7 +1475,7 @@ static int qgroup_account_snapshot(struct btrfs_trans_handle *trans, ret = btrfs_run_delayed_refs(trans, (unsigned long)-1); if (ret) { btrfs_abort_transaction(trans, ret); - goto out; + return ret; } /* @@ -1868,31 +1867,6 @@ int btrfs_transaction_blocked(struct btrfs_fs_info *info) return ret; } -/* - * wait for the current transaction commit to start and block subsequent - * transaction joins - */ -static void wait_current_trans_commit_start(struct btrfs_fs_info *fs_info, - struct btrfs_transaction *trans) -{ - wait_event(fs_info->transaction_blocked_wait, - trans->state >= TRANS_STATE_COMMIT_START || - TRANS_ABORTED(trans)); -} - -/* - * wait for the current transaction to start and then become unblocked. - * caller holds ref. - */ -static void wait_current_trans_commit_start_and_unblock( - struct btrfs_fs_info *fs_info, - struct btrfs_transaction *trans) -{ - wait_event(fs_info->transaction_wait, - trans->state >= TRANS_STATE_UNBLOCKED || - TRANS_ABORTED(trans)); -} - /* * commit transactions asynchronously. once btrfs_commit_transaction_async * returns, any subsequent transaction will not be allowed to join. @@ -1920,8 +1894,7 @@ static void do_async_commit(struct work_struct *work) kfree(ac); } -int btrfs_commit_transaction_async(struct btrfs_trans_handle *trans, - int wait_for_unblock) +int btrfs_commit_transaction_async(struct btrfs_trans_handle *trans) { struct btrfs_fs_info *fs_info = trans->fs_info; struct btrfs_async_commit *ac; @@ -1953,13 +1926,13 @@ int btrfs_commit_transaction_async(struct btrfs_trans_handle *trans, __sb_writers_release(fs_info->sb, SB_FREEZE_FS); schedule_work(&ac->work); - - /* wait for transaction to start and unblock */ - if (wait_for_unblock) - wait_current_trans_commit_start_and_unblock(fs_info, cur_trans); - else - wait_current_trans_commit_start(fs_info, cur_trans); - + /* + * Wait for the current transaction commit to start and block + * subsequent transaction joins + */ + wait_event(fs_info->transaction_blocked_wait, + cur_trans->state >= TRANS_STATE_COMMIT_START || + TRANS_ABORTED(cur_trans)); if (current->journal_info == trans) current->journal_info = NULL; @@ -2074,14 +2047,6 @@ int btrfs_commit_transaction(struct btrfs_trans_handle *trans) ASSERT(refcount_read(&trans->use_count) == 1); - /* - * Some places just start a transaction to commit it. We need to make - * sure that if this commit fails that the abort code actually marks the - * transaction as failed, so set trans->dirty to make the abort code do - * the right thing. - */ - trans->dirty = true; - /* Stop the commit early if ->aborted is set */ if (TRANS_ABORTED(cur_trans)) { ret = cur_trans->aborted; diff --git a/fs/btrfs/transaction.h b/fs/btrfs/transaction.h index 364cfbb4c5c5..07d76029f598 100644 --- a/fs/btrfs/transaction.h +++ b/fs/btrfs/transaction.h @@ -122,8 +122,6 @@ struct btrfs_transaction { #define TRANS_EXTWRITERS (__TRANS_START | __TRANS_ATTACH) -#define BTRFS_SEND_TRANS_STUB ((void *)1) - struct btrfs_trans_handle { u64 transid; u64 bytes_reserved; @@ -143,7 +141,6 @@ struct btrfs_trans_handle { bool allocating_chunk; bool can_flush_pending_bgs; bool reloc_reserved; - bool dirty; bool in_fsync; struct btrfs_root *root; struct btrfs_fs_info *fs_info; @@ -227,8 +224,7 @@ void btrfs_add_dead_root(struct btrfs_root *root); int btrfs_defrag_root(struct btrfs_root *root); int btrfs_clean_one_deleted_snapshot(struct btrfs_root *root); int btrfs_commit_transaction(struct btrfs_trans_handle *trans); -int btrfs_commit_transaction_async(struct btrfs_trans_handle *trans, - int wait_for_unblock); +int btrfs_commit_transaction_async(struct btrfs_trans_handle *trans); int btrfs_end_transaction_throttle(struct btrfs_trans_handle *trans); bool btrfs_should_end_transaction(struct btrfs_trans_handle *trans); void btrfs_throttle(struct btrfs_fs_info *fs_info); diff --git a/fs/btrfs/tree-log.c b/fs/btrfs/tree-log.c index dbcf8bb2f3b9..cab451d19547 100644 --- a/fs/btrfs/tree-log.c +++ b/fs/btrfs/tree-log.c @@ -4468,7 +4468,8 @@ static int btrfs_log_prealloc_extents(struct btrfs_trans_handle *trans, ret = btrfs_truncate_inode_items(trans, root->log_root, inode, truncate_offset, - BTRFS_EXTENT_DATA_KEY); + BTRFS_EXTENT_DATA_KEY, + NULL); } while (ret == -EAGAIN); if (ret) goto out; @@ -5416,7 +5417,7 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, &inode->runtime_flags); while(1) { ret = btrfs_truncate_inode_items(trans, - log, inode, 0, 0); + log, inode, 0, 0, NULL); if (ret != -EAGAIN) break; } @@ -5466,13 +5467,23 @@ static int btrfs_log_inode(struct btrfs_trans_handle *trans, btrfs_release_path(dst_path); if (need_log_inode_item) { err = log_inode_item(trans, log, dst_path, inode); - if (!err && !xattrs_logged) { - err = btrfs_log_all_xattrs(trans, root, inode, path, - dst_path); - btrfs_release_path(path); - } if (err) goto out_unlock; + /* + * If we are doing a fast fsync and the inode was logged before + * in this transaction, we don't need to log the xattrs because + * they were logged before. If xattrs were added, changed or + * deleted since the last time we logged the inode, then we have + * already logged them because the inode had the runtime flag + * BTRFS_INODE_COPY_EVERYTHING set. + */ + if (!xattrs_logged && inode->logged_trans < trans->transid) { + err = btrfs_log_all_xattrs(trans, root, inode, path, + dst_path); + if (err) + goto out_unlock; + btrfs_release_path(path); + } } if (fast_search) { ret = btrfs_log_changed_extents(trans, root, inode, dst_path, @@ -6371,6 +6382,7 @@ int btrfs_recover_log_trees(struct btrfs_root *log_root_tree) error: if (wc.trans) btrfs_end_transaction(wc.trans); + clear_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags); btrfs_free_path(path); return ret; } diff --git a/fs/btrfs/volumes.c b/fs/btrfs/volumes.c index 47d27059d064..782e16795bc4 100644 --- a/fs/btrfs/volumes.c +++ b/fs/btrfs/volumes.c @@ -717,7 +717,7 @@ static struct btrfs_fs_devices *find_fsid_changed( /* * Handles the case where scanned device is part of an fs that had - * multiple successful changes of FSID but curently device didn't + * multiple successful changes of FSID but currently device didn't * observe it. Meaning our fsid will be different than theirs. We need * to handle two subcases : * 1 - The fs still continues to have different METADATA/FSID uuids. @@ -1550,7 +1550,7 @@ static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start, * check to ensure dev extents are not double allocated. * This makes the function safe to allocate dev extents but may not report * correct usable device space, as device extent freed in current transaction - * is not reported as avaiable. + * is not reported as available. */ static int find_free_dev_extent_start(struct btrfs_device *device, u64 num_bytes, u64 search_start, u64 *start, @@ -4217,14 +4217,6 @@ int btrfs_balance(struct btrfs_fs_info *fs_info, btrfs_bg_type_to_raid_name(data_target)); } - if (fs_info->send_in_progress) { - btrfs_warn_rl(fs_info, -"cannot run balance while send operations are in progress (%d in progress)", - fs_info->send_in_progress); - ret = -EAGAIN; - goto out; - } - ret = insert_balance_item(fs_info, bctl); if (ret && ret != -EEXIST) goto out; @@ -6127,17 +6119,17 @@ static bool need_full_stripe(enum btrfs_map_op op) * @em: mapping containing the logical extent * @op: type of operation - write or read * @logical: address that we want to figure out the geometry of - * @len: the length of IO we are going to perform, starting at @logical * @io_geom: pointer used to return values * * Returns < 0 in case a chunk for the given logical address cannot be found, * usually shouldn't happen unless @logical is corrupted, 0 otherwise. */ int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *em, - enum btrfs_map_op op, u64 logical, u64 len, + enum btrfs_map_op op, u64 logical, struct btrfs_io_geometry *io_geom) { struct map_lookup *map; + u64 len; u64 offset; u64 stripe_offset; u64 stripe_nr; @@ -6152,7 +6144,7 @@ int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *em, offset = logical - em->start; /* Len of a stripe in a chunk */ stripe_len = map->stripe_len; - /* Stripe wher this block falls in */ + /* Stripe where this block falls in */ stripe_nr = div64_u64(offset, stripe_len); /* Offset of stripe in the chunk */ stripe_offset = stripe_nr * stripe_len; @@ -6243,7 +6235,7 @@ static int __btrfs_map_block(struct btrfs_fs_info *fs_info, em = btrfs_get_chunk_map(fs_info, logical, *length); ASSERT(!IS_ERR(em)); - ret = btrfs_get_io_geometry(fs_info, em, op, logical, *length, &geom); + ret = btrfs_get_io_geometry(fs_info, em, op, logical, &geom); if (ret < 0) return ret; @@ -6670,8 +6662,6 @@ blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio, * * If devid and uuid are both specified, the match must be exact, otherwise * only devid is used. - * - * If @seed is true, traverse through the seed devices. */ struct btrfs_device *btrfs_find_device(struct btrfs_fs_devices *fs_devices, u64 devid, u8 *uuid, u8 *fsid) @@ -7865,7 +7855,7 @@ static int verify_one_dev_extent(struct btrfs_fs_info *fs_info, ret = -EUCLEAN; } - /* Make sure no dev extent is beyond device bondary */ + /* Make sure no dev extent is beyond device boundary */ dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL); if (!dev) { btrfs_err(fs_info, "failed to find devid %llu", devid); diff --git a/fs/btrfs/volumes.h b/fs/btrfs/volumes.h index 9c0d84e5ec06..c7fc7caf575c 100644 --- a/fs/btrfs/volumes.h +++ b/fs/btrfs/volumes.h @@ -143,6 +143,9 @@ struct btrfs_device { struct completion kobj_unregister; /* For sysfs/FSID/devinfo/devid/ */ struct kobject devid_kobj; + + /* Bandwidth limit for scrub, in bytes */ + u64 scrub_speed_max; }; /* @@ -443,7 +446,7 @@ int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op, u64 logical, u64 *length, struct btrfs_bio **bbio_ret); int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, struct extent_map *map, - enum btrfs_map_op op, u64 logical, u64 len, + enum btrfs_map_op op, u64 logical, struct btrfs_io_geometry *io_geom); int btrfs_read_sys_array(struct btrfs_fs_info *fs_info); int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info); diff --git a/fs/btrfs/zoned.c b/fs/btrfs/zoned.c index f1f3b10d1dbb..297c0b1c0634 100644 --- a/fs/btrfs/zoned.c +++ b/fs/btrfs/zoned.c @@ -81,7 +81,7 @@ static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, * *: Special case, no superblock is written * 0: Use write pointer of zones[0] * 1: Use write pointer of zones[1] - * C: Compare super blcoks from zones[0] and zones[1], use the latest + * C: Compare super blocks from zones[0] and zones[1], use the latest * one determined by generation * x: Invalid state */ @@ -433,7 +433,7 @@ int btrfs_get_dev_zone_info(struct btrfs_device *device) } /* - * If zones[0] is conventional, always use the beggining of the + * If zones[0] is conventional, always use the beginning of the * zone to record superblock. No need to validate in that case. */ if (zone_info->sb_zones[BTRFS_NR_SB_LOG_ZONES * i].type == @@ -1140,6 +1140,10 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) } if (zone.type == BLK_ZONE_TYPE_CONVENTIONAL) { + btrfs_err_in_rcu(fs_info, + "zoned: unexpected conventional zone %llu on device %s (devid %llu)", + zone.start << SECTOR_SHIFT, + rcu_str_deref(device->name), device->devid); ret = -EIO; goto out; } @@ -1200,6 +1204,13 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) switch (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) { case 0: /* single */ + if (alloc_offsets[0] == WP_MISSING_DEV) { + btrfs_err(fs_info, + "zoned: cannot recover write pointer for zone %llu", + physical); + ret = -EIO; + goto out; + } cache->alloc_offset = alloc_offsets[0]; break; case BTRFS_BLOCK_GROUP_DUP: @@ -1217,6 +1228,13 @@ int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) } out: + if (cache->alloc_offset > fs_info->zone_size) { + btrfs_err(fs_info, + "zoned: invalid write pointer %llu in block group %llu", + cache->alloc_offset, cache->start); + ret = -EIO; + } + /* An extent is allocated after the write pointer */ if (!ret && num_conventional && last_alloc > cache->alloc_offset) { btrfs_err(fs_info, @@ -1515,3 +1533,24 @@ int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, length = wp - physical_pos; return btrfs_zoned_issue_zeroout(tgt_dev, physical_pos, length); } + +struct btrfs_device *btrfs_zoned_get_device(struct btrfs_fs_info *fs_info, + u64 logical, u64 length) +{ + struct btrfs_device *device; + struct extent_map *em; + struct map_lookup *map; + + em = btrfs_get_chunk_map(fs_info, logical, length); + if (IS_ERR(em)) + return ERR_CAST(em); + + map = em->map_lookup; + /* We only support single profile for now */ + ASSERT(map->num_stripes == 1); + device = map->stripes[0].dev; + + free_extent_map(em); + + return device; +} diff --git a/fs/btrfs/zoned.h b/fs/btrfs/zoned.h index e55d32595c2c..b0ae2608cb6b 100644 --- a/fs/btrfs/zoned.h +++ b/fs/btrfs/zoned.h @@ -65,6 +65,8 @@ void btrfs_revert_meta_write_pointer(struct btrfs_block_group *cache, int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length); int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, u64 physical_start, u64 physical_pos); +struct btrfs_device *btrfs_zoned_get_device(struct btrfs_fs_info *fs_info, + u64 logical, u64 length); #else /* CONFIG_BLK_DEV_ZONED */ static inline int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) @@ -191,6 +193,13 @@ static inline int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, return -EOPNOTSUPP; } +static inline struct btrfs_device *btrfs_zoned_get_device( + struct btrfs_fs_info *fs_info, + u64 logical, u64 length) +{ + return ERR_PTR(-EOPNOTSUPP); +} + #endif static inline bool btrfs_dev_is_sequential(struct btrfs_device *device, u64 pos) diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 6ca7d16593ff..d00455440d08 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -344,13 +344,9 @@ int fscrypt_fname_disk_to_usr(const struct inode *inode, offsetof(struct fscrypt_nokey_name, sha256)); BUILD_BUG_ON(BASE64_CHARS(FSCRYPT_NOKEY_NAME_MAX) > NAME_MAX); - if (hash) { - nokey_name.dirhash[0] = hash; - nokey_name.dirhash[1] = minor_hash; - } else { - nokey_name.dirhash[0] = 0; - nokey_name.dirhash[1] = 0; - } + nokey_name.dirhash[0] = hash; + nokey_name.dirhash[1] = minor_hash; + if (iname->len <= sizeof(nokey_name.bytes)) { memcpy(nokey_name.bytes, iname->name, iname->len); size = offsetof(struct fscrypt_nokey_name, bytes[iname->len]); diff --git a/fs/crypto/keysetup.c b/fs/crypto/keysetup.c index 163dba6c0c3f..32dc97e908fd 100644 --- a/fs/crypto/keysetup.c +++ b/fs/crypto/keysetup.c @@ -241,15 +241,40 @@ static int setup_per_mode_enc_key(struct fscrypt_info *ci, return err; } +/* + * Derive a SipHash key from the given fscrypt master key and the given + * application-specific information string. + * + * Note that the KDF produces a byte array, but the SipHash APIs expect the key + * as a pair of 64-bit words. Therefore, on big endian CPUs we have to do an + * endianness swap in order to get the same results as on little endian CPUs. + */ +static int fscrypt_derive_siphash_key(const struct fscrypt_master_key *mk, + u8 context, const u8 *info, + unsigned int infolen, siphash_key_t *key) +{ + int err; + + err = fscrypt_hkdf_expand(&mk->mk_secret.hkdf, context, info, infolen, + (u8 *)key, sizeof(*key)); + if (err) + return err; + + BUILD_BUG_ON(sizeof(*key) != 16); + BUILD_BUG_ON(ARRAY_SIZE(key->key) != 2); + le64_to_cpus(&key->key[0]); + le64_to_cpus(&key->key[1]); + return 0; +} + int fscrypt_derive_dirhash_key(struct fscrypt_info *ci, const struct fscrypt_master_key *mk) { int err; - err = fscrypt_hkdf_expand(&mk->mk_secret.hkdf, HKDF_CONTEXT_DIRHASH_KEY, - ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE, - (u8 *)&ci->ci_dirhash_key, - sizeof(ci->ci_dirhash_key)); + err = fscrypt_derive_siphash_key(mk, HKDF_CONTEXT_DIRHASH_KEY, + ci->ci_nonce, FSCRYPT_FILE_NONCE_SIZE, + &ci->ci_dirhash_key); if (err) return err; ci->ci_dirhash_key_initialized = true; @@ -284,10 +309,9 @@ static int fscrypt_setup_iv_ino_lblk_32_key(struct fscrypt_info *ci, if (mk->mk_ino_hash_key_initialized) goto unlock; - err = fscrypt_hkdf_expand(&mk->mk_secret.hkdf, - HKDF_CONTEXT_INODE_HASH_KEY, NULL, 0, - (u8 *)&mk->mk_ino_hash_key, - sizeof(mk->mk_ino_hash_key)); + err = fscrypt_derive_siphash_key(mk, + HKDF_CONTEXT_INODE_HASH_KEY, + NULL, 0, &mk->mk_ino_hash_key); if (err) goto unlock; /* pairs with smp_load_acquire() above */ diff --git a/fs/erofs/Kconfig b/fs/erofs/Kconfig index 858b3339f381..906af0c1998c 100644 --- a/fs/erofs/Kconfig +++ b/fs/erofs/Kconfig @@ -75,4 +75,3 @@ config EROFS_FS_ZIP Enable fixed-sized output compression for EROFS. If you don't want to enable compression feature, say N. - diff --git a/fs/erofs/compress.h b/fs/erofs/compress.h index aea129ddda74..3701c72bacb2 100644 --- a/fs/erofs/compress.h +++ b/fs/erofs/compress.h @@ -2,7 +2,6 @@ /* * Copyright (C) 2019 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #ifndef __EROFS_FS_COMPRESS_H #define __EROFS_FS_COMPRESS_H @@ -85,4 +84,3 @@ int z_erofs_decompress(struct z_erofs_decompress_req *rq, struct list_head *pagepool); #endif - diff --git a/fs/erofs/data.c b/fs/erofs/data.c index ebac756cb2a3..3787a5fb0a42 100644 --- a/fs/erofs/data.c +++ b/fs/erofs/data.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "internal.h" #include @@ -315,4 +314,3 @@ const struct address_space_operations erofs_raw_access_aops = { .readahead = erofs_raw_access_readahead, .bmap = erofs_bmap, }; - diff --git a/fs/erofs/decompressor.c b/fs/erofs/decompressor.c index 88e33addf229..a5bc4b1b7813 100644 --- a/fs/erofs/decompressor.c +++ b/fs/erofs/decompressor.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2019 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "compress.h" #include @@ -407,4 +406,3 @@ int z_erofs_decompress(struct z_erofs_decompress_req *rq, return z_erofs_shifted_transform(rq, pagepool); return z_erofs_decompress_generic(rq, pagepool); } - diff --git a/fs/erofs/dir.c b/fs/erofs/dir.c index 2776bb832127..eee9b0b31b63 100644 --- a/fs/erofs/dir.c +++ b/fs/erofs/dir.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "internal.h" @@ -139,4 +138,3 @@ const struct file_operations erofs_dir_fops = { .read = generic_read_dir, .iterate_shared = erofs_readdir, }; - diff --git a/fs/erofs/erofs_fs.h b/fs/erofs/erofs_fs.h index 8739d3adf51f..0f8da74570b4 100644 --- a/fs/erofs/erofs_fs.h +++ b/fs/erofs/erofs_fs.h @@ -4,7 +4,6 @@ * * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #ifndef __EROFS_FS_H #define __EROFS_FS_H @@ -348,4 +347,3 @@ static inline void erofs_check_ondisk_layout_definitions(void) } #endif - diff --git a/fs/erofs/inode.c b/fs/erofs/inode.c index 7ed2d7391692..aa8a0d770ba3 100644 --- a/fs/erofs/inode.c +++ b/fs/erofs/inode.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "xattr.h" @@ -374,4 +373,3 @@ const struct inode_operations erofs_fast_symlink_iops = { .listxattr = erofs_listxattr, .get_acl = erofs_get_acl, }; - diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index f92e3e32b9f4..543c2ff97d30 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #ifndef __EROFS_INTERNAL_H #define __EROFS_INTERNAL_H @@ -469,4 +468,3 @@ static inline int z_erofs_load_lz4_config(struct super_block *sb, #define EFSCORRUPTED EUCLEAN /* Filesystem is corrupted */ #endif /* __EROFS_INTERNAL_H */ - diff --git a/fs/erofs/namei.c b/fs/erofs/namei.c index 3a81e1f7fc06..a8271ce5e13f 100644 --- a/fs/erofs/namei.c +++ b/fs/erofs/namei.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "xattr.h" @@ -247,4 +246,3 @@ const struct inode_operations erofs_dir_iops = { .listxattr = erofs_listxattr, .get_acl = erofs_get_acl, }; - diff --git a/fs/erofs/super.c b/fs/erofs/super.c index bbf3bbd908e0..8fc6c04b54f4 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include #include @@ -285,6 +284,7 @@ static int erofs_read_superblock(struct super_block *sb) goto out; } + ret = -EINVAL; blkszbits = dsb->blkszbits; /* 9(512 bytes) + LOG_SECTORS_PER_BLOCK == LOG_BLOCK_SIZE */ if (blkszbits != LOG_BLOCK_SIZE) { @@ -751,4 +751,3 @@ module_exit(erofs_module_exit); MODULE_DESCRIPTION("Enhanced ROM File System"); MODULE_AUTHOR("Gao Xiang, Chao Yu, Miao Xie, CONSUMER BG, HUAWEI Inc."); MODULE_LICENSE("GPL"); - diff --git a/fs/erofs/tagptr.h b/fs/erofs/tagptr.h index a72897c86744..64ceb7270b5c 100644 --- a/fs/erofs/tagptr.h +++ b/fs/erofs/tagptr.h @@ -1,8 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* * A tagged pointer implementation - * - * Copyright (C) 2018 Gao Xiang */ #ifndef __EROFS_FS_TAGPTR_H #define __EROFS_FS_TAGPTR_H @@ -107,4 +105,3 @@ tagptr_init(o, cmpxchg(&ptptr->v, o.v, n.v)); }) *ptptr; }) #endif /* __EROFS_FS_TAGPTR_H */ - diff --git a/fs/erofs/utils.c b/fs/erofs/utils.c index 6758c5b19f7c..bd86067a63f7 100644 --- a/fs/erofs/utils.c +++ b/fs/erofs/utils.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "internal.h" #include @@ -278,4 +277,3 @@ void erofs_exit_shrinker(void) unregister_shrinker(&erofs_shrinker_info); } #endif /* !CONFIG_EROFS_FS_ZIP */ - diff --git a/fs/erofs/xattr.c b/fs/erofs/xattr.c index 47314a26767a..8dd54b420a1d 100644 --- a/fs/erofs/xattr.c +++ b/fs/erofs/xattr.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include #include "xattr.h" @@ -709,4 +708,3 @@ struct posix_acl *erofs_get_acl(struct inode *inode, int type) return acl; } #endif - diff --git a/fs/erofs/xattr.h b/fs/erofs/xattr.h index 815304bd335f..366dcb400525 100644 --- a/fs/erofs/xattr.h +++ b/fs/erofs/xattr.h @@ -2,7 +2,6 @@ /* * Copyright (C) 2017-2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #ifndef __EROFS_XATTR_H #define __EROFS_XATTR_H diff --git a/fs/erofs/zdata.c b/fs/erofs/zdata.c index 78e4b598ecca..cb4d0889eca9 100644 --- a/fs/erofs/zdata.c +++ b/fs/erofs/zdata.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "zdata.h" #include "compress.h" @@ -380,7 +379,6 @@ static int z_erofs_attach_page(struct z_erofs_collector *clt, enum z_erofs_page_type type) { int ret; - bool occupied; /* give priority for inplaceio */ if (clt->mode >= COLLECT_PRIMARY && @@ -388,8 +386,7 @@ static int z_erofs_attach_page(struct z_erofs_collector *clt, z_erofs_try_inplace_io(clt, page)) return 0; - ret = z_erofs_pagevec_enqueue(&clt->vector, - page, type, &occupied); + ret = z_erofs_pagevec_enqueue(&clt->vector, page, type); clt->cl->vcnt += (unsigned int)ret; return ret ? 0 : -EAGAIN; @@ -1471,4 +1468,3 @@ const struct address_space_operations z_erofs_aops = { .readpage = z_erofs_readpage, .readahead = z_erofs_readahead, }; - diff --git a/fs/erofs/zdata.h b/fs/erofs/zdata.h index 942ee69dff6a..3a008f1b9f78 100644 --- a/fs/erofs/zdata.h +++ b/fs/erofs/zdata.h @@ -2,7 +2,6 @@ /* * Copyright (C) 2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #ifndef __EROFS_FS_ZDATA_H #define __EROFS_FS_ZDATA_H diff --git a/fs/erofs/zmap.c b/fs/erofs/zmap.c index efaf32596b97..f68aea4baed7 100644 --- a/fs/erofs/zmap.c +++ b/fs/erofs/zmap.c @@ -2,7 +2,6 @@ /* * Copyright (C) 2018-2019 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #include "internal.h" #include @@ -597,4 +596,3 @@ int z_erofs_map_blocks_iter(struct inode *inode, DBG_BUGON(err < 0 && err != -ENOMEM); return err; } - diff --git a/fs/erofs/zpvec.h b/fs/erofs/zpvec.h index 1d67cbd38704..dfd7fe0503bb 100644 --- a/fs/erofs/zpvec.h +++ b/fs/erofs/zpvec.h @@ -2,7 +2,6 @@ /* * Copyright (C) 2018 HUAWEI, Inc. * https://www.huawei.com/ - * Created by Gao Xiang */ #ifndef __EROFS_FS_ZPVEC_H #define __EROFS_FS_ZPVEC_H @@ -107,10 +106,8 @@ static inline void z_erofs_pagevec_ctor_init(struct z_erofs_pagevec_ctor *ctor, static inline bool z_erofs_pagevec_enqueue(struct z_erofs_pagevec_ctor *ctor, struct page *page, - enum z_erofs_page_type type, - bool *occupied) + enum z_erofs_page_type type) { - *occupied = false; if (!ctor->next && type) if (ctor->index + 1 == ctor->nr) return false; @@ -125,7 +122,6 @@ static inline bool z_erofs_pagevec_enqueue(struct z_erofs_pagevec_ctor *ctor, /* should remind that collector->next never equal to 1, 2 */ if (type == (uintptr_t)ctor->next) { ctor->next = page; - *occupied = true; } ctor->pages[ctor->index++] = tagptr_fold(erofs_vtptr_t, page, type); return true; @@ -154,4 +150,3 @@ z_erofs_pagevec_dequeue(struct z_erofs_pagevec_ctor *ctor, return tagptr_unfold_ptr(t); } #endif - diff --git a/fs/pstore/blk.c b/fs/pstore/blk.c index 4bb8a344957a..04ce58c939a0 100644 --- a/fs/pstore/blk.c +++ b/fs/pstore/blk.c @@ -8,15 +8,16 @@ #include #include -#include "../../block/blk.h" #include #include #include #include #include #include +#include +#include +#include #include -#include static long kmsg_size = CONFIG_PSTORE_BLK_KMSG_SIZE; module_param(kmsg_size, long, 0400); @@ -57,27 +58,7 @@ MODULE_PARM_DESC(best_effort, "use best effort to write (i.e. do not require sto /* * blkdev - the block device to use for pstore storage - * - * Usually, this will be a partition of a block device. - * - * blkdev accepts the following variants: - * 1) device number in hexadecimal representation, - * with no leading 0x, for example b302. - * 2) /dev/ represents the device number of disk - * 3) /dev/ represents the device number - * of partition - device number of disk plus the partition number - * 4) /dev/p - same as the above, that form is - * used when disk name of partitioned disk ends on a digit. - * 5) PARTUUID=00112233-4455-6677-8899-AABBCCDDEEFF representing the - * unique id of a partition if the partition table provides it. - * The UUID may be either an EFI/GPT UUID, or refer to an MSDOS - * partition using the format SSSSSSSS-PP, where SSSSSSSS is a zero- - * filled hex representation of the 32-bit "NT disk signature", and PP - * is a zero-filled hex representation of the 1-based partition number. - * 6) PARTUUID=/PARTNROFF= to select a partition in relation to - * a partition with a known unique id. - * 7) : major and minor number of the device separated by - * a colon. + * See Documentation/admin-guide/pstore-blk.rst for details. */ static char blkdev[80] = CONFIG_PSTORE_BLK_BLKDEV; module_param_string(blkdev, blkdev, 80, 0400); @@ -88,14 +69,8 @@ MODULE_PARM_DESC(blkdev, "block device for pstore storage"); * during the register/unregister functions. */ static DEFINE_MUTEX(pstore_blk_lock); -static struct block_device *psblk_bdev; -static struct pstore_zone_info *pstore_zone_info; - -struct bdev_info { - dev_t devt; - sector_t nr_sects; - sector_t start_sect; -}; +static struct file *psblk_file; +static struct pstore_device_info *pstore_device_info; #define check_size(name, alignsize) ({ \ long _##name_ = (name); \ @@ -108,57 +83,63 @@ struct bdev_info { _##name_; \ }) +#define verify_size(name, alignsize, enabled) { \ + long _##name_; \ + if (enabled) \ + _##name_ = check_size(name, alignsize); \ + else \ + _##name_ = 0; \ + /* Synchronize module parameters with resuls. */ \ + name = _##name_ / 1024; \ + dev->zone.name = _##name_; \ +} + static int __register_pstore_device(struct pstore_device_info *dev) { int ret; lockdep_assert_held(&pstore_blk_lock); - if (!dev || !dev->total_size || !dev->read || !dev->write) + if (!dev) { + pr_err("NULL device info\n"); return -EINVAL; + } + if (!dev->zone.total_size) { + pr_err("zero sized device\n"); + return -EINVAL; + } + if (!dev->zone.read) { + pr_err("no read handler for device\n"); + return -EINVAL; + } + if (!dev->zone.write) { + pr_err("no write handler for device\n"); + return -EINVAL; + } /* someone already registered before */ - if (pstore_zone_info) + if (pstore_device_info) return -EBUSY; - pstore_zone_info = kzalloc(sizeof(struct pstore_zone_info), GFP_KERNEL); - if (!pstore_zone_info) - return -ENOMEM; - /* zero means not limit on which backends to attempt to store. */ if (!dev->flags) dev->flags = UINT_MAX; -#define verify_size(name, alignsize, enabled) { \ - long _##name_; \ - if (enabled) \ - _##name_ = check_size(name, alignsize); \ - else \ - _##name_ = 0; \ - name = _##name_ / 1024; \ - pstore_zone_info->name = _##name_; \ - } - + /* Copy in module parameters. */ verify_size(kmsg_size, 4096, dev->flags & PSTORE_FLAGS_DMESG); verify_size(pmsg_size, 4096, dev->flags & PSTORE_FLAGS_PMSG); verify_size(console_size, 4096, dev->flags & PSTORE_FLAGS_CONSOLE); verify_size(ftrace_size, 4096, dev->flags & PSTORE_FLAGS_FTRACE); -#undef verify_size + dev->zone.max_reason = max_reason; - pstore_zone_info->total_size = dev->total_size; - pstore_zone_info->max_reason = max_reason; - pstore_zone_info->read = dev->read; - pstore_zone_info->write = dev->write; - pstore_zone_info->erase = dev->erase; - pstore_zone_info->panic_write = dev->panic_write; - pstore_zone_info->name = KBUILD_MODNAME; - pstore_zone_info->owner = THIS_MODULE; + /* Initialize required zone ownership details. */ + dev->zone.name = KBUILD_MODNAME; + dev->zone.owner = THIS_MODULE; + + ret = register_pstore_zone(&dev->zone); + if (ret == 0) + pstore_device_info = dev; - ret = register_pstore_zone(pstore_zone_info); - if (ret) { - kfree(pstore_zone_info); - pstore_zone_info = NULL; - } return ret; } /** @@ -185,10 +166,9 @@ EXPORT_SYMBOL_GPL(register_pstore_device); static void __unregister_pstore_device(struct pstore_device_info *dev) { lockdep_assert_held(&pstore_blk_lock); - if (pstore_zone_info && pstore_zone_info->read == dev->read) { - unregister_pstore_zone(pstore_zone_info); - kfree(pstore_zone_info); - pstore_zone_info = NULL; + if (pstore_device_info && pstore_device_info == dev) { + unregister_pstore_zone(&dev->zone); + pstore_device_info = NULL; } } @@ -205,206 +185,61 @@ void unregister_pstore_device(struct pstore_device_info *dev) } EXPORT_SYMBOL_GPL(unregister_pstore_device); -/** - * psblk_get_bdev() - open block device - * - * @holder: Exclusive holder identifier - * @info: Information about bdev to fill in - * - * Return: pointer to block device on success and others on error. - * - * On success, the returned block_device has reference count of one. - */ -static struct block_device *psblk_get_bdev(void *holder, - struct bdev_info *info) -{ - struct block_device *bdev = ERR_PTR(-ENODEV); - fmode_t mode = FMODE_READ | FMODE_WRITE; - sector_t nr_sects; - - lockdep_assert_held(&pstore_blk_lock); - - if (pstore_zone_info) - return ERR_PTR(-EBUSY); - - if (!blkdev[0]) - return ERR_PTR(-ENODEV); - - if (holder) - mode |= FMODE_EXCL; - bdev = blkdev_get_by_path(blkdev, mode, holder); - if (IS_ERR(bdev)) { - dev_t devt; - - devt = name_to_dev_t(blkdev); - if (devt == 0) - return ERR_PTR(-ENODEV); - bdev = blkdev_get_by_dev(devt, mode, holder); - if (IS_ERR(bdev)) - return bdev; - } - - nr_sects = bdev_nr_sectors(bdev); - if (!nr_sects) { - pr_err("not enough space for '%s'\n", blkdev); - blkdev_put(bdev, mode); - return ERR_PTR(-ENOSPC); - } - - if (info) { - info->devt = bdev->bd_dev; - info->nr_sects = nr_sects; - info->start_sect = get_start_sect(bdev); - } - - return bdev; -} - -static void psblk_put_bdev(struct block_device *bdev, void *holder) -{ - fmode_t mode = FMODE_READ | FMODE_WRITE; - - lockdep_assert_held(&pstore_blk_lock); - - if (!bdev) - return; - - if (holder) - mode |= FMODE_EXCL; - blkdev_put(bdev, mode); -} - static ssize_t psblk_generic_blk_read(char *buf, size_t bytes, loff_t pos) { - struct block_device *bdev = psblk_bdev; - struct file file; - struct kiocb kiocb; - struct iov_iter iter; - struct kvec iov = {.iov_base = buf, .iov_len = bytes}; - - if (!bdev) - return -ENODEV; - - memset(&file, 0, sizeof(struct file)); - file.f_mapping = bdev->bd_inode->i_mapping; - file.f_flags = O_DSYNC | __O_SYNC | O_NOATIME; - file.f_inode = bdev->bd_inode; - file_ra_state_init(&file.f_ra, file.f_mapping); - - init_sync_kiocb(&kiocb, &file); - kiocb.ki_pos = pos; - iov_iter_kvec(&iter, READ, &iov, 1, bytes); - - return generic_file_read_iter(&kiocb, &iter); + return kernel_read(psblk_file, buf, bytes, &pos); } static ssize_t psblk_generic_blk_write(const char *buf, size_t bytes, loff_t pos) { - struct block_device *bdev = psblk_bdev; - struct iov_iter iter; - struct kiocb kiocb; - struct file file; - ssize_t ret; - struct kvec iov = {.iov_base = (void *)buf, .iov_len = bytes}; - - if (!bdev) - return -ENODEV; - /* Console/Ftrace backend may handle buffer until flush dirty zones */ if (in_interrupt() || irqs_disabled()) return -EBUSY; - - memset(&file, 0, sizeof(struct file)); - file.f_mapping = bdev->bd_inode->i_mapping; - file.f_flags = O_DSYNC | __O_SYNC | O_NOATIME; - file.f_inode = bdev->bd_inode; - - init_sync_kiocb(&kiocb, &file); - kiocb.ki_pos = pos; - iov_iter_kvec(&iter, WRITE, &iov, 1, bytes); - - inode_lock(bdev->bd_inode); - ret = generic_write_checks(&kiocb, &iter); - if (ret > 0) - ret = generic_perform_write(&file, &iter, pos); - inode_unlock(bdev->bd_inode); - - if (likely(ret > 0)) { - const struct file_operations f_op = {.fsync = blkdev_fsync}; - - file.f_op = &f_op; - kiocb.ki_pos += ret; - ret = generic_write_sync(&kiocb, ret); - } - return ret; + return kernel_write(psblk_file, buf, bytes, &pos); } /* * This takes its configuration only from the module parameters now. - * See psblk_get_bdev() and blkdev. */ -static int __register_pstore_blk(void) +static int __register_pstore_blk(struct pstore_device_info *dev, + const char *devpath) { - char bdev_name[BDEVNAME_SIZE]; - struct block_device *bdev; - struct pstore_device_info dev; - struct bdev_info binfo; - void *holder = blkdev; + struct inode *inode; int ret = -ENODEV; lockdep_assert_held(&pstore_blk_lock); - /* hold bdev exclusively */ - memset(&binfo, 0, sizeof(binfo)); - bdev = psblk_get_bdev(holder, &binfo); - if (IS_ERR(bdev)) { - pr_err("failed to open '%s'!\n", blkdev); - return PTR_ERR(bdev); + psblk_file = filp_open(devpath, O_RDWR | O_DSYNC | O_NOATIME | O_EXCL, 0); + if (IS_ERR(psblk_file)) { + ret = PTR_ERR(psblk_file); + pr_err("failed to open '%s': %d!\n", devpath, ret); + goto err; } - /* only allow driver matching the @blkdev */ - if (!binfo.devt) { - pr_debug("no major\n"); - ret = -ENODEV; - goto err_put_bdev; + inode = file_inode(psblk_file); + if (!S_ISBLK(inode->i_mode)) { + pr_err("'%s' is not block device!\n", devpath); + goto err_fput; } - /* psblk_bdev must be assigned before register to pstore/blk */ - psblk_bdev = bdev; + inode = I_BDEV(psblk_file->f_mapping->host)->bd_inode; + dev->zone.total_size = i_size_read(inode); - memset(&dev, 0, sizeof(dev)); - dev.total_size = binfo.nr_sects << SECTOR_SHIFT; - dev.read = psblk_generic_blk_read; - dev.write = psblk_generic_blk_write; - - ret = __register_pstore_device(&dev); + ret = __register_pstore_device(dev); if (ret) - goto err_put_bdev; + goto err_fput; - bdevname(bdev, bdev_name); - pr_info("attached %s (no dedicated panic_write!)\n", bdev_name); return 0; -err_put_bdev: - psblk_bdev = NULL; - psblk_put_bdev(bdev, holder); +err_fput: + fput(psblk_file); +err: + psblk_file = NULL; + return ret; } -static void __unregister_pstore_blk(unsigned int major) -{ - struct pstore_device_info dev = { .read = psblk_generic_blk_read }; - void *holder = blkdev; - - lockdep_assert_held(&pstore_blk_lock); - if (psblk_bdev && MAJOR(psblk_bdev->bd_dev) == major) { - __unregister_pstore_device(&dev); - psblk_put_bdev(psblk_bdev, holder); - psblk_bdev = NULL; - } -} - /* get information of pstore/blk */ int pstore_blk_get_config(struct pstore_blk_config *info) { @@ -419,13 +254,93 @@ int pstore_blk_get_config(struct pstore_blk_config *info) } EXPORT_SYMBOL_GPL(pstore_blk_get_config); + +#ifndef MODULE +static const char devname[] = "/dev/pstore-blk"; +static __init const char *early_boot_devpath(const char *initial_devname) +{ + /* + * During early boot the real root file system hasn't been + * mounted yet, and no device nodes are present yet. Use the + * same scheme to find the device that we use for mounting + * the root file system. + */ + dev_t dev = name_to_dev_t(initial_devname); + + if (!dev) { + pr_err("failed to resolve '%s'!\n", initial_devname); + return initial_devname; + } + + init_unlink(devname); + init_mknod(devname, S_IFBLK | 0600, new_encode_dev(dev)); + + return devname; +} +#else +static inline const char *early_boot_devpath(const char *initial_devname) +{ + return initial_devname; +} +#endif + +static int __init __best_effort_init(void) +{ + struct pstore_device_info *best_effort_dev; + int ret; + + /* No best-effort mode requested. */ + if (!best_effort) + return 0; + + /* Reject an empty blkdev. */ + if (!blkdev[0]) { + pr_err("blkdev empty with best_effort=Y\n"); + return -EINVAL; + } + + best_effort_dev = kzalloc(sizeof(*best_effort_dev), GFP_KERNEL); + if (!best_effort_dev) + return -ENOMEM; + + best_effort_dev->zone.read = psblk_generic_blk_read; + best_effort_dev->zone.write = psblk_generic_blk_write; + + ret = __register_pstore_blk(best_effort_dev, + early_boot_devpath(blkdev)); + if (ret) + kfree(best_effort_dev); + else + pr_info("attached %s (%zu) (no dedicated panic_write!)\n", + blkdev, best_effort_dev->zone.total_size); + + return ret; +} + +static void __exit __best_effort_exit(void) +{ + /* + * Currently, the only user of psblk_file is best_effort, so + * we can assume that pstore_device_info is associated with it. + * Once there are "real" blk devices, there will need to be a + * dedicated pstore_blk_info, etc. + */ + if (psblk_file) { + struct pstore_device_info *dev = pstore_device_info; + + __unregister_pstore_device(dev); + kfree(dev); + fput(psblk_file); + psblk_file = NULL; + } +} + static int __init pstore_blk_init(void) { - int ret = 0; + int ret; mutex_lock(&pstore_blk_lock); - if (!pstore_zone_info && best_effort && blkdev[0]) - ret = __register_pstore_blk(); + ret = __best_effort_init(); mutex_unlock(&pstore_blk_lock); return ret; @@ -435,15 +350,9 @@ late_initcall(pstore_blk_init); static void __exit pstore_blk_exit(void) { mutex_lock(&pstore_blk_lock); - if (psblk_bdev) - __unregister_pstore_blk(MAJOR(psblk_bdev->bd_dev)); - else { - struct pstore_device_info dev = { }; - - if (pstore_zone_info) - dev.read = pstore_zone_info->read; - __unregister_pstore_device(&dev); - } + __best_effort_exit(); + /* If we've been asked to unload, unregister any remaining device. */ + __unregister_pstore_device(pstore_device_info); mutex_unlock(&pstore_blk_lock); } module_exit(pstore_blk_exit); diff --git a/fs/reiserfs/namei.c b/fs/reiserfs/namei.c index 017db70d0f48..3d7a35d6a18b 100644 --- a/fs/reiserfs/namei.c +++ b/fs/reiserfs/namei.c @@ -132,6 +132,7 @@ int search_by_entry_key(struct super_block *sb, const struct cpu_key *key, return IO_ERROR; } PATH_LAST_POSITION(path)--; + break; case ITEM_FOUND: break; diff --git a/fs/xfs/libxfs/xfs_ag_resv.c b/fs/xfs/libxfs/xfs_ag_resv.c index bbfea8022a3b..e0d5cdc57cc2 100644 --- a/fs/xfs/libxfs/xfs_ag_resv.c +++ b/fs/xfs/libxfs/xfs_ag_resv.c @@ -366,7 +366,7 @@ xfs_ag_resv_alloc_extent( break; default: ASSERT(0); - /* fall through */ + fallthrough; case XFS_AG_RESV_NONE: field = args->wasdel ? XFS_TRANS_SB_RES_FDBLOCKS : XFS_TRANS_SB_FDBLOCKS; @@ -408,7 +408,7 @@ xfs_ag_resv_free_extent( break; default: ASSERT(0); - /* fall through */ + fallthrough; case XFS_AG_RESV_NONE: xfs_trans_mod_sb(tp, XFS_TRANS_SB_FDBLOCKS, (int64_t)len); return; diff --git a/fs/xfs/libxfs/xfs_alloc.c b/fs/xfs/libxfs/xfs_alloc.c index 82b7cbb1f24f..af3d5f9271f6 100644 --- a/fs/xfs/libxfs/xfs_alloc.c +++ b/fs/xfs/libxfs/xfs_alloc.c @@ -3174,7 +3174,7 @@ xfs_alloc_vextent( } args->agbno = XFS_FSB_TO_AGBNO(mp, args->fsbno); args->type = XFS_ALLOCTYPE_NEAR_BNO; - /* FALLTHROUGH */ + fallthrough; case XFS_ALLOCTYPE_FIRST_AG: /* * Rotate through the allocation groups looking for a winner. diff --git a/fs/xfs/libxfs/xfs_da_btree.c b/fs/xfs/libxfs/xfs_da_btree.c index 83ac9771bfb5..747ec77912c3 100644 --- a/fs/xfs/libxfs/xfs_da_btree.c +++ b/fs/xfs/libxfs/xfs_da_btree.c @@ -282,7 +282,7 @@ xfs_da3_node_read_verify( __this_address); break; } - /* fall through */ + fallthrough; case XFS_DA_NODE_MAGIC: fa = xfs_da3_node_verify(bp); if (fa) diff --git a/fs/xfs/scrub/agheader.c b/fs/xfs/scrub/agheader.c index 7a2f9b5f2db5..f96e84793cc9 100644 --- a/fs/xfs/scrub/agheader.c +++ b/fs/xfs/scrub/agheader.c @@ -86,6 +86,7 @@ xchk_superblock( case -ENOSYS: case -EFBIG: error = -EFSCORRUPTED; + fallthrough; default: break; } diff --git a/fs/xfs/scrub/bmap.c b/fs/xfs/scrub/bmap.c index b5ebf1d1b4db..77d5c4a0f09f 100644 --- a/fs/xfs/scrub/bmap.c +++ b/fs/xfs/scrub/bmap.c @@ -271,7 +271,7 @@ xchk_bmap_iextent_xref( case XFS_DATA_FORK: if (xfs_is_reflink_inode(info->sc->ip)) break; - /* fall through */ + fallthrough; case XFS_ATTR_FORK: xchk_xref_is_not_shared(info->sc, agbno, irec->br_blockcount); diff --git a/fs/xfs/scrub/btree.c b/fs/xfs/scrub/btree.c index a94bd8122c60..bd1172358964 100644 --- a/fs/xfs/scrub/btree.c +++ b/fs/xfs/scrub/btree.c @@ -44,7 +44,7 @@ __xchk_btree_process_error( /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; - /* fall through */ + fallthrough; default: if (cur->bc_flags & XFS_BTREE_ROOT_IN_INODE) trace_xchk_ifork_btree_op_error(sc, cur, level, diff --git a/fs/xfs/scrub/common.c b/fs/xfs/scrub/common.c index be38c960da85..6cc92291c46f 100644 --- a/fs/xfs/scrub/common.c +++ b/fs/xfs/scrub/common.c @@ -83,7 +83,7 @@ __xchk_process_error( /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; - /* fall through */ + fallthrough; default: trace_xchk_op_error(sc, agno, bno, *error, ret_ip); @@ -136,7 +136,7 @@ __xchk_fblock_process_error( /* Note the badness but don't abort. */ sc->sm->sm_flags |= errflag; *error = 0; - /* fall through */ + fallthrough; default: trace_xchk_file_op_error(sc, whichfork, offset, *error, ret_ip); @@ -696,7 +696,7 @@ xchk_get_inode( if (error) return -ENOENT; error = -EFSCORRUPTED; - /* fall through */ + fallthrough; default: trace_xchk_op_error(sc, XFS_INO_TO_AGNO(mp, sc->sm->sm_ino), diff --git a/fs/xfs/scrub/dabtree.c b/fs/xfs/scrub/dabtree.c index 653f3280e1c1..9f0dbb47c82c 100644 --- a/fs/xfs/scrub/dabtree.c +++ b/fs/xfs/scrub/dabtree.c @@ -47,7 +47,7 @@ xchk_da_process_error( /* Note the badness but don't abort. */ sc->sm->sm_flags |= XFS_SCRUB_OFLAG_CORRUPT; *error = 0; - /* fall through */ + fallthrough; default: trace_xchk_file_op_error(sc, ds->dargs.whichfork, xfs_dir2_da_to_db(ds->dargs.geo, diff --git a/fs/xfs/scrub/repair.c b/fs/xfs/scrub/repair.c index c2857d854c83..b8202dd08939 100644 --- a/fs/xfs/scrub/repair.c +++ b/fs/xfs/scrub/repair.c @@ -947,7 +947,7 @@ xrep_ino_dqattach( xrep_force_quotacheck(sc, XFS_DQTYPE_GROUP); if (XFS_IS_PQUOTA_ON(sc->mp) && !sc->ip->i_pdquot) xrep_force_quotacheck(sc, XFS_DQTYPE_PROJ); - /* fall through */ + fallthrough; case -ESRCH: error = 0; break; diff --git a/fs/xfs/xfs_bmap_util.c b/fs/xfs/xfs_bmap_util.c index 0936f3a96fe6..2988efa97e14 100644 --- a/fs/xfs/xfs_bmap_util.c +++ b/fs/xfs/xfs_bmap_util.c @@ -286,7 +286,7 @@ xfs_bmap_count_blocks( */ *count += btblocks - 1; - /* fall through */ + fallthrough; case XFS_DINODE_FMT_EXTENTS: *nextents = xfs_bmap_count_leaves(ifp, count); break; diff --git a/fs/xfs/xfs_export.c b/fs/xfs/xfs_export.c index 465fd9e048d4..1da59bdff245 100644 --- a/fs/xfs/xfs_export.c +++ b/fs/xfs/xfs_export.c @@ -84,7 +84,7 @@ xfs_fs_encode_fh( case FILEID_INO32_GEN_PARENT: fid->i32.parent_ino = XFS_I(parent)->i_ino; fid->i32.parent_gen = parent->i_generation; - /*FALLTHRU*/ + fallthrough; case FILEID_INO32_GEN: fid->i32.ino = XFS_I(inode)->i_ino; fid->i32.gen = inode->i_generation; @@ -92,7 +92,7 @@ xfs_fs_encode_fh( case FILEID_INO32_GEN_PARENT | XFS_FILEID_TYPE_64FLAG: fid64->parent_ino = XFS_I(parent)->i_ino; fid64->parent_gen = parent->i_generation; - /*FALLTHRU*/ + fallthrough; case FILEID_INO32_GEN | XFS_FILEID_TYPE_64FLAG: fid64->ino = XFS_I(inode)->i_ino; fid64->gen = inode->i_generation; diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c index 396ef36dcd0a..3c0749ab9e40 100644 --- a/fs/xfs/xfs_file.c +++ b/fs/xfs/xfs_file.c @@ -863,7 +863,7 @@ xfs_break_layouts( error = xfs_break_dax_layouts(inode, &retry); if (error || retry) break; - /* fall through */ + fallthrough; case BREAK_WRITE: error = xfs_break_leased_layouts(inode, iolock, &retry); break; diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index e4c2da4566f1..1db99a909b23 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -877,7 +877,7 @@ xfs_init_new_inode( xfs_inode_inherit_flags(ip, pip); if (pip && (pip->i_diflags2 & XFS_DIFLAG2_ANY)) xfs_inode_inherit_flags2(ip, pip); - /* FALLTHROUGH */ + fallthrough; case S_IFLNK: ip->i_df.if_format = XFS_DINODE_FMT_EXTENTS; ip->i_df.if_bytes = 0; diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index 1fe4c1fc0aea..7b1796cede10 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -558,7 +558,7 @@ xfs_ioc_attrmulti_one( case ATTR_OP_REMOVE: value = NULL; *len = 0; - /* fall through */ + fallthrough; case ATTR_OP_SET: error = mnt_want_write_file(parfilp); if (error) @@ -1511,7 +1511,7 @@ xfs_ioc_getbmap( switch (cmd) { case XFS_IOC_GETBMAPA: bmx.bmv_iflags = BMV_IF_ATTRFORK; - /*FALLTHRU*/ + fallthrough; case XFS_IOC_GETBMAP: /* struct getbmap is a strict subset of struct getbmapx. */ recsize = sizeof(struct getbmap); diff --git a/fs/xfs/xfs_iomap.c b/fs/xfs/xfs_iomap.c index d154f42e2dc6..d8cd2583dedb 100644 --- a/fs/xfs/xfs_iomap.c +++ b/fs/xfs/xfs_iomap.c @@ -1036,7 +1036,7 @@ xfs_buffered_write_iomap_begin( prealloc_blocks = 0; goto retry; } - /*FALLTHRU*/ + fallthrough; default: goto out_unlock; } diff --git a/fs/xfs/xfs_log.c b/fs/xfs/xfs_log.c index c19a82adea1e..a002425377b5 100644 --- a/fs/xfs/xfs_log.c +++ b/fs/xfs/xfs_log.c @@ -2626,6 +2626,7 @@ xlog_covered_state( case XLOG_STATE_COVER_IDLE: if (iclogs_changed == 1) return XLOG_STATE_COVER_IDLE; + fallthrough; case XLOG_STATE_COVER_NEED: case XLOG_STATE_COVER_NEED2: break; diff --git a/fs/xfs/xfs_trans_buf.c b/fs/xfs/xfs_trans_buf.c index 9aced0a00003..d11d032da0b4 100644 --- a/fs/xfs/xfs_trans_buf.c +++ b/fs/xfs/xfs_trans_buf.c @@ -294,7 +294,7 @@ xfs_trans_read_buf_map( default: if (tp && (tp->t_flags & XFS_TRANS_DIRTY)) xfs_force_shutdown(tp->t_mountp, SHUTDOWN_META_IO_ERROR); - /* fall through */ + fallthrough; case -ENOMEM: case -EAGAIN: return error; diff --git a/include/asm-generic/hyperv-tlfs.h b/include/asm-generic/hyperv-tlfs.h index 515c3fb06ab3..56348a541c50 100644 --- a/include/asm-generic/hyperv-tlfs.h +++ b/include/asm-generic/hyperv-tlfs.h @@ -194,6 +194,7 @@ enum HV_GENERIC_SET_FORMAT { #define HV_STATUS_INVALID_HYPERCALL_INPUT 3 #define HV_STATUS_INVALID_ALIGNMENT 4 #define HV_STATUS_INVALID_PARAMETER 5 +#define HV_STATUS_ACCESS_DENIED 6 #define HV_STATUS_OPERATION_DENIED 8 #define HV_STATUS_INSUFFICIENT_MEMORY 11 #define HV_STATUS_INVALID_PORT_ID 17 diff --git a/include/crypto/aead.h b/include/crypto/aead.h index e728469c4ccc..5af914c1ab8e 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -490,7 +490,7 @@ static inline void aead_request_set_callback(struct aead_request *req, * The memory structure for cipher operation has the following structure: * * - AEAD encryption input: assoc data || plaintext - * - AEAD encryption output: assoc data || cipherntext || auth tag + * - AEAD encryption output: assoc data || ciphertext || auth tag * - AEAD decryption input: assoc data || ciphertext || auth tag * - AEAD decryption output: assoc data || plaintext * diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 86f0748009af..5f6841c73e5a 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -96,6 +96,15 @@ struct scatter_walk { unsigned int offset; }; +struct crypto_attr_alg { + char name[CRYPTO_MAX_ALG_NAME]; +}; + +struct crypto_attr_type { + u32 type; + u32 mask; +}; + void crypto_mod_put(struct crypto_alg *alg); int crypto_register_template(struct crypto_template *tmpl); @@ -118,7 +127,6 @@ void *crypto_spawn_tfm2(struct crypto_spawn *spawn); struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb); int crypto_check_attr_type(struct rtattr **tb, u32 type, u32 *mask_ret); const char *crypto_attr_alg_name(struct rtattr *rta); -int crypto_attr_u32(struct rtattr *rta, u32 *num); int crypto_inst_setname(struct crypto_instance *inst, const char *name, struct crypto_alg *alg); diff --git a/include/crypto/engine.h b/include/crypto/engine.h index 3f06e40d063a..26cac19b0f46 100644 --- a/include/crypto/engine.h +++ b/include/crypto/engine.h @@ -28,7 +28,7 @@ * of a failed backlog request * crypto-engine, in head position to keep order * @list: link with the global crypto engine list - * @queue_lock: spinlock to syncronise access to request queue + * @queue_lock: spinlock to synchronise access to request queue * @queue: the crypto queue of the engine * @rt: whether this queue is set to run as a realtime task * @prepare_crypt_hardware: a request will soon arrive from the queue diff --git a/include/crypto/hash.h b/include/crypto/hash.h index b2bc1e46e86a..f140e4643949 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -458,7 +458,7 @@ int crypto_ahash_finup(struct ahash_request *req); * * Return: * 0 if the message digest was successfully calculated; - * -EINPROGRESS if data is feeded into hardware (DMA) or queued for later; + * -EINPROGRESS if data is fed into hardware (DMA) or queued for later; * -EBUSY if queue is full and request should be resubmitted later; * other < 0 if an error occurred */ diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 0a288dddcf5b..25806141db59 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -75,13 +75,7 @@ void crypto_unregister_ahashes(struct ahash_alg *algs, int count); int ahash_register_instance(struct crypto_template *tmpl, struct ahash_instance *inst); -int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, - unsigned int keylen); - -static inline bool crypto_shash_alg_has_setkey(struct shash_alg *alg) -{ - return alg->setkey != shash_no_setkey; -} +bool crypto_shash_alg_has_setkey(struct shash_alg *alg); static inline bool crypto_shash_alg_needs_key(struct shash_alg *alg) { diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index ec621180ef09..e602d848fc1a 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -72,6 +72,9 @@ struct vgic_global { bool has_gicv4; bool has_gicv4_1; + /* Pseudo GICv3 from outer space */ + bool no_hw_deactivation; + /* GIC system register CPU interface */ struct static_key_false gicv3_cpuif; @@ -89,6 +92,26 @@ enum vgic_irq_config { VGIC_CONFIG_LEVEL }; +/* + * Per-irq ops overriding some common behavious. + * + * Always called in non-preemptible section and the functions can use + * kvm_arm_get_running_vcpu() to get the vcpu pointer for private IRQs. + */ +struct irq_ops { + /* Per interrupt flags for special-cased interrupts */ + unsigned long flags; + +#define VGIC_IRQ_SW_RESAMPLE BIT(0) /* Clear the active state for resampling */ + + /* + * Callback function pointer to in-kernel devices that can tell us the + * state of the input level of mapped level-triggered IRQ faster than + * peaking into the physical GIC. + */ + bool (*get_input_level)(int vintid); +}; + struct vgic_irq { raw_spinlock_t irq_lock; /* Protects the content of the struct */ struct list_head lpi_list; /* Used to link all LPIs together */ @@ -126,21 +149,17 @@ struct vgic_irq { u8 group; /* 0 == group 0, 1 == group 1 */ enum vgic_irq_config config; /* Level or edge */ - /* - * Callback function pointer to in-kernel devices that can tell us the - * state of the input level of mapped level-triggered IRQ faster than - * peaking into the physical GIC. - * - * Always called in non-preemptible section and the functions can use - * kvm_arm_get_running_vcpu() to get the vcpu pointer for private - * IRQs. - */ - bool (*get_input_level)(int vintid); + struct irq_ops *ops; void *owner; /* Opaque pointer to reserve an interrupt for in-kernel devices. */ }; +static inline bool vgic_irq_needs_resampling(struct vgic_irq *irq) +{ + return irq->ops && (irq->ops->flags & VGIC_IRQ_SW_RESAMPLE); +} + struct vgic_register_region; struct vgic_its; @@ -352,7 +371,7 @@ void kvm_vgic_init_cpu_hardware(void); int kvm_vgic_inject_irq(struct kvm *kvm, int cpuid, unsigned int intid, bool level, void *owner); int kvm_vgic_map_phys_irq(struct kvm_vcpu *vcpu, unsigned int host_irq, - u32 vintid, bool (*get_input_level)(int vindid)); + u32 vintid, struct irq_ops *ops); int kvm_vgic_unmap_phys_irq(struct kvm_vcpu *vcpu, unsigned int vintid); bool kvm_vgic_map_is_active(struct kvm_vcpu *vcpu, unsigned int vintid); diff --git a/include/linux/arm-smccc.h b/include/linux/arm-smccc.h index 6861489a1890..7d1cabe15262 100644 --- a/include/linux/arm-smccc.h +++ b/include/linux/arm-smccc.h @@ -63,6 +63,9 @@ #define ARM_SMCCC_VERSION_1_0 0x10000 #define ARM_SMCCC_VERSION_1_1 0x10001 #define ARM_SMCCC_VERSION_1_2 0x10002 +#define ARM_SMCCC_VERSION_1_3 0x10003 + +#define ARM_SMCCC_1_3_SVE_HINT 0x10000 #define ARM_SMCCC_VERSION_FUNC_ID \ ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL, \ @@ -216,6 +219,8 @@ u32 arm_smccc_get_version(void); void __init arm_smccc_version_init(u32 version, enum arm_smccc_conduit conduit); +extern u64 smccc_has_sve_hint; + /** * struct arm_smccc_res - Result from SMC/HVC call * @a0-a3 result values from registers 0 to 3 @@ -227,6 +232,61 @@ struct arm_smccc_res { unsigned long a3; }; +#ifdef CONFIG_ARM64 +/** + * struct arm_smccc_1_2_regs - Arguments for or Results from SMC/HVC call + * @a0-a17 argument values from registers 0 to 17 + */ +struct arm_smccc_1_2_regs { + unsigned long a0; + unsigned long a1; + unsigned long a2; + unsigned long a3; + unsigned long a4; + unsigned long a5; + unsigned long a6; + unsigned long a7; + unsigned long a8; + unsigned long a9; + unsigned long a10; + unsigned long a11; + unsigned long a12; + unsigned long a13; + unsigned long a14; + unsigned long a15; + unsigned long a16; + unsigned long a17; +}; + +/** + * arm_smccc_1_2_hvc() - make HVC calls + * @args: arguments passed via struct arm_smccc_1_2_regs + * @res: result values via struct arm_smccc_1_2_regs + * + * This function is used to make HVC calls following SMC Calling Convention + * v1.2 or above. The content of the supplied param are copied from the + * structure to registers prior to the HVC instruction. The return values + * are updated with the content from registers on return from the HVC + * instruction. + */ +asmlinkage void arm_smccc_1_2_hvc(const struct arm_smccc_1_2_regs *args, + struct arm_smccc_1_2_regs *res); + +/** + * arm_smccc_1_2_smc() - make SMC calls + * @args: arguments passed via struct arm_smccc_1_2_regs + * @res: result values via struct arm_smccc_1_2_regs + * + * This function is used to make SMC calls following SMC Calling Convention + * v1.2 or above. The content of the supplied param are copied from the + * structure to registers prior to the SMC instruction. The return values + * are updated with the content from registers on return from the SMC + * instruction. + */ +asmlinkage void arm_smccc_1_2_smc(const struct arm_smccc_1_2_regs *args, + struct arm_smccc_1_2_regs *res); +#endif + /** * struct arm_smccc_quirk - Contains quirk information * @id: quirk identification @@ -240,6 +300,15 @@ struct arm_smccc_quirk { } state; }; +/** + * __arm_smccc_sve_check() - Set the SVE hint bit when doing SMC calls + * + * Sets the SMCCC hint bit to indicate if there is live state in the SVE + * registers, this modifies x0 in place and should never be called from C + * code. + */ +asmlinkage unsigned long __arm_smccc_sve_check(unsigned long x0); + /** * __arm_smccc_smc() - make SMC calls * @a0-a7: arguments passed in registers 0 to 7 @@ -297,6 +366,20 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, #endif +/* nVHE hypervisor doesn't have a current thread so needs separate checks */ +#if defined(CONFIG_ARM64_SVE) && !defined(__KVM_NVHE_HYPERVISOR__) + +#define SMCCC_SVE_CHECK ALTERNATIVE("nop \n", "bl __arm_smccc_sve_check \n", \ + ARM64_SVE) +#define smccc_sve_clobbers "x16", "x30", "cc", + +#else + +#define SMCCC_SVE_CHECK +#define smccc_sve_clobbers + +#endif + #define ___count_args(_0, _1, _2, _3, _4, _5, _6, _7, _8, x, ...) x #define __count_args(...) \ @@ -364,7 +447,7 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, #define ___constraints(count) \ : __constraint_read_ ## count \ - : "memory" + : smccc_sve_clobbers "memory" #define __constraints(count) ___constraints(count) /* @@ -379,7 +462,8 @@ asmlinkage void __arm_smccc_hvc(unsigned long a0, unsigned long a1, register unsigned long r2 asm("r2"); \ register unsigned long r3 asm("r3"); \ __declare_args(__count_args(__VA_ARGS__), __VA_ARGS__); \ - asm volatile(inst "\n" : \ + asm volatile(SMCCC_SVE_CHECK \ + inst "\n" : \ "=r" (r0), "=r" (r1), "=r" (r2), "=r" (r3) \ __constraints(__count_args(__VA_ARGS__))); \ if (___res) \ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index da5e0d74bb2f..855869e1fd32 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -643,32 +643,6 @@ struct crypto_comp { struct crypto_tfm base; }; -enum { - CRYPTOA_UNSPEC, - CRYPTOA_ALG, - CRYPTOA_TYPE, - CRYPTOA_U32, - __CRYPTOA_MAX, -}; - -#define CRYPTOA_MAX (__CRYPTOA_MAX - 1) - -/* Maximum number of (rtattr) parameters for each template. */ -#define CRYPTO_MAX_ATTRS 32 - -struct crypto_attr_alg { - char name[CRYPTO_MAX_ALG_NAME]; -}; - -struct crypto_attr_type { - u32 type; - u32 mask; -}; - -struct crypto_attr_u32 { - u32 num; -}; - /* * Transform user interface. */ diff --git a/include/linux/device.h b/include/linux/device.h index f1a00040fa53..959cb9d2c9ab 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -399,7 +399,7 @@ struct dev_links_info { * along with subsystem-level and driver-level callbacks. * @em_pd: device's energy model performance domain * @pins: For device pin management. - * See Documentation/driver-api/pinctl.rst for details. + * See Documentation/driver-api/pin-control.rst for details. * @msi_list: Hosts MSI descriptors * @msi_domain: The generic MSI domain this device is using. * @numa_node: NUMA node this device is close to. diff --git a/include/linux/evm.h b/include/linux/evm.h index 8302bc29bb35..4c374be70247 100644 --- a/include/linux/evm.h +++ b/include/linux/evm.h @@ -23,18 +23,25 @@ extern enum integrity_status evm_verifyxattr(struct dentry *dentry, struct integrity_iint_cache *iint); extern int evm_inode_setattr(struct dentry *dentry, struct iattr *attr); extern void evm_inode_post_setattr(struct dentry *dentry, int ia_valid); -extern int evm_inode_setxattr(struct dentry *dentry, const char *name, +extern int evm_inode_setxattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, size_t size); extern void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, size_t xattr_value_len); -extern int evm_inode_removexattr(struct dentry *dentry, const char *xattr_name); +extern int evm_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *xattr_name); extern void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name); extern int evm_inode_init_security(struct inode *inode, const struct xattr *xattr_array, struct xattr *evm); +extern bool evm_revalidate_status(const char *xattr_name); +extern int evm_protected_xattr_if_enabled(const char *req_xattr_name); +extern int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, + int buffer_size, char type, + bool canonical_fmt); #ifdef CONFIG_FS_POSIX_ACL extern int posix_xattr_acl(const char *xattrname); #else @@ -71,7 +78,8 @@ static inline void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) return; } -static inline int evm_inode_setxattr(struct dentry *dentry, const char *name, +static inline int evm_inode_setxattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *name, const void *value, size_t size) { return 0; @@ -85,7 +93,8 @@ static inline void evm_inode_post_setxattr(struct dentry *dentry, return; } -static inline int evm_inode_removexattr(struct dentry *dentry, +static inline int evm_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *xattr_name) { return 0; @@ -104,5 +113,22 @@ static inline int evm_inode_init_security(struct inode *inode, return 0; } +static inline bool evm_revalidate_status(const char *xattr_name) +{ + return false; +} + +static inline int evm_protected_xattr_if_enabled(const char *req_xattr_name) +{ + return false; +} + +static inline int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, + int buffer_size, char type, + bool canonical_fmt) +{ + return -EOPNOTSUPP; +} + #endif /* CONFIG_EVM */ #endif /* LINUX_EVM_H */ diff --git a/include/linux/gfp.h b/include/linux/gfp.h index 2003fd7dff1a..8eca80fe154d 100644 --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -53,8 +53,10 @@ struct vm_area_struct; #define ___GFP_HARDWALL 0x100000u #define ___GFP_THISNODE 0x200000u #define ___GFP_ACCOUNT 0x400000u +#define ___GFP_ZEROTAGS 0x800000u +#define ___GFP_SKIP_KASAN_POISON 0x1000000u #ifdef CONFIG_LOCKDEP -#define ___GFP_NOLOCKDEP 0x800000u +#define ___GFP_NOLOCKDEP 0x2000000u #else #define ___GFP_NOLOCKDEP 0 #endif @@ -229,16 +231,25 @@ struct vm_area_struct; * %__GFP_COMP address compound page metadata. * * %__GFP_ZERO returns a zeroed page on success. + * + * %__GFP_ZEROTAGS returns a page with zeroed memory tags on success, if + * __GFP_ZERO is set. + * + * %__GFP_SKIP_KASAN_POISON returns a page which does not need to be poisoned + * on deallocation. Typically used for userspace pages. Currently only has an + * effect in HW tags mode. */ #define __GFP_NOWARN ((__force gfp_t)___GFP_NOWARN) #define __GFP_COMP ((__force gfp_t)___GFP_COMP) #define __GFP_ZERO ((__force gfp_t)___GFP_ZERO) +#define __GFP_ZEROTAGS ((__force gfp_t)___GFP_ZEROTAGS) +#define __GFP_SKIP_KASAN_POISON ((__force gfp_t)___GFP_SKIP_KASAN_POISON) /* Disable lockdep for GFP context tracking */ #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP) /* Room for N __GFP_FOO bits */ -#define __GFP_BITS_SHIFT (23 + IS_ENABLED(CONFIG_LOCKDEP)) +#define __GFP_BITS_SHIFT (25 + IS_ENABLED(CONFIG_LOCKDEP)) #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /** @@ -319,7 +330,8 @@ struct vm_area_struct; #define GFP_DMA __GFP_DMA #define GFP_DMA32 __GFP_DMA32 #define GFP_HIGHUSER (GFP_USER | __GFP_HIGHMEM) -#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE) +#define GFP_HIGHUSER_MOVABLE (GFP_HIGHUSER | __GFP_MOVABLE | \ + __GFP_SKIP_KASAN_POISON) #define GFP_TRANSHUGE_LIGHT ((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \ __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM) #define GFP_TRANSHUGE (GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM) diff --git a/include/linux/highmem.h b/include/linux/highmem.h index 832b49b50c7b..8c6e8e996c87 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h @@ -152,28 +152,24 @@ static inline void clear_user_highpage(struct page *page, unsigned long vaddr) } #endif -#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +#ifndef __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE /** - * __alloc_zeroed_user_highpage - Allocate a zeroed HIGHMEM page for a VMA with caller-specified movable GFP flags - * @movableflags: The GFP flags related to the pages future ability to move like __GFP_MOVABLE + * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move * @vma: The VMA the page is to be allocated for * @vaddr: The virtual address the page will be inserted into * - * This function will allocate a page for a VMA but the caller is expected - * to specify via movableflags whether the page will be movable in the - * future or not + * This function will allocate a page for a VMA that the caller knows will + * be able to migrate in the future using move_pages() or reclaimed * * An architecture may override this function by defining - * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE and providing their own + * __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE_MOVABLE and providing their own * implementation. */ static inline struct page * -__alloc_zeroed_user_highpage(gfp_t movableflags, - struct vm_area_struct *vma, - unsigned long vaddr) +alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, + unsigned long vaddr) { - struct page *page = alloc_page_vma(GFP_HIGHUSER | movableflags, - vma, vaddr); + struct page *page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, vaddr); if (page) clear_user_highpage(page, vaddr); @@ -182,21 +178,6 @@ __alloc_zeroed_user_highpage(gfp_t movableflags, } #endif -/** - * alloc_zeroed_user_highpage_movable - Allocate a zeroed HIGHMEM page for a VMA that the caller knows can move - * @vma: The VMA the page is to be allocated for - * @vaddr: The virtual address the page will be inserted into - * - * This function will allocate a page for a VMA that the caller knows will - * be able to migrate in the future using move_pages() or reclaimed - */ -static inline struct page * -alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, - unsigned long vaddr) -{ - return __alloc_zeroed_user_highpage(__GFP_MOVABLE, vma, vaddr); -} - static inline void clear_highpage(struct page *page) { void *kaddr = kmap_atomic(page); @@ -204,6 +185,14 @@ static inline void clear_highpage(struct page *page) kunmap_atomic(kaddr); } +#ifndef __HAVE_ARCH_TAG_CLEAR_HIGHPAGE + +static inline void tag_clear_highpage(struct page *page) +{ +} + +#endif + /* * If we pass in a base or tail page, we can zero up to PAGE_SIZE. * If we pass in a head page, we can zero up to the size of the compound page. diff --git a/include/linux/integrity.h b/include/linux/integrity.h index 2271939c5c31..2ea0f2f65ab6 100644 --- a/include/linux/integrity.h +++ b/include/linux/integrity.h @@ -13,6 +13,7 @@ enum integrity_status { INTEGRITY_PASS = 0, INTEGRITY_PASS_IMMUTABLE, INTEGRITY_FAIL, + INTEGRITY_FAIL_IMMUTABLE, INTEGRITY_NOLABEL, INTEGRITY_NOXATTRS, INTEGRITY_UNKNOWN, diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 4777850a6dc7..35a374241515 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -319,39 +319,8 @@ struct irq_affinity_desc { extern cpumask_var_t irq_default_affinity; -/* Internal implementation. Use the helpers below */ -extern int __irq_set_affinity(unsigned int irq, const struct cpumask *cpumask, - bool force); - -/** - * irq_set_affinity - Set the irq affinity of a given irq - * @irq: Interrupt to set affinity - * @cpumask: cpumask - * - * Fails if cpumask does not contain an online CPU - */ -static inline int -irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) -{ - return __irq_set_affinity(irq, cpumask, false); -} - -/** - * irq_force_affinity - Force the irq affinity of a given irq - * @irq: Interrupt to set affinity - * @cpumask: cpumask - * - * Same as irq_set_affinity, but without checking the mask against - * online cpus. - * - * Solely for low level cpu hotplug code, where we need to make per - * cpu interrupts affine before the cpu becomes online. - */ -static inline int -irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) -{ - return __irq_set_affinity(irq, cpumask, true); -} +extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask); +extern int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask); extern int irq_can_set_affinity(unsigned int irq); extern int irq_select_affinity(unsigned int irq); diff --git a/include/linux/irqchip/arm-gic-common.h b/include/linux/irqchip/arm-gic-common.h index fa8c0455c352..1177f3a1aed5 100644 --- a/include/linux/irqchip/arm-gic-common.h +++ b/include/linux/irqchip/arm-gic-common.h @@ -7,8 +7,7 @@ #ifndef __LINUX_IRQCHIP_ARM_GIC_COMMON_H #define __LINUX_IRQCHIP_ARM_GIC_COMMON_H -#include -#include +#include #define GICD_INT_DEF_PRI 0xa0 #define GICD_INT_DEF_PRI_X4 ((GICD_INT_DEF_PRI << 24) |\ @@ -16,28 +15,6 @@ (GICD_INT_DEF_PRI << 8) |\ GICD_INT_DEF_PRI) -enum gic_type { - GIC_V2, - GIC_V3, -}; - -struct gic_kvm_info { - /* GIC type */ - enum gic_type type; - /* Virtual CPU interface */ - struct resource vcpu; - /* Interrupt number */ - unsigned int maint_irq; - /* Virtual control interface */ - struct resource vctrl; - /* vlpi support */ - bool has_v4; - /* rvpeid support */ - bool has_v4_1; -}; - -const struct gic_kvm_info *gic_get_kvm_info(void); - struct irq_domain; struct fwnode_handle; int gicv2m_init(struct fwnode_handle *parent_handle, diff --git a/include/linux/irqchip/arm-vgic-info.h b/include/linux/irqchip/arm-vgic-info.h new file mode 100644 index 000000000000..a75b2c7de69d --- /dev/null +++ b/include/linux/irqchip/arm-vgic-info.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * include/linux/irqchip/arm-vgic-info.h + * + * Copyright (C) 2016 ARM Limited, All Rights Reserved. + */ +#ifndef __LINUX_IRQCHIP_ARM_VGIC_INFO_H +#define __LINUX_IRQCHIP_ARM_VGIC_INFO_H + +#include +#include + +enum gic_type { + /* Full GICv2 */ + GIC_V2, + /* Full GICv3, optionally with v2 compat */ + GIC_V3, +}; + +struct gic_kvm_info { + /* GIC type */ + enum gic_type type; + /* Virtual CPU interface */ + struct resource vcpu; + /* Interrupt number */ + unsigned int maint_irq; + /* No interrupt mask, no need to use the above field */ + bool no_maint_irq_mask; + /* Virtual control interface */ + struct resource vctrl; + /* vlpi support */ + bool has_v4; + /* rvpeid support */ + bool has_v4_1; + /* Deactivation impared, subpar stuff */ + bool no_hw_deactivation; +}; + +#ifdef CONFIG_KVM +void vgic_set_kvm_info(const struct gic_kvm_info *info); +#else +static inline void vgic_set_kvm_info(const struct gic_kvm_info *info) {} +#endif + +#endif diff --git a/include/linux/kasan.h b/include/linux/kasan.h index b1678a61e6a7..a1c7ce5f3e4f 100644 --- a/include/linux/kasan.h +++ b/include/linux/kasan.h @@ -2,6 +2,7 @@ #ifndef _LINUX_KASAN_H #define _LINUX_KASAN_H +#include #include #include @@ -79,14 +80,6 @@ static inline void kasan_disable_current(void) {} #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */ -#ifdef CONFIG_KASAN - -struct kasan_cache { - int alloc_meta_offset; - int free_meta_offset; - bool is_kmalloc; -}; - #ifdef CONFIG_KASAN_HW_TAGS DECLARE_STATIC_KEY_FALSE(kasan_flag_enabled); @@ -101,11 +94,14 @@ static inline bool kasan_has_integrated_init(void) return kasan_enabled(); } +void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags); +void kasan_free_pages(struct page *page, unsigned int order); + #else /* CONFIG_KASAN_HW_TAGS */ static inline bool kasan_enabled(void) { - return true; + return IS_ENABLED(CONFIG_KASAN); } static inline bool kasan_has_integrated_init(void) @@ -113,8 +109,30 @@ static inline bool kasan_has_integrated_init(void) return false; } +static __always_inline void kasan_alloc_pages(struct page *page, + unsigned int order, gfp_t flags) +{ + /* Only available for integrated init. */ + BUILD_BUG(); +} + +static __always_inline void kasan_free_pages(struct page *page, + unsigned int order) +{ + /* Only available for integrated init. */ + BUILD_BUG(); +} + #endif /* CONFIG_KASAN_HW_TAGS */ +#ifdef CONFIG_KASAN + +struct kasan_cache { + int alloc_meta_offset; + int free_meta_offset; + bool is_kmalloc; +}; + slab_flags_t __kasan_never_merge(void); static __always_inline slab_flags_t kasan_never_merge(void) { @@ -130,20 +148,20 @@ static __always_inline void kasan_unpoison_range(const void *addr, size_t size) __kasan_unpoison_range(addr, size); } -void __kasan_alloc_pages(struct page *page, unsigned int order, bool init); -static __always_inline void kasan_alloc_pages(struct page *page, +void __kasan_poison_pages(struct page *page, unsigned int order, bool init); +static __always_inline void kasan_poison_pages(struct page *page, unsigned int order, bool init) { if (kasan_enabled()) - __kasan_alloc_pages(page, order, init); + __kasan_poison_pages(page, order, init); } -void __kasan_free_pages(struct page *page, unsigned int order, bool init); -static __always_inline void kasan_free_pages(struct page *page, - unsigned int order, bool init) +void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init); +static __always_inline void kasan_unpoison_pages(struct page *page, + unsigned int order, bool init) { if (kasan_enabled()) - __kasan_free_pages(page, order, init); + __kasan_unpoison_pages(page, order, init); } void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size, @@ -285,21 +303,15 @@ void kasan_restore_multi_shot(bool enabled); #else /* CONFIG_KASAN */ -static inline bool kasan_enabled(void) -{ - return false; -} -static inline bool kasan_has_integrated_init(void) -{ - return false; -} static inline slab_flags_t kasan_never_merge(void) { return 0; } static inline void kasan_unpoison_range(const void *address, size_t size) {} -static inline void kasan_alloc_pages(struct page *page, unsigned int order, bool init) {} -static inline void kasan_free_pages(struct page *page, unsigned int order, bool init) {} +static inline void kasan_poison_pages(struct page *page, unsigned int order, + bool init) {} +static inline void kasan_unpoison_pages(struct page *page, unsigned int order, + bool init) {} static inline void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags) {} diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 8583ed3ff344..ae7735b490b4 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include @@ -304,7 +305,6 @@ struct kvm_vcpu { struct pid __rcu *pid; int sigset_active; sigset_t sigset; - struct kvm_vcpu_stat stat; unsigned int halt_poll_ns; bool valid_wakeup; @@ -341,6 +341,8 @@ struct kvm_vcpu { bool preempted; bool ready; struct kvm_vcpu_arch arch; + struct kvm_vcpu_stat stat; + char stats_id[KVM_STATS_NAME_SIZE]; struct kvm_dirty_ring dirty_ring; }; @@ -523,6 +525,15 @@ struct kvm { #endif /* KVM_HAVE_MMU_RWLOCK */ struct mutex slots_lock; + + /* + * Protects the arch-specific fields of struct kvm_memory_slots in + * use by the VM. To be used under the slots_lock (above) or in a + * kvm->srcu critical section where acquiring the slots_lock would + * lead to deadlock with the synchronize_srcu in + * install_new_memslots. + */ + struct mutex slots_arch_lock; struct mm_struct *mm; /* userspace tied to this vm */ struct kvm_memslots __rcu *memslots[KVM_ADDRESS_SPACE_NUM]; struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; @@ -585,6 +596,11 @@ struct kvm { pid_t userspace_pid; unsigned int max_halt_poll_ns; u32 dirty_ring_size; + +#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER + struct notifier_block pm_notifier; +#endif + char stats_id[KVM_STATS_NAME_SIZE]; }; #define kvm_err(fmt, ...) \ @@ -998,6 +1014,10 @@ int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); +#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER +int kvm_arch_pm_notifier(struct kvm *kvm, unsigned long state); +#endif + #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry); #endif @@ -1244,26 +1264,104 @@ enum kvm_stat_kind { struct kvm_stat_data { struct kvm *kvm; - struct kvm_stats_debugfs_item *dbgfs_item; -}; - -struct kvm_stats_debugfs_item { - const char *name; - int offset; + const struct _kvm_stats_desc *desc; enum kvm_stat_kind kind; - int mode; }; -#define KVM_DBGFS_GET_MODE(dbgfs_item) \ - ((dbgfs_item)->mode ? (dbgfs_item)->mode : 0644) +struct _kvm_stats_desc { + struct kvm_stats_desc desc; + char name[KVM_STATS_NAME_SIZE]; +}; -#define VM_STAT(n, x, ...) \ - { n, offsetof(struct kvm, stat.x), KVM_STAT_VM, ## __VA_ARGS__ } -#define VCPU_STAT(n, x, ...) \ - { n, offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU, ## __VA_ARGS__ } +#define STATS_DESC_COMMON(type, unit, base, exp) \ + .flags = type | unit | base | \ + BUILD_BUG_ON_ZERO(type & ~KVM_STATS_TYPE_MASK) | \ + BUILD_BUG_ON_ZERO(unit & ~KVM_STATS_UNIT_MASK) | \ + BUILD_BUG_ON_ZERO(base & ~KVM_STATS_BASE_MASK), \ + .exponent = exp, \ + .size = 1 + +#define VM_GENERIC_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vm_stat, generic.stat) \ + }, \ + .name = #stat, \ + } +#define VCPU_GENERIC_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vcpu_stat, generic.stat) \ + }, \ + .name = #stat, \ + } +#define VM_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vm_stat, stat) \ + }, \ + .name = #stat, \ + } +#define VCPU_STATS_DESC(stat, type, unit, base, exp) \ + { \ + { \ + STATS_DESC_COMMON(type, unit, base, exp), \ + .offset = offsetof(struct kvm_vcpu_stat, stat) \ + }, \ + .name = #stat, \ + } +/* SCOPE: VM, VM_GENERIC, VCPU, VCPU_GENERIC */ +#define STATS_DESC(SCOPE, stat, type, unit, base, exp) \ + SCOPE##_STATS_DESC(stat, type, unit, base, exp) + +#define STATS_DESC_CUMULATIVE(SCOPE, name, unit, base, exponent) \ + STATS_DESC(SCOPE, name, KVM_STATS_TYPE_CUMULATIVE, unit, base, exponent) +#define STATS_DESC_INSTANT(SCOPE, name, unit, base, exponent) \ + STATS_DESC(SCOPE, name, KVM_STATS_TYPE_INSTANT, unit, base, exponent) +#define STATS_DESC_PEAK(SCOPE, name, unit, base, exponent) \ + STATS_DESC(SCOPE, name, KVM_STATS_TYPE_PEAK, unit, base, exponent) + +/* Cumulative counter, read/write */ +#define STATS_DESC_COUNTER(SCOPE, name) \ + STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_NONE, \ + KVM_STATS_BASE_POW10, 0) +/* Instantaneous counter, read only */ +#define STATS_DESC_ICOUNTER(SCOPE, name) \ + STATS_DESC_INSTANT(SCOPE, name, KVM_STATS_UNIT_NONE, \ + KVM_STATS_BASE_POW10, 0) +/* Peak counter, read/write */ +#define STATS_DESC_PCOUNTER(SCOPE, name) \ + STATS_DESC_PEAK(SCOPE, name, KVM_STATS_UNIT_NONE, \ + KVM_STATS_BASE_POW10, 0) + +/* Cumulative time in nanosecond */ +#define STATS_DESC_TIME_NSEC(SCOPE, name) \ + STATS_DESC_CUMULATIVE(SCOPE, name, KVM_STATS_UNIT_SECONDS, \ + KVM_STATS_BASE_POW10, -9) + +#define KVM_GENERIC_VM_STATS() \ + STATS_DESC_COUNTER(VM_GENERIC, remote_tlb_flush) + +#define KVM_GENERIC_VCPU_STATS() \ + STATS_DESC_COUNTER(VCPU_GENERIC, halt_successful_poll), \ + STATS_DESC_COUNTER(VCPU_GENERIC, halt_attempted_poll), \ + STATS_DESC_COUNTER(VCPU_GENERIC, halt_poll_invalid), \ + STATS_DESC_COUNTER(VCPU_GENERIC, halt_wakeup), \ + STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_success_ns), \ + STATS_DESC_TIME_NSEC(VCPU_GENERIC, halt_poll_fail_ns) -extern struct kvm_stats_debugfs_item debugfs_entries[]; extern struct dentry *kvm_debugfs_dir; +ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, + const struct _kvm_stats_desc *desc, + void *stats, size_t size_stats, + char __user *user_buffer, size_t size, loff_t *offset); +extern const struct kvm_stats_header kvm_vm_stats_header; +extern const struct _kvm_stats_desc kvm_vm_stats_desc[]; +extern const struct kvm_stats_header kvm_vcpu_stats_header; +extern const struct _kvm_stats_desc kvm_vcpu_stats_desc[]; #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER) static inline int mmu_notifier_retry(struct kvm *kvm, unsigned long mmu_seq) diff --git a/include/linux/kvm_types.h b/include/linux/kvm_types.h index a7580f69dda0..ed6a985c5680 100644 --- a/include/linux/kvm_types.h +++ b/include/linux/kvm_types.h @@ -76,5 +76,19 @@ struct kvm_mmu_memory_cache { }; #endif +struct kvm_vm_stat_generic { + u64 remote_tlb_flush; +}; + +struct kvm_vcpu_stat_generic { + u64 halt_successful_poll; + u64 halt_attempted_poll; + u64 halt_poll_invalid; + u64 halt_wakeup; + u64 halt_poll_success_ns; + u64 halt_poll_fail_ns; +}; + +#define KVM_STATS_NAME_SIZE 48 #endif /* __KVM_TYPES_H__ */ diff --git a/include/linux/math64.h b/include/linux/math64.h index 66deb1fdc2ef..2928f03d6d46 100644 --- a/include/linux/math64.h +++ b/include/linux/math64.h @@ -3,6 +3,7 @@ #define _LINUX_MATH64_H #include +#include #include #include @@ -234,6 +235,24 @@ static inline u64 mul_u64_u64_shr(u64 a, u64 b, unsigned int shift) #endif +#ifndef mul_s64_u64_shr +static inline u64 mul_s64_u64_shr(s64 a, u64 b, unsigned int shift) +{ + u64 ret; + + /* + * Extract the sign before the multiplication and put it back + * afterwards if needed. + */ + ret = mul_u64_u64_shr(abs(a), b, shift); + + if (a < 0) + ret = -((s64) ret); + + return ret; +} +#endif /* mul_s64_u64_shr */ + #ifndef mul_u64_u32_div static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor) { diff --git a/include/linux/mfd/madera/pdata.h b/include/linux/mfd/madera/pdata.h index 601cbbc10370..32e3470708ed 100644 --- a/include/linux/mfd/madera/pdata.h +++ b/include/linux/mfd/madera/pdata.h @@ -31,7 +31,7 @@ struct pinctrl_map; * @irq_flags: Mode for primary IRQ (defaults to active low) * @gpio_base: Base GPIO number * @gpio_configs: Array of GPIO configurations (See - * Documentation/driver-api/pinctl.rst) + * Documentation/driver-api/pin-control.rst) * @n_gpio_configs: Number of entries in gpio_configs * @gpsw: General purpose switch mode setting. Depends on the external * hardware connected to the switch. (See the SW1_MODE field diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 04a34c08e0a6..40e2c5000585 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -137,6 +137,9 @@ enum pageflags { #endif #ifdef CONFIG_64BIT PG_arch_2, +#endif +#ifdef CONFIG_KASAN_HW_TAGS + PG_skip_kasan_poison, #endif __NR_PAGEFLAGS, @@ -443,6 +446,12 @@ TESTCLEARFLAG(Young, young, PF_ANY) PAGEFLAG(Idle, idle, PF_ANY) #endif +#ifdef CONFIG_KASAN_HW_TAGS +PAGEFLAG(SkipKASanPoison, skip_kasan_poison, PF_HEAD) +#else +PAGEFLAG_FALSE(SkipKASanPoison) +#endif + /* * PageReported() is used to track reported free pages within the Buddy * allocator. We can use the non-atomic version of the test and set diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h index f5a6a2f069ed..2d510ad750ed 100644 --- a/include/linux/perf_event.h +++ b/include/linux/perf_event.h @@ -1576,6 +1576,12 @@ static struct perf_pmu_events_attr _var = { \ .event_str = _str, \ }; +#define PMU_EVENT_ATTR_ID(_name, _show, _id) \ + (&((struct perf_pmu_events_attr[]) { \ + { .attr = __ATTR(_name, 0444, _show, NULL), \ + .id = _id, } \ + })[0].attr.attr) + #define PMU_FORMAT_ATTR(_name, _format) \ static ssize_t \ _name##_show(struct device *dev, \ diff --git a/include/linux/pinctrl/pinconf-generic.h b/include/linux/pinctrl/pinconf-generic.h index e18ab3d5908f..5a96602a3316 100644 --- a/include/linux/pinctrl/pinconf-generic.h +++ b/include/linux/pinctrl/pinconf-generic.h @@ -89,7 +89,7 @@ struct pinctrl_map; * it. * @PIN_CONFIG_OUTPUT: this will configure the pin as an output and drive a * value on the line. Use argument 1 to indicate high level, argument 0 to - * indicate low level. (Please see Documentation/driver-api/pinctl.rst, + * indicate low level. (Please see Documentation/driver-api/pin-control.rst, * section "GPIO mode pitfalls" for a discussion around this parameter.) * @PIN_CONFIG_PERSIST_STATE: retain pin state across sleep or controller reset * @PIN_CONFIG_POWER_SOURCE: if the pin can select between different power diff --git a/include/linux/platform_profile.h b/include/linux/platform_profile.h index a6329003aee7..e5cbb6841f3a 100644 --- a/include/linux/platform_profile.h +++ b/include/linux/platform_profile.h @@ -2,7 +2,7 @@ /* * Platform profile sysfs interface * - * See Documentation/ABI/testing/sysfs-platform_profile.rst for more + * See Documentation/userspace-api/sysfs-platform_profile.rst for more * information. */ diff --git a/include/linux/pstore_blk.h b/include/linux/pstore_blk.h index 99564f93d774..924ca07aafbd 100644 --- a/include/linux/pstore_blk.h +++ b/include/linux/pstore_blk.h @@ -10,36 +10,15 @@ /** * struct pstore_device_info - back-end pstore/blk driver structure. * - * @total_size: The total size in bytes pstore/blk can use. It must be greater - * than 4096 and be multiple of 4096. * @flags: Refer to macro starting with PSTORE_FLAGS defined in * linux/pstore.h. It means what front-ends this device support. * Zero means all backends for compatible. - * @read: The general read operation. Both of the function parameters - * @size and @offset are relative value to bock device (not the - * whole disk). - * On success, the number of bytes should be returned, others - * means error. - * @write: The same as @read, but the following error number: - * -EBUSY means try to write again later. - * -ENOMSG means to try next zone. - * @erase: The general erase operation for device with special removing - * job. Both of the function parameters @size and @offset are - * relative value to storage. - * Return 0 on success and others on failure. - * @panic_write:The write operation only used for panic case. It's optional - * if you do not care panic log. The parameters are relative - * value to storage. - * On success, the number of bytes should be returned, others - * excluding -ENOMSG mean error. -ENOMSG means to try next zone. + * @zone: The struct pstore_zone_info details. + * */ struct pstore_device_info { - unsigned long total_size; unsigned int flags; - pstore_zone_read_op read; - pstore_zone_write_op write; - pstore_zone_erase_op erase; - pstore_zone_write_op panic_write; + struct pstore_zone_info zone; }; int register_pstore_device(struct pstore_device_info *dev); diff --git a/include/linux/sched.h b/include/linux/sched.h index 6c0c3246040b..f9107a2701d0 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2036,6 +2036,8 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) #endif /* CONFIG_SMP */ +extern bool sched_task_on_rq(struct task_struct *p); + /* * In order to reduce various lock holder preemption latencies provide an * interface to see if a vCPU is currently running or not. diff --git a/include/linux/tick.h b/include/linux/tick.h index 1a0ff88fa107..bfd571f18cfd 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -186,13 +186,17 @@ static inline bool tick_nohz_full_enabled(void) return tick_nohz_full_running; } -static inline bool tick_nohz_full_cpu(int cpu) -{ - if (!tick_nohz_full_enabled()) - return false; - - return cpumask_test_cpu(cpu, tick_nohz_full_mask); -} +/* + * Check if a CPU is part of the nohz_full subset. Arrange for evaluating + * the cpu expression (typically smp_processor_id()) _after_ the static + * key. + */ +#define tick_nohz_full_cpu(_cpu) ({ \ + bool __ret = false; \ + if (tick_nohz_full_enabled()) \ + __ret = cpumask_test_cpu((_cpu), tick_nohz_full_mask); \ + __ret; \ +}) static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { @@ -208,7 +212,7 @@ extern void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit); extern void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit); -extern void tick_nohz_dep_set_signal(struct signal_struct *signal, +extern void tick_nohz_dep_set_signal(struct task_struct *tsk, enum tick_dep_bits bit); extern void tick_nohz_dep_clear_signal(struct signal_struct *signal, enum tick_dep_bits bit); @@ -253,11 +257,11 @@ static inline void tick_dep_clear_task(struct task_struct *tsk, if (tick_nohz_full_enabled()) tick_nohz_dep_clear_task(tsk, bit); } -static inline void tick_dep_set_signal(struct signal_struct *signal, +static inline void tick_dep_set_signal(struct task_struct *tsk, enum tick_dep_bits bit) { if (tick_nohz_full_enabled()) - tick_nohz_dep_set_signal(signal, bit); + tick_nohz_dep_set_signal(tsk, bit); } static inline void tick_dep_clear_signal(struct signal_struct *signal, enum tick_dep_bits bit) @@ -285,7 +289,7 @@ static inline void tick_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit) { } static inline void tick_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit) { } -static inline void tick_dep_set_signal(struct signal_struct *signal, +static inline void tick_dep_set_signal(struct task_struct *tsk, enum tick_dep_bits bit) { } static inline void tick_dep_clear_signal(struct signal_struct *signal, enum tick_dep_bits bit) { } diff --git a/include/media/hevc-ctrls.h b/include/media/hevc-ctrls.h index b4cb2ef02f17..53c0038c792b 100644 --- a/include/media/hevc-ctrls.h +++ b/include/media/hevc-ctrls.h @@ -19,6 +19,7 @@ #define V4L2_CID_MPEG_VIDEO_HEVC_SPS (V4L2_CID_CODEC_BASE + 1008) #define V4L2_CID_MPEG_VIDEO_HEVC_PPS (V4L2_CID_CODEC_BASE + 1009) #define V4L2_CID_MPEG_VIDEO_HEVC_SLICE_PARAMS (V4L2_CID_CODEC_BASE + 1010) +#define V4L2_CID_MPEG_VIDEO_HEVC_DECODE_PARAMS (V4L2_CID_CODEC_BASE + 1012) #define V4L2_CID_MPEG_VIDEO_HEVC_DECODE_MODE (V4L2_CID_CODEC_BASE + 1015) #define V4L2_CID_MPEG_VIDEO_HEVC_START_CODE (V4L2_CID_CODEC_BASE + 1016) @@ -26,6 +27,7 @@ #define V4L2_CTRL_TYPE_HEVC_SPS 0x0120 #define V4L2_CTRL_TYPE_HEVC_PPS 0x0121 #define V4L2_CTRL_TYPE_HEVC_SLICE_PARAMS 0x0122 +#define V4L2_CTRL_TYPE_HEVC_DECODE_PARAMS 0x0124 enum v4l2_mpeg_video_hevc_decode_mode { V4L2_MPEG_VIDEO_HEVC_DECODE_MODE_SLICE_BASED, @@ -75,13 +77,12 @@ struct v4l2_ctrl_hevc_sps { __u8 num_short_term_ref_pic_sets; __u8 num_long_term_ref_pics_sps; __u8 chroma_format_idc; - - __u8 padding; + __u8 sps_max_sub_layers_minus1; __u64 flags; }; -#define V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT (1ULL << 0) +#define V4L2_HEVC_PPS_FLAG_DEPENDENT_SLICE_SEGMENT_ENABLED (1ULL << 0) #define V4L2_HEVC_PPS_FLAG_OUTPUT_FLAG_PRESENT (1ULL << 1) #define V4L2_HEVC_PPS_FLAG_SIGN_DATA_HIDING_ENABLED (1ULL << 2) #define V4L2_HEVC_PPS_FLAG_CABAC_INIT_PRESENT (1ULL << 3) @@ -100,10 +101,14 @@ struct v4l2_ctrl_hevc_sps { #define V4L2_HEVC_PPS_FLAG_PPS_DISABLE_DEBLOCKING_FILTER (1ULL << 16) #define V4L2_HEVC_PPS_FLAG_LISTS_MODIFICATION_PRESENT (1ULL << 17) #define V4L2_HEVC_PPS_FLAG_SLICE_SEGMENT_HEADER_EXTENSION_PRESENT (1ULL << 18) +#define V4L2_HEVC_PPS_FLAG_DEBLOCKING_FILTER_CONTROL_PRESENT (1ULL << 19) +#define V4L2_HEVC_PPS_FLAG_UNIFORM_SPACING (1ULL << 20) struct v4l2_ctrl_hevc_pps { /* ISO/IEC 23008-2, ITU-T Rec. H.265: Picture parameter set */ __u8 num_extra_slice_header_bits; + __u8 num_ref_idx_l0_default_active_minus1; + __u8 num_ref_idx_l1_default_active_minus1; __s8 init_qp_minus26; __u8 diff_cu_qp_delta_depth; __s8 pps_cb_qp_offset; @@ -160,6 +165,7 @@ struct v4l2_hevc_pred_weight_table { #define V4L2_HEVC_SLICE_PARAMS_FLAG_USE_INTEGER_MV (1ULL << 6) #define V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_DEBLOCKING_FILTER_DISABLED (1ULL << 7) #define V4L2_HEVC_SLICE_PARAMS_FLAG_SLICE_LOOP_FILTER_ACROSS_SLICES_ENABLED (1ULL << 8) +#define V4L2_HEVC_SLICE_PARAMS_FLAG_DEPENDENT_SLICE_SEGMENT (1ULL << 9) struct v4l2_ctrl_hevc_slice_params { __u32 bit_size; @@ -190,18 +196,10 @@ struct v4l2_ctrl_hevc_slice_params { __u8 pic_struct; /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ - __u8 num_active_dpb_entries; __u8 ref_idx_l0[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; __u8 ref_idx_l1[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; - __u8 num_rps_poc_st_curr_before; - __u8 num_rps_poc_st_curr_after; - __u8 num_rps_poc_lt_curr; - - __u8 padding; - - /* ISO/IEC 23008-2, ITU-T Rec. H.265: General slice segment header */ - struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __u8 padding[5]; /* ISO/IEC 23008-2, ITU-T Rec. H.265: Weighted prediction parameter */ struct v4l2_hevc_pred_weight_table pred_weight_table; @@ -209,4 +207,34 @@ struct v4l2_ctrl_hevc_slice_params { __u64 flags; }; +#define V4L2_HEVC_DECODE_PARAM_FLAG_IRAP_PIC 0x1 +#define V4L2_HEVC_DECODE_PARAM_FLAG_IDR_PIC 0x2 +#define V4L2_HEVC_DECODE_PARAM_FLAG_NO_OUTPUT_OF_PRIOR 0x4 + +struct v4l2_ctrl_hevc_decode_params { + __s32 pic_order_cnt_val; + __u8 num_active_dpb_entries; + struct v4l2_hevc_dpb_entry dpb[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __u8 num_poc_st_curr_before; + __u8 num_poc_st_curr_after; + __u8 num_poc_lt_curr; + __u8 poc_st_curr_before[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __u8 poc_st_curr_after[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __u8 poc_lt_curr[V4L2_HEVC_DPB_ENTRIES_NUM_MAX]; + __u64 flags; +}; + +/* MPEG-class control IDs specific to the Hantro driver as defined by V4L2 */ +#define V4L2_CID_CODEC_HANTRO_BASE (V4L2_CTRL_CLASS_CODEC | 0x1200) +/* + * V4L2_CID_HANTRO_HEVC_SLICE_HEADER_SKIP - + * the number of data (in bits) to skip in the + * slice segment header. + * If non-IDR, the bits to be skipped go from syntax element "pic_output_flag" + * to before syntax element "slice_temporal_mvp_enabled_flag". + * If IDR, the skipped bits are just "pic_output_flag" + * (separate_colour_plane_flag is not supported). + */ +#define V4L2_CID_HANTRO_HEVC_SLICE_HEADER_SKIP (V4L2_CID_CODEC_HANTRO_BASE + 0) + #endif diff --git a/include/media/media-dev-allocator.h b/include/media/media-dev-allocator.h index b35ea6062596..2ab54d426c64 100644 --- a/include/media/media-dev-allocator.h +++ b/include/media/media-dev-allocator.h @@ -19,7 +19,7 @@ struct usb_device; -#if defined(CONFIG_MEDIA_CONTROLLER) && defined(CONFIG_USB) +#if defined(CONFIG_MEDIA_CONTROLLER) && IS_ENABLED(CONFIG_USB) /** * media_device_usb_allocate() - Allocate and return struct &media device * diff --git a/include/media/mpeg2-ctrls.h b/include/media/mpeg2-ctrls.h deleted file mode 100644 index 2a4ae6701166..000000000000 --- a/include/media/mpeg2-ctrls.h +++ /dev/null @@ -1,82 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * These are the MPEG2 state controls for use with stateless MPEG-2 - * codec drivers. - * - * It turns out that these structs are not stable yet and will undergo - * more changes. So keep them private until they are stable and ready to - * become part of the official public API. - */ - -#ifndef _MPEG2_CTRLS_H_ -#define _MPEG2_CTRLS_H_ - -#define V4L2_CID_MPEG_VIDEO_MPEG2_SLICE_PARAMS (V4L2_CID_CODEC_BASE+250) -#define V4L2_CID_MPEG_VIDEO_MPEG2_QUANTIZATION (V4L2_CID_CODEC_BASE+251) - -/* enum v4l2_ctrl_type type values */ -#define V4L2_CTRL_TYPE_MPEG2_SLICE_PARAMS 0x0103 -#define V4L2_CTRL_TYPE_MPEG2_QUANTIZATION 0x0104 - -#define V4L2_MPEG2_PICTURE_CODING_TYPE_I 1 -#define V4L2_MPEG2_PICTURE_CODING_TYPE_P 2 -#define V4L2_MPEG2_PICTURE_CODING_TYPE_B 3 -#define V4L2_MPEG2_PICTURE_CODING_TYPE_D 4 - -struct v4l2_mpeg2_sequence { - /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence header */ - __u16 horizontal_size; - __u16 vertical_size; - __u32 vbv_buffer_size; - - /* ISO/IEC 13818-2, ITU-T Rec. H.262: Sequence extension */ - __u16 profile_and_level_indication; - __u8 progressive_sequence; - __u8 chroma_format; -}; - -struct v4l2_mpeg2_picture { - /* ISO/IEC 13818-2, ITU-T Rec. H.262: Picture header */ - __u8 picture_coding_type; - - /* ISO/IEC 13818-2, ITU-T Rec. H.262: Picture coding extension */ - __u8 f_code[2][2]; - __u8 intra_dc_precision; - __u8 picture_structure; - __u8 top_field_first; - __u8 frame_pred_frame_dct; - __u8 concealment_motion_vectors; - __u8 q_scale_type; - __u8 intra_vlc_format; - __u8 alternate_scan; - __u8 repeat_first_field; - __u16 progressive_frame; -}; - -struct v4l2_ctrl_mpeg2_slice_params { - __u32 bit_size; - __u32 data_bit_offset; - __u64 backward_ref_ts; - __u64 forward_ref_ts; - - struct v4l2_mpeg2_sequence sequence; - struct v4l2_mpeg2_picture picture; - - /* ISO/IEC 13818-2, ITU-T Rec. H.262: Slice */ - __u32 quantiser_scale_code; -}; - -struct v4l2_ctrl_mpeg2_quantization { - /* ISO/IEC 13818-2, ITU-T Rec. H.262: Quant matrix extension */ - __u8 load_intra_quantiser_matrix; - __u8 load_non_intra_quantiser_matrix; - __u8 load_chroma_intra_quantiser_matrix; - __u8 load_chroma_non_intra_quantiser_matrix; - - __u8 intra_quantiser_matrix[64]; - __u8 non_intra_quantiser_matrix[64]; - __u8 chroma_intra_quantiser_matrix[64]; - __u8 chroma_non_intra_quantiser_matrix[64]; -}; - -#endif diff --git a/include/media/rc-map.h b/include/media/rc-map.h index b5585d14fff4..793b54342dff 100644 --- a/include/media/rc-map.h +++ b/include/media/rc-map.h @@ -231,6 +231,7 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_CEC "rc-cec" #define RC_MAP_CINERGY "rc-cinergy" #define RC_MAP_CINERGY_1400 "rc-cinergy-1400" +#define RC_MAP_CT_90405 "rc-ct-90405" #define RC_MAP_D680_DMB "rc-d680-dmb" #define RC_MAP_DELOCK_61959 "rc-delock-61959" #define RC_MAP_DIB0700_NEC_TABLE "rc-dib0700-nec" @@ -312,7 +313,6 @@ struct rc_map *rc_map_get(const char *name); #define RC_MAP_SNAPSTREAM_FIREFLY "rc-snapstream-firefly" #define RC_MAP_STREAMZAP "rc-streamzap" #define RC_MAP_SU3000 "rc-su3000" -#define RC_MAP_TANGO "rc-tango" #define RC_MAP_TANIX_TX3MINI "rc-tanix-tx3mini" #define RC_MAP_TANIX_TX5MAX "rc-tanix-tx5max" #define RC_MAP_TBS_NEC "rc-tbs-nec" diff --git a/include/media/v4l2-ctrls.h b/include/media/v4l2-ctrls.h index a5953b812878..575b59fbac77 100644 --- a/include/media/v4l2-ctrls.h +++ b/include/media/v4l2-ctrls.h @@ -17,7 +17,6 @@ * Include the stateless codec compound control definitions. * This will move to the public headers once this API is fully stable. */ -#include #include /* forward references */ @@ -40,8 +39,9 @@ struct video_device; * @p_u16: Pointer to a 16-bit unsigned value. * @p_u32: Pointer to a 32-bit unsigned value. * @p_char: Pointer to a string. - * @p_mpeg2_slice_params: Pointer to a MPEG2 slice parameters structure. - * @p_mpeg2_quantization: Pointer to a MPEG2 quantization data structure. + * @p_mpeg2_sequence: Pointer to a MPEG2 sequence structure. + * @p_mpeg2_picture: Pointer to a MPEG2 picture structure. + * @p_mpeg2_quantisation: Pointer to a MPEG2 quantisation data structure. * @p_fwht_params: Pointer to a FWHT stateless parameters structure. * @p_h264_sps: Pointer to a struct v4l2_ctrl_h264_sps. * @p_h264_pps: Pointer to a struct v4l2_ctrl_h264_pps. @@ -66,8 +66,9 @@ union v4l2_ctrl_ptr { u16 *p_u16; u32 *p_u32; char *p_char; - struct v4l2_ctrl_mpeg2_slice_params *p_mpeg2_slice_params; - struct v4l2_ctrl_mpeg2_quantization *p_mpeg2_quantization; + struct v4l2_ctrl_mpeg2_sequence *p_mpeg2_sequence; + struct v4l2_ctrl_mpeg2_picture *p_mpeg2_picture; + struct v4l2_ctrl_mpeg2_quantisation *p_mpeg2_quantisation; struct v4l2_ctrl_fwht_params *p_fwht_params; struct v4l2_ctrl_h264_sps *p_h264_sps; struct v4l2_ctrl_h264_pps *p_h264_pps; diff --git a/include/media/v4l2-subdev.h b/include/media/v4l2-subdev.h index d0e9a5bdb08b..95f8bfd63273 100644 --- a/include/media/v4l2-subdev.h +++ b/include/media/v4l2-subdev.h @@ -162,6 +162,9 @@ struct v4l2_subdev_io_pin_config { * @s_gpio: set GPIO pins. Very simple right now, might need to be extended with * a direction argument if needed. * + * @command: called by in-kernel drivers in order to call functions internal + * to subdev drivers driver that have a separate callback. + * * @ioctl: called at the end of ioctl() syscall handler at the V4L2 core. * used to provide support for private ioctls used on the driver. * @@ -193,6 +196,7 @@ struct v4l2_subdev_core_ops { int (*load_fw)(struct v4l2_subdev *sd); int (*reset)(struct v4l2_subdev *sd, u32 val); int (*s_gpio)(struct v4l2_subdev *sd, u32 val); + long (*command)(struct v4l2_subdev *sd, unsigned int cmd, void *arg); long (*ioctl)(struct v4l2_subdev *sd, unsigned int cmd, void *arg); #ifdef CONFIG_COMPAT long (*compat_ioctl32)(struct v4l2_subdev *sd, unsigned int cmd, @@ -623,6 +627,19 @@ struct v4l2_subdev_pad_config { struct v4l2_rect try_compose; }; +/** + * struct v4l2_subdev_state - Used for storing subdev state information. + * + * @pads: &struct v4l2_subdev_pad_config array + * + * This structure only needs to be passed to the pad op if the 'which' field + * of the main argument is set to %V4L2_SUBDEV_FORMAT_TRY. For + * %V4L2_SUBDEV_FORMAT_ACTIVE it is safe to pass %NULL. + */ +struct v4l2_subdev_state { + struct v4l2_subdev_pad_config *pads; +}; + /** * struct v4l2_subdev_pad_ops - v4l2-subdev pad level operations * @@ -687,27 +704,27 @@ struct v4l2_subdev_pad_config { */ struct v4l2_subdev_pad_ops { int (*init_cfg)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg); + struct v4l2_subdev_state *state); int (*enum_mbus_code)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_mbus_code_enum *code); int (*enum_frame_size)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_frame_size_enum *fse); int (*enum_frame_interval)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_frame_interval_enum *fie); int (*get_fmt)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_format *format); int (*set_fmt)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_format *format); int (*get_selection)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_selection *sel); int (*set_selection)(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, struct v4l2_subdev_selection *sel); int (*get_edid)(struct v4l2_subdev *sd, struct v4l2_edid *edid); int (*set_edid)(struct v4l2_subdev *sd, struct v4l2_edid *edid); @@ -918,14 +935,14 @@ struct v4l2_subdev { * struct v4l2_subdev_fh - Used for storing subdev information per file handle * * @vfh: pointer to &struct v4l2_fh - * @pad: pointer to &struct v4l2_subdev_pad_config + * @state: pointer to &struct v4l2_subdev_state * @owner: module pointer to the owner of this file handle */ struct v4l2_subdev_fh { struct v4l2_fh vfh; struct module *owner; #if defined(CONFIG_VIDEO_V4L2_SUBDEV_API) - struct v4l2_subdev_pad_config *pad; + struct v4l2_subdev_state *state; #endif }; @@ -945,17 +962,17 @@ struct v4l2_subdev_fh { * &struct v4l2_subdev_pad_config->try_fmt * * @sd: pointer to &struct v4l2_subdev - * @cfg: pointer to &struct v4l2_subdev_pad_config array. - * @pad: index of the pad in the @cfg array. + * @state: pointer to &struct v4l2_subdev_state + * @pad: index of the pad in the &struct v4l2_subdev_state->pads array */ static inline struct v4l2_mbus_framefmt * v4l2_subdev_get_try_format(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, unsigned int pad) { if (WARN_ON(pad >= sd->entity.num_pads)) pad = 0; - return &cfg[pad].try_fmt; + return &state->pads[pad].try_fmt; } /** @@ -963,17 +980,17 @@ v4l2_subdev_get_try_format(struct v4l2_subdev *sd, * &struct v4l2_subdev_pad_config->try_crop * * @sd: pointer to &struct v4l2_subdev - * @cfg: pointer to &struct v4l2_subdev_pad_config array. - * @pad: index of the pad in the @cfg array. + * @state: pointer to &struct v4l2_subdev_state. + * @pad: index of the pad in the &struct v4l2_subdev_state->pads array. */ static inline struct v4l2_rect * v4l2_subdev_get_try_crop(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, unsigned int pad) { if (WARN_ON(pad >= sd->entity.num_pads)) pad = 0; - return &cfg[pad].try_crop; + return &state->pads[pad].try_crop; } /** @@ -981,17 +998,17 @@ v4l2_subdev_get_try_crop(struct v4l2_subdev *sd, * &struct v4l2_subdev_pad_config->try_compose * * @sd: pointer to &struct v4l2_subdev - * @cfg: pointer to &struct v4l2_subdev_pad_config array. - * @pad: index of the pad in the @cfg array. + * @state: pointer to &struct v4l2_subdev_state. + * @pad: index of the pad in the &struct v4l2_subdev_state->pads array. */ static inline struct v4l2_rect * v4l2_subdev_get_try_compose(struct v4l2_subdev *sd, - struct v4l2_subdev_pad_config *cfg, + struct v4l2_subdev_state *state, unsigned int pad) { if (WARN_ON(pad >= sd->entity.num_pads)) pad = 0; - return &cfg[pad].try_compose; + return &state->pads[pad].try_compose; } #endif @@ -1093,20 +1110,21 @@ int v4l2_subdev_link_validate_default(struct v4l2_subdev *sd, int v4l2_subdev_link_validate(struct media_link *link); /** - * v4l2_subdev_alloc_pad_config - Allocates memory for pad config + * v4l2_subdev_alloc_state - allocate v4l2_subdev_state * - * @sd: pointer to struct v4l2_subdev + * @sd: pointer to &struct v4l2_subdev for which the state is being allocated. + * + * Must call v4l2_subdev_free_state() when state is no longer needed. */ -struct -v4l2_subdev_pad_config *v4l2_subdev_alloc_pad_config(struct v4l2_subdev *sd); +struct v4l2_subdev_state *v4l2_subdev_alloc_state(struct v4l2_subdev *sd); /** - * v4l2_subdev_free_pad_config - Frees memory allocated by - * v4l2_subdev_alloc_pad_config(). + * v4l2_subdev_free_state - free a v4l2_subdev_state * - * @cfg: pointer to &struct v4l2_subdev_pad_config + * @state: v4l2_subdev_state to be freed. */ -void v4l2_subdev_free_pad_config(struct v4l2_subdev_pad_config *cfg); +void v4l2_subdev_free_state(struct v4l2_subdev_state *state); + #endif /* CONFIG_MEDIA_CONTROLLER */ /** diff --git a/include/media/videobuf2-v4l2.h b/include/media/videobuf2-v4l2.h index c203047eb834..b66585e304e2 100644 --- a/include/media/videobuf2-v4l2.h +++ b/include/media/videobuf2-v4l2.h @@ -261,6 +261,22 @@ int __must_check vb2_queue_init_name(struct vb2_queue *q, const char *name); */ void vb2_queue_release(struct vb2_queue *q); +/** + * vb2_queue_change_type() - change the type of an inactive vb2_queue + * @q: pointer to &struct vb2_queue with videobuf2 queue. + * @type: the type to change to (V4L2_BUF_TYPE_VIDEO_*) + * + * This function changes the type of the vb2_queue. This is only possible + * if the queue is not busy (i.e. no buffers have been allocated). + * + * vb2_queue_change_type() can be used to support multiple buffer types using + * the same queue. The driver can implement v4l2_ioctl_ops.vidioc_reqbufs and + * v4l2_ioctl_ops.vidioc_create_bufs functions and call vb2_queue_change_type() + * before calling vb2_ioctl_reqbufs() or vb2_ioctl_create_bufs(), and thus + * "lock" the buffer type until the buffers have been released. + */ +int vb2_queue_change_type(struct vb2_queue *q, unsigned int type); + /** * vb2_poll() - implements poll userspace operation * @q: pointer to &struct vb2_queue with videobuf2 queue. diff --git a/include/trace/events/btrfs.h b/include/trace/events/btrfs.h index a41dd8a0c730..c7237317a8b9 100644 --- a/include/trace/events/btrfs.h +++ b/include/trace/events/btrfs.h @@ -99,8 +99,7 @@ struct btrfs_space_info; EM( ALLOC_CHUNK, "ALLOC_CHUNK") \ EM( ALLOC_CHUNK_FORCE, "ALLOC_CHUNK_FORCE") \ EM( RUN_DELAYED_IPUTS, "RUN_DELAYED_IPUTS") \ - EM( COMMIT_TRANS, "COMMIT_TRANS") \ - EMe(FORCE_COMMIT_TRANS, "FORCE_COMMIT_TRANS") + EMe(COMMIT_TRANS, "COMMIT_TRANS") /* * First define the enums in the above macros to be exported to userspace via @@ -654,34 +653,30 @@ DEFINE_EVENT(btrfs__writepage, __extent_writepage, TRACE_EVENT(btrfs_writepage_end_io_hook, - TP_PROTO(const struct page *page, u64 start, u64 end, int uptodate), + TP_PROTO(const struct btrfs_inode *inode, u64 start, u64 end, + int uptodate), - TP_ARGS(page, start, end, uptodate), + TP_ARGS(inode, start, end, uptodate), TP_STRUCT__entry_btrfs( __field( u64, ino ) - __field( unsigned long, index ) __field( u64, start ) __field( u64, end ) __field( int, uptodate ) __field( u64, root_objectid ) ), - TP_fast_assign_btrfs(btrfs_sb(page->mapping->host->i_sb), - __entry->ino = btrfs_ino(BTRFS_I(page->mapping->host)); - __entry->index = page->index; + TP_fast_assign_btrfs(inode->root->fs_info, + __entry->ino = btrfs_ino(inode); __entry->start = start; __entry->end = end; __entry->uptodate = uptodate; - __entry->root_objectid = - BTRFS_I(page->mapping->host)->root->root_key.objectid; + __entry->root_objectid = inode->root->root_key.objectid; ), - TP_printk_btrfs("root=%llu(%s) ino=%llu page_index=%lu start=%llu " - "end=%llu uptodate=%d", + TP_printk_btrfs("root=%llu(%s) ino=%llu start=%llu end=%llu uptodate=%d", show_root_type(__entry->root_objectid), - __entry->ino, __entry->index, - __entry->start, + __entry->ino, __entry->start, __entry->end, __entry->uptodate) ); diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h index 629c7a0eaff2..390270e00a1d 100644 --- a/include/trace/events/mmflags.h +++ b/include/trace/events/mmflags.h @@ -85,6 +85,12 @@ #define IF_HAVE_PG_ARCH_2(flag,string) #endif +#ifdef CONFIG_KASAN_HW_TAGS +#define IF_HAVE_PG_SKIP_KASAN_POISON(flag,string) ,{1UL << flag, string} +#else +#define IF_HAVE_PG_SKIP_KASAN_POISON(flag,string) +#endif + #define __def_pageflag_names \ {1UL << PG_locked, "locked" }, \ {1UL << PG_waiters, "waiters" }, \ @@ -112,7 +118,8 @@ IF_HAVE_PG_UNCACHED(PG_uncached, "uncached" ) \ IF_HAVE_PG_HWPOISON(PG_hwpoison, "hwpoison" ) \ IF_HAVE_PG_IDLE(PG_young, "young" ) \ IF_HAVE_PG_IDLE(PG_idle, "idle" ) \ -IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" ) +IF_HAVE_PG_ARCH_2(PG_arch_2, "arch_2" ) \ +IF_HAVE_PG_SKIP_KASAN_POISON(PG_skip_kasan_poison, "skip_kasan_poison") #define show_page_flags(flags) \ (flags) ? __print_flags(flags, "|", \ diff --git a/include/uapi/linux/btrfs.h b/include/uapi/linux/btrfs.h index 5df73001aad4..22cd037123fa 100644 --- a/include/uapi/linux/btrfs.h +++ b/include/uapi/linux/btrfs.h @@ -154,7 +154,7 @@ struct btrfs_scrub_progress { __u64 tree_bytes_scrubbed; /* # of tree bytes scrubbed */ __u64 read_errors; /* # of read errors encountered (EIO) */ __u64 csum_errors; /* # of failed csum checks */ - __u64 verify_errors; /* # of occurences, where the metadata + __u64 verify_errors; /* # of occurrences, where the metadata * of a tree block did not match the * expected values, like generation or * logical */ @@ -174,7 +174,7 @@ struct btrfs_scrub_progress { __u64 last_physical; /* last physical address scrubbed. In * case a scrub was aborted, this can * be used to restart the scrub */ - __u64 unverified_errors; /* # of occurences where a read for a + __u64 unverified_errors; /* # of occurrences where a read for a * full (64k) bio failed, but the re- * check succeeded for each 4k piece. * Intermittent error. */ diff --git a/include/uapi/linux/btrfs_tree.h b/include/uapi/linux/btrfs_tree.h index 58d7cff9afb1..ccdb40fe40dc 100644 --- a/include/uapi/linux/btrfs_tree.h +++ b/include/uapi/linux/btrfs_tree.h @@ -59,7 +59,7 @@ /* for storing balance parameters in the root tree */ #define BTRFS_BALANCE_OBJECTID -4ULL -/* orhpan objectid for tracking unlinked/truncated files */ +/* orphan objectid for tracking unlinked/truncated files */ #define BTRFS_ORPHAN_OBJECTID -5ULL /* does write ahead logging to speed up fsyncs */ @@ -275,7 +275,7 @@ #define BTRFS_PERSISTENT_ITEM_KEY 249 /* - * Persistantly stores the device replace state in the device tree. + * Persistently stores the device replace state in the device tree. * The key is built like this: (0, BTRFS_DEV_REPLACE_KEY, 0). */ #define BTRFS_DEV_REPLACE_KEY 250 diff --git a/include/uapi/linux/cec-funcs.h b/include/uapi/linux/cec-funcs.h index 37590027b604..c3baaea0b8ef 100644 --- a/include/uapi/linux/cec-funcs.h +++ b/include/uapi/linux/cec-funcs.h @@ -1665,7 +1665,7 @@ static inline void cec_ops_report_current_latency(const struct cec_msg *msg, if (*audio_out_compensated == 3 && msg->len >= 7) *audio_out_delay = msg->msg[6]; else - *audio_out_delay = 0; + *audio_out_delay = 1; } static inline void cec_msg_request_current_latency(struct cec_msg *msg, diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 79d9c44d1ad7..d9e4aabcb31a 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -280,6 +280,9 @@ struct kvm_xen_exit { /* Encounter unexpected vm-exit reason */ #define KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON 4 +/* Flags that describe what fields in emulation_failure hold valid data. */ +#define KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES (1ULL << 0) + /* for KVM_RUN, returned by mmap(vcpu_fd, offset=0) */ struct kvm_run { /* in */ @@ -383,6 +386,25 @@ struct kvm_run { __u32 ndata; __u64 data[16]; } internal; + /* + * KVM_INTERNAL_ERROR_EMULATION + * + * "struct emulation_failure" is an overlay of "struct internal" + * that is used for the KVM_INTERNAL_ERROR_EMULATION sub-type of + * KVM_EXIT_INTERNAL_ERROR. Note, unlike other internal error + * sub-types, this struct is ABI! It also needs to be backwards + * compatible with "struct internal". Take special care that + * "ndata" is correct, that new fields are enumerated in "flags", + * and that each flag enumerates fields that are 64-bit aligned + * and sized (so that ndata+internal.data[] is valid/accurate). + */ + struct { + __u32 suberror; + __u32 ndata; + __u64 flags; + __u8 insn_size; + __u8 insn_bytes[15]; + } emulation_failure; /* KVM_EXIT_OSI */ struct { __u64 gprs[32]; @@ -1083,6 +1105,13 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_SGX_ATTRIBUTE 196 #define KVM_CAP_VM_COPY_ENC_CONTEXT_FROM 197 #define KVM_CAP_PTP_KVM 198 +#define KVM_CAP_HYPERV_ENFORCE_CPUID 199 +#define KVM_CAP_SREGS2 200 +#define KVM_CAP_EXIT_HYPERCALL 201 +#define KVM_CAP_PPC_RPT_INVALIDATE 202 +#define KVM_CAP_BINARY_STATS_FD 203 +#define KVM_CAP_EXIT_ON_EMULATION_FAILURE 204 +#define KVM_CAP_ARM_MTE 205 #ifdef KVM_CAP_IRQ_ROUTING @@ -1428,6 +1457,7 @@ struct kvm_s390_ucas_mapping { /* Available with KVM_CAP_PMU_EVENT_FILTER */ #define KVM_SET_PMU_EVENT_FILTER _IOW(KVMIO, 0xb2, struct kvm_pmu_event_filter) #define KVM_PPC_SVM_OFF _IO(KVMIO, 0xb3) +#define KVM_ARM_MTE_COPY_TAGS _IOR(KVMIO, 0xb4, struct kvm_arm_copy_mte_tags) /* ioctl for vm fd */ #define KVM_CREATE_DEVICE _IOWR(KVMIO, 0xe0, struct kvm_create_device) @@ -1621,6 +1651,9 @@ struct kvm_xen_hvm_attr { #define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr) #define KVM_XEN_VCPU_SET_ATTR _IOW(KVMIO, 0xcb, struct kvm_xen_vcpu_attr) +#define KVM_GET_SREGS2 _IOR(KVMIO, 0xcc, struct kvm_sregs2) +#define KVM_SET_SREGS2 _IOW(KVMIO, 0xcd, struct kvm_sregs2) + struct kvm_xen_vcpu_attr { __u16 type; __u16 pad[3]; @@ -1899,4 +1932,76 @@ struct kvm_dirty_gfn { #define KVM_BUS_LOCK_DETECTION_OFF (1 << 0) #define KVM_BUS_LOCK_DETECTION_EXIT (1 << 1) +/** + * struct kvm_stats_header - Header of per vm/vcpu binary statistics data. + * @flags: Some extra information for header, always 0 for now. + * @name_size: The size in bytes of the memory which contains statistics + * name string including trailing '\0'. The memory is allocated + * at the send of statistics descriptor. + * @num_desc: The number of statistics the vm or vcpu has. + * @id_offset: The offset of the vm/vcpu stats' id string in the file pointed + * by vm/vcpu stats fd. + * @desc_offset: The offset of the vm/vcpu stats' descriptor block in the file + * pointd by vm/vcpu stats fd. + * @data_offset: The offset of the vm/vcpu stats' data block in the file + * pointed by vm/vcpu stats fd. + * + * This is the header userspace needs to read from stats fd before any other + * readings. It is used by userspace to discover all the information about the + * vm/vcpu's binary statistics. + * Userspace reads this header from the start of the vm/vcpu's stats fd. + */ +struct kvm_stats_header { + __u32 flags; + __u32 name_size; + __u32 num_desc; + __u32 id_offset; + __u32 desc_offset; + __u32 data_offset; +}; + +#define KVM_STATS_TYPE_SHIFT 0 +#define KVM_STATS_TYPE_MASK (0xF << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_CUMULATIVE (0x0 << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_INSTANT (0x1 << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_PEAK (0x2 << KVM_STATS_TYPE_SHIFT) +#define KVM_STATS_TYPE_MAX KVM_STATS_TYPE_PEAK + +#define KVM_STATS_UNIT_SHIFT 4 +#define KVM_STATS_UNIT_MASK (0xF << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_NONE (0x0 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_BYTES (0x1 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_SECONDS (0x2 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_CYCLES (0x3 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_MAX KVM_STATS_UNIT_CYCLES + +#define KVM_STATS_BASE_SHIFT 8 +#define KVM_STATS_BASE_MASK (0xF << KVM_STATS_BASE_SHIFT) +#define KVM_STATS_BASE_POW10 (0x0 << KVM_STATS_BASE_SHIFT) +#define KVM_STATS_BASE_POW2 (0x1 << KVM_STATS_BASE_SHIFT) +#define KVM_STATS_BASE_MAX KVM_STATS_BASE_POW2 + +/** + * struct kvm_stats_desc - Descriptor of a KVM statistics. + * @flags: Annotations of the stats, like type, unit, etc. + * @exponent: Used together with @flags to determine the unit. + * @size: The number of data items for this stats. + * Every data item is of type __u64. + * @offset: The offset of the stats to the start of stat structure in + * struture kvm or kvm_vcpu. + * @unused: Unused field for future usage. Always 0 for now. + * @name: The name string for the stats. Its size is indicated by the + * &kvm_stats_header->name_size. + */ +struct kvm_stats_desc { + __u32 flags; + __s16 exponent; + __u16 size; + __u32 offset; + __u32 unused; + char name[]; +}; + +#define KVM_GET_STATS_FD _IO(KVMIO, 0xce) + #endif /* __LINUX_KVM_H */ diff --git a/include/uapi/linux/kvm_para.h b/include/uapi/linux/kvm_para.h index 8b86609849b9..960c7e93d1a9 100644 --- a/include/uapi/linux/kvm_para.h +++ b/include/uapi/linux/kvm_para.h @@ -29,6 +29,7 @@ #define KVM_HC_CLOCK_PAIRING 9 #define KVM_HC_SEND_IPI 10 #define KVM_HC_SCHED_YIELD 11 +#define KVM_HC_MAP_GPA_RANGE 12 /* * hypercalls use architecture specific diff --git a/include/uapi/linux/lirc.h b/include/uapi/linux/lirc.h index c45a4eaea667..9919f2062b14 100644 --- a/include/uapi/linux/lirc.h +++ b/include/uapi/linux/lirc.h @@ -1,7 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ /* * lirc.h - linux infrared remote control header file - * last modified 2010/07/13 by Jarod Wilson */ #ifndef _LINUX_LIRC_H diff --git a/include/uapi/linux/seccomp.h b/include/uapi/linux/seccomp.h index 6ba18b82a02e..78074254ab98 100644 --- a/include/uapi/linux/seccomp.h +++ b/include/uapi/linux/seccomp.h @@ -115,6 +115,7 @@ struct seccomp_notif_resp { /* valid flags for seccomp_notif_addfd */ #define SECCOMP_ADDFD_FLAG_SETFD (1UL << 0) /* Specify remote fd */ +#define SECCOMP_ADDFD_FLAG_SEND (1UL << 1) /* Addfd and return it, atomically */ /** * struct seccomp_notif_addfd diff --git a/include/uapi/linux/v4l2-controls.h b/include/uapi/linux/v4l2-controls.h index d43bec5f1afd..fdf97a6d7d18 100644 --- a/include/uapi/linux/v4l2-controls.h +++ b/include/uapi/linux/v4l2-controls.h @@ -50,6 +50,7 @@ #ifndef __LINUX_V4L2_CONTROLS_H #define __LINUX_V4L2_CONTROLS_H +#include #include /* Control classes */ @@ -1602,30 +1603,30 @@ struct v4l2_ctrl_h264_decode_params { #define V4L2_FWHT_VERSION 3 /* Set if this is an interlaced format */ -#define V4L2_FWHT_FL_IS_INTERLACED BIT(0) +#define V4L2_FWHT_FL_IS_INTERLACED _BITUL(0) /* Set if this is a bottom-first (NTSC) interlaced format */ -#define V4L2_FWHT_FL_IS_BOTTOM_FIRST BIT(1) +#define V4L2_FWHT_FL_IS_BOTTOM_FIRST _BITUL(1) /* Set if each 'frame' contains just one field */ -#define V4L2_FWHT_FL_IS_ALTERNATE BIT(2) +#define V4L2_FWHT_FL_IS_ALTERNATE _BITUL(2) /* * If V4L2_FWHT_FL_IS_ALTERNATE was set, then this is set if this * 'frame' is the bottom field, else it is the top field. */ -#define V4L2_FWHT_FL_IS_BOTTOM_FIELD BIT(3) +#define V4L2_FWHT_FL_IS_BOTTOM_FIELD _BITUL(3) /* Set if the Y' plane is uncompressed */ -#define V4L2_FWHT_FL_LUMA_IS_UNCOMPRESSED BIT(4) +#define V4L2_FWHT_FL_LUMA_IS_UNCOMPRESSED _BITUL(4) /* Set if the Cb plane is uncompressed */ -#define V4L2_FWHT_FL_CB_IS_UNCOMPRESSED BIT(5) +#define V4L2_FWHT_FL_CB_IS_UNCOMPRESSED _BITUL(5) /* Set if the Cr plane is uncompressed */ -#define V4L2_FWHT_FL_CR_IS_UNCOMPRESSED BIT(6) +#define V4L2_FWHT_FL_CR_IS_UNCOMPRESSED _BITUL(6) /* Set if the chroma plane is full height, if cleared it is half height */ -#define V4L2_FWHT_FL_CHROMA_FULL_HEIGHT BIT(7) +#define V4L2_FWHT_FL_CHROMA_FULL_HEIGHT _BITUL(7) /* Set if the chroma plane is full width, if cleared it is half width */ -#define V4L2_FWHT_FL_CHROMA_FULL_WIDTH BIT(8) +#define V4L2_FWHT_FL_CHROMA_FULL_WIDTH _BITUL(8) /* Set if the alpha plane is uncompressed */ -#define V4L2_FWHT_FL_ALPHA_IS_UNCOMPRESSED BIT(9) +#define V4L2_FWHT_FL_ALPHA_IS_UNCOMPRESSED _BITUL(9) /* Set if this is an I Frame */ -#define V4L2_FWHT_FL_I_FRAME BIT(10) +#define V4L2_FWHT_FL_I_FRAME _BITUL(10) /* A 4-values flag - the number of components - 1 */ #define V4L2_FWHT_FL_COMPONENTS_NUM_MSK GENMASK(18, 16) @@ -1862,6 +1863,118 @@ struct v4l2_ctrl_vp8_frame { __u64 flags; }; +/* Stateless MPEG-2 controls */ + +#define V4L2_MPEG2_SEQ_FLAG_PROGRESSIVE 0x01 + +#define V4L2_CID_STATELESS_MPEG2_SEQUENCE (V4L2_CID_CODEC_STATELESS_BASE+220) +/** + * struct v4l2_ctrl_mpeg2_sequence - MPEG-2 sequence header + * + * All the members on this structure match the sequence header and sequence + * extension syntaxes as specified by the MPEG-2 specification. + * + * Fields horizontal_size, vertical_size and vbv_buffer_size are a + * combination of respective _value and extension syntax elements, + * as described in section 6.3.3 "Sequence header". + * + * @horizontal_size: combination of elements horizontal_size_value and + * horizontal_size_extension. + * @vertical_size: combination of elements vertical_size_value and + * vertical_size_extension. + * @vbv_buffer_size: combination of elements vbv_buffer_size_value and + * vbv_buffer_size_extension. + * @profile_and_level_indication: see MPEG-2 specification. + * @chroma_format: see MPEG-2 specification. + * @flags: see V4L2_MPEG2_SEQ_FLAG_{}. + */ +struct v4l2_ctrl_mpeg2_sequence { + __u16 horizontal_size; + __u16 vertical_size; + __u32 vbv_buffer_size; + __u16 profile_and_level_indication; + __u8 chroma_format; + __u8 flags; +}; + +#define V4L2_MPEG2_PIC_CODING_TYPE_I 1 +#define V4L2_MPEG2_PIC_CODING_TYPE_P 2 +#define V4L2_MPEG2_PIC_CODING_TYPE_B 3 +#define V4L2_MPEG2_PIC_CODING_TYPE_D 4 + +#define V4L2_MPEG2_PIC_TOP_FIELD 0x1 +#define V4L2_MPEG2_PIC_BOTTOM_FIELD 0x2 +#define V4L2_MPEG2_PIC_FRAME 0x3 + +#define V4L2_MPEG2_PIC_FLAG_TOP_FIELD_FIRST 0x0001 +#define V4L2_MPEG2_PIC_FLAG_FRAME_PRED_DCT 0x0002 +#define V4L2_MPEG2_PIC_FLAG_CONCEALMENT_MV 0x0004 +#define V4L2_MPEG2_PIC_FLAG_Q_SCALE_TYPE 0x0008 +#define V4L2_MPEG2_PIC_FLAG_INTRA_VLC 0x0010 +#define V4L2_MPEG2_PIC_FLAG_ALT_SCAN 0x0020 +#define V4L2_MPEG2_PIC_FLAG_REPEAT_FIRST 0x0040 +#define V4L2_MPEG2_PIC_FLAG_PROGRESSIVE 0x0080 + +#define V4L2_CID_STATELESS_MPEG2_PICTURE (V4L2_CID_CODEC_STATELESS_BASE+221) +/** + * struct v4l2_ctrl_mpeg2_picture - MPEG-2 picture header + * + * All the members on this structure match the picture header and picture + * coding extension syntaxes as specified by the MPEG-2 specification. + * + * @backward_ref_ts: timestamp of the V4L2 capture buffer to use as + * reference for backward prediction. + * @forward_ref_ts: timestamp of the V4L2 capture buffer to use as + * reference for forward prediction. These timestamp refers to the + * timestamp field in struct v4l2_buffer. Use v4l2_timeval_to_ns() + * to convert the struct timeval to a __u64. + * @flags: see V4L2_MPEG2_PIC_FLAG_{}. + * @f_code: see MPEG-2 specification. + * @picture_coding_type: see MPEG-2 specification. + * @picture_structure: see V4L2_MPEG2_PIC_{}_FIELD. + * @intra_dc_precision: see MPEG-2 specification. + * @reserved: padding field. Should be zeroed by applications. + */ +struct v4l2_ctrl_mpeg2_picture { + __u64 backward_ref_ts; + __u64 forward_ref_ts; + __u32 flags; + __u8 f_code[2][2]; + __u8 picture_coding_type; + __u8 picture_structure; + __u8 intra_dc_precision; + __u8 reserved[5]; +}; + +#define V4L2_CID_STATELESS_MPEG2_QUANTISATION (V4L2_CID_CODEC_STATELESS_BASE+222) +/** + * struct v4l2_ctrl_mpeg2_quantisation - MPEG-2 quantisation + * + * Quantisation matrices as specified by section 6.3.7 + * "Quant matrix extension". + * + * @intra_quantiser_matrix: The quantisation matrix coefficients + * for intra-coded frames, in zigzag scanning order. It is relevant + * for both luma and chroma components, although it can be superseded + * by the chroma-specific matrix for non-4:2:0 YUV formats. + * @non_intra_quantiser_matrix: The quantisation matrix coefficients + * for non-intra-coded frames, in zigzag scanning order. It is relevant + * for both luma and chroma components, although it can be superseded + * by the chroma-specific matrix for non-4:2:0 YUV formats. + * @chroma_intra_quantiser_matrix: The quantisation matrix coefficients + * for the chominance component of intra-coded frames, in zigzag scanning + * order. Only relevant for 4:2:2 and 4:4:4 YUV formats. + * @chroma_non_intra_quantiser_matrix: The quantisation matrix coefficients + * for the chrominance component of non-intra-coded frames, in zigzag scanning + * order. Only relevant for 4:2:2 and 4:4:4 YUV formats. + */ +struct v4l2_ctrl_mpeg2_quantisation { + __u8 intra_quantiser_matrix[64]; + __u8 non_intra_quantiser_matrix[64]; + __u8 chroma_intra_quantiser_matrix[64]; + __u8 chroma_non_intra_quantiser_matrix[64]; +}; + #define V4L2_CID_COLORIMETRY_CLASS_BASE (V4L2_CTRL_CLASS_COLORIMETRY | 0x900) #define V4L2_CID_COLORIMETRY_CLASS (V4L2_CTRL_CLASS_COLORIMETRY | 1) diff --git a/include/uapi/linux/videodev2.h b/include/uapi/linux/videodev2.h index d84bad19816b..7396e1145c06 100644 --- a/include/uapi/linux/videodev2.h +++ b/include/uapi/linux/videodev2.h @@ -1747,6 +1747,9 @@ struct v4l2_ext_control { struct v4l2_ctrl_h264_decode_params __user *p_h264_decode_params; struct v4l2_ctrl_fwht_params __user *p_fwht_params; struct v4l2_ctrl_vp8_frame __user *p_vp8_frame; + struct v4l2_ctrl_mpeg2_sequence __user *p_mpeg2_sequence; + struct v4l2_ctrl_mpeg2_picture __user *p_mpeg2_picture; + struct v4l2_ctrl_mpeg2_quantisation __user *p_mpeg2_quantisation; void __user *ptr; }; } __attribute__ ((packed)); @@ -1807,6 +1810,10 @@ enum v4l2_ctrl_type { V4L2_CTRL_TYPE_FWHT_PARAMS = 0x0220, V4L2_CTRL_TYPE_VP8_FRAME = 0x0240, + + V4L2_CTRL_TYPE_MPEG2_QUANTISATION = 0x0250, + V4L2_CTRL_TYPE_MPEG2_SEQUENCE = 0x0251, + V4L2_CTRL_TYPE_MPEG2_PICTURE = 0x0252, }; /* Used in the VIDIOC_QUERYCTRL ioctl for querying controls */ diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 4c14356543d9..a847dd2044c8 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -441,7 +441,8 @@ int irq_update_affinity_desc(unsigned int irq, return ret; } -int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force) +static int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, + bool force) { struct irq_desc *desc = irq_to_desc(irq); unsigned long flags; @@ -456,6 +457,36 @@ int __irq_set_affinity(unsigned int irq, const struct cpumask *mask, bool force) return ret; } +/** + * irq_set_affinity - Set the irq affinity of a given irq + * @irq: Interrupt to set affinity + * @cpumask: cpumask + * + * Fails if cpumask does not contain an online CPU + */ +int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask) +{ + return __irq_set_affinity(irq, cpumask, false); +} +EXPORT_SYMBOL_GPL(irq_set_affinity); + +/** + * irq_force_affinity - Force the irq affinity of a given irq + * @irq: Interrupt to set affinity + * @cpumask: cpumask + * + * Same as irq_set_affinity, but without checking the mask against + * online cpus. + * + * Solely for low level cpu hotplug code, where we need to make per + * cpu interrupts affine before the cpu becomes online. + */ +int irq_force_affinity(unsigned int irq, const struct cpumask *cpumask) +{ + return __irq_set_affinity(irq, cpumask, true); +} +EXPORT_SYMBOL_GPL(irq_force_affinity); + int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m) { unsigned long flags; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b51bdc840c1e..1dd248dbd1cb 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1940,6 +1940,11 @@ static inline void uclamp_post_fork(struct task_struct *p) { } static inline void init_uclamp(void) { } #endif /* CONFIG_UCLAMP_TASK */ +bool sched_task_on_rq(struct task_struct *p) +{ + return task_on_rq_queued(p); +} + static inline void enqueue_task(struct rq *rq, struct task_struct *p, int flags) { if (!(flags & ENQUEUE_NOCLOCK)) @@ -4582,6 +4587,7 @@ static struct rq *finish_task_switch(struct task_struct *prev) vtime_task_switch(prev); perf_event_task_sched_in(prev, current); finish_task(prev); + tick_nohz_task_switch(); finish_lock_switch(rq); finish_arch_post_lock_switch(); kcov_finish_switch(current); @@ -4627,7 +4633,6 @@ static struct rq *finish_task_switch(struct task_struct *prev) put_task_struct_rcu_user(prev); } - tick_nohz_task_switch(); return rq; } diff --git a/kernel/seccomp.c b/kernel/seccomp.c index 9f58049ac16d..057e17f3215d 100644 --- a/kernel/seccomp.c +++ b/kernel/seccomp.c @@ -107,6 +107,7 @@ struct seccomp_knotif { * installing process should allocate the fd as normal. * @flags: The flags for the new file descriptor. At the moment, only O_CLOEXEC * is allowed. + * @ioctl_flags: The flags used for the seccomp_addfd ioctl. * @ret: The return value of the installing process. It is set to the fd num * upon success (>= 0). * @completion: Indicates that the installing process has completed fd @@ -118,6 +119,7 @@ struct seccomp_kaddfd { struct file *file; int fd; unsigned int flags; + __u32 ioctl_flags; union { bool setfd; @@ -1065,18 +1067,37 @@ static u64 seccomp_next_notify_id(struct seccomp_filter *filter) return filter->notif->next_id++; } -static void seccomp_handle_addfd(struct seccomp_kaddfd *addfd) +static void seccomp_handle_addfd(struct seccomp_kaddfd *addfd, struct seccomp_knotif *n) { + int fd; + /* * Remove the notification, and reset the list pointers, indicating * that it has been handled. */ list_del_init(&addfd->list); if (!addfd->setfd) - addfd->ret = receive_fd(addfd->file, addfd->flags); + fd = receive_fd(addfd->file, addfd->flags); else - addfd->ret = receive_fd_replace(addfd->fd, addfd->file, - addfd->flags); + fd = receive_fd_replace(addfd->fd, addfd->file, addfd->flags); + addfd->ret = fd; + + if (addfd->ioctl_flags & SECCOMP_ADDFD_FLAG_SEND) { + /* If we fail reset and return an error to the notifier */ + if (fd < 0) { + n->state = SECCOMP_NOTIFY_SENT; + } else { + /* Return the FD we just added */ + n->flags = 0; + n->error = 0; + n->val = fd; + } + } + + /* + * Mark the notification as completed. From this point, addfd mem + * might be invalidated and we can't safely read it anymore. + */ complete(&addfd->completion); } @@ -1120,7 +1141,7 @@ static int seccomp_do_user_notification(int this_syscall, struct seccomp_kaddfd, list); /* Check if we were woken up by a addfd message */ if (addfd) - seccomp_handle_addfd(addfd); + seccomp_handle_addfd(addfd, &n); } while (n.state != SECCOMP_NOTIFY_REPLIED); @@ -1581,7 +1602,7 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, if (addfd.newfd_flags & ~O_CLOEXEC) return -EINVAL; - if (addfd.flags & ~SECCOMP_ADDFD_FLAG_SETFD) + if (addfd.flags & ~(SECCOMP_ADDFD_FLAG_SETFD | SECCOMP_ADDFD_FLAG_SEND)) return -EINVAL; if (addfd.newfd && !(addfd.flags & SECCOMP_ADDFD_FLAG_SETFD)) @@ -1591,6 +1612,7 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, if (!kaddfd.file) return -EBADF; + kaddfd.ioctl_flags = addfd.flags; kaddfd.flags = addfd.newfd_flags; kaddfd.setfd = addfd.flags & SECCOMP_ADDFD_FLAG_SETFD; kaddfd.fd = addfd.newfd; @@ -1616,6 +1638,23 @@ static long seccomp_notify_addfd(struct seccomp_filter *filter, goto out_unlock; } + if (addfd.flags & SECCOMP_ADDFD_FLAG_SEND) { + /* + * Disallow queuing an atomic addfd + send reply while there are + * some addfd requests still to process. + * + * There is no clear reason to support it and allows us to keep + * the loop on the other side straight-forward. + */ + if (!list_empty(&knotif->addfd)) { + ret = -EBUSY; + goto out_unlock; + } + + /* Allow exactly only one reply */ + knotif->state = SECCOMP_NOTIFY_REPLIED; + } + list_add(&kaddfd.list, &knotif->addfd); complete(&knotif->ready); mutex_unlock(&filter->notify_lock); diff --git a/kernel/time/Kconfig b/kernel/time/Kconfig index 83e158d016ba..7df71ef0e1fd 100644 --- a/kernel/time/Kconfig +++ b/kernel/time/Kconfig @@ -117,13 +117,14 @@ config NO_HZ_FULL the task mostly runs in userspace and has few kernel activity. You need to fill up the nohz_full boot parameter with the - desired range of dynticks CPUs. + desired range of dynticks CPUs to use it. This is implemented at + the expense of some overhead in user <-> kernel transitions: + syscalls, exceptions and interrupts. - This is implemented at the expense of some overhead in user <-> kernel - transitions: syscalls, exceptions and interrupts. Even when it's - dynamically off. + By default, without passing the nohz_full parameter, this behaves just + like NO_HZ_IDLE. - Say N. + If you're a distro say Y. endchoice diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c index 3bb96a8b49c9..29a5e54e6e10 100644 --- a/kernel/time/posix-cpu-timers.c +++ b/kernel/time/posix-cpu-timers.c @@ -523,7 +523,7 @@ static void arm_timer(struct k_itimer *timer, struct task_struct *p) if (CPUCLOCK_PERTHREAD(timer->it_clock)) tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER); else - tick_dep_set_signal(p->signal, TICK_DEP_BIT_POSIX_TIMER); + tick_dep_set_signal(p, TICK_DEP_BIT_POSIX_TIMER); } /* @@ -1358,7 +1358,7 @@ void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid, if (*newval < *nextevt) *nextevt = *newval; - tick_dep_set_signal(tsk->signal, TICK_DEP_BIT_POSIX_TIMER); + tick_dep_set_signal(tsk, TICK_DEP_BIT_POSIX_TIMER); } static int do_cpu_nanosleep(const clockid_t which_clock, int flags, diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index bf233132ef92..4f024ef0676d 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -326,6 +326,46 @@ void tick_nohz_full_kick_cpu(int cpu) irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu); } +static void tick_nohz_kick_task(struct task_struct *tsk) +{ + int cpu; + + /* + * If the task is not running, run_posix_cpu_timers() + * has nothing to elapse, IPI can then be spared. + * + * activate_task() STORE p->tick_dep_mask + * STORE p->on_rq + * __schedule() (switch to task 'p') smp_mb() (atomic_fetch_or()) + * LOCK rq->lock LOAD p->on_rq + * smp_mb__after_spin_lock() + * tick_nohz_task_switch() + * LOAD p->tick_dep_mask + */ + if (!sched_task_on_rq(tsk)) + return; + + /* + * If the task concurrently migrates to another CPU, + * we guarantee it sees the new tick dependency upon + * schedule. + * + * set_task_cpu(p, cpu); + * STORE p->cpu = @cpu + * __schedule() (switch to task 'p') + * LOCK rq->lock + * smp_mb__after_spin_lock() STORE p->tick_dep_mask + * tick_nohz_task_switch() smp_mb() (atomic_fetch_or()) + * LOAD p->tick_dep_mask LOAD p->cpu + */ + cpu = task_cpu(tsk); + + preempt_disable(); + if (cpu_online(cpu)) + tick_nohz_full_kick_cpu(cpu); + preempt_enable(); +} + /* * Kick all full dynticks CPUs in order to force these to re-evaluate * their dependency on the tick and restart it if necessary. @@ -408,19 +448,8 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu); */ void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit) { - if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) { - if (tsk == current) { - preempt_disable(); - tick_nohz_full_kick(); - preempt_enable(); - } else { - /* - * Some future tick_nohz_full_kick_task() - * should optimize this. - */ - tick_nohz_full_kick_all(); - } - } + if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) + tick_nohz_kick_task(tsk); } EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task); @@ -434,9 +463,20 @@ EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task); * Set a per-taskgroup tick dependency. Posix CPU timers need this in order to elapse * per process timers. */ -void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit) +void tick_nohz_dep_set_signal(struct task_struct *tsk, + enum tick_dep_bits bit) { - tick_nohz_dep_set_all(&sig->tick_dep_mask, bit); + int prev; + struct signal_struct *sig = tsk->signal; + + prev = atomic_fetch_or(BIT(bit), &sig->tick_dep_mask); + if (!prev) { + struct task_struct *t; + + lockdep_assert_held(&tsk->sighand->siglock); + __for_each_thread(sig, t) + tick_nohz_kick_task(t); + } } void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit) @@ -451,13 +491,10 @@ void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bi */ void __tick_nohz_task_switch(void) { - unsigned long flags; struct tick_sched *ts; - local_irq_save(flags); - if (!tick_nohz_full_cpu(smp_processor_id())) - goto out; + return; ts = this_cpu_ptr(&tick_cpu_sched); @@ -466,8 +503,6 @@ void __tick_nohz_task_switch(void) atomic_read(¤t->signal->tick_dep_mask)) tick_nohz_full_kick(); } -out: - local_irq_restore(flags); } /* Get the boot-time nohz CPU list from the kernel parameters. */ @@ -925,27 +960,31 @@ static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now) * Cancel the scheduled timer and restore the tick */ ts->tick_stopped = 0; - ts->idle_exittime = now; - tick_nohz_restart(ts, now); } -static void tick_nohz_full_update_tick(struct tick_sched *ts) +static void __tick_nohz_full_update_tick(struct tick_sched *ts, + ktime_t now) { #ifdef CONFIG_NO_HZ_FULL int cpu = smp_processor_id(); - if (!tick_nohz_full_cpu(cpu)) + if (can_stop_full_tick(cpu, ts)) + tick_nohz_stop_sched_tick(ts, cpu); + else if (ts->tick_stopped) + tick_nohz_restart_sched_tick(ts, now); +#endif +} + +static void tick_nohz_full_update_tick(struct tick_sched *ts) +{ + if (!tick_nohz_full_cpu(smp_processor_id())) return; if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE) return; - if (can_stop_full_tick(cpu, ts)) - tick_nohz_stop_sched_tick(ts, cpu); - else if (ts->tick_stopped) - tick_nohz_restart_sched_tick(ts, ktime_get()); -#endif + __tick_nohz_full_update_tick(ts, ktime_get()); } static bool can_stop_idle_tick(int cpu, struct tick_sched *ts) @@ -1194,11 +1233,13 @@ unsigned long tick_nohz_get_idle_calls(void) return ts->idle_calls; } -static void tick_nohz_account_idle_ticks(struct tick_sched *ts) +static void tick_nohz_account_idle_time(struct tick_sched *ts, + ktime_t now) { -#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE unsigned long ticks; + ts->idle_exittime = now; + if (vtime_accounting_enabled_this_cpu()) return; /* @@ -1212,21 +1253,27 @@ static void tick_nohz_account_idle_ticks(struct tick_sched *ts) */ if (ticks && ticks < LONG_MAX) account_idle_ticks(ticks); -#endif -} - -static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now) -{ - tick_nohz_restart_sched_tick(ts, now); - tick_nohz_account_idle_ticks(ts); } void tick_nohz_idle_restart_tick(void) { struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched); - if (ts->tick_stopped) - __tick_nohz_idle_restart_tick(ts, ktime_get()); + if (ts->tick_stopped) { + ktime_t now = ktime_get(); + tick_nohz_restart_sched_tick(ts, now); + tick_nohz_account_idle_time(ts, now); + } +} + +static void tick_nohz_idle_update_tick(struct tick_sched *ts, ktime_t now) +{ + if (tick_nohz_full_cpu(smp_processor_id())) + __tick_nohz_full_update_tick(ts, now); + else + tick_nohz_restart_sched_tick(ts, now); + + tick_nohz_account_idle_time(ts, now); } /** @@ -1258,7 +1305,7 @@ void tick_nohz_idle_exit(void) tick_nohz_stop_idle(ts, now); if (tick_stopped) - __tick_nohz_idle_restart_tick(ts, now); + tick_nohz_idle_update_tick(ts, now); local_irq_enable(); } diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 6bb87f2acd4e..0ecd293af344 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -97,7 +97,7 @@ slab_flags_t __kasan_never_merge(void) return 0; } -void __kasan_alloc_pages(struct page *page, unsigned int order, bool init) +void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init) { u8 tag; unsigned long i; @@ -111,7 +111,7 @@ void __kasan_alloc_pages(struct page *page, unsigned int order, bool init) kasan_unpoison(page_address(page), PAGE_SIZE << order, init); } -void __kasan_free_pages(struct page *page, unsigned int order, bool init) +void __kasan_poison_pages(struct page *page, unsigned int order, bool init) { if (likely(!PageHighMem(page))) kasan_poison(page_address(page), PAGE_SIZE << order, diff --git a/mm/kasan/hw_tags.c b/mm/kasan/hw_tags.c index 4004388b4e4b..ed5e5b833d61 100644 --- a/mm/kasan/hw_tags.c +++ b/mm/kasan/hw_tags.c @@ -238,6 +238,38 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, return &alloc_meta->free_track[0]; } +void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags) +{ + /* + * This condition should match the one in post_alloc_hook() in + * page_alloc.c. + */ + bool init = !want_init_on_free() && want_init_on_alloc(flags); + + if (flags & __GFP_SKIP_KASAN_POISON) + SetPageSkipKASanPoison(page); + + if (flags & __GFP_ZEROTAGS) { + int i; + + for (i = 0; i != 1 << order; ++i) + tag_clear_highpage(page + i); + } else { + kasan_unpoison_pages(page, order, init); + } +} + +void kasan_free_pages(struct page *page, unsigned int order) +{ + /* + * This condition should match the one in free_pages_prepare() in + * page_alloc.c. + */ + bool init = want_init_on_free(); + + kasan_poison_pages(page, order, init); +} + #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) void kasan_set_tagging_report_once(bool state) diff --git a/mm/kasan/sw_tags.c b/mm/kasan/sw_tags.c index 9df8e7f69e87..9362938abbfa 100644 --- a/mm/kasan/sw_tags.c +++ b/mm/kasan/sw_tags.c @@ -207,3 +207,10 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, return &alloc_meta->free_track[i]; } + +void kasan_tag_mismatch(unsigned long addr, unsigned long access_info, + unsigned long ret_ip) +{ + kasan_report(addr, 1 << (access_info & 0xf), access_info & 0x10, + ret_ip); +} diff --git a/mm/mempool.c b/mm/mempool.c index a258cf4de575..0b8afbec3e35 100644 --- a/mm/mempool.c +++ b/mm/mempool.c @@ -106,7 +106,8 @@ static __always_inline void kasan_poison_element(mempool_t *pool, void *element) if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) kasan_slab_free_mempool(element); else if (pool->alloc == mempool_alloc_pages) - kasan_free_pages(element, (unsigned long)pool->pool_data, false); + kasan_poison_pages(element, (unsigned long)pool->pool_data, + false); } static void kasan_unpoison_element(mempool_t *pool, void *element) @@ -114,7 +115,8 @@ static void kasan_unpoison_element(mempool_t *pool, void *element) if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) kasan_unpoison_range(element, __ksize(element)); else if (pool->alloc == mempool_alloc_pages) - kasan_alloc_pages(element, (unsigned long)pool->pool_data, false); + kasan_unpoison_pages(element, (unsigned long)pool->pool_data, + false); } static __always_inline void add_element(mempool_t *pool, void *element) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index 63f947a3ff47..b09d61589f61 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -394,7 +394,7 @@ int page_group_by_mobility_disabled __read_mostly; static DEFINE_STATIC_KEY_TRUE(deferred_pages); /* - * Calling kasan_free_pages() only after deferred memory initialization + * Calling kasan_poison_pages() only after deferred memory initialization * has completed. Poisoning pages during deferred memory init will greatly * lengthen the process and cause problem in large memory systems as the * deferred pages initialization is done with interrupt disabled. @@ -406,15 +406,12 @@ static DEFINE_STATIC_KEY_TRUE(deferred_pages); * on-demand allocation and then freed again before the deferred pages * initialization is done, but this is not likely to happen. */ -static inline void kasan_free_nondeferred_pages(struct page *page, int order, - bool init, fpi_t fpi_flags) +static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) { - if (static_branch_unlikely(&deferred_pages)) - return; - if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && - (fpi_flags & FPI_SKIP_KASAN_POISON)) - return; - kasan_free_pages(page, order, init); + return static_branch_unlikely(&deferred_pages) || + (!IS_ENABLED(CONFIG_KASAN_GENERIC) && + (fpi_flags & FPI_SKIP_KASAN_POISON)) || + PageSkipKASanPoison(page); } /* Returns true if the struct page for the pfn is uninitialised */ @@ -465,13 +462,11 @@ defer_init(int nid, unsigned long pfn, unsigned long end_pfn) return false; } #else -static inline void kasan_free_nondeferred_pages(struct page *page, int order, - bool init, fpi_t fpi_flags) +static inline bool should_skip_kasan_poison(struct page *page, fpi_t fpi_flags) { - if (!IS_ENABLED(CONFIG_KASAN_GENERIC) && - (fpi_flags & FPI_SKIP_KASAN_POISON)) - return; - kasan_free_pages(page, order, init); + return (!IS_ENABLED(CONFIG_KASAN_GENERIC) && + (fpi_flags & FPI_SKIP_KASAN_POISON)) || + PageSkipKASanPoison(page); } static inline bool early_page_uninitialised(unsigned long pfn) @@ -1238,10 +1233,16 @@ static int free_tail_pages_check(struct page *head_page, struct page *page) return ret; } -static void kernel_init_free_pages(struct page *page, int numpages) +static void kernel_init_free_pages(struct page *page, int numpages, bool zero_tags) { int i; + if (zero_tags) { + for (i = 0; i < numpages; i++) + tag_clear_highpage(page + i); + return; + } + /* s390's use of memset() could override KASAN redzones. */ kasan_disable_current(); for (i = 0; i < numpages; i++) { @@ -1257,7 +1258,7 @@ static __always_inline bool free_pages_prepare(struct page *page, unsigned int order, bool check_free, fpi_t fpi_flags) { int bad = 0; - bool init; + bool skip_kasan_poison = should_skip_kasan_poison(page, fpi_flags); VM_BUG_ON_PAGE(PageTail(page), page); @@ -1326,10 +1327,17 @@ static __always_inline bool free_pages_prepare(struct page *page, * With hardware tag-based KASAN, memory tags must be set before the * page becomes unavailable via debug_pagealloc or arch_free_page. */ - init = want_init_on_free(); - if (init && !kasan_has_integrated_init()) - kernel_init_free_pages(page, 1 << order); - kasan_free_nondeferred_pages(page, order, init, fpi_flags); + if (kasan_has_integrated_init()) { + if (!skip_kasan_poison) + kasan_free_pages(page, order); + } else { + bool init = want_init_on_free(); + + if (init) + kernel_init_free_pages(page, 1 << order, false); + if (!skip_kasan_poison) + kasan_poison_pages(page, order, init); + } /* * arch_free_page() can make the page's contents inaccessible. s390 @@ -2336,8 +2344,6 @@ static bool check_new_pages(struct page *page, unsigned int order) inline void post_alloc_hook(struct page *page, unsigned int order, gfp_t gfp_flags) { - bool init; - set_page_private(page, 0); set_page_refcounted(page); @@ -2356,10 +2362,16 @@ inline void post_alloc_hook(struct page *page, unsigned int order, * kasan_alloc_pages and kernel_init_free_pages must be * kept together to avoid discrepancies in behavior. */ - init = !want_init_on_free() && want_init_on_alloc(gfp_flags); - kasan_alloc_pages(page, order, init); - if (init && !kasan_has_integrated_init()) - kernel_init_free_pages(page, 1 << order); + if (kasan_has_integrated_init()) { + kasan_alloc_pages(page, order, gfp_flags); + } else { + bool init = !want_init_on_free() && want_init_on_alloc(gfp_flags); + + kasan_unpoison_pages(page, order, init); + if (init) + kernel_init_free_pages(page, 1 << order, + gfp_flags & __GFP_ZEROTAGS); + } set_page_owner(page, order, gfp_flags); } diff --git a/net/ipv4/ah4.c b/net/ipv4/ah4.c index 36ed85bf2ad5..fab0958c41be 100644 --- a/net/ipv4/ah4.c +++ b/net/ipv4/ah4.c @@ -450,6 +450,7 @@ static int ah4_err(struct sk_buff *skb, u32 info) case ICMP_DEST_UNREACH: if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) return 0; + break; case ICMP_REDIRECT: break; default: diff --git a/net/ipv4/esp4.c b/net/ipv4/esp4.c index 35803ab7ac80..8e3b445a8c21 100644 --- a/net/ipv4/esp4.c +++ b/net/ipv4/esp4.c @@ -982,6 +982,7 @@ static int esp4_err(struct sk_buff *skb, u32 info) case ICMP_DEST_UNREACH: if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) return 0; + break; case ICMP_REDIRECT: break; default: diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index a632b66bc13a..4c0c33e4710d 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -1874,6 +1874,7 @@ static int call_fib_nh_notifiers(struct fib_nh *nh, (nh->fib_nh_flags & RTNH_F_DEAD)) return call_fib4_notifiers(dev_net(nh->fib_nh_dev), event_type, &info.info); + break; default: break; } diff --git a/net/ipv4/ip_vti.c b/net/ipv4/ip_vti.c index 31c6c6d99d5e..eb560eecee08 100644 --- a/net/ipv4/ip_vti.c +++ b/net/ipv4/ip_vti.c @@ -351,6 +351,7 @@ static int vti4_err(struct sk_buff *skb, u32 info) case ICMP_DEST_UNREACH: if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) return 0; + break; case ICMP_REDIRECT: break; default: diff --git a/net/ipv4/ipcomp.c b/net/ipv4/ipcomp.c index b42683212c65..bbb56f5e06dd 100644 --- a/net/ipv4/ipcomp.c +++ b/net/ipv4/ipcomp.c @@ -31,6 +31,7 @@ static int ipcomp4_err(struct sk_buff *skb, u32 info) case ICMP_DEST_UNREACH: if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) return 0; + break; case ICMP_REDIRECT: break; default: diff --git a/net/netrom/nr_route.c b/net/netrom/nr_route.c index 78da5eab252a..de0456073dc0 100644 --- a/net/netrom/nr_route.c +++ b/net/netrom/nr_route.c @@ -266,6 +266,7 @@ static int __must_check nr_add_node(ax25_address *nr, const char *mnemonic, fallthrough; case 2: re_sort_routes(nr_node, 0, 1); + break; case 1: break; } @@ -359,6 +360,7 @@ static int nr_del_node(ax25_address *callsign, ax25_address *neighbour, struct n fallthrough; case 1: nr_node->routes[1] = nr_node->routes[2]; + fallthrough; case 2: break; } @@ -482,6 +484,7 @@ static int nr_dec_obs(void) fallthrough; case 1: s->routes[1] = s->routes[2]; + break; case 2: break; } @@ -529,6 +532,7 @@ void nr_rt_device_down(struct net_device *dev) fallthrough; case 1: t->routes[1] = t->routes[2]; + break; case 2: break; } diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 330ba68828e7..d56941d51e20 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -1656,6 +1656,7 @@ static int fanout_add(struct sock *sk, struct fanout_args *args) case PACKET_FANOUT_ROLLOVER: if (type_flags & PACKET_FANOUT_FLAG_ROLLOVER) return -EINVAL; + break; case PACKET_FANOUT_HASH: case PACKET_FANOUT_LB: case PACKET_FANOUT_CPU: diff --git a/net/rds/tcp_connect.c b/net/rds/tcp_connect.c index 4e64598176b0..5461d77fff4f 100644 --- a/net/rds/tcp_connect.c +++ b/net/rds/tcp_connect.c @@ -78,6 +78,7 @@ void rds_tcp_state_change(struct sock *sk) case TCP_CLOSE_WAIT: case TCP_CLOSE: rds_conn_path_drop(cp, false); + break; default: break; } diff --git a/net/rds/threads.c b/net/rds/threads.c index 32dc50f0a303..1f424cbfcbb4 100644 --- a/net/rds/threads.c +++ b/net/rds/threads.c @@ -208,6 +208,7 @@ void rds_send_worker(struct work_struct *work) case -ENOMEM: rds_stats_inc(s_send_delayed_retry); queue_delayed_work(rds_wq, &cp->cp_send_w, 2); + break; default: break; } @@ -232,6 +233,7 @@ void rds_recv_worker(struct work_struct *work) case -ENOMEM: rds_stats_inc(s_recv_delayed_retry); queue_delayed_work(rds_wq, &cp->cp_recv_w, 2); + break; default: break; } diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c index 41671af6b33f..2b5f89713e36 100644 --- a/net/rxrpc/af_rxrpc.c +++ b/net/rxrpc/af_rxrpc.c @@ -471,6 +471,7 @@ static int rxrpc_connect(struct socket *sock, struct sockaddr *addr, switch (rx->sk.sk_state) { case RXRPC_UNBOUND: rx->sk.sk_state = RXRPC_CLIENT_UNBOUND; + break; case RXRPC_CLIENT_UNBOUND: case RXRPC_CLIENT_BOUND: break; diff --git a/net/sctp/input.c b/net/sctp/input.c index d508f6f3dd08..5ceaf75105ba 100644 --- a/net/sctp/input.c +++ b/net/sctp/input.c @@ -633,7 +633,7 @@ int sctp_v4_err(struct sk_buff *skb, __u32 info) break; case ICMP_REDIRECT: sctp_icmp_redirect(sk, transport, skb); - /* Fall through to out_unlock. */ + goto out_unlock; default: goto out_unlock; } @@ -1236,6 +1236,7 @@ static struct sctp_association *__sctp_rcv_walk_lookup(struct net *net, net, ch, laddr, sctp_hdr(skb)->source, transportp); + break; default: break; } diff --git a/net/tipc/link.c b/net/tipc/link.c index c44b4bfaaee6..1b7a487c8841 100644 --- a/net/tipc/link.c +++ b/net/tipc/link.c @@ -654,6 +654,7 @@ int tipc_link_fsm_evt(struct tipc_link *l, int evt) break; case LINK_FAILOVER_BEGIN_EVT: l->state = LINK_FAILINGOVER; + break; case LINK_FAILURE_EVT: case LINK_RESET_EVT: case LINK_ESTABLISH_EVT: diff --git a/net/xfrm/xfrm_interface.c b/net/xfrm/xfrm_interface.c index 8831f5a9e992..41de46b5ffa9 100644 --- a/net/xfrm/xfrm_interface.c +++ b/net/xfrm/xfrm_interface.c @@ -432,6 +432,7 @@ static int xfrmi4_err(struct sk_buff *skb, u32 info) case ICMP_DEST_UNREACH: if (icmp_hdr(skb)->code != ICMP_FRAG_NEEDED) return 0; + break; case ICMP_REDIRECT: break; default: diff --git a/samples/kprobes/kprobe_example.c b/samples/kprobes/kprobe_example.c index 4b2f31828951..f991a66b5b02 100644 --- a/samples/kprobes/kprobe_example.c +++ b/samples/kprobes/kprobe_example.c @@ -10,6 +10,8 @@ * whenever kernel_clone() is invoked to create a new process. */ +#define pr_fmt(fmt) "%s: " fmt, __func__ + #include #include #include @@ -27,32 +29,31 @@ static struct kprobe kp = { static int __kprobes handler_pre(struct kprobe *p, struct pt_regs *regs) { #ifdef CONFIG_X86 - pr_info("<%s> pre_handler: p->addr = 0x%p, ip = %lx, flags = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, ip = %lx, flags = 0x%lx\n", p->symbol_name, p->addr, regs->ip, regs->flags); #endif #ifdef CONFIG_PPC - pr_info("<%s> pre_handler: p->addr = 0x%p, nip = 0x%lx, msr = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, nip = 0x%lx, msr = 0x%lx\n", p->symbol_name, p->addr, regs->nip, regs->msr); #endif #ifdef CONFIG_MIPS - pr_info("<%s> pre_handler: p->addr = 0x%p, epc = 0x%lx, status = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, epc = 0x%lx, status = 0x%lx\n", p->symbol_name, p->addr, regs->cp0_epc, regs->cp0_status); #endif #ifdef CONFIG_ARM64 - pr_info("<%s> pre_handler: p->addr = 0x%p, pc = 0x%lx," - " pstate = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, pc = 0x%lx, pstate = 0x%lx\n", p->symbol_name, p->addr, (long)regs->pc, (long)regs->pstate); #endif #ifdef CONFIG_ARM - pr_info("<%s> pre_handler: p->addr = 0x%p, pc = 0x%lx, cpsr = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, pc = 0x%lx, cpsr = 0x%lx\n", p->symbol_name, p->addr, (long)regs->ARM_pc, (long)regs->ARM_cpsr); #endif #ifdef CONFIG_RISCV - pr_info("<%s> pre_handler: p->addr = 0x%p, pc = 0x%lx, status = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, pc = 0x%lx, status = 0x%lx\n", p->symbol_name, p->addr, regs->epc, regs->status); #endif #ifdef CONFIG_S390 - pr_info("<%s> pre_handler: p->addr, 0x%p, ip = 0x%lx, flags = 0x%lx\n", + pr_info("<%s> p->addr, 0x%p, ip = 0x%lx, flags = 0x%lx\n", p->symbol_name, p->addr, regs->psw.addr, regs->flags); #endif @@ -65,31 +66,31 @@ static void __kprobes handler_post(struct kprobe *p, struct pt_regs *regs, unsigned long flags) { #ifdef CONFIG_X86 - pr_info("<%s> post_handler: p->addr = 0x%p, flags = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, flags = 0x%lx\n", p->symbol_name, p->addr, regs->flags); #endif #ifdef CONFIG_PPC - pr_info("<%s> post_handler: p->addr = 0x%p, msr = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, msr = 0x%lx\n", p->symbol_name, p->addr, regs->msr); #endif #ifdef CONFIG_MIPS - pr_info("<%s> post_handler: p->addr = 0x%p, status = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, status = 0x%lx\n", p->symbol_name, p->addr, regs->cp0_status); #endif #ifdef CONFIG_ARM64 - pr_info("<%s> post_handler: p->addr = 0x%p, pstate = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, pstate = 0x%lx\n", p->symbol_name, p->addr, (long)regs->pstate); #endif #ifdef CONFIG_ARM - pr_info("<%s> post_handler: p->addr = 0x%p, cpsr = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, cpsr = 0x%lx\n", p->symbol_name, p->addr, (long)regs->ARM_cpsr); #endif #ifdef CONFIG_RISCV - pr_info("<%s> post_handler: p->addr = 0x%p, status = 0x%lx\n", + pr_info("<%s> p->addr = 0x%p, status = 0x%lx\n", p->symbol_name, p->addr, regs->status); #endif #ifdef CONFIG_S390 - pr_info("<%s> pre_handler: p->addr, 0x%p, flags = 0x%lx\n", + pr_info("<%s> p->addr, 0x%p, flags = 0x%lx\n", p->symbol_name, p->addr, regs->flags); #endif } diff --git a/scripts/Makefile.kasan b/scripts/Makefile.kasan index 3d791908ed36..801c415bac59 100644 --- a/scripts/Makefile.kasan +++ b/scripts/Makefile.kasan @@ -50,6 +50,7 @@ endif CFLAGS_KASAN := -fsanitize=kernel-hwaddress \ $(call cc-param,hwasan-instrument-stack=$(stack_enable)) \ $(call cc-param,hwasan-use-short-granules=0) \ + $(call cc-param,hwasan-inline-all-checks=0) \ $(instrumentation_flags) endif # CONFIG_KASAN_SW_TAGS diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index c71832b2312b..7187ea5e5149 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -24,7 +24,7 @@ my $help = 0; my $fix = 0; my $warn = 0; -if (! -d ".git") { +if (! -e ".git") { printf "Warning: can't check if file exists, as this is not a git tree\n"; exit 0; } diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 4840e748fca8..7c4a6a507ac4 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -406,6 +406,8 @@ my $doc_inline_sect = '\s*\*\s*(@\s*[\w][\w\.]*\s*):(.*)'; my $doc_inline_end = '^\s*\*/\s*$'; my $doc_inline_oneline = '^\s*/\*\*\s*(@[\w\s]+):\s*(.*)\s*\*/\s*$'; my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;'; +my $function_pointer = qr{([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)}; +my $attribute = qr{__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)}i; my %parameterdescs; my %parameterdesc_start_lines; @@ -694,7 +696,7 @@ sub output_function_man(%) { $post = ");"; } $type = $args{'parametertypes'}{$parameter}; - if ($type =~ m/([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)/) { + if ($type =~ m/$function_pointer/) { # pointer-to-function print ".BI \"" . $parenth . $1 . "\" " . " \") (" . $2 . ")" . $post . "\"\n"; } else { @@ -974,7 +976,7 @@ sub output_function_rst(%) { $count++; $type = $args{'parametertypes'}{$parameter}; - if ($type =~ m/([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)/) { + if ($type =~ m/$function_pointer/) { # pointer-to-function print $1 . $parameter . ") (" . $2 . ")"; } else { @@ -1211,7 +1213,9 @@ sub dump_struct($$) { my $members; my $type = qr{struct|union}; # For capturing struct/union definition body, i.e. "{members*}qualifiers*" - my $definition_body = qr{\{(.*)\}(?:\s*(?:__packed|__aligned|____cacheline_aligned_in_smp|____cacheline_aligned|__attribute__\s*\(\([a-z0-9,_\s\(\)]*\)\)))*}; + my $qualifiers = qr{$attribute|__packed|__aligned|____cacheline_aligned_in_smp|____cacheline_aligned}; + my $definition_body = qr{\{(.*)\}\s*$qualifiers*}; + my $struct_members = qr{($type)([^\{\};]+)\{([^\{\}]*)\}([^\{\}\;]*)\;}; if ($x =~ /($type)\s+(\w+)\s*$definition_body/) { $decl_type = $1; @@ -1235,27 +1239,27 @@ sub dump_struct($$) { # strip comments: $members =~ s/\/\*.*?\*\///gos; # strip attributes - $members =~ s/\s*__attribute__\s*\(\([a-z0-9,_\*\s\(\)]*\)\)/ /gi; + $members =~ s/\s*$attribute/ /gi; $members =~ s/\s*__aligned\s*\([^;]*\)/ /gos; $members =~ s/\s*__packed\s*/ /gos; $members =~ s/\s*CRYPTO_MINALIGN_ATTR/ /gos; $members =~ s/\s*____cacheline_aligned_in_smp/ /gos; $members =~ s/\s*____cacheline_aligned/ /gos; + my $args = qr{([^,)]+)}; # replace DECLARE_BITMAP $members =~ s/__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)/DECLARE_BITMAP($1, __ETHTOOL_LINK_MODE_MASK_NBITS)/gos; - $members =~ s/DECLARE_BITMAP\s*\(([^,)]+),\s*([^,)]+)\)/unsigned long $1\[BITS_TO_LONGS($2)\]/gos; + $members =~ s/DECLARE_BITMAP\s*\($args,\s*$args\)/unsigned long $1\[BITS_TO_LONGS($2)\]/gos; # replace DECLARE_HASHTABLE - $members =~ s/DECLARE_HASHTABLE\s*\(([^,)]+),\s*([^,)]+)\)/unsigned long $1\[1 << (($2) - 1)\]/gos; + $members =~ s/DECLARE_HASHTABLE\s*\($args,\s*$args\)/unsigned long $1\[1 << (($2) - 1)\]/gos; # replace DECLARE_KFIFO - $members =~ s/DECLARE_KFIFO\s*\(([^,)]+),\s*([^,)]+),\s*([^,)]+)\)/$2 \*$1/gos; + $members =~ s/DECLARE_KFIFO\s*\($args,\s*$args,\s*$args\)/$2 \*$1/gos; # replace DECLARE_KFIFO_PTR - $members =~ s/DECLARE_KFIFO_PTR\s*\(([^,)]+),\s*([^,)]+)\)/$2 \*$1/gos; - + $members =~ s/DECLARE_KFIFO_PTR\s*\($args,\s*$args\)/$2 \*$1/gos; my $declaration = $members; # Split nested struct/union elements as newer ones - while ($members =~ m/(struct|union)([^\{\};]+)\{([^\{\}]*)\}([^\{\}\;]*)\;/) { + while ($members =~ m/$struct_members/) { my $newmember; my $maintype = $1; my $ids = $4; @@ -1315,7 +1319,7 @@ sub dump_struct($$) { } } } - $members =~ s/(struct|union)([^\{\};]+)\{([^\{\}]*)\}([^\{\}\;]*)\;/$newmember/; + $members =~ s/$struct_members/$newmember/; } # Ignore other nested elements, like enums @@ -1555,8 +1559,9 @@ sub create_parameterlist($$$$) { my $param; # temporarily replace commas inside function pointer definition - while ($args =~ /(\([^\),]+),/) { - $args =~ s/(\([^\),]+),/$1#/g; + my $arg_expr = qr{\([^\),]+}; + while ($args =~ /$arg_expr,/) { + $args =~ s/($arg_expr),/$1#/g; } foreach my $arg (split($splitter, $args)) { @@ -1707,7 +1712,7 @@ sub check_sections($$$$$) { foreach $px (0 .. $#prms) { $prm_clean = $prms[$px]; $prm_clean =~ s/\[.*\]//; - $prm_clean =~ s/__attribute__\s*\(\([a-z,_\*\s\(\)]*\)\)//i; + $prm_clean =~ s/$attribute//i; # ignore array size in a parameter string; # however, the original param string may contain # spaces, e.g.: addr[6 + 2] @@ -1809,8 +1814,14 @@ sub dump_function($$) { # - parport_register_device (function pointer parameters) # - atomic_set (macro) # - pci_match_device, __copy_to_user (long return type) + my $name = qr{[a-zA-Z0-9_~:]+}; + my $prototype_end1 = qr{[^\(]*}; + my $prototype_end2 = qr{[^\{]*}; + my $prototype_end = qr{\(($prototype_end1|$prototype_end2)\)}; + my $type1 = qr{[\w\s]+}; + my $type2 = qr{$type1\*+}; - if ($define && $prototype =~ m/^()([a-zA-Z0-9_~:]+)\s+/) { + if ($define && $prototype =~ m/^()($name)\s+/) { # This is an object-like macro, it has no return type and no parameter # list. # Function-like macros are not allowed to have spaces between @@ -1818,23 +1829,9 @@ sub dump_function($$) { $return_type = $1; $declaration_name = $2; $noret = 1; - } elsif ($prototype =~ m/^()([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\(]*)\)/ || - $prototype =~ m/^()([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s+\w+)\s+([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s+\w+\s+\w+\s*\*+)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/ || - $prototype =~ m/^(\w+\s+\w+\s*\*+\s*\w+\s*\*+\s*)\s*([a-zA-Z0-9_~:]+)\s*\(([^\{]*)\)/) { + } elsif ($prototype =~ m/^()($name)\s*$prototype_end/ || + $prototype =~ m/^($type1)\s+($name)\s*$prototype_end/ || + $prototype =~ m/^($type2+)\s*($name)\s*$prototype_end/) { $return_type = $1; $declaration_name = $2; my $args = $3; @@ -2111,12 +2108,12 @@ sub process_name($$) { } elsif (/$doc_decl/o) { $identifier = $1; my $is_kernel_comment = 0; - my $decl_start = qr{\s*\*}; + my $decl_start = qr{$doc_com}; # test for pointer declaration type, foo * bar() - desc my $fn_type = qr{\w+\s*\*\s*}; my $parenthesis = qr{\(\w*\)}; my $decl_end = qr{[-:].*}; - if (/^$decl_start\s*([\w\s]+?)$parenthesis?\s*$decl_end?$/) { + if (/^$decl_start([\w\s]+?)$parenthesis?\s*$decl_end?$/) { $identifier = $1; } if ($identifier =~ m/^(struct|union|enum|typedef)\b\s*(\S*)/) { @@ -2126,8 +2123,8 @@ sub process_name($$) { } # Look for foo() or static void foo() - description; or misspelt # identifier - elsif (/^$decl_start\s*$fn_type?(\w+)\s*$parenthesis?\s*$decl_end?$/ || - /^$decl_start\s*$fn_type?(\w+.*)$parenthesis?\s*$decl_end$/) { + elsif (/^$decl_start$fn_type?(\w+)\s*$parenthesis?\s*$decl_end?$/ || + /^$decl_start$fn_type?(\w+.*)$parenthesis?\s*$decl_end$/) { $identifier = $1; $decl_type = 'function'; $identifier =~ s/^define\s+//; diff --git a/scripts/sphinx-pre-install b/scripts/sphinx-pre-install index fe92020d67e3..288e86a9d1e5 100755 --- a/scripts/sphinx-pre-install +++ b/scripts/sphinx-pre-install @@ -22,16 +22,18 @@ my $need = 0; my $optional = 0; my $need_symlink = 0; my $need_sphinx = 0; -my $need_venv = 0; +my $need_pip = 0; my $need_virtualenv = 0; my $rec_sphinx_upgrade = 0; my $install = ""; my $virtenv_dir = ""; my $python_cmd = ""; +my $activate_cmd; my $min_version; my $cur_version; my $rec_version = "1.7.9"; # PDF won't build here my $min_pdf_version = "2.4.4"; # Min version where pdf builds +my $latest_avail_ver; # # Command line arguments @@ -319,10 +321,7 @@ sub check_sphinx() return; } - if ($cur_version lt $rec_version) { - $rec_sphinx_upgrade = 1; - return; - } + return if ($cur_version lt $rec_version); # On version check mode, just assume Sphinx has all mandatory deps exit (0) if ($version_check); @@ -701,6 +700,162 @@ sub deactivate_help() printf "\tdeactivate\n"; } +sub get_virtenv() +{ + my $ver; + my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; + my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; + + @activates = sort {$b cmp $a} @activates; + + foreach my $f (@activates) { + next if ($f lt $min_activate); + + my $sphinx_cmd = $f; + $sphinx_cmd =~ s/activate/sphinx-build/; + next if (! -f $sphinx_cmd); + + my $ver = get_sphinx_version($sphinx_cmd); + if ($need_sphinx && ($ver ge $min_version)) { + return ($f, $ver); + } elsif ($ver gt $cur_version) { + return ($f, $ver); + } + } + return ("", ""); +} + +sub recommend_sphinx_upgrade() +{ + my $venv_ver; + + # Avoid running sphinx-builds from venv if $cur_version is good + if ($cur_version && ($cur_version ge $rec_version)) { + $latest_avail_ver = $cur_version; + return; + } + + # Get the highest version from sphinx_*/bin/sphinx-build and the + # corresponding command to activate the venv/virtenv + $activate_cmd = get_virtenv(); + + # Store the highest version from Sphinx existing virtualenvs + if (($activate_cmd ne "") && ($venv_ver gt $cur_version)) { + $latest_avail_ver = $venv_ver; + } else { + $latest_avail_ver = $cur_version if ($cur_version); + } + + # As we don't know package version of Sphinx, and there's no + # virtual environments, don't check if upgrades are needed + if (!$virtualenv) { + return if (!$latest_avail_ver); + } + + # Either there are already a virtual env or a new one should be created + $need_pip = 1; + + # Return if the reason is due to an upgrade or not + if ($latest_avail_ver lt $rec_version) { + $rec_sphinx_upgrade = 1; + } +} + +# +# The logic here is complex, as it have to deal with different versions: +# - minimal supported version; +# - minimal PDF version; +# - recommended version. +# It also needs to work fine with both distro's package and venv/virtualenv +sub recommend_sphinx_version($) +{ + my $virtualenv_cmd = shift; + + if ($latest_avail_ver lt $min_pdf_version) { + print "note: If you want pdf, you need at least Sphinx $min_pdf_version.\n"; + } + + # Version is OK. Nothing to do. + return if ($cur_version && ($cur_version ge $rec_version)); + + if (!$need_sphinx) { + # sphinx-build is present and its version is >= $min_version + + #only recommend enabling a newer virtenv version if makes sense. + if ($latest_avail_ver gt $cur_version) { + printf "\nYou may also use the newer Sphinx version $latest_avail_ver with:\n"; + printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + return if ($latest_avail_ver ge $rec_version); + } + + if (!$virtualenv) { + # No sphinx either via package or via virtenv. As we can't + # Compare the versions here, just return, recommending the + # user to install it from the package distro. + return if (!$latest_avail_ver); + + # User doesn't want a virtenv recommendation, but he already + # installed one via virtenv with a newer version. + # So, print commands to enable it + if ($latest_avail_ver gt $cur_version) { + printf "\nYou may also use the Sphinx virtualenv version $latest_avail_ver with:\n"; + printf "\tdeactivate\n" if ($ENV{'PWD'} =~ /${virtenv_prefix}/); + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + print "\n"; + } else { + $need++ if ($need_sphinx); + } + + # Suggest newer versions if current ones are too old + if ($latest_avail_ver && $cur_version ge $min_version) { + # If there's a good enough version, ask the user to enable it + if ($latest_avail_ver ge $rec_version) { + printf "\nNeed to activate Sphinx (version $latest_avail_ver) on virtualenv with:\n"; + printf "\t. $activate_cmd\n"; + deactivate_help(); + + return; + } + + # Version is above the minimal required one, but may be + # below the recommended one. So, print warnings/notes + + if ($latest_avail_ver lt $rec_version) { + print "Warning: It is recommended at least Sphinx version $rec_version.\n"; + } + } + + # At this point, either it needs Sphinx or upgrade is recommended, + # both via pip + + if ($rec_sphinx_upgrade) { + if (!$virtualenv) { + print "Instead of install/upgrade Python Sphinx pkg, you could use pip/pypi with:\n\n"; + } else { + print "To upgrade Sphinx, use:\n\n"; + } + } else { + print "Sphinx needs to be installed either as a package or via pip/pypi with:\n"; + } + + $python_cmd = find_python_no_venv(); + + printf "\t$virtualenv_cmd $virtenv_dir\n"; + + printf "\t. $virtenv_dir/bin/activate\n"; + printf "\tpip install -r $requirement_file\n"; + deactivate_help(); +} + sub check_needs() { # Check if Sphinx is already accessible from current environment @@ -722,15 +877,14 @@ sub check_needs() if ($virtualenv) { my $tmp = qx($python_cmd --version 2>&1); if ($tmp =~ m/(\d+\.)(\d+\.)/) { - if ($1 >= 3 && $2 >= 3) { - $need_venv = 1; # python 3.3 or upper - } else { - $need_virtualenv = 1; - } if ($1 < 3) { # Fail if it finds python2 (or worse) die "Python 3 is required to build the kernel docs\n"; } + if ($1 == 3 && $2 < 3) { + # Need Python 3.3 or upper for venv + $need_virtualenv = 1; + } } else { die "Warning: couldn't identify $python_cmd version!"; } @@ -739,14 +893,22 @@ sub check_needs() } } - # Set virtualenv command line, if python < 3.3 + recommend_sphinx_upgrade(); + my $virtualenv_cmd; - if ($need_virtualenv) { - $virtualenv_cmd = findprog("virtualenv-3"); - $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); - if (!$virtualenv_cmd) { - check_program("virtualenv", 0); - $virtualenv_cmd = "virtualenv"; + + if ($need_pip) { + # Set virtualenv command line, if python < 3.3 + if ($need_virtualenv) { + $virtualenv_cmd = findprog("virtualenv-3"); + $virtualenv_cmd = findprog("virtualenv-3.5") if (!$virtualenv_cmd); + if (!$virtualenv_cmd) { + check_program("virtualenv", 0); + $virtualenv_cmd = "virtualenv"; + } + } else { + $virtualenv_cmd = "$python_cmd -m venv"; + check_python_module("ensurepip", 0); } } @@ -763,10 +925,6 @@ sub check_needs() check_program("rsvg-convert", 2) if ($pdf); check_program("latexmk", 2) if ($pdf); - if ($need_sphinx || $rec_sphinx_upgrade) { - check_python_module("ensurepip", 0) if ($need_venv); - } - # Do distro-specific checks and output distro-install commands check_distros(); @@ -784,67 +942,7 @@ sub check_needs() which("sphinx-build-3"); } - # NOTE: if the system has a too old Sphinx version installed, - # it will recommend installing a newer version using virtualenv - - if ($need_sphinx || $rec_sphinx_upgrade) { - my $min_activate = "$ENV{'PWD'}/${virtenv_prefix}${min_version}/bin/activate"; - my @activates = glob "$ENV{'PWD'}/${virtenv_prefix}*/bin/activate"; - - if ($cur_version lt $rec_version) { - print "Warning: It is recommended at least Sphinx version $rec_version.\n"; - print " If you want pdf, you need at least $min_pdf_version.\n"; - } - if ($cur_version lt $min_pdf_version) { - print "Note: It is recommended at least Sphinx version $min_pdf_version if you need PDF support.\n"; - } - @activates = sort {$b cmp $a} @activates; - my ($activate, $ver); - foreach my $f (@activates) { - next if ($f lt $min_activate); - - my $sphinx_cmd = $f; - $sphinx_cmd =~ s/activate/sphinx-build/; - next if (! -f $sphinx_cmd); - - $ver = get_sphinx_version($sphinx_cmd); - if ($need_sphinx && ($ver ge $min_version)) { - $activate = $f; - last; - } elsif ($ver gt $cur_version) { - $activate = $f; - last; - } - } - if ($activate ne "") { - if ($need_sphinx) { - printf "\nNeed to activate Sphinx (version $ver) on virtualenv with:\n"; - printf "\t. $activate\n"; - deactivate_help(); - exit (1); - } else { - printf "\nYou may also use a newer Sphinx (version $ver) with:\n"; - printf "\tdeactivate && . $activate\n"; - } - } else { - my $rec_activate = "$virtenv_dir/bin/activate"; - - print "To upgrade Sphinx, use:\n\n" if ($rec_sphinx_upgrade); - - $python_cmd = find_python_no_venv(); - - if ($need_venv) { - printf "\t$python_cmd -m venv $virtenv_dir\n"; - } else { - printf "\t$virtualenv_cmd $virtenv_dir\n"; - } - printf "\t. $rec_activate\n"; - printf "\tpip install -r $requirement_file\n"; - deactivate_help(); - - $need++ if (!$rec_sphinx_upgrade); - } - } + recommend_sphinx_version($virtualenv_cmd); printf "\n"; print "All optional dependencies are met.\n" if (!$optional); diff --git a/security/integrity/evm/evm.h b/security/integrity/evm/evm.h index f2fef2b5ed51..0d44f41d16f8 100644 --- a/security/integrity/evm/evm.h +++ b/security/integrity/evm/evm.h @@ -29,6 +29,7 @@ struct xattr_list { struct list_head list; char *name; + bool enabled; }; extern int evm_initialized; diff --git a/security/integrity/evm/evm_crypto.c b/security/integrity/evm/evm_crypto.c index d76b006cbcc4..0450d79afdc8 100644 --- a/security/integrity/evm/evm_crypto.c +++ b/security/integrity/evm/evm_crypto.c @@ -10,6 +10,8 @@ * Using root's kernel master key (kmk), calculate the HMAC */ +#define pr_fmt(fmt) "EVM: "fmt + #include #include #include @@ -175,6 +177,30 @@ static void hmac_add_misc(struct shash_desc *desc, struct inode *inode, type != EVM_XATTR_PORTABLE_DIGSIG) crypto_shash_update(desc, (u8 *)&inode->i_sb->s_uuid, UUID_SIZE); crypto_shash_final(desc, digest); + + pr_debug("hmac_misc: (%zu) [%*phN]\n", sizeof(struct h_misc), + (int)sizeof(struct h_misc), &hmac_misc); +} + +/* + * Dump large security xattr values as a continuous ascii hexademical string. + * (pr_debug is limited to 64 bytes.) + */ +static void dump_security_xattr(const char *prefix, const void *src, + size_t count) +{ +#if defined(DEBUG) || defined(CONFIG_DYNAMIC_DEBUG) + char *asciihex, *p; + + p = asciihex = kmalloc(count * 2 + 1, GFP_KERNEL); + if (!asciihex) + return; + + p = bin2hex(p, src, count); + *p = 0; + pr_debug("%s: (%zu) %.*s\n", prefix, count, (int)count * 2, asciihex); + kfree(asciihex); +#endif } /* @@ -196,7 +222,7 @@ static int evm_calc_hmac_or_hash(struct dentry *dentry, size_t xattr_size = 0; char *xattr_value = NULL; int error; - int size; + int size, user_space_size; bool ima_present = false; if (!(inode->i_opflags & IOP_XATTR) || @@ -216,6 +242,13 @@ static int evm_calc_hmac_or_hash(struct dentry *dentry, if (strcmp(xattr->name, XATTR_NAME_IMA) == 0) is_ima = true; + /* + * Skip non-enabled xattrs for locally calculated + * signatures/HMACs. + */ + if (type != EVM_XATTR_PORTABLE_DIGSIG && !xattr->enabled) + continue; + if ((req_xattr_name && req_xattr_value) && !strcmp(xattr->name, req_xattr_name)) { error = 0; @@ -223,6 +256,16 @@ static int evm_calc_hmac_or_hash(struct dentry *dentry, req_xattr_value_len); if (is_ima) ima_present = true; + + if (req_xattr_value_len < 64) + pr_debug("%s: (%zu) [%*phN]\n", req_xattr_name, + req_xattr_value_len, + (int)req_xattr_value_len, + req_xattr_value); + else + dump_security_xattr(req_xattr_name, + req_xattr_value, + req_xattr_value_len); continue; } size = vfs_getxattr_alloc(&init_user_ns, dentry, xattr->name, @@ -234,11 +277,24 @@ static int evm_calc_hmac_or_hash(struct dentry *dentry, if (size < 0) continue; + user_space_size = vfs_getxattr(&init_user_ns, dentry, + xattr->name, NULL, 0); + if (user_space_size != size) + pr_debug("file %s: xattr %s size mismatch (kernel: %d, user: %d)\n", + dentry->d_name.name, xattr->name, size, + user_space_size); error = 0; xattr_size = size; crypto_shash_update(desc, (const u8 *)xattr_value, xattr_size); if (is_ima) ima_present = true; + + if (xattr_size < 64) + pr_debug("%s: (%zu) [%*phN]", xattr->name, xattr_size, + (int)xattr_size, xattr_value); + else + dump_security_xattr(xattr->name, xattr_value, + xattr_size); } hmac_add_misc(desc, inode, type, data->digest); diff --git a/security/integrity/evm/evm_main.c b/security/integrity/evm/evm_main.c index 0de367aaa2d3..1c8435dfabee 100644 --- a/security/integrity/evm/evm_main.c +++ b/security/integrity/evm/evm_main.c @@ -11,6 +11,8 @@ * evm_inode_removexattr, and evm_verifyxattr */ +#define pr_fmt(fmt) "EVM: "fmt + #include #include #include @@ -18,6 +20,7 @@ #include #include #include +#include #include #include @@ -27,29 +30,50 @@ int evm_initialized; static const char * const integrity_status_msg[] = { - "pass", "pass_immutable", "fail", "no_label", "no_xattrs", "unknown" + "pass", "pass_immutable", "fail", "fail_immutable", "no_label", + "no_xattrs", "unknown" }; int evm_hmac_attrs; static struct xattr_list evm_config_default_xattrnames[] = { + {.name = XATTR_NAME_SELINUX, #ifdef CONFIG_SECURITY_SELINUX - {.name = XATTR_NAME_SELINUX}, + .enabled = true #endif + }, + {.name = XATTR_NAME_SMACK, #ifdef CONFIG_SECURITY_SMACK - {.name = XATTR_NAME_SMACK}, + .enabled = true +#endif + }, + {.name = XATTR_NAME_SMACKEXEC, #ifdef CONFIG_EVM_EXTRA_SMACK_XATTRS - {.name = XATTR_NAME_SMACKEXEC}, - {.name = XATTR_NAME_SMACKTRANSMUTE}, - {.name = XATTR_NAME_SMACKMMAP}, + .enabled = true #endif + }, + {.name = XATTR_NAME_SMACKTRANSMUTE, +#ifdef CONFIG_EVM_EXTRA_SMACK_XATTRS + .enabled = true #endif + }, + {.name = XATTR_NAME_SMACKMMAP, +#ifdef CONFIG_EVM_EXTRA_SMACK_XATTRS + .enabled = true +#endif + }, + {.name = XATTR_NAME_APPARMOR, #ifdef CONFIG_SECURITY_APPARMOR - {.name = XATTR_NAME_APPARMOR}, + .enabled = true #endif + }, + {.name = XATTR_NAME_IMA, #ifdef CONFIG_IMA_APPRAISE - {.name = XATTR_NAME_IMA}, + .enabled = true #endif - {.name = XATTR_NAME_CAPS}, + }, + {.name = XATTR_NAME_CAPS, + .enabled = true + }, }; LIST_HEAD(evm_config_xattrnames); @@ -74,7 +98,9 @@ static void __init evm_init_config(void) pr_info("Initialising EVM extended attributes:\n"); for (i = 0; i < xattrs; i++) { - pr_info("%s\n", evm_config_default_xattrnames[i].name); + pr_info("%s%s\n", evm_config_default_xattrnames[i].name, + !evm_config_default_xattrnames[i].enabled ? + " (disabled)" : ""); list_add_tail(&evm_config_default_xattrnames[i].list, &evm_config_xattrnames); } @@ -90,6 +116,24 @@ static bool evm_key_loaded(void) return (bool)(evm_initialized & EVM_KEY_MASK); } +/* + * This function determines whether or not it is safe to ignore verification + * errors, based on the ability of EVM to calculate HMACs. If the HMAC key + * is not loaded, and it cannot be loaded in the future due to the + * EVM_SETUP_COMPLETE initialization flag, allowing an operation despite the + * attrs/xattrs being found invalid will not make them valid. + */ +static bool evm_hmac_disabled(void) +{ + if (evm_initialized & EVM_INIT_HMAC) + return false; + + if (!(evm_initialized & EVM_SETUP_COMPLETE)) + return false; + + return true; +} + static int evm_find_protected_xattrs(struct dentry *dentry) { struct inode *inode = d_backing_inode(dentry); @@ -137,7 +181,7 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, enum integrity_status evm_status = INTEGRITY_PASS; struct evm_digest digest; struct inode *inode; - int rc, xattr_len; + int rc, xattr_len, evm_immutable = 0; if (iint && (iint->evm_status == INTEGRITY_PASS || iint->evm_status == INTEGRITY_PASS_IMMUTABLE)) @@ -182,8 +226,10 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, if (rc) rc = -EINVAL; break; - case EVM_IMA_XATTR_DIGSIG: case EVM_XATTR_PORTABLE_DIGSIG: + evm_immutable = 1; + fallthrough; + case EVM_IMA_XATTR_DIGSIG: /* accept xattr with non-empty signature field */ if (xattr_len <= sizeof(struct signature_v2_hdr)) { evm_status = INTEGRITY_FAIL; @@ -220,9 +266,16 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, break; } - if (rc) - evm_status = (rc == -ENODATA) ? - INTEGRITY_NOXATTRS : INTEGRITY_FAIL; + if (rc) { + if (rc == -ENODATA) + evm_status = INTEGRITY_NOXATTRS; + else if (evm_immutable) + evm_status = INTEGRITY_FAIL_IMMUTABLE; + else + evm_status = INTEGRITY_FAIL; + } + pr_debug("digest: (%d) [%*phN]\n", digest.hdr.length, digest.hdr.length, + digest.digest); out: if (iint) iint->evm_status = evm_status; @@ -230,7 +283,8 @@ static enum integrity_status evm_verify_hmac(struct dentry *dentry, return evm_status; } -static int evm_protected_xattr(const char *req_xattr_name) +static int evm_protected_xattr_common(const char *req_xattr_name, + bool all_xattrs) { int namelen; int found = 0; @@ -238,6 +292,9 @@ static int evm_protected_xattr(const char *req_xattr_name) namelen = strlen(req_xattr_name); list_for_each_entry_lockless(xattr, &evm_config_xattrnames, list) { + if (!all_xattrs && !xattr->enabled) + continue; + if ((strlen(xattr->name) == namelen) && (strncmp(req_xattr_name, xattr->name, namelen) == 0)) { found = 1; @@ -254,6 +311,85 @@ static int evm_protected_xattr(const char *req_xattr_name) return found; } +static int evm_protected_xattr(const char *req_xattr_name) +{ + return evm_protected_xattr_common(req_xattr_name, false); +} + +int evm_protected_xattr_if_enabled(const char *req_xattr_name) +{ + return evm_protected_xattr_common(req_xattr_name, true); +} + +/** + * evm_read_protected_xattrs - read EVM protected xattr names, lengths, values + * @dentry: dentry of the read xattrs + * @inode: inode of the read xattrs + * @buffer: buffer xattr names, lengths or values are copied to + * @buffer_size: size of buffer + * @type: n: names, l: lengths, v: values + * @canonical_fmt: data format (true: little endian, false: native format) + * + * Read protected xattr names (separated by |), lengths (u32) or values for a + * given dentry and return the total size of copied data. If buffer is NULL, + * just return the total size. + * + * Returns the total size on success, a negative value on error. + */ +int evm_read_protected_xattrs(struct dentry *dentry, u8 *buffer, + int buffer_size, char type, bool canonical_fmt) +{ + struct xattr_list *xattr; + int rc, size, total_size = 0; + + list_for_each_entry_lockless(xattr, &evm_config_xattrnames, list) { + rc = __vfs_getxattr(dentry, d_backing_inode(dentry), + xattr->name, NULL, 0); + if (rc < 0 && rc == -ENODATA) + continue; + else if (rc < 0) + return rc; + + switch (type) { + case 'n': + size = strlen(xattr->name) + 1; + if (buffer) { + if (total_size) + *(buffer + total_size - 1) = '|'; + + memcpy(buffer + total_size, xattr->name, size); + } + break; + case 'l': + size = sizeof(u32); + if (buffer) { + if (canonical_fmt) + rc = (__force int)cpu_to_le32(rc); + + *(u32 *)(buffer + total_size) = rc; + } + break; + case 'v': + size = rc; + if (buffer) { + rc = __vfs_getxattr(dentry, + d_backing_inode(dentry), xattr->name, + buffer + total_size, + buffer_size - total_size); + if (rc < 0) + return rc; + } + break; + default: + return -EINVAL; + } + + total_size += size; + } + + return total_size; +} + /** * evm_verifyxattr - verify the integrity of the requested xattr * @dentry: object of the verify xattr @@ -304,6 +440,92 @@ static enum integrity_status evm_verify_current_integrity(struct dentry *dentry) return evm_verify_hmac(dentry, NULL, NULL, 0, NULL); } +/* + * evm_xattr_acl_change - check if passed ACL changes the inode mode + * @mnt_userns: user namespace of the idmapped mount + * @dentry: pointer to the affected dentry + * @xattr_name: requested xattr + * @xattr_value: requested xattr value + * @xattr_value_len: requested xattr value length + * + * Check if passed ACL changes the inode mode, which is protected by EVM. + * + * Returns 1 if passed ACL causes inode mode change, 0 otherwise. + */ +static int evm_xattr_acl_change(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *xattr_name, + const void *xattr_value, size_t xattr_value_len) +{ +#ifdef CONFIG_FS_POSIX_ACL + umode_t mode; + struct posix_acl *acl = NULL, *acl_res; + struct inode *inode = d_backing_inode(dentry); + int rc; + + /* + * user_ns is not relevant here, ACL_USER/ACL_GROUP don't have impact + * on the inode mode (see posix_acl_equiv_mode()). + */ + acl = posix_acl_from_xattr(&init_user_ns, xattr_value, xattr_value_len); + if (IS_ERR_OR_NULL(acl)) + return 1; + + acl_res = acl; + /* + * Passing mnt_userns is necessary to correctly determine the GID in + * an idmapped mount, as the GID is used to clear the setgid bit in + * the inode mode. + */ + rc = posix_acl_update_mode(mnt_userns, inode, &mode, &acl_res); + + posix_acl_release(acl); + + if (rc) + return 1; + + if (inode->i_mode != mode) + return 1; +#endif + return 0; +} + +/* + * evm_xattr_change - check if passed xattr value differs from current value + * @mnt_userns: user namespace of the idmapped mount + * @dentry: pointer to the affected dentry + * @xattr_name: requested xattr + * @xattr_value: requested xattr value + * @xattr_value_len: requested xattr value length + * + * Check if passed xattr value differs from current value. + * + * Returns 1 if passed xattr value differs from current value, 0 otherwise. + */ +static int evm_xattr_change(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *xattr_name, + const void *xattr_value, size_t xattr_value_len) +{ + char *xattr_data = NULL; + int rc = 0; + + if (posix_xattr_acl(xattr_name)) + return evm_xattr_acl_change(mnt_userns, dentry, xattr_name, + xattr_value, xattr_value_len); + + rc = vfs_getxattr_alloc(&init_user_ns, dentry, xattr_name, &xattr_data, + 0, GFP_NOFS); + if (rc < 0) + return 1; + + if (rc == xattr_value_len) + rc = !!memcmp(xattr_value, xattr_data, rc); + else + rc = 1; + + kfree(xattr_data); + return rc; +} + /* * evm_protect_xattr - protect the EVM extended attribute * @@ -316,7 +538,8 @@ static enum integrity_status evm_verify_current_integrity(struct dentry *dentry) * For posix xattr acls only, permit security.evm, even if it currently * doesn't exist, to be updated unless the EVM signature is immutable. */ -static int evm_protect_xattr(struct dentry *dentry, const char *xattr_name, +static int evm_protect_xattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *xattr_name, const void *xattr_value, size_t xattr_value_len) { enum integrity_status evm_status; @@ -338,6 +561,10 @@ static int evm_protect_xattr(struct dentry *dentry, const char *xattr_name, if (evm_status == INTEGRITY_NOXATTRS) { struct integrity_iint_cache *iint; + /* Exception if the HMAC is not going to be calculated. */ + if (evm_hmac_disabled()) + return 0; + iint = integrity_iint_find(d_backing_inode(dentry)); if (iint && (iint->flags & IMA_NEW_FILE)) return 0; @@ -354,7 +581,25 @@ static int evm_protect_xattr(struct dentry *dentry, const char *xattr_name, -EPERM, 0); } out: - if (evm_status != INTEGRITY_PASS) + /* Exception if the HMAC is not going to be calculated. */ + if (evm_hmac_disabled() && (evm_status == INTEGRITY_NOLABEL || + evm_status == INTEGRITY_UNKNOWN)) + return 0; + + /* + * Writing other xattrs is safe for portable signatures, as portable + * signatures are immutable and can never be updated. + */ + if (evm_status == INTEGRITY_FAIL_IMMUTABLE) + return 0; + + if (evm_status == INTEGRITY_PASS_IMMUTABLE && + !evm_xattr_change(mnt_userns, dentry, xattr_name, xattr_value, + xattr_value_len)) + return 0; + + if (evm_status != INTEGRITY_PASS && + evm_status != INTEGRITY_PASS_IMMUTABLE) integrity_audit_msg(AUDIT_INTEGRITY_METADATA, d_backing_inode(dentry), dentry->d_name.name, "appraise_metadata", integrity_status_msg[evm_status], @@ -364,6 +609,7 @@ static int evm_protect_xattr(struct dentry *dentry, const char *xattr_name, /** * evm_inode_setxattr - protect the EVM extended attribute + * @mnt_userns: user namespace of the idmapped mount * @dentry: pointer to the affected dentry * @xattr_name: pointer to the affected extended attribute name * @xattr_value: pointer to the new extended attribute value @@ -375,8 +621,9 @@ static int evm_protect_xattr(struct dentry *dentry, const char *xattr_name, * userspace from writing HMAC value. Writing 'security.evm' requires * requires CAP_SYS_ADMIN privileges. */ -int evm_inode_setxattr(struct dentry *dentry, const char *xattr_name, - const void *xattr_value, size_t xattr_value_len) +int evm_inode_setxattr(struct user_namespace *mnt_userns, struct dentry *dentry, + const char *xattr_name, const void *xattr_value, + size_t xattr_value_len) { const struct evm_ima_xattr_data *xattr_data = xattr_value; @@ -393,19 +640,21 @@ int evm_inode_setxattr(struct dentry *dentry, const char *xattr_name, xattr_data->type != EVM_XATTR_PORTABLE_DIGSIG) return -EPERM; } - return evm_protect_xattr(dentry, xattr_name, xattr_value, + return evm_protect_xattr(mnt_userns, dentry, xattr_name, xattr_value, xattr_value_len); } /** * evm_inode_removexattr - protect the EVM extended attribute + * @mnt_userns: user namespace of the idmapped mount * @dentry: pointer to the affected dentry * @xattr_name: pointer to the affected extended attribute name * * Removing 'security.evm' requires CAP_SYS_ADMIN privileges and that * the current value is valid. */ -int evm_inode_removexattr(struct dentry *dentry, const char *xattr_name) +int evm_inode_removexattr(struct user_namespace *mnt_userns, + struct dentry *dentry, const char *xattr_name) { /* Policy permits modification of the protected xattrs even though * there's no HMAC key loaded @@ -413,7 +662,7 @@ int evm_inode_removexattr(struct dentry *dentry, const char *xattr_name) if (evm_initialized & EVM_ALLOW_METADATA_WRITES) return 0; - return evm_protect_xattr(dentry, xattr_name, NULL, 0); + return evm_protect_xattr(mnt_userns, dentry, xattr_name, NULL, 0); } static void evm_reset_status(struct inode *inode) @@ -425,6 +674,31 @@ static void evm_reset_status(struct inode *inode) iint->evm_status = INTEGRITY_UNKNOWN; } +/** + * evm_revalidate_status - report whether EVM status re-validation is necessary + * @xattr_name: pointer to the affected extended attribute name + * + * Report whether callers of evm_verifyxattr() should re-validate the + * EVM status. + * + * Return true if re-validation is necessary, false otherwise. + */ +bool evm_revalidate_status(const char *xattr_name) +{ + if (!evm_key_loaded()) + return false; + + /* evm_inode_post_setattr() passes NULL */ + if (!xattr_name) + return true; + + if (!evm_protected_xattr(xattr_name) && !posix_xattr_acl(xattr_name) && + strcmp(xattr_name, XATTR_NAME_EVM)) + return false; + + return true; +} + /** * evm_inode_post_setxattr - update 'security.evm' to reflect the changes * @dentry: pointer to the affected dentry @@ -441,12 +715,17 @@ static void evm_reset_status(struct inode *inode) void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, size_t xattr_value_len) { - if (!evm_key_loaded() || (!evm_protected_xattr(xattr_name) - && !posix_xattr_acl(xattr_name))) + if (!evm_revalidate_status(xattr_name)) return; evm_reset_status(dentry->d_inode); + if (!strcmp(xattr_name, XATTR_NAME_EVM)) + return; + + if (!(evm_initialized & EVM_INIT_HMAC)) + return; + evm_update_evmxattr(dentry, xattr_name, xattr_value, xattr_value_len); } @@ -462,14 +741,33 @@ void evm_inode_post_setxattr(struct dentry *dentry, const char *xattr_name, */ void evm_inode_post_removexattr(struct dentry *dentry, const char *xattr_name) { - if (!evm_key_loaded() || !evm_protected_xattr(xattr_name)) + if (!evm_revalidate_status(xattr_name)) return; evm_reset_status(dentry->d_inode); + if (!strcmp(xattr_name, XATTR_NAME_EVM)) + return; + + if (!(evm_initialized & EVM_INIT_HMAC)) + return; + evm_update_evmxattr(dentry, xattr_name, NULL, 0); } +static int evm_attr_change(struct dentry *dentry, struct iattr *attr) +{ + struct inode *inode = d_backing_inode(dentry); + unsigned int ia_valid = attr->ia_valid; + + if ((!(ia_valid & ATTR_UID) || uid_eq(attr->ia_uid, inode->i_uid)) && + (!(ia_valid & ATTR_GID) || gid_eq(attr->ia_gid, inode->i_gid)) && + (!(ia_valid & ATTR_MODE) || attr->ia_mode == inode->i_mode)) + return 0; + + return 1; +} + /** * evm_inode_setattr - prevent updating an invalid EVM extended attribute * @dentry: pointer to the affected dentry @@ -491,9 +789,21 @@ int evm_inode_setattr(struct dentry *dentry, struct iattr *attr) if (!(ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID))) return 0; evm_status = evm_verify_current_integrity(dentry); + /* + * Writing attrs is safe for portable signatures, as portable signatures + * are immutable and can never be updated. + */ if ((evm_status == INTEGRITY_PASS) || - (evm_status == INTEGRITY_NOXATTRS)) + (evm_status == INTEGRITY_NOXATTRS) || + (evm_status == INTEGRITY_FAIL_IMMUTABLE) || + (evm_hmac_disabled() && (evm_status == INTEGRITY_NOLABEL || + evm_status == INTEGRITY_UNKNOWN))) return 0; + + if (evm_status == INTEGRITY_PASS_IMMUTABLE && + !evm_attr_change(dentry, attr)) + return 0; + integrity_audit_msg(AUDIT_INTEGRITY_METADATA, d_backing_inode(dentry), dentry->d_name.name, "appraise_metadata", integrity_status_msg[evm_status], -EPERM, 0); @@ -513,7 +823,12 @@ int evm_inode_setattr(struct dentry *dentry, struct iattr *attr) */ void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) { - if (!evm_key_loaded()) + if (!evm_revalidate_status(NULL)) + return; + + evm_reset_status(dentry->d_inode); + + if (!(evm_initialized & EVM_INIT_HMAC)) return; if (ia_valid & (ATTR_MODE | ATTR_UID | ATTR_GID)) @@ -521,7 +836,7 @@ void evm_inode_post_setattr(struct dentry *dentry, int ia_valid) } /* - * evm_inode_init_security - initializes security.evm + * evm_inode_init_security - initializes security.evm HMAC value */ int evm_inode_init_security(struct inode *inode, const struct xattr *lsm_xattr, @@ -530,7 +845,8 @@ int evm_inode_init_security(struct inode *inode, struct evm_xattr *xattr_data; int rc; - if (!evm_key_loaded() || !evm_protected_xattr(lsm_xattr->name)) + if (!(evm_initialized & EVM_INIT_HMAC) || + !evm_protected_xattr(lsm_xattr->name)) return 0; xattr_data = kzalloc(sizeof(*xattr_data), GFP_NOFS); diff --git a/security/integrity/evm/evm_secfs.c b/security/integrity/evm/evm_secfs.c index bbc85637e18b..8a9db7dfca7e 100644 --- a/security/integrity/evm/evm_secfs.c +++ b/security/integrity/evm/evm_secfs.c @@ -66,12 +66,13 @@ static ssize_t evm_read_key(struct file *filp, char __user *buf, static ssize_t evm_write_key(struct file *file, const char __user *buf, size_t count, loff_t *ppos) { - int i, ret; + unsigned int i; + int ret; if (!capable(CAP_SYS_ADMIN) || (evm_initialized & EVM_SETUP_COMPLETE)) return -EPERM; - ret = kstrtoint_from_user(buf, count, 0, &i); + ret = kstrtouint_from_user(buf, count, 0, &i); if (ret) return ret; @@ -80,12 +81,12 @@ static ssize_t evm_write_key(struct file *file, const char __user *buf, if (!i || (i & ~EVM_INIT_MASK) != 0) return -EINVAL; - /* Don't allow a request to freshly enable metadata writes if - * keys are loaded. + /* + * Don't allow a request to enable metadata writes if + * an HMAC key is loaded. */ if ((i & EVM_ALLOW_METADATA_WRITES) && - ((evm_initialized & EVM_KEY_MASK) != 0) && - !(evm_initialized & EVM_ALLOW_METADATA_WRITES)) + (evm_initialized & EVM_INIT_HMAC) != 0) return -EPERM; if (i & EVM_INIT_HMAC) { @@ -138,8 +139,12 @@ static ssize_t evm_read_xattrs(struct file *filp, char __user *buf, if (rc) return -ERESTARTSYS; - list_for_each_entry(xattr, &evm_config_xattrnames, list) + list_for_each_entry(xattr, &evm_config_xattrnames, list) { + if (!xattr->enabled) + continue; + size += strlen(xattr->name) + 1; + } temp = kmalloc(size + 1, GFP_KERNEL); if (!temp) { @@ -148,6 +153,9 @@ static ssize_t evm_read_xattrs(struct file *filp, char __user *buf, } list_for_each_entry(xattr, &evm_config_xattrnames, list) { + if (!xattr->enabled) + continue; + sprintf(temp + offset, "%s\n", xattr->name); offset += strlen(xattr->name) + 1; } @@ -189,7 +197,7 @@ static ssize_t evm_write_xattrs(struct file *file, const char __user *buf, ab = audit_log_start(audit_context(), GFP_KERNEL, AUDIT_INTEGRITY_EVM_XATTR); - if (!ab) + if (!ab && IS_ENABLED(CONFIG_AUDIT)) return -ENOMEM; xattr = kmalloc(sizeof(struct xattr_list), GFP_KERNEL); @@ -198,6 +206,7 @@ static ssize_t evm_write_xattrs(struct file *file, const char __user *buf, goto out; } + xattr->enabled = true; xattr->name = memdup_user_nul(buf, count); if (IS_ERR(xattr->name)) { err = PTR_ERR(xattr->name); @@ -244,6 +253,10 @@ static ssize_t evm_write_xattrs(struct file *file, const char __user *buf, list_for_each_entry(tmp, &evm_config_xattrnames, list) { if (strcmp(xattr->name, tmp->name) == 0) { err = -EEXIST; + if (!tmp->enabled) { + tmp->enabled = true; + err = count; + } mutex_unlock(&xattr_list_mutex); goto out; } @@ -255,7 +268,7 @@ static ssize_t evm_write_xattrs(struct file *file, const char __user *buf, audit_log_end(ab); return count; out: - audit_log_format(ab, " res=%d", err); + audit_log_format(ab, " res=%d", (err < 0) ? err : 0); audit_log_end(ab); if (xattr) { kfree(xattr->name); diff --git a/security/integrity/iint.c b/security/integrity/iint.c index fca8a9409e4a..8638976f7990 100644 --- a/security/integrity/iint.c +++ b/security/integrity/iint.c @@ -208,7 +208,9 @@ int integrity_kernel_read(struct file *file, loff_t offset, void __init integrity_load_keys(void) { ima_load_x509(); - evm_load_x509(); + + if (!IS_ENABLED(CONFIG_IMA_LOAD_X509)) + evm_load_x509(); } static int __init integrity_fs_init(void) diff --git a/security/integrity/ima/Kconfig b/security/integrity/ima/Kconfig index 12e9250c1bec..d0ceada99243 100644 --- a/security/integrity/ima/Kconfig +++ b/security/integrity/ima/Kconfig @@ -334,3 +334,10 @@ config IMA_SECURE_AND_OR_TRUSTED_BOOT help This option is selected by architectures to enable secure and/or trusted boot based on IMA runtime policies. + +config IMA_DISABLE_HTABLE + bool "Disable htable to allow measurement of duplicate records" + depends on IMA + default n + help + This option disables htable to allow measurement of duplicate records. diff --git a/security/integrity/ima/ima_appraise.c b/security/integrity/ima/ima_appraise.c index 4e5eb0236278..ef9dcfce45d4 100644 --- a/security/integrity/ima/ima_appraise.c +++ b/security/integrity/ima/ima_appraise.c @@ -242,12 +242,16 @@ static int xattr_verify(enum ima_hooks func, struct integrity_iint_cache *iint, hash_start = 1; fallthrough; case IMA_XATTR_DIGEST: - if (iint->flags & IMA_DIGSIG_REQUIRED) { - *cause = "IMA-signature-required"; - *status = INTEGRITY_FAIL; - break; + if (*status != INTEGRITY_PASS_IMMUTABLE) { + if (iint->flags & IMA_DIGSIG_REQUIRED) { + *cause = "IMA-signature-required"; + *status = INTEGRITY_FAIL; + break; + } + clear_bit(IMA_DIGSIG, &iint->atomic_flags); + } else { + set_bit(IMA_DIGSIG, &iint->atomic_flags); } - clear_bit(IMA_DIGSIG, &iint->atomic_flags); if (xattr_len - sizeof(xattr_value->type) - hash_start >= iint->ima_hash->length) /* @@ -416,6 +420,10 @@ int ima_appraise_measurement(enum ima_hooks func, case INTEGRITY_NOLABEL: /* No security.evm xattr. */ cause = "missing-HMAC"; goto out; + case INTEGRITY_FAIL_IMMUTABLE: + set_bit(IMA_DIGSIG, &iint->atomic_flags); + cause = "invalid-fail-immutable"; + goto out; case INTEGRITY_FAIL: /* Invalid HMAC/signature. */ cause = "invalid-HMAC"; goto out; @@ -459,9 +467,12 @@ int ima_appraise_measurement(enum ima_hooks func, status = INTEGRITY_PASS; } - /* Permit new files with file signatures, but without data. */ + /* + * Permit new files with file/EVM portable signatures, but + * without data. + */ if (inode->i_size == 0 && iint->flags & IMA_NEW_FILE && - xattr_value && xattr_value->type == EVM_IMA_XATTR_DIGSIG) { + test_bit(IMA_DIGSIG, &iint->atomic_flags)) { status = INTEGRITY_PASS; } @@ -522,8 +533,6 @@ void ima_inode_post_setattr(struct user_namespace *mnt_userns, return; action = ima_must_appraise(mnt_userns, inode, MAY_ACCESS, POST_SETATTR); - if (!action) - __vfs_removexattr(&init_user_ns, dentry, XATTR_NAME_IMA); iint = integrity_iint_find(inode); if (iint) { set_bit(IMA_CHANGE_ATTR, &iint->atomic_flags); @@ -570,6 +579,7 @@ int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, const void *xattr_value, size_t xattr_value_len) { const struct evm_ima_xattr_data *xvalue = xattr_value; + int digsig = 0; int result; result = ima_protect_xattr(dentry, xattr_name, xattr_value, @@ -577,9 +587,14 @@ int ima_inode_setxattr(struct dentry *dentry, const char *xattr_name, if (result == 1) { if (!xattr_value_len || (xvalue->type >= IMA_XATTR_LAST)) return -EINVAL; - ima_reset_appraise_flags(d_backing_inode(dentry), - xvalue->type == EVM_IMA_XATTR_DIGSIG); - result = 0; + digsig = (xvalue->type == EVM_IMA_XATTR_DIGSIG); + } else if (!strcmp(xattr_name, XATTR_NAME_EVM) && xattr_value_len > 0) { + digsig = (xvalue->type == EVM_XATTR_PORTABLE_DIGSIG); + } + if (result == 1 || evm_revalidate_status(xattr_name)) { + ima_reset_appraise_flags(d_backing_inode(dentry), digsig); + if (result == 1) + result = 0; } return result; } @@ -589,9 +604,10 @@ int ima_inode_removexattr(struct dentry *dentry, const char *xattr_name) int result; result = ima_protect_xattr(dentry, xattr_name, NULL, 0); - if (result == 1) { + if (result == 1 || evm_revalidate_status(xattr_name)) { ima_reset_appraise_flags(d_backing_inode(dentry), 0); - result = 0; + if (result == 1) + result = 0; } return result; } diff --git a/security/integrity/ima/ima_asymmetric_keys.c b/security/integrity/ima/ima_asymmetric_keys.c index 1fb0b0e09559..c985418698a4 100644 --- a/security/integrity/ima/ima_asymmetric_keys.c +++ b/security/integrity/ima/ima_asymmetric_keys.c @@ -11,6 +11,7 @@ #include #include +#include #include "ima.h" /** diff --git a/security/integrity/ima/ima_crypto.c b/security/integrity/ima/ima_crypto.c index f6a7e9643b54..a7206cc1d7d1 100644 --- a/security/integrity/ima/ima_crypto.c +++ b/security/integrity/ima/ima_crypto.c @@ -598,8 +598,8 @@ static int ima_calc_field_array_hash_tfm(struct ima_field_data *field_data, u8 buffer[IMA_EVENT_NAME_LEN_MAX + 1] = { 0 }; u8 *data_to_hash = field_data[i].data; u32 datalen = field_data[i].len; - u32 datalen_to_hash = - !ima_canonical_fmt ? datalen : cpu_to_le32(datalen); + u32 datalen_to_hash = !ima_canonical_fmt ? + datalen : (__force u32)cpu_to_le32(datalen); if (strcmp(td->name, IMA_TEMPLATE_IMA_NAME) != 0) { rc = crypto_shash_update(shash, diff --git a/security/integrity/ima/ima_fs.c b/security/integrity/ima/ima_fs.c index ea8ff8a07b36..3d8e9d5db5aa 100644 --- a/security/integrity/ima/ima_fs.c +++ b/security/integrity/ima/ima_fs.c @@ -147,7 +147,7 @@ int ima_measurements_show(struct seq_file *m, void *v) * PCR used defaults to the same (config option) in * little-endian format, unless set in policy */ - pcr = !ima_canonical_fmt ? e->pcr : cpu_to_le32(e->pcr); + pcr = !ima_canonical_fmt ? e->pcr : (__force u32)cpu_to_le32(e->pcr); ima_putc(m, &pcr, sizeof(e->pcr)); /* 2nd: template digest */ @@ -155,7 +155,7 @@ int ima_measurements_show(struct seq_file *m, void *v) /* 3rd: template name size */ namelen = !ima_canonical_fmt ? strlen(template_name) : - cpu_to_le32(strlen(template_name)); + (__force u32)cpu_to_le32(strlen(template_name)); ima_putc(m, &namelen, sizeof(namelen)); /* 4th: template name */ @@ -167,7 +167,7 @@ int ima_measurements_show(struct seq_file *m, void *v) if (!is_ima_template) { template_data_len = !ima_canonical_fmt ? e->template_data_len : - cpu_to_le32(e->template_data_len); + (__force u32)cpu_to_le32(e->template_data_len); ima_putc(m, &template_data_len, sizeof(e->template_data_len)); } diff --git a/security/integrity/ima/ima_init.c b/security/integrity/ima/ima_init.c index 6e8742916d1d..5076a7d9d23e 100644 --- a/security/integrity/ima/ima_init.c +++ b/security/integrity/ima/ima_init.c @@ -108,6 +108,10 @@ void __init ima_load_x509(void) ima_policy_flag &= ~unset_flags; integrity_load_x509(INTEGRITY_KEYRING_IMA, CONFIG_IMA_X509_PATH); + + /* load also EVM key to avoid appraisal */ + evm_load_x509(); + ima_policy_flag |= unset_flags; } #endif diff --git a/security/integrity/ima/ima_kexec.c b/security/integrity/ima/ima_kexec.c index 667887665823..f799cc278a9a 100644 --- a/security/integrity/ima/ima_kexec.c +++ b/security/integrity/ima/ima_kexec.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "ima.h" #ifdef CONFIG_IMA_KEXEC diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 906c1d8e0b71..287b90509006 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -433,7 +433,7 @@ int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) inode = file_inode(vma->vm_file); action = ima_get_action(file_mnt_user_ns(vma->vm_file), inode, current_cred(), secid, MAY_EXEC, MMAP_CHECK, - &pcr, &template, 0); + &pcr, &template, NULL); /* Is the mmap'ed file in policy? */ if (!(action & (IMA_MEASURE | IMA_APPRAISE_SUBMASK))) diff --git a/security/integrity/ima/ima_queue.c b/security/integrity/ima/ima_queue.c index c096ef8945c7..532da87ce519 100644 --- a/security/integrity/ima/ima_queue.c +++ b/security/integrity/ima/ima_queue.c @@ -168,7 +168,7 @@ int ima_add_template_entry(struct ima_template_entry *entry, int violation, int result = 0, tpmresult = 0; mutex_lock(&ima_extend_list_mutex); - if (!violation) { + if (!violation && !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)) { if (ima_lookup_digest_entry(digest, entry->pcr)) { audit_cause = "hash_exists"; result = -EEXIST; @@ -176,7 +176,8 @@ int ima_add_template_entry(struct ima_template_entry *entry, int violation, } } - result = ima_add_digest_entry(entry, 1); + result = ima_add_digest_entry(entry, + !IS_ENABLED(CONFIG_IMA_DISABLE_HTABLE)); if (result < 0) { audit_cause = "ENOMEM"; audit_info = 0; diff --git a/security/integrity/ima/ima_template.c b/security/integrity/ima/ima_template.c index 4e081e650047..694560396be0 100644 --- a/security/integrity/ima/ima_template.c +++ b/security/integrity/ima/ima_template.c @@ -22,6 +22,8 @@ static struct ima_template_desc builtin_templates[] = { {.name = "ima-sig", .fmt = "d-ng|n-ng|sig"}, {.name = "ima-buf", .fmt = "d-ng|n-ng|buf"}, {.name = "ima-modsig", .fmt = "d-ng|n-ng|sig|d-modsig|modsig"}, + {.name = "evm-sig", + .fmt = "d-ng|n-ng|evmsig|xattrnames|xattrlengths|xattrvalues|iuid|igid|imode"}, {.name = "", .fmt = ""}, /* placeholder for a custom format */ }; @@ -45,6 +47,23 @@ static const struct ima_template_field supported_fields[] = { .field_show = ima_show_template_digest_ng}, {.field_id = "modsig", .field_init = ima_eventmodsig_init, .field_show = ima_show_template_sig}, + {.field_id = "evmsig", .field_init = ima_eventevmsig_init, + .field_show = ima_show_template_sig}, + {.field_id = "iuid", .field_init = ima_eventinodeuid_init, + .field_show = ima_show_template_uint}, + {.field_id = "igid", .field_init = ima_eventinodegid_init, + .field_show = ima_show_template_uint}, + {.field_id = "imode", .field_init = ima_eventinodemode_init, + .field_show = ima_show_template_uint}, + {.field_id = "xattrnames", + .field_init = ima_eventinodexattrnames_init, + .field_show = ima_show_template_string}, + {.field_id = "xattrlengths", + .field_init = ima_eventinodexattrlengths_init, + .field_show = ima_show_template_sig}, + {.field_id = "xattrvalues", + .field_init = ima_eventinodexattrvalues_init, + .field_show = ima_show_template_sig}, }; /* @@ -52,7 +71,8 @@ static const struct ima_template_field supported_fields[] = { * need to be accounted for since they shouldn't be defined in the same template * description as 'd-ng' and 'n-ng' respectively. */ -#define MAX_TEMPLATE_NAME_LEN sizeof("d-ng|n-ng|sig|buf|d-modisg|modsig") +#define MAX_TEMPLATE_NAME_LEN \ + sizeof("d-ng|n-ng|evmsig|xattrnames|xattrlengths|xattrvalues|iuid|igid|imode") static struct ima_template_desc *ima_template; static struct ima_template_desc *ima_buf_template; @@ -403,9 +423,9 @@ int ima_restore_measurement_list(loff_t size, void *buf) return 0; if (ima_canonical_fmt) { - khdr->version = le16_to_cpu(khdr->version); - khdr->count = le64_to_cpu(khdr->count); - khdr->buffer_size = le64_to_cpu(khdr->buffer_size); + khdr->version = le16_to_cpu((__force __le16)khdr->version); + khdr->count = le64_to_cpu((__force __le64)khdr->count); + khdr->buffer_size = le64_to_cpu((__force __le64)khdr->buffer_size); } if (khdr->version != 1) { @@ -495,7 +515,7 @@ int ima_restore_measurement_list(loff_t size, void *buf) } entry->pcr = !ima_canonical_fmt ? *(u32 *)(hdr[HDR_PCR].data) : - le32_to_cpu(*(u32 *)(hdr[HDR_PCR].data)); + le32_to_cpu(*(__le32 *)(hdr[HDR_PCR].data)); ret = ima_restore_measurement_entry(entry); if (ret < 0) break; diff --git a/security/integrity/ima/ima_template_lib.c b/security/integrity/ima/ima_template_lib.c index c022ee9e2a4e..ca017cae73eb 100644 --- a/security/integrity/ima/ima_template_lib.c +++ b/security/integrity/ima/ima_template_lib.c @@ -10,6 +10,8 @@ */ #include "ima_template_lib.h" +#include +#include static bool ima_template_hash_algo_allowed(u8 algo) { @@ -23,7 +25,8 @@ enum data_formats { DATA_FMT_DIGEST = 0, DATA_FMT_DIGEST_WITH_ALGO, DATA_FMT_STRING, - DATA_FMT_HEX + DATA_FMT_HEX, + DATA_FMT_UINT }; static int ima_write_template_field_data(const void *data, const u32 datalen, @@ -87,6 +90,36 @@ static void ima_show_template_data_ascii(struct seq_file *m, case DATA_FMT_STRING: seq_printf(m, "%s", buf_ptr); break; + case DATA_FMT_UINT: + switch (field_data->len) { + case sizeof(u8): + seq_printf(m, "%u", *(u8 *)buf_ptr); + break; + case sizeof(u16): + if (ima_canonical_fmt) + seq_printf(m, "%u", + le16_to_cpu(*(__le16 *)buf_ptr)); + else + seq_printf(m, "%u", *(u16 *)buf_ptr); + break; + case sizeof(u32): + if (ima_canonical_fmt) + seq_printf(m, "%u", + le32_to_cpu(*(__le32 *)buf_ptr)); + else + seq_printf(m, "%u", *(u32 *)buf_ptr); + break; + case sizeof(u64): + if (ima_canonical_fmt) + seq_printf(m, "%llu", + le64_to_cpu(*(__le64 *)buf_ptr)); + else + seq_printf(m, "%llu", *(u64 *)buf_ptr); + break; + default: + break; + } + break; default: break; } @@ -101,7 +134,8 @@ static void ima_show_template_data_binary(struct seq_file *m, strlen(field_data->data) : field_data->len; if (show != IMA_SHOW_BINARY_NO_FIELD_LEN) { - u32 field_len = !ima_canonical_fmt ? len : cpu_to_le32(len); + u32 field_len = !ima_canonical_fmt ? + len : (__force u32)cpu_to_le32(len); ima_putc(m, &field_len, sizeof(field_len)); } @@ -162,6 +196,12 @@ void ima_show_template_buf(struct seq_file *m, enum ima_show_type show, ima_show_template_field_data(m, show, DATA_FMT_HEX, field_data); } +void ima_show_template_uint(struct seq_file *m, enum ima_show_type show, + struct ima_field_data *field_data) +{ + ima_show_template_field_data(m, show, DATA_FMT_UINT, field_data); +} + /** * ima_parse_buf() - Parses lengths and data from an input buffer * @bufstartp: Buffer start address. @@ -188,9 +228,10 @@ int ima_parse_buf(void *bufstartp, void *bufendp, void **bufcurp, if (bufp > (bufendp - sizeof(u32))) break; - fields[i].len = *(u32 *)bufp; if (ima_canonical_fmt) - fields[i].len = le32_to_cpu(fields[i].len); + fields[i].len = le32_to_cpu(*(__le32 *)bufp); + else + fields[i].len = *(u32 *)bufp; bufp += sizeof(u32); } @@ -438,7 +479,7 @@ int ima_eventsig_init(struct ima_event_data *event_data, struct evm_ima_xattr_data *xattr_value = event_data->xattr_value; if ((!xattr_value) || (xattr_value->type != EVM_IMA_XATTR_DIGSIG)) - return 0; + return ima_eventevmsig_init(event_data, field_data); return ima_write_template_field_data(xattr_value, event_data->xattr_len, DATA_FMT_HEX, field_data); @@ -484,3 +525,163 @@ int ima_eventmodsig_init(struct ima_event_data *event_data, return ima_write_template_field_data(data, data_len, DATA_FMT_HEX, field_data); } + +/* + * ima_eventevmsig_init - include the EVM portable signature as part of the + * template data + */ +int ima_eventevmsig_init(struct ima_event_data *event_data, + struct ima_field_data *field_data) +{ + struct evm_ima_xattr_data *xattr_data = NULL; + int rc = 0; + + if (!event_data->file) + return 0; + + rc = vfs_getxattr_alloc(&init_user_ns, file_dentry(event_data->file), + XATTR_NAME_EVM, (char **)&xattr_data, 0, + GFP_NOFS); + if (rc <= 0) + return 0; + + if (xattr_data->type != EVM_XATTR_PORTABLE_DIGSIG) { + kfree(xattr_data); + return 0; + } + + rc = ima_write_template_field_data((char *)xattr_data, rc, DATA_FMT_HEX, + field_data); + kfree(xattr_data); + return rc; +} + +static int ima_eventinodedac_init_common(struct ima_event_data *event_data, + struct ima_field_data *field_data, + bool get_uid) +{ + unsigned int id; + + if (!event_data->file) + return 0; + + if (get_uid) + id = i_uid_read(file_inode(event_data->file)); + else + id = i_gid_read(file_inode(event_data->file)); + + if (ima_canonical_fmt) { + if (sizeof(id) == sizeof(u16)) + id = (__force u16)cpu_to_le16(id); + else + id = (__force u32)cpu_to_le32(id); + } + + return ima_write_template_field_data((void *)&id, sizeof(id), + DATA_FMT_UINT, field_data); +} + +/* + * ima_eventinodeuid_init - include the inode UID as part of the template + * data + */ +int ima_eventinodeuid_init(struct ima_event_data *event_data, + struct ima_field_data *field_data) +{ + return ima_eventinodedac_init_common(event_data, field_data, true); +} + +/* + * ima_eventinodegid_init - include the inode GID as part of the template + * data + */ +int ima_eventinodegid_init(struct ima_event_data *event_data, + struct ima_field_data *field_data) +{ + return ima_eventinodedac_init_common(event_data, field_data, false); +} + +/* + * ima_eventinodemode_init - include the inode mode as part of the template + * data + */ +int ima_eventinodemode_init(struct ima_event_data *event_data, + struct ima_field_data *field_data) +{ + struct inode *inode; + u16 mode; + + if (!event_data->file) + return 0; + + inode = file_inode(event_data->file); + mode = inode->i_mode; + if (ima_canonical_fmt) + mode = (__force u16)cpu_to_le16(mode); + + return ima_write_template_field_data((char *)&mode, sizeof(mode), + DATA_FMT_UINT, field_data); +} + +static int ima_eventinodexattrs_init_common(struct ima_event_data *event_data, + struct ima_field_data *field_data, + char type) +{ + u8 *buffer = NULL; + int rc; + + if (!event_data->file) + return 0; + + rc = evm_read_protected_xattrs(file_dentry(event_data->file), NULL, 0, + type, ima_canonical_fmt); + if (rc < 0) + return 0; + + buffer = kmalloc(rc, GFP_KERNEL); + if (!buffer) + return 0; + + rc = evm_read_protected_xattrs(file_dentry(event_data->file), buffer, + rc, type, ima_canonical_fmt); + if (rc < 0) { + rc = 0; + goto out; + } + + rc = ima_write_template_field_data((char *)buffer, rc, DATA_FMT_HEX, + field_data); +out: + kfree(buffer); + return rc; +} + +/* + * ima_eventinodexattrnames_init - include a list of xattr names as part of the + * template data + */ +int ima_eventinodexattrnames_init(struct ima_event_data *event_data, + struct ima_field_data *field_data) +{ + return ima_eventinodexattrs_init_common(event_data, field_data, 'n'); +} + +/* + * ima_eventinodexattrlengths_init - include a list of xattr lengths as part of + * the template data + */ +int ima_eventinodexattrlengths_init(struct ima_event_data *event_data, + struct ima_field_data *field_data) +{ + return ima_eventinodexattrs_init_common(event_data, field_data, 'l'); +} + +/* + * ima_eventinodexattrvalues_init - include a list of xattr values as part of + * the template data + */ +int ima_eventinodexattrvalues_init(struct ima_event_data *event_data, + struct ima_field_data *field_data) +{ + return ima_eventinodexattrs_init_common(event_data, field_data, 'v'); +} diff --git a/security/integrity/ima/ima_template_lib.h b/security/integrity/ima/ima_template_lib.h index 6b3b880637a0..c71f1de95753 100644 --- a/security/integrity/ima/ima_template_lib.h +++ b/security/integrity/ima/ima_template_lib.h @@ -27,6 +27,8 @@ void ima_show_template_sig(struct seq_file *m, enum ima_show_type show, struct ima_field_data *field_data); void ima_show_template_buf(struct seq_file *m, enum ima_show_type show, struct ima_field_data *field_data); +void ima_show_template_uint(struct seq_file *m, enum ima_show_type show, + struct ima_field_data *field_data); int ima_parse_buf(void *bufstartp, void *bufendp, void **bufcurp, int maxfields, struct ima_field_data *fields, int *curfields, unsigned long *len_mask, int enforce_mask, char *bufname); @@ -46,4 +48,18 @@ int ima_eventbuf_init(struct ima_event_data *event_data, struct ima_field_data *field_data); int ima_eventmodsig_init(struct ima_event_data *event_data, struct ima_field_data *field_data); +int ima_eventevmsig_init(struct ima_event_data *event_data, + struct ima_field_data *field_data); +int ima_eventinodeuid_init(struct ima_event_data *event_data, + struct ima_field_data *field_data); +int ima_eventinodegid_init(struct ima_event_data *event_data, + struct ima_field_data *field_data); +int ima_eventinodemode_init(struct ima_event_data *event_data, + struct ima_field_data *field_data); +int ima_eventinodexattrnames_init(struct ima_event_data *event_data, + struct ima_field_data *field_data); +int ima_eventinodexattrlengths_init(struct ima_event_data *event_data, + struct ima_field_data *field_data); +int ima_eventinodexattrvalues_init(struct ima_event_data *event_data, + struct ima_field_data *field_data); #endif /* __LINUX_IMA_TEMPLATE_LIB_H */ diff --git a/security/security.c b/security/security.c index b38155b2de83..e9f8010a2341 100644 --- a/security/security.c +++ b/security/security.c @@ -1354,7 +1354,7 @@ int security_inode_setxattr(struct user_namespace *mnt_userns, ret = ima_inode_setxattr(dentry, name, value, size); if (ret) return ret; - return evm_inode_setxattr(dentry, name, value, size); + return evm_inode_setxattr(mnt_userns, dentry, name, value, size); } void security_inode_post_setxattr(struct dentry *dentry, const char *name, @@ -1399,7 +1399,7 @@ int security_inode_removexattr(struct user_namespace *mnt_userns, ret = ima_inode_removexattr(dentry, name); if (ret) return ret; - return evm_inode_removexattr(dentry, name); + return evm_inode_removexattr(mnt_userns, dentry, name); } int security_inode_need_killpriv(struct dentry *dentry) diff --git a/security/tomoyo/audit.c b/security/tomoyo/audit.c index b51bad121c11..d79bf07e16be 100644 --- a/security/tomoyo/audit.c +++ b/security/tomoyo/audit.c @@ -320,6 +320,7 @@ static unsigned int tomoyo_log_count; * @ns: Pointer to "struct tomoyo_policy_namespace". * @profile: Profile number. * @index: Index number of functionality. + * @matched_acl: Pointer to "struct tomoyo_acl_info". * @is_granted: True if granted log, false otherwise. * * Returns true if this request should be audited, false otherwise. diff --git a/security/tomoyo/securityfs_if.c b/security/tomoyo/securityfs_if.c index 065f4941c4d8..a2705798476f 100644 --- a/security/tomoyo/securityfs_if.c +++ b/security/tomoyo/securityfs_if.c @@ -139,6 +139,7 @@ static int tomoyo_open(struct inode *inode, struct file *file) /** * tomoyo_release - close() for /sys/kernel/security/tomoyo/ interface. * + * @inode: Pointer to "struct inode". * @file: Pointer to "struct file". * */ diff --git a/security/tomoyo/tomoyo.c b/security/tomoyo/tomoyo.c index 1f3cd432d830..b6a31901f289 100644 --- a/security/tomoyo/tomoyo.c +++ b/security/tomoyo/tomoyo.c @@ -63,7 +63,7 @@ static void tomoyo_bprm_committed_creds(struct linux_binprm *bprm) #ifndef CONFIG_SECURITY_TOMOYO_OMIT_USERSPACE_LOADER /** - * tomoyo_bprm_for_exec - Target for security_bprm_creds_for_exec(). + * tomoyo_bprm_creds_for_exec - Target for security_bprm_creds_for_exec(). * * @bprm: Pointer to "struct linux_binprm". * @@ -113,8 +113,7 @@ static int tomoyo_bprm_check_security(struct linux_binprm *bprm) /** * tomoyo_inode_getattr - Target for security_inode_getattr(). * - * @mnt: Pointer to "struct vfsmount". - * @dentry: Pointer to "struct dentry". + * @path: Pointer to "struct path". * * Returns 0 on success, negative value otherwise. */ @@ -300,8 +299,7 @@ static int tomoyo_file_fcntl(struct file *file, unsigned int cmd, /** * tomoyo_file_open - Target for security_file_open(). * - * @f: Pointer to "struct file". - * @cred: Pointer to "struct cred". + * @f: Pointer to "struct file". * * Returns 0 on success, negative value otherwise. */ @@ -487,8 +485,8 @@ struct lsm_blob_sizes tomoyo_blob_sizes __lsm_ro_after_init = { /** * tomoyo_task_alloc - Target for security_task_alloc(). * - * @task: Pointer to "struct task_struct". - * @flags: clone() flags. + * @task: Pointer to "struct task_struct". + * @clone_flags: clone() flags. * * Returns 0. */ diff --git a/security/tomoyo/util.c b/security/tomoyo/util.c index e89cac913583..1da2e3722b12 100644 --- a/security/tomoyo/util.c +++ b/security/tomoyo/util.c @@ -83,8 +83,8 @@ const u8 tomoyo_index2category[TOMOYO_MAX_MAC_INDEX] = { /** * tomoyo_convert_time - Convert time_t to YYYY/MM/DD hh/mm/ss. * - * @time: Seconds since 1970/01/01 00:00:00. - * @stamp: Pointer to "struct tomoyo_time". + * @time64: Seconds since 1970/01/01 00:00:00. + * @stamp: Pointer to "struct tomoyo_time". * * Returns nothing. */ diff --git a/tools/arch/x86/include/asm/asm.h b/tools/arch/x86/include/asm/asm.h new file mode 100644 index 000000000000..3ad3da9a7d97 --- /dev/null +++ b/tools/arch/x86/include/asm/asm.h @@ -0,0 +1,193 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_ASM_H +#define _ASM_X86_ASM_H + +#ifdef __ASSEMBLY__ +# define __ASM_FORM(x, ...) x,## __VA_ARGS__ +# define __ASM_FORM_RAW(x, ...) x,## __VA_ARGS__ +# define __ASM_FORM_COMMA(x, ...) x,## __VA_ARGS__, +#else +#include +# define __ASM_FORM(x, ...) " " __stringify(x,##__VA_ARGS__) " " +# define __ASM_FORM_RAW(x, ...) __stringify(x,##__VA_ARGS__) +# define __ASM_FORM_COMMA(x, ...) " " __stringify(x,##__VA_ARGS__) "," +#endif + +#define _ASM_BYTES(x, ...) __ASM_FORM(.byte x,##__VA_ARGS__ ;) + +#ifndef __x86_64__ +/* 32 bit */ +# define __ASM_SEL(a,b) __ASM_FORM(a) +# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(a) +#else +/* 64 bit */ +# define __ASM_SEL(a,b) __ASM_FORM(b) +# define __ASM_SEL_RAW(a,b) __ASM_FORM_RAW(b) +#endif + +#define __ASM_SIZE(inst, ...) __ASM_SEL(inst##l##__VA_ARGS__, \ + inst##q##__VA_ARGS__) +#define __ASM_REG(reg) __ASM_SEL_RAW(e##reg, r##reg) + +#define _ASM_PTR __ASM_SEL(.long, .quad) +#define _ASM_ALIGN __ASM_SEL(.balign 4, .balign 8) + +#define _ASM_MOV __ASM_SIZE(mov) +#define _ASM_INC __ASM_SIZE(inc) +#define _ASM_DEC __ASM_SIZE(dec) +#define _ASM_ADD __ASM_SIZE(add) +#define _ASM_SUB __ASM_SIZE(sub) +#define _ASM_XADD __ASM_SIZE(xadd) +#define _ASM_MUL __ASM_SIZE(mul) + +#define _ASM_AX __ASM_REG(ax) +#define _ASM_BX __ASM_REG(bx) +#define _ASM_CX __ASM_REG(cx) +#define _ASM_DX __ASM_REG(dx) +#define _ASM_SP __ASM_REG(sp) +#define _ASM_BP __ASM_REG(bp) +#define _ASM_SI __ASM_REG(si) +#define _ASM_DI __ASM_REG(di) + +#ifndef __x86_64__ +/* 32 bit */ + +#define _ASM_ARG1 _ASM_AX +#define _ASM_ARG2 _ASM_DX +#define _ASM_ARG3 _ASM_CX + +#define _ASM_ARG1L eax +#define _ASM_ARG2L edx +#define _ASM_ARG3L ecx + +#define _ASM_ARG1W ax +#define _ASM_ARG2W dx +#define _ASM_ARG3W cx + +#define _ASM_ARG1B al +#define _ASM_ARG2B dl +#define _ASM_ARG3B cl + +#else +/* 64 bit */ + +#define _ASM_ARG1 _ASM_DI +#define _ASM_ARG2 _ASM_SI +#define _ASM_ARG3 _ASM_DX +#define _ASM_ARG4 _ASM_CX +#define _ASM_ARG5 r8 +#define _ASM_ARG6 r9 + +#define _ASM_ARG1Q rdi +#define _ASM_ARG2Q rsi +#define _ASM_ARG3Q rdx +#define _ASM_ARG4Q rcx +#define _ASM_ARG5Q r8 +#define _ASM_ARG6Q r9 + +#define _ASM_ARG1L edi +#define _ASM_ARG2L esi +#define _ASM_ARG3L edx +#define _ASM_ARG4L ecx +#define _ASM_ARG5L r8d +#define _ASM_ARG6L r9d + +#define _ASM_ARG1W di +#define _ASM_ARG2W si +#define _ASM_ARG3W dx +#define _ASM_ARG4W cx +#define _ASM_ARG5W r8w +#define _ASM_ARG6W r9w + +#define _ASM_ARG1B dil +#define _ASM_ARG2B sil +#define _ASM_ARG3B dl +#define _ASM_ARG4B cl +#define _ASM_ARG5B r8b +#define _ASM_ARG6B r9b + +#endif + +/* + * Macros to generate condition code outputs from inline assembly, + * The output operand must be type "bool". + */ +#ifdef __GCC_ASM_FLAG_OUTPUTS__ +# define CC_SET(c) "\n\t/* output condition code " #c "*/\n" +# define CC_OUT(c) "=@cc" #c +#else +# define CC_SET(c) "\n\tset" #c " %[_cc_" #c "]\n" +# define CC_OUT(c) [_cc_ ## c] "=qm" +#endif + +#ifdef __KERNEL__ + +/* Exception table entry */ +#ifdef __ASSEMBLY__ +# define _ASM_EXTABLE_HANDLE(from, to, handler) \ + .pushsection "__ex_table","a" ; \ + .balign 4 ; \ + .long (from) - . ; \ + .long (to) - . ; \ + .long (handler) - . ; \ + .popsection + +# define _ASM_EXTABLE(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_default) + +# define _ASM_EXTABLE_UA(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess) + +# define _ASM_EXTABLE_CPY(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_copy) + +# define _ASM_EXTABLE_FAULT(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) + +# ifdef CONFIG_KPROBES +# define _ASM_NOKPROBE(entry) \ + .pushsection "_kprobe_blacklist","aw" ; \ + _ASM_ALIGN ; \ + _ASM_PTR (entry); \ + .popsection +# else +# define _ASM_NOKPROBE(entry) +# endif + +#else /* ! __ASSEMBLY__ */ +# define _EXPAND_EXTABLE_HANDLE(x) #x +# define _ASM_EXTABLE_HANDLE(from, to, handler) \ + " .pushsection \"__ex_table\",\"a\"\n" \ + " .balign 4\n" \ + " .long (" #from ") - .\n" \ + " .long (" #to ") - .\n" \ + " .long (" _EXPAND_EXTABLE_HANDLE(handler) ") - .\n" \ + " .popsection\n" + +# define _ASM_EXTABLE(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_default) + +# define _ASM_EXTABLE_UA(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_uaccess) + +# define _ASM_EXTABLE_CPY(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_copy) + +# define _ASM_EXTABLE_FAULT(from, to) \ + _ASM_EXTABLE_HANDLE(from, to, ex_handler_fault) + +/* For C file, we already have NOKPROBE_SYMBOL macro */ + +/* + * This output constraint should be used for any inline asm which has a "call" + * instruction. Otherwise the asm may be inserted before the frame pointer + * gets set up by the containing function. If you forget to do this, objtool + * may print a "call without frame pointer save/setup" warning. + */ +register unsigned long current_stack_pointer asm(_ASM_SP); +#define ASM_CALL_CONSTRAINT "+r" (current_stack_pointer) +#endif /* __ASSEMBLY__ */ + +#endif /* __KERNEL__ */ + +#endif /* _ASM_X86_ASM_H */ diff --git a/tools/arch/x86/include/asm/nops.h b/tools/arch/x86/include/asm/nops.h index c1e5e818ba16..c5573eaa5bb9 100644 --- a/tools/arch/x86/include/asm/nops.h +++ b/tools/arch/x86/include/asm/nops.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_NOPS_H #define _ASM_X86_NOPS_H +#include + /* * Define nops for use with alternative() and for tracing. */ @@ -57,20 +59,14 @@ #endif /* CONFIG_64BIT */ -#ifdef __ASSEMBLY__ -#define _ASM_MK_NOP(x) .byte x -#else -#define _ASM_MK_NOP(x) ".byte " __stringify(x) "\n" -#endif - -#define ASM_NOP1 _ASM_MK_NOP(BYTES_NOP1) -#define ASM_NOP2 _ASM_MK_NOP(BYTES_NOP2) -#define ASM_NOP3 _ASM_MK_NOP(BYTES_NOP3) -#define ASM_NOP4 _ASM_MK_NOP(BYTES_NOP4) -#define ASM_NOP5 _ASM_MK_NOP(BYTES_NOP5) -#define ASM_NOP6 _ASM_MK_NOP(BYTES_NOP6) -#define ASM_NOP7 _ASM_MK_NOP(BYTES_NOP7) -#define ASM_NOP8 _ASM_MK_NOP(BYTES_NOP8) +#define ASM_NOP1 _ASM_BYTES(BYTES_NOP1) +#define ASM_NOP2 _ASM_BYTES(BYTES_NOP2) +#define ASM_NOP3 _ASM_BYTES(BYTES_NOP3) +#define ASM_NOP4 _ASM_BYTES(BYTES_NOP4) +#define ASM_NOP5 _ASM_BYTES(BYTES_NOP5) +#define ASM_NOP6 _ASM_BYTES(BYTES_NOP6) +#define ASM_NOP7 _ASM_BYTES(BYTES_NOP7) +#define ASM_NOP8 _ASM_BYTES(BYTES_NOP8) #define ASM_NOP_MAX 8 diff --git a/tools/debugging/kernel-chktaint b/tools/debugging/kernel-chktaint index 719f18b1edf0..f1af27ce9f20 100755 --- a/tools/debugging/kernel-chktaint +++ b/tools/debugging/kernel-chktaint @@ -196,7 +196,7 @@ else fi echo "For a more detailed explanation of the various taint flags see" -echo " Documentation/admin-guide/tainted-kernels.rst in the the Linux kernel sources" +echo " Documentation/admin-guide/tainted-kernels.rst in the Linux kernel sources" echo " or https://kernel.org/doc/html/latest/admin-guide/tainted-kernels.html" echo "Raw taint value as int/string: $taint/'$out'" #EOF# diff --git a/tools/lib/traceevent/plugins/plugin_kvm.c b/tools/lib/traceevent/plugins/plugin_kvm.c index 51ceeb9147eb..9ce7b4b68e3f 100644 --- a/tools/lib/traceevent/plugins/plugin_kvm.c +++ b/tools/lib/traceevent/plugins/plugin_kvm.c @@ -366,7 +366,7 @@ union kvm_mmu_page_role { unsigned direct:1; unsigned access:3; unsigned invalid:1; - unsigned nxe:1; + unsigned efer_nx:1; unsigned cr0_wp:1; unsigned smep_and_not_wp:1; unsigned smap_and_not_wp:1; @@ -403,7 +403,7 @@ static int kvm_mmu_print_role(struct trace_seq *s, struct tep_record *record, access_str[role.access], role.invalid ? " invalid" : "", role.cr4_pae ? "" : "!", - role.nxe ? "" : "!", + role.efer_nx ? "" : "!", role.cr0_wp ? "" : "!", role.smep_and_not_wp ? " smep" : "", role.smap_and_not_wp ? " smap" : "", diff --git a/tools/testing/selftests/arm64/fp/sve-probe-vls.c b/tools/testing/selftests/arm64/fp/sve-probe-vls.c index b29cbc642c57..76e138525d55 100644 --- a/tools/testing/selftests/arm64/fp/sve-probe-vls.c +++ b/tools/testing/selftests/arm64/fp/sve-probe-vls.c @@ -25,7 +25,7 @@ int main(int argc, char **argv) ksft_set_plan(2); if (!(getauxval(AT_HWCAP) & HWCAP_SVE)) - ksft_exit_skip("SVE not available"); + ksft_exit_skip("SVE not available\n"); /* * Enumerate up to SVE_VQ_MAX vector lengths diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index 524c857a049c..06a351b4f93b 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only +/aarch64/debug-exceptions /aarch64/get-reg-list -/aarch64/get-reg-list-sve /aarch64/vgic_init /s390x/memop /s390x/resets @@ -8,12 +8,15 @@ /x86_64/cr4_cpuid_sync_test /x86_64/debug_regs /x86_64/evmcs_test +/x86_64/emulator_error_test /x86_64/get_cpuid_test /x86_64/get_msr_index_features /x86_64/kvm_pv_test /x86_64/hyperv_clock /x86_64/hyperv_cpuid +/x86_64/hyperv_features /x86_64/mmio_warning_test +/x86_64/mmu_role_test /x86_64/platform_info_test /x86_64/set_boot_cpu_id /x86_64/set_sregs_test @@ -29,6 +32,7 @@ /x86_64/vmx_preemption_timer_test /x86_64/vmx_set_nested_state_test /x86_64/vmx_tsc_adjust_test +/x86_64/vmx_nested_tsc_scaling_test /x86_64/xapic_ipi_test /x86_64/xen_shinfo_test /x86_64/xen_vmcall_test @@ -44,3 +48,4 @@ /memslot_perf_test /set_memory_region_test /steal_time +/kvm_binary_stats_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index daaee1888b12..b853be2ae3c6 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -34,18 +34,21 @@ ifeq ($(ARCH),s390) endif LIBKVM = lib/assert.c lib/elf.c lib/io.c lib/kvm_util.c lib/rbtree.c lib/sparsebit.c lib/test_util.c lib/guest_modes.c lib/perf_test_util.c -LIBKVM_x86_64 = lib/x86_64/processor.c lib/x86_64/vmx.c lib/x86_64/svm.c lib/x86_64/ucall.c lib/x86_64/handlers.S -LIBKVM_aarch64 = lib/aarch64/processor.c lib/aarch64/ucall.c +LIBKVM_x86_64 = lib/x86_64/apic.c lib/x86_64/processor.c lib/x86_64/vmx.c lib/x86_64/svm.c lib/x86_64/ucall.c lib/x86_64/handlers.S +LIBKVM_aarch64 = lib/aarch64/processor.c lib/aarch64/ucall.c lib/aarch64/handlers.S LIBKVM_s390x = lib/s390x/processor.c lib/s390x/ucall.c lib/s390x/diag318_test_handler.c TEST_GEN_PROGS_x86_64 = x86_64/cr4_cpuid_sync_test TEST_GEN_PROGS_x86_64 += x86_64/get_msr_index_features TEST_GEN_PROGS_x86_64 += x86_64/evmcs_test +TEST_GEN_PROGS_x86_64 += x86_64/emulator_error_test TEST_GEN_PROGS_x86_64 += x86_64/get_cpuid_test TEST_GEN_PROGS_x86_64 += x86_64/hyperv_clock TEST_GEN_PROGS_x86_64 += x86_64/hyperv_cpuid +TEST_GEN_PROGS_x86_64 += x86_64/hyperv_features TEST_GEN_PROGS_x86_64 += x86_64/kvm_pv_test TEST_GEN_PROGS_x86_64 += x86_64/mmio_warning_test +TEST_GEN_PROGS_x86_64 += x86_64/mmu_role_test TEST_GEN_PROGS_x86_64 += x86_64/platform_info_test TEST_GEN_PROGS_x86_64 += x86_64/set_boot_cpu_id TEST_GEN_PROGS_x86_64 += x86_64/set_sregs_test @@ -60,6 +63,7 @@ TEST_GEN_PROGS_x86_64 += x86_64/vmx_close_while_nested_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_dirty_log_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_set_nested_state_test TEST_GEN_PROGS_x86_64 += x86_64/vmx_tsc_adjust_test +TEST_GEN_PROGS_x86_64 += x86_64/vmx_nested_tsc_scaling_test TEST_GEN_PROGS_x86_64 += x86_64/xapic_ipi_test TEST_GEN_PROGS_x86_64 += x86_64/xss_msr_test TEST_GEN_PROGS_x86_64 += x86_64/debug_regs @@ -77,9 +81,10 @@ TEST_GEN_PROGS_x86_64 += memslot_modification_stress_test TEST_GEN_PROGS_x86_64 += memslot_perf_test TEST_GEN_PROGS_x86_64 += set_memory_region_test TEST_GEN_PROGS_x86_64 += steal_time +TEST_GEN_PROGS_x86_64 += kvm_binary_stats_test +TEST_GEN_PROGS_aarch64 += aarch64/debug-exceptions TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list -TEST_GEN_PROGS_aarch64 += aarch64/get-reg-list-sve TEST_GEN_PROGS_aarch64 += aarch64/vgic_init TEST_GEN_PROGS_aarch64 += demand_paging_test TEST_GEN_PROGS_aarch64 += dirty_log_test @@ -88,6 +93,7 @@ TEST_GEN_PROGS_aarch64 += kvm_create_max_vcpus TEST_GEN_PROGS_aarch64 += kvm_page_table_test TEST_GEN_PROGS_aarch64 += set_memory_region_test TEST_GEN_PROGS_aarch64 += steal_time +TEST_GEN_PROGS_aarch64 += kvm_binary_stats_test TEST_GEN_PROGS_s390x = s390x/memop TEST_GEN_PROGS_s390x += s390x/resets @@ -97,6 +103,7 @@ TEST_GEN_PROGS_s390x += dirty_log_test TEST_GEN_PROGS_s390x += kvm_create_max_vcpus TEST_GEN_PROGS_s390x += kvm_page_table_test TEST_GEN_PROGS_s390x += set_memory_region_test +TEST_GEN_PROGS_s390x += kvm_binary_stats_test TEST_GEN_PROGS += $(TEST_GEN_PROGS_$(UNAME_M)) LIBKVM += $(LIBKVM_$(UNAME_M)) diff --git a/tools/testing/selftests/kvm/aarch64/debug-exceptions.c b/tools/testing/selftests/kvm/aarch64/debug-exceptions.c new file mode 100644 index 000000000000..e5e6c92b60da --- /dev/null +++ b/tools/testing/selftests/kvm/aarch64/debug-exceptions.c @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +#define VCPU_ID 0 + +#define MDSCR_KDE (1 << 13) +#define MDSCR_MDE (1 << 15) +#define MDSCR_SS (1 << 0) + +#define DBGBCR_LEN8 (0xff << 5) +#define DBGBCR_EXEC (0x0 << 3) +#define DBGBCR_EL1 (0x1 << 1) +#define DBGBCR_E (0x1 << 0) + +#define DBGWCR_LEN8 (0xff << 5) +#define DBGWCR_RD (0x1 << 3) +#define DBGWCR_WR (0x2 << 3) +#define DBGWCR_EL1 (0x1 << 1) +#define DBGWCR_E (0x1 << 0) + +#define SPSR_D (1 << 9) +#define SPSR_SS (1 << 21) + +extern unsigned char sw_bp, hw_bp, bp_svc, bp_brk, hw_wp, ss_start; +static volatile uint64_t sw_bp_addr, hw_bp_addr; +static volatile uint64_t wp_addr, wp_data_addr; +static volatile uint64_t svc_addr; +static volatile uint64_t ss_addr[4], ss_idx; +#define PC(v) ((uint64_t)&(v)) + +static void reset_debug_state(void) +{ + asm volatile("msr daifset, #8"); + + write_sysreg(osdlr_el1, 0); + write_sysreg(oslar_el1, 0); + isb(); + + write_sysreg(mdscr_el1, 0); + /* This test only uses the first bp and wp slot. */ + write_sysreg(dbgbvr0_el1, 0); + write_sysreg(dbgbcr0_el1, 0); + write_sysreg(dbgwcr0_el1, 0); + write_sysreg(dbgwvr0_el1, 0); + isb(); +} + +static void install_wp(uint64_t addr) +{ + uint32_t wcr; + uint32_t mdscr; + + wcr = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | DBGWCR_EL1 | DBGWCR_E; + write_sysreg(dbgwcr0_el1, wcr); + write_sysreg(dbgwvr0_el1, addr); + isb(); + + asm volatile("msr daifclr, #8"); + + mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_MDE; + write_sysreg(mdscr_el1, mdscr); + isb(); +} + +static void install_hw_bp(uint64_t addr) +{ + uint32_t bcr; + uint32_t mdscr; + + bcr = DBGBCR_LEN8 | DBGBCR_EXEC | DBGBCR_EL1 | DBGBCR_E; + write_sysreg(dbgbcr0_el1, bcr); + write_sysreg(dbgbvr0_el1, addr); + isb(); + + asm volatile("msr daifclr, #8"); + + mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_MDE; + write_sysreg(mdscr_el1, mdscr); + isb(); +} + +static void install_ss(void) +{ + uint32_t mdscr; + + asm volatile("msr daifclr, #8"); + + mdscr = read_sysreg(mdscr_el1) | MDSCR_KDE | MDSCR_SS; + write_sysreg(mdscr_el1, mdscr); + isb(); +} + +static volatile char write_data; + +static void guest_code(void) +{ + GUEST_SYNC(0); + + /* Software-breakpoint */ + asm volatile("sw_bp: brk #0"); + GUEST_ASSERT_EQ(sw_bp_addr, PC(sw_bp)); + + GUEST_SYNC(1); + + /* Hardware-breakpoint */ + reset_debug_state(); + install_hw_bp(PC(hw_bp)); + asm volatile("hw_bp: nop"); + GUEST_ASSERT_EQ(hw_bp_addr, PC(hw_bp)); + + GUEST_SYNC(2); + + /* Hardware-breakpoint + svc */ + reset_debug_state(); + install_hw_bp(PC(bp_svc)); + asm volatile("bp_svc: svc #0"); + GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_svc)); + GUEST_ASSERT_EQ(svc_addr, PC(bp_svc) + 4); + + GUEST_SYNC(3); + + /* Hardware-breakpoint + software-breakpoint */ + reset_debug_state(); + install_hw_bp(PC(bp_brk)); + asm volatile("bp_brk: brk #0"); + GUEST_ASSERT_EQ(sw_bp_addr, PC(bp_brk)); + GUEST_ASSERT_EQ(hw_bp_addr, PC(bp_brk)); + + GUEST_SYNC(4); + + /* Watchpoint */ + reset_debug_state(); + install_wp(PC(write_data)); + write_data = 'x'; + GUEST_ASSERT_EQ(write_data, 'x'); + GUEST_ASSERT_EQ(wp_data_addr, PC(write_data)); + + GUEST_SYNC(5); + + /* Single-step */ + reset_debug_state(); + install_ss(); + ss_idx = 0; + asm volatile("ss_start:\n" + "mrs x0, esr_el1\n" + "add x0, x0, #1\n" + "msr daifset, #8\n" + : : : "x0"); + GUEST_ASSERT_EQ(ss_addr[0], PC(ss_start)); + GUEST_ASSERT_EQ(ss_addr[1], PC(ss_start) + 4); + GUEST_ASSERT_EQ(ss_addr[2], PC(ss_start) + 8); + + GUEST_DONE(); +} + +static void guest_sw_bp_handler(struct ex_regs *regs) +{ + sw_bp_addr = regs->pc; + regs->pc += 4; +} + +static void guest_hw_bp_handler(struct ex_regs *regs) +{ + hw_bp_addr = regs->pc; + regs->pstate |= SPSR_D; +} + +static void guest_wp_handler(struct ex_regs *regs) +{ + wp_data_addr = read_sysreg(far_el1); + wp_addr = regs->pc; + regs->pstate |= SPSR_D; +} + +static void guest_ss_handler(struct ex_regs *regs) +{ + GUEST_ASSERT_1(ss_idx < 4, ss_idx); + ss_addr[ss_idx++] = regs->pc; + regs->pstate |= SPSR_SS; +} + +static void guest_svc_handler(struct ex_regs *regs) +{ + svc_addr = regs->pc; +} + +static int debug_version(struct kvm_vm *vm) +{ + uint64_t id_aa64dfr0; + + get_reg(vm, VCPU_ID, ARM64_SYS_REG(ID_AA64DFR0_EL1), &id_aa64dfr0); + return id_aa64dfr0 & 0xf; +} + +int main(int argc, char *argv[]) +{ + struct kvm_vm *vm; + struct ucall uc; + int stage; + + vm = vm_create_default(VCPU_ID, 0, guest_code); + ucall_init(vm, NULL); + + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vm, VCPU_ID); + + if (debug_version(vm) < 6) { + print_skip("Armv8 debug architecture not supported."); + kvm_vm_free(vm); + exit(KSFT_SKIP); + } + + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, + ESR_EC_BRK_INS, guest_sw_bp_handler); + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, + ESR_EC_HW_BP_CURRENT, guest_hw_bp_handler); + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, + ESR_EC_WP_CURRENT, guest_wp_handler); + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, + ESR_EC_SSTEP_CURRENT, guest_ss_handler); + vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, + ESR_EC_SVC64, guest_svc_handler); + + for (stage = 0; stage < 7; stage++) { + vcpu_run(vm, VCPU_ID); + + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_SYNC: + TEST_ASSERT(uc.args[1] == stage, + "Stage %d: Unexpected sync ucall, got %lx", + stage, (ulong)uc.args[1]); + break; + case UCALL_ABORT: + TEST_FAIL("%s at %s:%ld\n\tvalues: %#lx, %#lx", + (const char *)uc.args[0], + __FILE__, uc.args[1], uc.args[2], uc.args[3]); + break; + case UCALL_DONE: + goto done; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } + +done: + kvm_vm_free(vm); + return 0; +} diff --git a/tools/testing/selftests/kvm/aarch64/get-reg-list-sve.c b/tools/testing/selftests/kvm/aarch64/get-reg-list-sve.c deleted file mode 100644 index efba76682b4b..000000000000 --- a/tools/testing/selftests/kvm/aarch64/get-reg-list-sve.c +++ /dev/null @@ -1,3 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -#define REG_LIST_SVE -#include "get-reg-list.c" diff --git a/tools/testing/selftests/kvm/aarch64/get-reg-list.c b/tools/testing/selftests/kvm/aarch64/get-reg-list.c index 486932164cf2..a16c8f05366c 100644 --- a/tools/testing/selftests/kvm/aarch64/get-reg-list.c +++ b/tools/testing/selftests/kvm/aarch64/get-reg-list.c @@ -27,17 +27,37 @@ #include #include #include +#include +#include +#include #include "kvm_util.h" #include "test_util.h" #include "processor.h" -#ifdef REG_LIST_SVE -#define reg_list_sve() (true) -#else -#define reg_list_sve() (false) -#endif +static struct kvm_reg_list *reg_list; +static __u64 *blessed_reg, blessed_n; -#define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_COPROC_MASK) +struct reg_sublist { + const char *name; + long capability; + int feature; + bool finalize; + __u64 *regs; + __u64 regs_n; + __u64 *rejects_set; + __u64 rejects_set_n; +}; + +struct vcpu_config { + char *name; + struct reg_sublist sublists[]; +}; + +static struct vcpu_config *vcpu_configs[]; +static int vcpu_configs_n; + +#define for_each_sublist(c, s) \ + for ((s) = &(c)->sublists[0]; (s)->regs; ++(s)) #define for_each_reg(i) \ for ((i) = 0; (i) < reg_list->n; ++(i)) @@ -54,12 +74,41 @@ for_each_reg_filtered(i) \ if (!find_reg(blessed_reg, blessed_n, reg_list->reg[i])) +static const char *config_name(struct vcpu_config *c) +{ + struct reg_sublist *s; + int len = 0; -static struct kvm_reg_list *reg_list; + if (c->name) + return c->name; -static __u64 base_regs[], vregs[], sve_regs[], rejects_set[]; -static __u64 base_regs_n, vregs_n, sve_regs_n, rejects_set_n; -static __u64 *blessed_reg, blessed_n; + for_each_sublist(c, s) + len += strlen(s->name) + 1; + + c->name = malloc(len); + + len = 0; + for_each_sublist(c, s) { + if (!strcmp(s->name, "base")) + continue; + strcat(c->name + len, s->name); + len += strlen(s->name) + 1; + c->name[len - 1] = '+'; + } + c->name[len - 1] = '\0'; + + return c->name; +} + +static bool has_cap(struct vcpu_config *c, long capability) +{ + struct reg_sublist *s; + + for_each_sublist(c, s) + if (s->capability == capability) + return true; + return false; +} static bool filter_reg(__u64 reg) { @@ -96,11 +145,13 @@ static const char *str_with_index(const char *template, __u64 index) return (const char *)str; } +#define REG_MASK (KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK | KVM_REG_ARM_COPROC_MASK) + #define CORE_REGS_XX_NR_WORDS 2 #define CORE_SPSR_XX_NR_WORDS 2 #define CORE_FPREGS_XX_NR_WORDS 4 -static const char *core_id_to_str(__u64 id) +static const char *core_id_to_str(struct vcpu_config *c, __u64 id) { __u64 core_off = id & ~REG_MASK, idx; @@ -111,7 +162,7 @@ static const char *core_id_to_str(__u64 id) case KVM_REG_ARM_CORE_REG(regs.regs[0]) ... KVM_REG_ARM_CORE_REG(regs.regs[30]): idx = (core_off - KVM_REG_ARM_CORE_REG(regs.regs[0])) / CORE_REGS_XX_NR_WORDS; - TEST_ASSERT(idx < 31, "Unexpected regs.regs index: %lld", idx); + TEST_ASSERT(idx < 31, "%s: Unexpected regs.regs index: %lld", config_name(c), idx); return str_with_index("KVM_REG_ARM_CORE_REG(regs.regs[##])", idx); case KVM_REG_ARM_CORE_REG(regs.sp): return "KVM_REG_ARM_CORE_REG(regs.sp)"; @@ -126,12 +177,12 @@ static const char *core_id_to_str(__u64 id) case KVM_REG_ARM_CORE_REG(spsr[0]) ... KVM_REG_ARM_CORE_REG(spsr[KVM_NR_SPSR - 1]): idx = (core_off - KVM_REG_ARM_CORE_REG(spsr[0])) / CORE_SPSR_XX_NR_WORDS; - TEST_ASSERT(idx < KVM_NR_SPSR, "Unexpected spsr index: %lld", idx); + TEST_ASSERT(idx < KVM_NR_SPSR, "%s: Unexpected spsr index: %lld", config_name(c), idx); return str_with_index("KVM_REG_ARM_CORE_REG(spsr[##])", idx); case KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]) ... KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]): idx = (core_off - KVM_REG_ARM_CORE_REG(fp_regs.vregs[0])) / CORE_FPREGS_XX_NR_WORDS; - TEST_ASSERT(idx < 32, "Unexpected fp_regs.vregs index: %lld", idx); + TEST_ASSERT(idx < 32, "%s: Unexpected fp_regs.vregs index: %lld", config_name(c), idx); return str_with_index("KVM_REG_ARM_CORE_REG(fp_regs.vregs[##])", idx); case KVM_REG_ARM_CORE_REG(fp_regs.fpsr): return "KVM_REG_ARM_CORE_REG(fp_regs.fpsr)"; @@ -139,11 +190,11 @@ static const char *core_id_to_str(__u64 id) return "KVM_REG_ARM_CORE_REG(fp_regs.fpcr)"; } - TEST_FAIL("Unknown core reg id: 0x%llx", id); + TEST_FAIL("%s: Unknown core reg id: 0x%llx", config_name(c), id); return NULL; } -static const char *sve_id_to_str(__u64 id) +static const char *sve_id_to_str(struct vcpu_config *c, __u64 id) { __u64 sve_off, n, i; @@ -153,37 +204,37 @@ static const char *sve_id_to_str(__u64 id) sve_off = id & ~(REG_MASK | ((1ULL << 5) - 1)); i = id & (KVM_ARM64_SVE_MAX_SLICES - 1); - TEST_ASSERT(i == 0, "Currently we don't expect slice > 0, reg id 0x%llx", id); + TEST_ASSERT(i == 0, "%s: Currently we don't expect slice > 0, reg id 0x%llx", config_name(c), id); switch (sve_off) { case KVM_REG_ARM64_SVE_ZREG_BASE ... KVM_REG_ARM64_SVE_ZREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_ZREGS - 1: n = (id >> 5) & (KVM_ARM64_SVE_NUM_ZREGS - 1); TEST_ASSERT(id == KVM_REG_ARM64_SVE_ZREG(n, 0), - "Unexpected bits set in SVE ZREG id: 0x%llx", id); + "%s: Unexpected bits set in SVE ZREG id: 0x%llx", config_name(c), id); return str_with_index("KVM_REG_ARM64_SVE_ZREG(##, 0)", n); case KVM_REG_ARM64_SVE_PREG_BASE ... KVM_REG_ARM64_SVE_PREG_BASE + (1ULL << 5) * KVM_ARM64_SVE_NUM_PREGS - 1: n = (id >> 5) & (KVM_ARM64_SVE_NUM_PREGS - 1); TEST_ASSERT(id == KVM_REG_ARM64_SVE_PREG(n, 0), - "Unexpected bits set in SVE PREG id: 0x%llx", id); + "%s: Unexpected bits set in SVE PREG id: 0x%llx", config_name(c), id); return str_with_index("KVM_REG_ARM64_SVE_PREG(##, 0)", n); case KVM_REG_ARM64_SVE_FFR_BASE: TEST_ASSERT(id == KVM_REG_ARM64_SVE_FFR(0), - "Unexpected bits set in SVE FFR id: 0x%llx", id); + "%s: Unexpected bits set in SVE FFR id: 0x%llx", config_name(c), id); return "KVM_REG_ARM64_SVE_FFR(0)"; } return NULL; } -static void print_reg(__u64 id) +static void print_reg(struct vcpu_config *c, __u64 id) { unsigned op0, op1, crn, crm, op2; const char *reg_size = NULL; TEST_ASSERT((id & KVM_REG_ARCH_MASK) == KVM_REG_ARM64, - "KVM_REG_ARM64 missing in reg id: 0x%llx", id); + "%s: KVM_REG_ARM64 missing in reg id: 0x%llx", config_name(c), id); switch (id & KVM_REG_SIZE_MASK) { case KVM_REG_SIZE_U8: @@ -214,17 +265,17 @@ static void print_reg(__u64 id) reg_size = "KVM_REG_SIZE_U2048"; break; default: - TEST_FAIL("Unexpected reg size: 0x%llx in reg id: 0x%llx", - (id & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT, id); + TEST_FAIL("%s: Unexpected reg size: 0x%llx in reg id: 0x%llx", + config_name(c), (id & KVM_REG_SIZE_MASK) >> KVM_REG_SIZE_SHIFT, id); } switch (id & KVM_REG_ARM_COPROC_MASK) { case KVM_REG_ARM_CORE: - printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_CORE | %s,\n", reg_size, core_id_to_str(id)); + printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_CORE | %s,\n", reg_size, core_id_to_str(c, id)); break; case KVM_REG_ARM_DEMUX: TEST_ASSERT(!(id & ~(REG_MASK | KVM_REG_ARM_DEMUX_ID_MASK | KVM_REG_ARM_DEMUX_VAL_MASK)), - "Unexpected bits set in DEMUX reg id: 0x%llx", id); + "%s: Unexpected bits set in DEMUX reg id: 0x%llx", config_name(c), id); printf("\tKVM_REG_ARM64 | %s | KVM_REG_ARM_DEMUX | KVM_REG_ARM_DEMUX_ID_CCSIDR | %lld,\n", reg_size, id & KVM_REG_ARM_DEMUX_VAL_MASK); break; @@ -235,23 +286,23 @@ static void print_reg(__u64 id) crm = (id & KVM_REG_ARM64_SYSREG_CRM_MASK) >> KVM_REG_ARM64_SYSREG_CRM_SHIFT; op2 = (id & KVM_REG_ARM64_SYSREG_OP2_MASK) >> KVM_REG_ARM64_SYSREG_OP2_SHIFT; TEST_ASSERT(id == ARM64_SYS_REG(op0, op1, crn, crm, op2), - "Unexpected bits set in SYSREG reg id: 0x%llx", id); + "%s: Unexpected bits set in SYSREG reg id: 0x%llx", config_name(c), id); printf("\tARM64_SYS_REG(%d, %d, %d, %d, %d),\n", op0, op1, crn, crm, op2); break; case KVM_REG_ARM_FW: TEST_ASSERT(id == KVM_REG_ARM_FW_REG(id & 0xffff), - "Unexpected bits set in FW reg id: 0x%llx", id); + "%s: Unexpected bits set in FW reg id: 0x%llx", config_name(c), id); printf("\tKVM_REG_ARM_FW_REG(%lld),\n", id & 0xffff); break; case KVM_REG_ARM64_SVE: - if (reg_list_sve()) - printf("\t%s,\n", sve_id_to_str(id)); + if (has_cap(c, KVM_CAP_ARM_SVE)) + printf("\t%s,\n", sve_id_to_str(c, id)); else - TEST_FAIL("KVM_REG_ARM64_SVE is an unexpected coproc type in reg id: 0x%llx", id); + TEST_FAIL("%s: KVM_REG_ARM64_SVE is an unexpected coproc type in reg id: 0x%llx", config_name(c), id); break; default: - TEST_FAIL("Unexpected coproc type: 0x%llx in reg id: 0x%llx", - (id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT, id); + TEST_FAIL("%s: Unexpected coproc type: 0x%llx in reg id: 0x%llx", + config_name(c), (id & KVM_REG_ARM_COPROC_MASK) >> KVM_REG_ARM_COPROC_SHIFT, id); } } @@ -312,56 +363,58 @@ static void core_reg_fixup(void) reg_list = tmp; } -static void prepare_vcpu_init(struct kvm_vcpu_init *init) +static void prepare_vcpu_init(struct vcpu_config *c, struct kvm_vcpu_init *init) { - if (reg_list_sve()) - init->features[0] |= 1 << KVM_ARM_VCPU_SVE; + struct reg_sublist *s; + + for_each_sublist(c, s) + if (s->capability) + init->features[s->feature / 32] |= 1 << (s->feature % 32); } -static void finalize_vcpu(struct kvm_vm *vm, uint32_t vcpuid) +static void finalize_vcpu(struct kvm_vm *vm, uint32_t vcpuid, struct vcpu_config *c) { + struct reg_sublist *s; int feature; - if (reg_list_sve()) { - feature = KVM_ARM_VCPU_SVE; - vcpu_ioctl(vm, vcpuid, KVM_ARM_VCPU_FINALIZE, &feature); + for_each_sublist(c, s) { + if (s->finalize) { + feature = s->feature; + vcpu_ioctl(vm, vcpuid, KVM_ARM_VCPU_FINALIZE, &feature); + } } } -static void check_supported(void) +static void check_supported(struct vcpu_config *c) { - if (reg_list_sve() && !kvm_check_cap(KVM_CAP_ARM_SVE)) { - fprintf(stderr, "SVE not available, skipping tests\n"); - exit(KSFT_SKIP); + struct reg_sublist *s; + + for_each_sublist(c, s) { + if (s->capability && !kvm_check_cap(s->capability)) { + fprintf(stderr, "%s: %s not available, skipping tests\n", config_name(c), s->name); + exit(KSFT_SKIP); + } } } -int main(int ac, char **av) +static bool print_list; +static bool print_filtered; +static bool fixup_core_regs; + +static void run_test(struct vcpu_config *c) { struct kvm_vcpu_init init = { .target = -1, }; - int new_regs = 0, missing_regs = 0, i; + int new_regs = 0, missing_regs = 0, i, n; int failed_get = 0, failed_set = 0, failed_reject = 0; - bool print_list = false, print_filtered = false, fixup_core_regs = false; struct kvm_vm *vm; - __u64 *vec_regs; + struct reg_sublist *s; - check_supported(); - - for (i = 1; i < ac; ++i) { - if (strcmp(av[i], "--core-reg-fixup") == 0) - fixup_core_regs = true; - else if (strcmp(av[i], "--list") == 0) - print_list = true; - else if (strcmp(av[i], "--list-filtered") == 0) - print_filtered = true; - else - TEST_FAIL("Unknown option: %s\n", av[i]); - } + check_supported(c); vm = vm_create(VM_MODE_DEFAULT, DEFAULT_GUEST_PHY_PAGES, O_RDWR); - prepare_vcpu_init(&init); + prepare_vcpu_init(c, &init); aarch64_vcpu_add_default(vm, 0, &init, NULL); - finalize_vcpu(vm, 0); + finalize_vcpu(vm, 0, c); reg_list = vcpu_get_reg_list(vm, 0); @@ -374,10 +427,10 @@ int main(int ac, char **av) __u64 id = reg_list->reg[i]; if ((print_list && !filter_reg(id)) || (print_filtered && filter_reg(id))) - print_reg(id); + print_reg(c, id); } putchar('\n'); - return 0; + return; } /* @@ -396,50 +449,52 @@ int main(int ac, char **av) .id = reg_list->reg[i], .addr = (__u64)&addr, }; + bool reject_reg = false; int ret; ret = _vcpu_ioctl(vm, 0, KVM_GET_ONE_REG, ®); if (ret) { - puts("Failed to get "); - print_reg(reg.id); + printf("%s: Failed to get ", config_name(c)); + print_reg(c, reg.id); putchar('\n'); ++failed_get; } /* rejects_set registers are rejected after KVM_ARM_VCPU_FINALIZE */ - if (find_reg(rejects_set, rejects_set_n, reg.id)) { - ret = _vcpu_ioctl(vm, 0, KVM_SET_ONE_REG, ®); - if (ret != -1 || errno != EPERM) { - printf("Failed to reject (ret=%d, errno=%d) ", ret, errno); - print_reg(reg.id); - putchar('\n'); - ++failed_reject; + for_each_sublist(c, s) { + if (s->rejects_set && find_reg(s->rejects_set, s->rejects_set_n, reg.id)) { + reject_reg = true; + ret = _vcpu_ioctl(vm, 0, KVM_SET_ONE_REG, ®); + if (ret != -1 || errno != EPERM) { + printf("%s: Failed to reject (ret=%d, errno=%d) ", config_name(c), ret, errno); + print_reg(c, reg.id); + putchar('\n'); + ++failed_reject; + } + break; } - continue; } - ret = _vcpu_ioctl(vm, 0, KVM_SET_ONE_REG, ®); - if (ret) { - puts("Failed to set "); - print_reg(reg.id); - putchar('\n'); - ++failed_set; + if (!reject_reg) { + ret = _vcpu_ioctl(vm, 0, KVM_SET_ONE_REG, ®); + if (ret) { + printf("%s: Failed to set ", config_name(c)); + print_reg(c, reg.id); + putchar('\n'); + ++failed_set; + } } } - if (reg_list_sve()) { - blessed_n = base_regs_n + sve_regs_n; - vec_regs = sve_regs; - } else { - blessed_n = base_regs_n + vregs_n; - vec_regs = vregs; - } - + for_each_sublist(c, s) + blessed_n += s->regs_n; blessed_reg = calloc(blessed_n, sizeof(__u64)); - for (i = 0; i < base_regs_n; ++i) - blessed_reg[i] = base_regs[i]; - for (i = 0; i < blessed_n - base_regs_n; ++i) - blessed_reg[base_regs_n + i] = vec_regs[i]; + + n = 0; + for_each_sublist(c, s) { + for (i = 0; i < s->regs_n; ++i) + blessed_reg[n++] = s->regs[i]; + } for_each_new_reg(i) ++new_regs; @@ -448,40 +503,141 @@ int main(int ac, char **av) ++missing_regs; if (new_regs || missing_regs) { - printf("Number blessed registers: %5lld\n", blessed_n); - printf("Number registers: %5lld\n", reg_list->n); + printf("%s: Number blessed registers: %5lld\n", config_name(c), blessed_n); + printf("%s: Number registers: %5lld\n", config_name(c), reg_list->n); } if (new_regs) { - printf("\nThere are %d new registers.\n" + printf("\n%s: There are %d new registers.\n" "Consider adding them to the blessed reg " - "list with the following lines:\n\n", new_regs); + "list with the following lines:\n\n", config_name(c), new_regs); for_each_new_reg(i) - print_reg(reg_list->reg[i]); + print_reg(c, reg_list->reg[i]); putchar('\n'); } if (missing_regs) { - printf("\nThere are %d missing registers.\n" - "The following lines are missing registers:\n\n", missing_regs); + printf("\n%s: There are %d missing registers.\n" + "The following lines are missing registers:\n\n", config_name(c), missing_regs); for_each_missing_reg(i) - print_reg(blessed_reg[i]); + print_reg(c, blessed_reg[i]); putchar('\n'); } TEST_ASSERT(!missing_regs && !failed_get && !failed_set && !failed_reject, - "There are %d missing registers; " + "%s: There are %d missing registers; " "%d registers failed get; %d registers failed set; %d registers failed reject", - missing_regs, failed_get, failed_set, failed_reject); + config_name(c), missing_regs, failed_get, failed_set, failed_reject); - return 0; + pr_info("%s: PASS\n", config_name(c)); + blessed_n = 0; + free(blessed_reg); + free(reg_list); + kvm_vm_free(vm); +} + +static void help(void) +{ + struct vcpu_config *c; + int i; + + printf( + "\n" + "usage: get-reg-list [--config=] [--list] [--list-filtered] [--core-reg-fixup]\n\n" + " --config= Used to select a specific vcpu configuration for the test/listing\n" + " '' may be\n"); + + for (i = 0; i < vcpu_configs_n; ++i) { + c = vcpu_configs[i]; + printf( + " '%s'\n", config_name(c)); + } + + printf( + "\n" + " --list Print the register list rather than test it (requires --config)\n" + " --list-filtered Print registers that would normally be filtered out (requires --config)\n" + " --core-reg-fixup Needed when running on old kernels with broken core reg listings\n" + "\n" + ); +} + +static struct vcpu_config *parse_config(const char *config) +{ + struct vcpu_config *c; + int i; + + if (config[8] != '=') + help(), exit(1); + + for (i = 0; i < vcpu_configs_n; ++i) { + c = vcpu_configs[i]; + if (strcmp(config_name(c), &config[9]) == 0) + break; + } + + if (i == vcpu_configs_n) + help(), exit(1); + + return c; +} + +int main(int ac, char **av) +{ + struct vcpu_config *c, *sel = NULL; + int i, ret = 0; + pid_t pid; + + for (i = 1; i < ac; ++i) { + if (strcmp(av[i], "--core-reg-fixup") == 0) + fixup_core_regs = true; + else if (strncmp(av[i], "--config", 8) == 0) + sel = parse_config(av[i]); + else if (strcmp(av[i], "--list") == 0) + print_list = true; + else if (strcmp(av[i], "--list-filtered") == 0) + print_filtered = true; + else if (strcmp(av[i], "--help") == 0 || strcmp(av[1], "-h") == 0) + help(), exit(0); + else + help(), exit(1); + } + + if (print_list || print_filtered) { + /* + * We only want to print the register list of a single config. + */ + if (!sel) + help(), exit(1); + } + + for (i = 0; i < vcpu_configs_n; ++i) { + c = vcpu_configs[i]; + if (sel && c != sel) + continue; + + pid = fork(); + + if (!pid) { + run_test(c); + exit(0); + } else { + int wstatus; + pid_t wpid = wait(&wstatus); + TEST_ASSERT(wpid == pid && WIFEXITED(wstatus), "wait: Unexpected return"); + if (WEXITSTATUS(wstatus) && WEXITSTATUS(wstatus) != KSFT_SKIP) + ret = KSFT_FAIL; + } + } + + return ret; } /* * The current blessed list was primed with the output of kernel version * v4.15 with --core-reg-fixup and then later updated with new registers. * - * The blessed list is up to date with kernel version v5.10-rc5 + * The blessed list is up to date with kernel version v5.13-rc3 */ static __u64 base_regs[] = { KVM_REG_ARM64 | KVM_REG_SIZE_U64 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(regs.regs[0]), @@ -673,8 +829,6 @@ static __u64 base_regs[] = { ARM64_SYS_REG(3, 0, 5, 2, 0), /* ESR_EL1 */ ARM64_SYS_REG(3, 0, 6, 0, 0), /* FAR_EL1 */ ARM64_SYS_REG(3, 0, 7, 4, 0), /* PAR_EL1 */ - ARM64_SYS_REG(3, 0, 9, 14, 1), /* PMINTENSET_EL1 */ - ARM64_SYS_REG(3, 0, 9, 14, 2), /* PMINTENCLR_EL1 */ ARM64_SYS_REG(3, 0, 10, 2, 0), /* MAIR_EL1 */ ARM64_SYS_REG(3, 0, 10, 3, 0), /* AMAIR_EL1 */ ARM64_SYS_REG(3, 0, 12, 0, 0), /* VBAR_EL1 */ @@ -683,6 +837,16 @@ static __u64 base_regs[] = { ARM64_SYS_REG(3, 0, 13, 0, 4), /* TPIDR_EL1 */ ARM64_SYS_REG(3, 0, 14, 1, 0), /* CNTKCTL_EL1 */ ARM64_SYS_REG(3, 2, 0, 0, 0), /* CSSELR_EL1 */ + ARM64_SYS_REG(3, 3, 13, 0, 2), /* TPIDR_EL0 */ + ARM64_SYS_REG(3, 3, 13, 0, 3), /* TPIDRRO_EL0 */ + ARM64_SYS_REG(3, 4, 3, 0, 0), /* DACR32_EL2 */ + ARM64_SYS_REG(3, 4, 5, 0, 1), /* IFSR32_EL2 */ + ARM64_SYS_REG(3, 4, 5, 3, 0), /* FPEXC32_EL2 */ +}; + +static __u64 pmu_regs[] = { + ARM64_SYS_REG(3, 0, 9, 14, 1), /* PMINTENSET_EL1 */ + ARM64_SYS_REG(3, 0, 9, 14, 2), /* PMINTENCLR_EL1 */ ARM64_SYS_REG(3, 3, 9, 12, 0), /* PMCR_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 1), /* PMCNTENSET_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 2), /* PMCNTENCLR_EL0 */ @@ -692,8 +856,6 @@ static __u64 base_regs[] = { ARM64_SYS_REG(3, 3, 9, 13, 0), /* PMCCNTR_EL0 */ ARM64_SYS_REG(3, 3, 9, 14, 0), /* PMUSERENR_EL0 */ ARM64_SYS_REG(3, 3, 9, 14, 3), /* PMOVSSET_EL0 */ - ARM64_SYS_REG(3, 3, 13, 0, 2), /* TPIDR_EL0 */ - ARM64_SYS_REG(3, 3, 13, 0, 3), /* TPIDRRO_EL0 */ ARM64_SYS_REG(3, 3, 14, 8, 0), ARM64_SYS_REG(3, 3, 14, 8, 1), ARM64_SYS_REG(3, 3, 14, 8, 2), @@ -757,11 +919,7 @@ static __u64 base_regs[] = { ARM64_SYS_REG(3, 3, 14, 15, 5), ARM64_SYS_REG(3, 3, 14, 15, 6), ARM64_SYS_REG(3, 3, 14, 15, 7), /* PMCCFILTR_EL0 */ - ARM64_SYS_REG(3, 4, 3, 0, 0), /* DACR32_EL2 */ - ARM64_SYS_REG(3, 4, 5, 0, 1), /* IFSR32_EL2 */ - ARM64_SYS_REG(3, 4, 5, 3, 0), /* FPEXC32_EL2 */ }; -static __u64 base_regs_n = ARRAY_SIZE(base_regs); static __u64 vregs[] = { KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[0]), @@ -797,7 +955,6 @@ static __u64 vregs[] = { KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[30]), KVM_REG_ARM64 | KVM_REG_SIZE_U128 | KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(fp_regs.vregs[31]), }; -static __u64 vregs_n = ARRAY_SIZE(vregs); static __u64 sve_regs[] = { KVM_REG_ARM64_SVE_VLS, @@ -852,11 +1009,57 @@ static __u64 sve_regs[] = { KVM_REG_ARM64_SVE_FFR(0), ARM64_SYS_REG(3, 0, 1, 2, 0), /* ZCR_EL1 */ }; -static __u64 sve_regs_n = ARRAY_SIZE(sve_regs); -static __u64 rejects_set[] = { -#ifdef REG_LIST_SVE +static __u64 sve_rejects_set[] = { KVM_REG_ARM64_SVE_VLS, -#endif }; -static __u64 rejects_set_n = ARRAY_SIZE(rejects_set); + +#define BASE_SUBLIST \ + { "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), } +#define VREGS_SUBLIST \ + { "vregs", .regs = vregs, .regs_n = ARRAY_SIZE(vregs), } +#define PMU_SUBLIST \ + { "pmu", .regs = pmu_regs, .regs_n = ARRAY_SIZE(pmu_regs), } +#define SVE_SUBLIST \ + { "sve", .capability = KVM_CAP_ARM_SVE, .feature = KVM_ARM_VCPU_SVE, .finalize = true, \ + .regs = sve_regs, .regs_n = ARRAY_SIZE(sve_regs), \ + .rejects_set = sve_rejects_set, .rejects_set_n = ARRAY_SIZE(sve_rejects_set), } + +static struct vcpu_config vregs_config = { + .sublists = { + BASE_SUBLIST, + VREGS_SUBLIST, + {0}, + }, +}; +static struct vcpu_config vregs_pmu_config = { + .sublists = { + BASE_SUBLIST, + VREGS_SUBLIST, + PMU_SUBLIST, + {0}, + }, +}; +static struct vcpu_config sve_config = { + .sublists = { + BASE_SUBLIST, + SVE_SUBLIST, + {0}, + }, +}; +static struct vcpu_config sve_pmu_config = { + .sublists = { + BASE_SUBLIST, + SVE_SUBLIST, + PMU_SUBLIST, + {0}, + }, +}; + +static struct vcpu_config *vcpu_configs[] = { + &vregs_config, + &vregs_pmu_config, + &sve_config, + &sve_pmu_config, +}; +static int vcpu_configs_n = ARRAY_SIZE(vcpu_configs); diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 81edbd23d371..5fe0140e407e 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -16,7 +16,6 @@ #include #include #include -#include #include #include "kvm_util.h" @@ -681,7 +680,7 @@ static struct kvm_vm *create_vm(enum vm_guest_mode mode, uint32_t vcpuid, pr_info("Testing guest mode: %s\n", vm_guest_mode_string(mode)); vm = vm_create(mode, DEFAULT_GUEST_PHY_PAGES + extra_pg_pages, O_RDWR); - kvm_vm_elf_load(vm, program_invocation_name, 0, 0); + kvm_vm_elf_load(vm, program_invocation_name); #ifdef __x86_64__ vm_create_irqchip(vm); #endif @@ -761,7 +760,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) KVM_MEM_LOG_DIRTY_PAGES); /* Do mapping for the dirty track memory slot */ - virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages, 0); + virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages); /* Cache the HVA pointer of the region */ host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem); diff --git a/tools/testing/selftests/kvm/hardware_disable_test.c b/tools/testing/selftests/kvm/hardware_disable_test.c index 4b8db3bce610..b21c69a56daa 100644 --- a/tools/testing/selftests/kvm/hardware_disable_test.c +++ b/tools/testing/selftests/kvm/hardware_disable_test.c @@ -105,7 +105,7 @@ static void run_test(uint32_t run) CPU_SET(i, &cpu_set); vm = vm_create(VM_MODE_DEFAULT, DEFAULT_GUEST_PHY_PAGES, O_RDWR); - kvm_vm_elf_load(vm, program_invocation_name, 0, 0); + kvm_vm_elf_load(vm, program_invocation_name); vm_create_irqchip(vm); pr_debug("%s: [%d] start vcpus\n", __func__, run); diff --git a/tools/testing/selftests/kvm/include/aarch64/processor.h b/tools/testing/selftests/kvm/include/aarch64/processor.h index b7fa0c8551db..27dc5c2e56b9 100644 --- a/tools/testing/selftests/kvm/include/aarch64/processor.h +++ b/tools/testing/selftests/kvm/include/aarch64/processor.h @@ -8,16 +8,20 @@ #define SELFTEST_KVM_PROCESSOR_H #include "kvm_util.h" +#include #define ARM64_CORE_REG(x) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ KVM_REG_ARM_CORE | KVM_REG_ARM_CORE_REG(x)) -#define CPACR_EL1 3, 0, 1, 0, 2 -#define TCR_EL1 3, 0, 2, 0, 2 -#define MAIR_EL1 3, 0, 10, 2, 0 -#define TTBR0_EL1 3, 0, 2, 0, 0 -#define SCTLR_EL1 3, 0, 1, 0, 0 +#define CPACR_EL1 3, 0, 1, 0, 2 +#define TCR_EL1 3, 0, 2, 0, 2 +#define MAIR_EL1 3, 0, 10, 2, 0 +#define TTBR0_EL1 3, 0, 2, 0, 0 +#define SCTLR_EL1 3, 0, 1, 0, 0 +#define VBAR_EL1 3, 0, 12, 0, 0 + +#define ID_AA64DFR0_EL1 3, 0, 0, 5, 0 /* * Default MAIR @@ -56,4 +60,73 @@ void aarch64_vcpu_setup(struct kvm_vm *vm, int vcpuid, struct kvm_vcpu_init *ini void aarch64_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_vcpu_init *init, void *guest_code); +struct ex_regs { + u64 regs[31]; + u64 sp; + u64 pc; + u64 pstate; +}; + +#define VECTOR_NUM 16 + +enum { + VECTOR_SYNC_CURRENT_SP0, + VECTOR_IRQ_CURRENT_SP0, + VECTOR_FIQ_CURRENT_SP0, + VECTOR_ERROR_CURRENT_SP0, + + VECTOR_SYNC_CURRENT, + VECTOR_IRQ_CURRENT, + VECTOR_FIQ_CURRENT, + VECTOR_ERROR_CURRENT, + + VECTOR_SYNC_LOWER_64, + VECTOR_IRQ_LOWER_64, + VECTOR_FIQ_LOWER_64, + VECTOR_ERROR_LOWER_64, + + VECTOR_SYNC_LOWER_32, + VECTOR_IRQ_LOWER_32, + VECTOR_FIQ_LOWER_32, + VECTOR_ERROR_LOWER_32, +}; + +#define VECTOR_IS_SYNC(v) ((v) == VECTOR_SYNC_CURRENT_SP0 || \ + (v) == VECTOR_SYNC_CURRENT || \ + (v) == VECTOR_SYNC_LOWER_64 || \ + (v) == VECTOR_SYNC_LOWER_32) + +#define ESR_EC_NUM 64 +#define ESR_EC_SHIFT 26 +#define ESR_EC_MASK (ESR_EC_NUM - 1) + +#define ESR_EC_SVC64 0x15 +#define ESR_EC_HW_BP_CURRENT 0x31 +#define ESR_EC_SSTEP_CURRENT 0x33 +#define ESR_EC_WP_CURRENT 0x35 +#define ESR_EC_BRK_INS 0x3c + +void vm_init_descriptor_tables(struct kvm_vm *vm); +void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid); + +typedef void(*handler_fn)(struct ex_regs *); +void vm_install_exception_handler(struct kvm_vm *vm, + int vector, handler_fn handler); +void vm_install_sync_handler(struct kvm_vm *vm, + int vector, int ec, handler_fn handler); + +#define write_sysreg(reg, val) \ +({ \ + u64 __val = (u64)(val); \ + asm volatile("msr " __stringify(reg) ", %x0" : : "rZ" (__val)); \ +}) + +#define read_sysreg(reg) \ +({ u64 val; \ + asm volatile("mrs %0, "__stringify(reg) : "=r"(val) : : "memory");\ + val; \ +}) + +#define isb() asm volatile("isb" : : : "memory") + #endif /* SELFTEST_KVM_PROCESSOR_H */ diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 35739567189e..615ab254899d 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -30,6 +30,7 @@ typedef uint64_t vm_vaddr_t; /* Virtual Machine (Guest) virtual address */ /* Minimum allocated guest virtual and physical addresses */ #define KVM_UTIL_MIN_VADDR 0x2000 +#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 #define DEFAULT_GUEST_PHY_PAGES 512 #define DEFAULT_GUEST_STACK_VADDR_MIN 0xab6000 @@ -98,8 +99,7 @@ uint32_t kvm_vm_reset_dirty_ring(struct kvm_vm *vm); int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, const vm_vaddr_t gva, size_t len); -void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename, - uint32_t data_memslot, uint32_t pgd_memslot); +void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename); void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent); @@ -141,10 +141,12 @@ void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags); void vm_mem_region_move(struct kvm_vm *vm, uint32_t slot, uint64_t new_gpa); void vm_mem_region_delete(struct kvm_vm *vm, uint32_t slot); void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid); -vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, - uint32_t data_memslot, uint32_t pgd_memslot); +vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min); +vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages); +vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm); + void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - unsigned int npages, uint32_t pgd_memslot); + unsigned int npages); void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa); void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva); vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva); @@ -237,7 +239,7 @@ int kvm_device_access(int dev_fd, uint32_t group, uint64_t attr, const char *exit_reason_str(unsigned int exit_reason); -void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot); +void virt_pgd_alloc(struct kvm_vm *vm); /* * VM Virtual Page Map @@ -255,13 +257,13 @@ void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot); * Within @vm, creates a virtual translation for the page starting * at @vaddr to the page starting at @paddr. */ -void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - uint32_t memslot); +void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr); vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, uint32_t memslot); vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num, vm_paddr_t paddr_min, uint32_t memslot); +vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm); /* * Create a VM with reasonable defaults @@ -351,6 +353,7 @@ enum { UCALL_SYNC, UCALL_ABORT, UCALL_DONE, + UCALL_UNHANDLED, }; #define UCALL_MAX_ARGS 6 @@ -369,26 +372,31 @@ uint64_t get_ucall(struct kvm_vm *vm, uint32_t vcpu_id, struct ucall *uc); ucall(UCALL_SYNC, 6, "hello", stage, arg1, arg2, arg3, arg4) #define GUEST_SYNC(stage) ucall(UCALL_SYNC, 2, "hello", stage) #define GUEST_DONE() ucall(UCALL_DONE, 0) -#define __GUEST_ASSERT(_condition, _nargs, _args...) do { \ - if (!(_condition)) \ - ucall(UCALL_ABORT, 2 + _nargs, \ - "Failed guest assert: " \ - #_condition, __LINE__, _args); \ +#define __GUEST_ASSERT(_condition, _condstr, _nargs, _args...) do { \ + if (!(_condition)) \ + ucall(UCALL_ABORT, 2 + _nargs, \ + "Failed guest assert: " \ + _condstr, __LINE__, _args); \ } while (0) #define GUEST_ASSERT(_condition) \ - __GUEST_ASSERT((_condition), 0, 0) + __GUEST_ASSERT(_condition, #_condition, 0, 0) #define GUEST_ASSERT_1(_condition, arg1) \ - __GUEST_ASSERT((_condition), 1, (arg1)) + __GUEST_ASSERT(_condition, #_condition, 1, (arg1)) #define GUEST_ASSERT_2(_condition, arg1, arg2) \ - __GUEST_ASSERT((_condition), 2, (arg1), (arg2)) + __GUEST_ASSERT(_condition, #_condition, 2, (arg1), (arg2)) #define GUEST_ASSERT_3(_condition, arg1, arg2, arg3) \ - __GUEST_ASSERT((_condition), 3, (arg1), (arg2), (arg3)) + __GUEST_ASSERT(_condition, #_condition, 3, (arg1), (arg2), (arg3)) #define GUEST_ASSERT_4(_condition, arg1, arg2, arg3, arg4) \ - __GUEST_ASSERT((_condition), 4, (arg1), (arg2), (arg3), (arg4)) + __GUEST_ASSERT(_condition, #_condition, 4, (arg1), (arg2), (arg3), (arg4)) + +#define GUEST_ASSERT_EQ(a, b) __GUEST_ASSERT((a) == (b), #a " == " #b, 2, a, b) + +int vm_get_stats_fd(struct kvm_vm *vm); +int vcpu_get_stats_fd(struct kvm_vm *vm, uint32_t vcpuid); #endif /* SELFTEST_KVM_UTIL_H */ diff --git a/tools/testing/selftests/kvm/include/x86_64/apic.h b/tools/testing/selftests/kvm/include/x86_64/apic.h new file mode 100644 index 000000000000..0be4757f1f20 --- /dev/null +++ b/tools/testing/selftests/kvm/include/x86_64/apic.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * tools/testing/selftests/kvm/include/x86_64/apic.h + * + * Copyright (C) 2021, Google LLC. + */ + +#ifndef SELFTEST_KVM_APIC_H +#define SELFTEST_KVM_APIC_H + +#include + +#include "processor.h" + +#define APIC_DEFAULT_GPA 0xfee00000ULL + +/* APIC base address MSR and fields */ +#define MSR_IA32_APICBASE 0x0000001b +#define MSR_IA32_APICBASE_BSP (1<<8) +#define MSR_IA32_APICBASE_EXTD (1<<10) +#define MSR_IA32_APICBASE_ENABLE (1<<11) +#define MSR_IA32_APICBASE_BASE (0xfffff<<12) +#define GET_APIC_BASE(x) (((x) >> 12) << 12) + +#define APIC_BASE_MSR 0x800 +#define X2APIC_ENABLE (1UL << 10) +#define APIC_ID 0x20 +#define APIC_LVR 0x30 +#define GET_APIC_ID_FIELD(x) (((x) >> 24) & 0xFF) +#define APIC_TASKPRI 0x80 +#define APIC_PROCPRI 0xA0 +#define APIC_EOI 0xB0 +#define APIC_SPIV 0xF0 +#define APIC_SPIV_FOCUS_DISABLED (1 << 9) +#define APIC_SPIV_APIC_ENABLED (1 << 8) +#define APIC_ICR 0x300 +#define APIC_DEST_SELF 0x40000 +#define APIC_DEST_ALLINC 0x80000 +#define APIC_DEST_ALLBUT 0xC0000 +#define APIC_ICR_RR_MASK 0x30000 +#define APIC_ICR_RR_INVALID 0x00000 +#define APIC_ICR_RR_INPROG 0x10000 +#define APIC_ICR_RR_VALID 0x20000 +#define APIC_INT_LEVELTRIG 0x08000 +#define APIC_INT_ASSERT 0x04000 +#define APIC_ICR_BUSY 0x01000 +#define APIC_DEST_LOGICAL 0x00800 +#define APIC_DEST_PHYSICAL 0x00000 +#define APIC_DM_FIXED 0x00000 +#define APIC_DM_FIXED_MASK 0x00700 +#define APIC_DM_LOWEST 0x00100 +#define APIC_DM_SMI 0x00200 +#define APIC_DM_REMRD 0x00300 +#define APIC_DM_NMI 0x00400 +#define APIC_DM_INIT 0x00500 +#define APIC_DM_STARTUP 0x00600 +#define APIC_DM_EXTINT 0x00700 +#define APIC_VECTOR_MASK 0x000FF +#define APIC_ICR2 0x310 +#define SET_APIC_DEST_FIELD(x) ((x) << 24) + +void apic_disable(void); +void xapic_enable(void); +void x2apic_enable(void); + +static inline uint32_t get_bsp_flag(void) +{ + return rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_BSP; +} + +static inline uint32_t xapic_read_reg(unsigned int reg) +{ + return ((volatile uint32_t *)APIC_DEFAULT_GPA)[reg >> 2]; +} + +static inline void xapic_write_reg(unsigned int reg, uint32_t val) +{ + ((volatile uint32_t *)APIC_DEFAULT_GPA)[reg >> 2] = val; +} + +static inline uint64_t x2apic_read_reg(unsigned int reg) +{ + return rdmsr(APIC_BASE_MSR + (reg >> 4)); +} + +static inline void x2apic_write_reg(unsigned int reg, uint64_t value) +{ + wrmsr(APIC_BASE_MSR + (reg >> 4), value); +} + +#endif /* SELFTEST_KVM_APIC_H */ diff --git a/tools/testing/selftests/kvm/include/evmcs.h b/tools/testing/selftests/kvm/include/x86_64/evmcs.h similarity index 99% rename from tools/testing/selftests/kvm/include/evmcs.h rename to tools/testing/selftests/kvm/include/x86_64/evmcs.h index a034438b6266..c9af97abd622 100644 --- a/tools/testing/selftests/kvm/include/evmcs.h +++ b/tools/testing/selftests/kvm/include/x86_64/evmcs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * tools/testing/selftests/kvm/include/vmx.h + * tools/testing/selftests/kvm/include/x86_64/evmcs.h * * Copyright (C) 2018, Red Hat, Inc. * diff --git a/tools/testing/selftests/kvm/include/x86_64/hyperv.h b/tools/testing/selftests/kvm/include/x86_64/hyperv.h new file mode 100644 index 000000000000..412eaee7884a --- /dev/null +++ b/tools/testing/selftests/kvm/include/x86_64/hyperv.h @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * tools/testing/selftests/kvm/include/x86_64/hyperv.h + * + * Copyright (C) 2021, Red Hat, Inc. + * + */ + +#ifndef SELFTEST_KVM_HYPERV_H +#define SELFTEST_KVM_HYPERV_H + +#define HYPERV_CPUID_VENDOR_AND_MAX_FUNCTIONS 0x40000000 +#define HYPERV_CPUID_INTERFACE 0x40000001 +#define HYPERV_CPUID_VERSION 0x40000002 +#define HYPERV_CPUID_FEATURES 0x40000003 +#define HYPERV_CPUID_ENLIGHTMENT_INFO 0x40000004 +#define HYPERV_CPUID_IMPLEMENT_LIMITS 0x40000005 +#define HYPERV_CPUID_CPU_MANAGEMENT_FEATURES 0x40000007 +#define HYPERV_CPUID_NESTED_FEATURES 0x4000000A +#define HYPERV_CPUID_SYNDBG_VENDOR_AND_MAX_FUNCTIONS 0x40000080 +#define HYPERV_CPUID_SYNDBG_INTERFACE 0x40000081 +#define HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES 0x40000082 + +#define HV_X64_MSR_GUEST_OS_ID 0x40000000 +#define HV_X64_MSR_HYPERCALL 0x40000001 +#define HV_X64_MSR_VP_INDEX 0x40000002 +#define HV_X64_MSR_RESET 0x40000003 +#define HV_X64_MSR_VP_RUNTIME 0x40000010 +#define HV_X64_MSR_TIME_REF_COUNT 0x40000020 +#define HV_X64_MSR_REFERENCE_TSC 0x40000021 +#define HV_X64_MSR_TSC_FREQUENCY 0x40000022 +#define HV_X64_MSR_APIC_FREQUENCY 0x40000023 +#define HV_X64_MSR_EOI 0x40000070 +#define HV_X64_MSR_ICR 0x40000071 +#define HV_X64_MSR_TPR 0x40000072 +#define HV_X64_MSR_VP_ASSIST_PAGE 0x40000073 +#define HV_X64_MSR_SCONTROL 0x40000080 +#define HV_X64_MSR_SVERSION 0x40000081 +#define HV_X64_MSR_SIEFP 0x40000082 +#define HV_X64_MSR_SIMP 0x40000083 +#define HV_X64_MSR_EOM 0x40000084 +#define HV_X64_MSR_SINT0 0x40000090 +#define HV_X64_MSR_SINT1 0x40000091 +#define HV_X64_MSR_SINT2 0x40000092 +#define HV_X64_MSR_SINT3 0x40000093 +#define HV_X64_MSR_SINT4 0x40000094 +#define HV_X64_MSR_SINT5 0x40000095 +#define HV_X64_MSR_SINT6 0x40000096 +#define HV_X64_MSR_SINT7 0x40000097 +#define HV_X64_MSR_SINT8 0x40000098 +#define HV_X64_MSR_SINT9 0x40000099 +#define HV_X64_MSR_SINT10 0x4000009A +#define HV_X64_MSR_SINT11 0x4000009B +#define HV_X64_MSR_SINT12 0x4000009C +#define HV_X64_MSR_SINT13 0x4000009D +#define HV_X64_MSR_SINT14 0x4000009E +#define HV_X64_MSR_SINT15 0x4000009F +#define HV_X64_MSR_STIMER0_CONFIG 0x400000B0 +#define HV_X64_MSR_STIMER0_COUNT 0x400000B1 +#define HV_X64_MSR_STIMER1_CONFIG 0x400000B2 +#define HV_X64_MSR_STIMER1_COUNT 0x400000B3 +#define HV_X64_MSR_STIMER2_CONFIG 0x400000B4 +#define HV_X64_MSR_STIMER2_COUNT 0x400000B5 +#define HV_X64_MSR_STIMER3_CONFIG 0x400000B6 +#define HV_X64_MSR_STIMER3_COUNT 0x400000B7 +#define HV_X64_MSR_GUEST_IDLE 0x400000F0 +#define HV_X64_MSR_CRASH_P0 0x40000100 +#define HV_X64_MSR_CRASH_P1 0x40000101 +#define HV_X64_MSR_CRASH_P2 0x40000102 +#define HV_X64_MSR_CRASH_P3 0x40000103 +#define HV_X64_MSR_CRASH_P4 0x40000104 +#define HV_X64_MSR_CRASH_CTL 0x40000105 +#define HV_X64_MSR_REENLIGHTENMENT_CONTROL 0x40000106 +#define HV_X64_MSR_TSC_EMULATION_CONTROL 0x40000107 +#define HV_X64_MSR_TSC_EMULATION_STATUS 0x40000108 +#define HV_X64_MSR_TSC_INVARIANT_CONTROL 0x40000118 + +#define HV_X64_MSR_SYNDBG_CONTROL 0x400000F1 +#define HV_X64_MSR_SYNDBG_STATUS 0x400000F2 +#define HV_X64_MSR_SYNDBG_SEND_BUFFER 0x400000F3 +#define HV_X64_MSR_SYNDBG_RECV_BUFFER 0x400000F4 +#define HV_X64_MSR_SYNDBG_PENDING_BUFFER 0x400000F5 +#define HV_X64_MSR_SYNDBG_OPTIONS 0x400000FF + +/* HYPERV_CPUID_FEATURES.EAX */ +#define HV_MSR_VP_RUNTIME_AVAILABLE BIT(0) +#define HV_MSR_TIME_REF_COUNT_AVAILABLE BIT(1) +#define HV_MSR_SYNIC_AVAILABLE BIT(2) +#define HV_MSR_SYNTIMER_AVAILABLE BIT(3) +#define HV_MSR_APIC_ACCESS_AVAILABLE BIT(4) +#define HV_MSR_HYPERCALL_AVAILABLE BIT(5) +#define HV_MSR_VP_INDEX_AVAILABLE BIT(6) +#define HV_MSR_RESET_AVAILABLE BIT(7) +#define HV_MSR_STAT_PAGES_AVAILABLE BIT(8) +#define HV_MSR_REFERENCE_TSC_AVAILABLE BIT(9) +#define HV_MSR_GUEST_IDLE_AVAILABLE BIT(10) +#define HV_ACCESS_FREQUENCY_MSRS BIT(11) +#define HV_ACCESS_REENLIGHTENMENT BIT(13) +#define HV_ACCESS_TSC_INVARIANT BIT(15) + +/* HYPERV_CPUID_FEATURES.EBX */ +#define HV_CREATE_PARTITIONS BIT(0) +#define HV_ACCESS_PARTITION_ID BIT(1) +#define HV_ACCESS_MEMORY_POOL BIT(2) +#define HV_ADJUST_MESSAGE_BUFFERS BIT(3) +#define HV_POST_MESSAGES BIT(4) +#define HV_SIGNAL_EVENTS BIT(5) +#define HV_CREATE_PORT BIT(6) +#define HV_CONNECT_PORT BIT(7) +#define HV_ACCESS_STATS BIT(8) +#define HV_DEBUGGING BIT(11) +#define HV_CPU_MANAGEMENT BIT(12) +#define HV_ISOLATION BIT(22) + +/* HYPERV_CPUID_FEATURES.EDX */ +#define HV_X64_MWAIT_AVAILABLE BIT(0) +#define HV_X64_GUEST_DEBUGGING_AVAILABLE BIT(1) +#define HV_X64_PERF_MONITOR_AVAILABLE BIT(2) +#define HV_X64_CPU_DYNAMIC_PARTITIONING_AVAILABLE BIT(3) +#define HV_X64_HYPERCALL_PARAMS_XMM_AVAILABLE BIT(4) +#define HV_X64_GUEST_IDLE_STATE_AVAILABLE BIT(5) +#define HV_FEATURE_FREQUENCY_MSRS_AVAILABLE BIT(8) +#define HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE BIT(10) +#define HV_FEATURE_DEBUG_MSRS_AVAILABLE BIT(11) +#define HV_STIMER_DIRECT_MODE_AVAILABLE BIT(19) + +/* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */ +#define HV_X64_AS_SWITCH_RECOMMENDED BIT(0) +#define HV_X64_LOCAL_TLB_FLUSH_RECOMMENDED BIT(1) +#define HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED BIT(2) +#define HV_X64_APIC_ACCESS_RECOMMENDED BIT(3) +#define HV_X64_SYSTEM_RESET_RECOMMENDED BIT(4) +#define HV_X64_RELAXED_TIMING_RECOMMENDED BIT(5) +#define HV_DEPRECATING_AEOI_RECOMMENDED BIT(9) +#define HV_X64_CLUSTER_IPI_RECOMMENDED BIT(10) +#define HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED BIT(11) +#define HV_X64_ENLIGHTENED_VMCS_RECOMMENDED BIT(14) + +/* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */ +#define HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING BIT(1) + +/* Hypercalls */ +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE 0x0002 +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST 0x0003 +#define HVCALL_NOTIFY_LONG_SPIN_WAIT 0x0008 +#define HVCALL_SEND_IPI 0x000b +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX 0x0013 +#define HVCALL_FLUSH_VIRTUAL_ADDRESS_LIST_EX 0x0014 +#define HVCALL_SEND_IPI_EX 0x0015 +#define HVCALL_GET_PARTITION_ID 0x0046 +#define HVCALL_DEPOSIT_MEMORY 0x0048 +#define HVCALL_CREATE_VP 0x004e +#define HVCALL_GET_VP_REGISTERS 0x0050 +#define HVCALL_SET_VP_REGISTERS 0x0051 +#define HVCALL_POST_MESSAGE 0x005c +#define HVCALL_SIGNAL_EVENT 0x005d +#define HVCALL_POST_DEBUG_DATA 0x0069 +#define HVCALL_RETRIEVE_DEBUG_DATA 0x006a +#define HVCALL_RESET_DEBUG_SESSION 0x006b +#define HVCALL_ADD_LOGICAL_PROCESSOR 0x0076 +#define HVCALL_MAP_DEVICE_INTERRUPT 0x007c +#define HVCALL_UNMAP_DEVICE_INTERRUPT 0x007d +#define HVCALL_RETARGET_INTERRUPT 0x007e +#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_SPACE 0x00af +#define HVCALL_FLUSH_GUEST_PHYSICAL_ADDRESS_LIST 0x00b0 + +#define HV_FLUSH_ALL_PROCESSORS BIT(0) +#define HV_FLUSH_ALL_VIRTUAL_ADDRESS_SPACES BIT(1) +#define HV_FLUSH_NON_GLOBAL_MAPPINGS_ONLY BIT(2) +#define HV_FLUSH_USE_EXTENDED_RANGE_FORMAT BIT(3) + +/* hypercall status code */ +#define HV_STATUS_SUCCESS 0 +#define HV_STATUS_INVALID_HYPERCALL_CODE 2 +#define HV_STATUS_INVALID_HYPERCALL_INPUT 3 +#define HV_STATUS_INVALID_ALIGNMENT 4 +#define HV_STATUS_INVALID_PARAMETER 5 +#define HV_STATUS_ACCESS_DENIED 6 +#define HV_STATUS_OPERATION_DENIED 8 +#define HV_STATUS_INSUFFICIENT_MEMORY 11 +#define HV_STATUS_INVALID_PORT_ID 17 +#define HV_STATUS_INVALID_CONNECTION_ID 18 +#define HV_STATUS_INSUFFICIENT_BUFFERS 19 + +#endif /* !SELFTEST_KVM_HYPERV_H */ diff --git a/tools/testing/selftests/kvm/include/x86_64/processor.h b/tools/testing/selftests/kvm/include/x86_64/processor.h index 0b30b4e15c38..242ae8e09a65 100644 --- a/tools/testing/selftests/kvm/include/x86_64/processor.h +++ b/tools/testing/selftests/kvm/include/x86_64/processor.h @@ -13,6 +13,8 @@ #include +#include "../kvm_util.h" + #define X86_EFLAGS_FIXED (1u << 1) #define X86_CR4_VME (1ul << 0) @@ -53,7 +55,8 @@ #define CPUID_PKU (1ul << 3) #define CPUID_LA57 (1ul << 16) -#define UNEXPECTED_VECTOR_PORT 0xfff0u +/* CPUID.0x8000_0001.EDX */ +#define CPUID_GBPAGES (1ul << 26) /* General Registers in 64-Bit Mode */ struct gpr64_regs { @@ -391,9 +394,13 @@ struct ex_regs { void vm_init_descriptor_tables(struct kvm_vm *vm); void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid); -void vm_handle_exception(struct kvm_vm *vm, int vector, +void vm_install_exception_handler(struct kvm_vm *vm, int vector, void (*handler)(struct ex_regs *)); +uint64_t vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr); +void vm_set_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr, + uint64_t pte); + /* * set_cpuid() - overwrites a matching cpuid entry with the provided value. * matches based on ent->function && ent->index. returns true @@ -410,6 +417,14 @@ struct kvm_cpuid2 *kvm_get_supported_hv_cpuid(void); void vcpu_set_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid); struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vm *vm, uint32_t vcpuid); +enum x86_page_size { + X86_PAGE_SIZE_4K = 0, + X86_PAGE_SIZE_2M, + X86_PAGE_SIZE_1G, +}; +void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, + enum x86_page_size page_size); + /* * Basic CPU control in CR0 */ @@ -425,53 +440,6 @@ struct kvm_cpuid2 *vcpu_get_supported_hv_cpuid(struct kvm_vm *vm, uint32_t vcpui #define X86_CR0_CD (1UL<<30) /* Cache Disable */ #define X86_CR0_PG (1UL<<31) /* Paging */ -#define APIC_DEFAULT_GPA 0xfee00000ULL - -/* APIC base address MSR and fields */ -#define MSR_IA32_APICBASE 0x0000001b -#define MSR_IA32_APICBASE_BSP (1<<8) -#define MSR_IA32_APICBASE_EXTD (1<<10) -#define MSR_IA32_APICBASE_ENABLE (1<<11) -#define MSR_IA32_APICBASE_BASE (0xfffff<<12) -#define GET_APIC_BASE(x) (((x) >> 12) << 12) - -#define APIC_BASE_MSR 0x800 -#define X2APIC_ENABLE (1UL << 10) -#define APIC_ID 0x20 -#define APIC_LVR 0x30 -#define GET_APIC_ID_FIELD(x) (((x) >> 24) & 0xFF) -#define APIC_TASKPRI 0x80 -#define APIC_PROCPRI 0xA0 -#define APIC_EOI 0xB0 -#define APIC_SPIV 0xF0 -#define APIC_SPIV_FOCUS_DISABLED (1 << 9) -#define APIC_SPIV_APIC_ENABLED (1 << 8) -#define APIC_ICR 0x300 -#define APIC_DEST_SELF 0x40000 -#define APIC_DEST_ALLINC 0x80000 -#define APIC_DEST_ALLBUT 0xC0000 -#define APIC_ICR_RR_MASK 0x30000 -#define APIC_ICR_RR_INVALID 0x00000 -#define APIC_ICR_RR_INPROG 0x10000 -#define APIC_ICR_RR_VALID 0x20000 -#define APIC_INT_LEVELTRIG 0x08000 -#define APIC_INT_ASSERT 0x04000 -#define APIC_ICR_BUSY 0x01000 -#define APIC_DEST_LOGICAL 0x00800 -#define APIC_DEST_PHYSICAL 0x00000 -#define APIC_DM_FIXED 0x00000 -#define APIC_DM_FIXED_MASK 0x00700 -#define APIC_DM_LOWEST 0x00100 -#define APIC_DM_SMI 0x00200 -#define APIC_DM_REMRD 0x00300 -#define APIC_DM_NMI 0x00400 -#define APIC_DM_INIT 0x00500 -#define APIC_DM_STARTUP 0x00600 -#define APIC_DM_EXTINT 0x00700 -#define APIC_VECTOR_MASK 0x000FF -#define APIC_ICR2 0x310 -#define SET_APIC_DEST_FIELD(x) ((x) << 24) - /* VMX_EPT_VPID_CAP bits */ #define VMX_EPT_VPID_CAP_AD_BITS (1ULL << 21) diff --git a/tools/testing/selftests/kvm/include/x86_64/vmx.h b/tools/testing/selftests/kvm/include/x86_64/vmx.h index 65eb1079a161..583ceb0d1457 100644 --- a/tools/testing/selftests/kvm/include/x86_64/vmx.h +++ b/tools/testing/selftests/kvm/include/x86_64/vmx.h @@ -10,6 +10,7 @@ #include #include "processor.h" +#include "apic.h" /* * Definitions of Primary Processor-Based VM-Execution Controls. @@ -607,15 +608,13 @@ bool nested_vmx_supported(void); void nested_vmx_check_supported(void); void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm, - uint64_t nested_paddr, uint64_t paddr, uint32_t eptp_memslot); + uint64_t nested_paddr, uint64_t paddr); void nested_map(struct vmx_pages *vmx, struct kvm_vm *vm, - uint64_t nested_paddr, uint64_t paddr, uint64_t size, - uint32_t eptp_memslot); + uint64_t nested_paddr, uint64_t paddr, uint64_t size); void nested_map_memslot(struct vmx_pages *vmx, struct kvm_vm *vm, - uint32_t memslot, uint32_t eptp_memslot); + uint32_t memslot); void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm, uint32_t eptp_memslot); -void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm, - uint32_t eptp_memslot); +void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm); #endif /* SELFTEST_KVM_VMX_H */ diff --git a/tools/testing/selftests/kvm/kvm_binary_stats_test.c b/tools/testing/selftests/kvm/kvm_binary_stats_test.c new file mode 100644 index 000000000000..5906bbc08483 --- /dev/null +++ b/tools/testing/selftests/kvm/kvm_binary_stats_test.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * kvm_binary_stats_test + * + * Copyright (C) 2021, Google LLC. + * + * Test the fd-based interface for KVM statistics. + */ + +#define _GNU_SOURCE /* for program_invocation_short_name */ +#include +#include +#include +#include +#include + +#include "test_util.h" + +#include "kvm_util.h" +#include "asm/kvm.h" +#include "linux/kvm.h" + +static void stats_test(int stats_fd) +{ + ssize_t ret; + int i; + size_t size_desc; + size_t size_data = 0; + struct kvm_stats_header *header; + char *id; + struct kvm_stats_desc *stats_desc; + u64 *stats_data; + struct kvm_stats_desc *pdesc; + + /* Read kvm stats header */ + header = malloc(sizeof(*header)); + TEST_ASSERT(header, "Allocate memory for stats header"); + + ret = read(stats_fd, header, sizeof(*header)); + TEST_ASSERT(ret == sizeof(*header), "Read stats header"); + size_desc = sizeof(*stats_desc) + header->name_size; + + /* Read kvm stats id string */ + id = malloc(header->name_size); + TEST_ASSERT(id, "Allocate memory for id string"); + ret = read(stats_fd, id, header->name_size); + TEST_ASSERT(ret == header->name_size, "Read id string"); + + /* Check id string, that should start with "kvm" */ + TEST_ASSERT(!strncmp(id, "kvm", 3) && strlen(id) < header->name_size, + "Invalid KVM stats type, id: %s", id); + + /* Sanity check for other fields in header */ + if (header->num_desc == 0) { + printf("No KVM stats defined!"); + return; + } + /* Check overlap */ + TEST_ASSERT(header->desc_offset > 0 && header->data_offset > 0 + && header->desc_offset >= sizeof(*header) + && header->data_offset >= sizeof(*header), + "Invalid offset fields in header"); + TEST_ASSERT(header->desc_offset > header->data_offset || + (header->desc_offset + size_desc * header->num_desc <= + header->data_offset), + "Descriptor block is overlapped with data block"); + + /* Allocate memory for stats descriptors */ + stats_desc = calloc(header->num_desc, size_desc); + TEST_ASSERT(stats_desc, "Allocate memory for stats descriptors"); + /* Read kvm stats descriptors */ + ret = pread(stats_fd, stats_desc, + size_desc * header->num_desc, header->desc_offset); + TEST_ASSERT(ret == size_desc * header->num_desc, + "Read KVM stats descriptors"); + + /* Sanity check for fields in descriptors */ + for (i = 0; i < header->num_desc; ++i) { + pdesc = (void *)stats_desc + i * size_desc; + /* Check type,unit,base boundaries */ + TEST_ASSERT((pdesc->flags & KVM_STATS_TYPE_MASK) + <= KVM_STATS_TYPE_MAX, "Unknown KVM stats type"); + TEST_ASSERT((pdesc->flags & KVM_STATS_UNIT_MASK) + <= KVM_STATS_UNIT_MAX, "Unknown KVM stats unit"); + TEST_ASSERT((pdesc->flags & KVM_STATS_BASE_MASK) + <= KVM_STATS_BASE_MAX, "Unknown KVM stats base"); + /* Check exponent for stats unit + * Exponent for counter should be greater than or equal to 0 + * Exponent for unit bytes should be greater than or equal to 0 + * Exponent for unit seconds should be less than or equal to 0 + * Exponent for unit clock cycles should be greater than or + * equal to 0 + */ + switch (pdesc->flags & KVM_STATS_UNIT_MASK) { + case KVM_STATS_UNIT_NONE: + case KVM_STATS_UNIT_BYTES: + case KVM_STATS_UNIT_CYCLES: + TEST_ASSERT(pdesc->exponent >= 0, + "Unsupported KVM stats unit"); + break; + case KVM_STATS_UNIT_SECONDS: + TEST_ASSERT(pdesc->exponent <= 0, + "Unsupported KVM stats unit"); + break; + } + /* Check name string */ + TEST_ASSERT(strlen(pdesc->name) < header->name_size, + "KVM stats name(%s) too long", pdesc->name); + /* Check size field, which should not be zero */ + TEST_ASSERT(pdesc->size, "KVM descriptor(%s) with size of 0", + pdesc->name); + size_data += pdesc->size * sizeof(*stats_data); + } + /* Check overlap */ + TEST_ASSERT(header->data_offset >= header->desc_offset + || header->data_offset + size_data <= header->desc_offset, + "Data block is overlapped with Descriptor block"); + /* Check validity of all stats data size */ + TEST_ASSERT(size_data >= header->num_desc * sizeof(*stats_data), + "Data size is not correct"); + /* Check stats offset */ + for (i = 0; i < header->num_desc; ++i) { + pdesc = (void *)stats_desc + i * size_desc; + TEST_ASSERT(pdesc->offset < size_data, + "Invalid offset (%u) for stats: %s", + pdesc->offset, pdesc->name); + } + + /* Allocate memory for stats data */ + stats_data = malloc(size_data); + TEST_ASSERT(stats_data, "Allocate memory for stats data"); + /* Read kvm stats data as a bulk */ + ret = pread(stats_fd, stats_data, size_data, header->data_offset); + TEST_ASSERT(ret == size_data, "Read KVM stats data"); + /* Read kvm stats data one by one */ + size_data = 0; + for (i = 0; i < header->num_desc; ++i) { + pdesc = (void *)stats_desc + i * size_desc; + ret = pread(stats_fd, stats_data, + pdesc->size * sizeof(*stats_data), + header->data_offset + size_data); + TEST_ASSERT(ret == pdesc->size * sizeof(*stats_data), + "Read data of KVM stats: %s", pdesc->name); + size_data += pdesc->size * sizeof(*stats_data); + } + + free(stats_data); + free(stats_desc); + free(id); + free(header); +} + + +static void vm_stats_test(struct kvm_vm *vm) +{ + int stats_fd; + + /* Get fd for VM stats */ + stats_fd = vm_get_stats_fd(vm); + TEST_ASSERT(stats_fd >= 0, "Get VM stats fd"); + + stats_test(stats_fd); + close(stats_fd); + TEST_ASSERT(fcntl(stats_fd, F_GETFD) == -1, "Stats fd not freed"); +} + +static void vcpu_stats_test(struct kvm_vm *vm, int vcpu_id) +{ + int stats_fd; + + /* Get fd for VCPU stats */ + stats_fd = vcpu_get_stats_fd(vm, vcpu_id); + TEST_ASSERT(stats_fd >= 0, "Get VCPU stats fd"); + + stats_test(stats_fd); + close(stats_fd); + TEST_ASSERT(fcntl(stats_fd, F_GETFD) == -1, "Stats fd not freed"); +} + +#define DEFAULT_NUM_VM 4 +#define DEFAULT_NUM_VCPU 4 + +/* + * Usage: kvm_bin_form_stats [#vm] [#vcpu] + * The first parameter #vm set the number of VMs being created. + * The second parameter #vcpu set the number of VCPUs being created. + * By default, DEFAULT_NUM_VM VM and DEFAULT_NUM_VCPU VCPU for the VM would be + * created for testing. + */ + +int main(int argc, char *argv[]) +{ + int i, j; + struct kvm_vm **vms; + int max_vm = DEFAULT_NUM_VM; + int max_vcpu = DEFAULT_NUM_VCPU; + + /* Get the number of VMs and VCPUs that would be created for testing. */ + if (argc > 1) { + max_vm = strtol(argv[1], NULL, 0); + if (max_vm <= 0) + max_vm = DEFAULT_NUM_VM; + } + if (argc > 2) { + max_vcpu = strtol(argv[2], NULL, 0); + if (max_vcpu <= 0) + max_vcpu = DEFAULT_NUM_VCPU; + } + + /* Check the extension for binary stats */ + if (kvm_check_cap(KVM_CAP_BINARY_STATS_FD) <= 0) { + print_skip("Binary form statistics interface is not supported"); + exit(KSFT_SKIP); + } + + /* Create VMs and VCPUs */ + vms = malloc(sizeof(vms[0]) * max_vm); + TEST_ASSERT(vms, "Allocate memory for storing VM pointers"); + for (i = 0; i < max_vm; ++i) { + vms[i] = vm_create(VM_MODE_DEFAULT, + DEFAULT_GUEST_PHY_PAGES, O_RDWR); + for (j = 0; j < max_vcpu; ++j) + vm_vcpu_add(vms[i], j); + } + + /* Check stats read for every VM and VCPU */ + for (i = 0; i < max_vm; ++i) { + vm_stats_test(vms[i]); + for (j = 0; j < max_vcpu; ++j) + vcpu_stats_test(vms[i], j); + } + + for (i = 0; i < max_vm; ++i) + kvm_vm_free(vms[i]); + free(vms); + return 0; +} diff --git a/tools/testing/selftests/kvm/kvm_page_table_test.c b/tools/testing/selftests/kvm/kvm_page_table_test.c index 82171f17c1d7..0d04a7db7f24 100644 --- a/tools/testing/selftests/kvm/kvm_page_table_test.c +++ b/tools/testing/selftests/kvm/kvm_page_table_test.c @@ -303,7 +303,7 @@ static struct kvm_vm *pre_init_before_test(enum vm_guest_mode mode, void *arg) TEST_MEM_SLOT_INDEX, guest_num_pages, 0); /* Do mapping(GVA->GPA) for the testing memory slot */ - virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages, 0); + virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages); /* Cache the HVA pointer of the region */ host_test_mem = addr_gpa2hva(vm, (vm_paddr_t)guest_test_phys_mem); diff --git a/tools/testing/selftests/kvm/lib/aarch64/handlers.S b/tools/testing/selftests/kvm/lib/aarch64/handlers.S new file mode 100644 index 000000000000..0e443eadfac6 --- /dev/null +++ b/tools/testing/selftests/kvm/lib/aarch64/handlers.S @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +.macro save_registers + add sp, sp, #-16 * 17 + + stp x0, x1, [sp, #16 * 0] + stp x2, x3, [sp, #16 * 1] + stp x4, x5, [sp, #16 * 2] + stp x6, x7, [sp, #16 * 3] + stp x8, x9, [sp, #16 * 4] + stp x10, x11, [sp, #16 * 5] + stp x12, x13, [sp, #16 * 6] + stp x14, x15, [sp, #16 * 7] + stp x16, x17, [sp, #16 * 8] + stp x18, x19, [sp, #16 * 9] + stp x20, x21, [sp, #16 * 10] + stp x22, x23, [sp, #16 * 11] + stp x24, x25, [sp, #16 * 12] + stp x26, x27, [sp, #16 * 13] + stp x28, x29, [sp, #16 * 14] + + /* + * This stores sp_el1 into ex_regs.sp so exception handlers can "look" + * at it. It will _not_ be used to restore the sp on return from the + * exception so handlers can not update it. + */ + add x1, sp, #16 * 17 + stp x30, x1, [sp, #16 * 15] /* x30, SP */ + + mrs x1, elr_el1 + mrs x2, spsr_el1 + stp x1, x2, [sp, #16 * 16] /* PC, PSTATE */ +.endm + +.macro restore_registers + ldp x1, x2, [sp, #16 * 16] /* PC, PSTATE */ + msr elr_el1, x1 + msr spsr_el1, x2 + + /* sp is not restored */ + ldp x30, xzr, [sp, #16 * 15] /* x30, SP */ + + ldp x28, x29, [sp, #16 * 14] + ldp x26, x27, [sp, #16 * 13] + ldp x24, x25, [sp, #16 * 12] + ldp x22, x23, [sp, #16 * 11] + ldp x20, x21, [sp, #16 * 10] + ldp x18, x19, [sp, #16 * 9] + ldp x16, x17, [sp, #16 * 8] + ldp x14, x15, [sp, #16 * 7] + ldp x12, x13, [sp, #16 * 6] + ldp x10, x11, [sp, #16 * 5] + ldp x8, x9, [sp, #16 * 4] + ldp x6, x7, [sp, #16 * 3] + ldp x4, x5, [sp, #16 * 2] + ldp x2, x3, [sp, #16 * 1] + ldp x0, x1, [sp, #16 * 0] + + add sp, sp, #16 * 17 + + eret +.endm + +.pushsection ".entry.text", "ax" +.balign 0x800 +.global vectors +vectors: +.popsection + +.set vector, 0 + +/* + * Build an exception handler for vector and append a jump to it into + * vectors (while making sure that it's 0x80 aligned). + */ +.macro HANDLER, label +handler_\label: + save_registers + mov x0, sp + mov x1, #vector + bl route_exception + restore_registers + +.pushsection ".entry.text", "ax" +.balign 0x80 + b handler_\label +.popsection + +.set vector, vector + 1 +.endm + +.macro HANDLER_INVALID +.pushsection ".entry.text", "ax" +.balign 0x80 +/* This will abort so no need to save and restore registers. */ + mov x0, #vector + mov x1, #0 /* ec */ + mov x2, #0 /* valid_ec */ + b kvm_exit_unexpected_exception +.popsection + +.set vector, vector + 1 +.endm + +/* + * Caution: be sure to not add anything between the declaration of vectors + * above and these macro calls that will build the vectors table below it. + */ + HANDLER_INVALID // Synchronous EL1t + HANDLER_INVALID // IRQ EL1t + HANDLER_INVALID // FIQ EL1t + HANDLER_INVALID // Error EL1t + + HANDLER el1h_sync // Synchronous EL1h + HANDLER el1h_irq // IRQ EL1h + HANDLER el1h_fiq // FIQ EL1h + HANDLER el1h_error // Error EL1h + + HANDLER el0_sync_64 // Synchronous 64-bit EL0 + HANDLER el0_irq_64 // IRQ 64-bit EL0 + HANDLER el0_fiq_64 // FIQ 64-bit EL0 + HANDLER el0_error_64 // Error 64-bit EL0 + + HANDLER el0_sync_32 // Synchronous 32-bit EL0 + HANDLER el0_irq_32 // IRQ 32-bit EL0 + HANDLER el0_fiq_32 // FIQ 32-bit EL0 + HANDLER el0_error_32 // Error 32-bit EL0 diff --git a/tools/testing/selftests/kvm/lib/aarch64/processor.c b/tools/testing/selftests/kvm/lib/aarch64/processor.c index cee92d477dc0..9f49f6caafe5 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/processor.c +++ b/tools/testing/selftests/kvm/lib/aarch64/processor.c @@ -6,14 +6,16 @@ */ #include +#include #include "kvm_util.h" #include "../kvm_util_internal.h" #include "processor.h" -#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 #define DEFAULT_ARM64_GUEST_STACK_VADDR_MIN 0xac0000 +static vm_vaddr_t exception_handlers; + static uint64_t page_align(struct kvm_vm *vm, uint64_t v) { return (v + vm->page_size) & ~(vm->page_size - 1); @@ -72,19 +74,19 @@ static uint64_t __maybe_unused ptrs_per_pte(struct kvm_vm *vm) return 1 << (vm->page_shift - 3); } -void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot) +void virt_pgd_alloc(struct kvm_vm *vm) { if (!vm->pgd_created) { vm_paddr_t paddr = vm_phy_pages_alloc(vm, page_align(vm, ptrs_per_pgd(vm) * 8) / vm->page_size, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot); + KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0); vm->pgd = paddr; vm->pgd_created = true; } } -void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - uint32_t pgd_memslot, uint64_t flags) +static void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, + uint64_t flags) { uint8_t attr_idx = flags & 7; uint64_t *ptep; @@ -104,25 +106,19 @@ void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, paddr, vm->max_gfn, vm->page_size); ptep = addr_gpa2hva(vm, vm->pgd) + pgd_index(vm, vaddr) * 8; - if (!*ptep) { - *ptep = vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot); - *ptep |= 3; - } + if (!*ptep) + *ptep = vm_alloc_page_table(vm) | 3; switch (vm->pgtable_levels) { case 4: ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pud_index(vm, vaddr) * 8; - if (!*ptep) { - *ptep = vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot); - *ptep |= 3; - } + if (!*ptep) + *ptep = vm_alloc_page_table(vm) | 3; /* fall through */ case 3: ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pmd_index(vm, vaddr) * 8; - if (!*ptep) { - *ptep = vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot); - *ptep |= 3; - } + if (!*ptep) + *ptep = vm_alloc_page_table(vm) | 3; /* fall through */ case 2: ptep = addr_gpa2hva(vm, pte_addr(vm, *ptep)) + pte_index(vm, vaddr) * 8; @@ -135,12 +131,11 @@ void _virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, *ptep |= (attr_idx << 2) | (1 << 10) /* Access Flag */; } -void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - uint32_t pgd_memslot) +void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) { uint64_t attr_idx = 4; /* NORMAL (See DEFAULT_MAIR_EL1) */ - _virt_pg_map(vm, vaddr, paddr, pgd_memslot, attr_idx); + _virt_pg_map(vm, vaddr, paddr, attr_idx); } vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) @@ -302,7 +297,7 @@ void aarch64_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, DEFAULT_STACK_PGS * vm->page_size : vm->page_size; uint64_t stack_vaddr = vm_vaddr_alloc(vm, stack_size, - DEFAULT_ARM64_GUEST_STACK_VADDR_MIN, 0, 0); + DEFAULT_ARM64_GUEST_STACK_VADDR_MIN); vm_vcpu_add(vm, vcpuid); aarch64_vcpu_setup(vm, vcpuid, init); @@ -334,6 +329,100 @@ void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...) va_end(ap); } +void kvm_exit_unexpected_exception(int vector, uint64_t ec, bool valid_ec) +{ + ucall(UCALL_UNHANDLED, 3, vector, ec, valid_ec); + while (1) + ; +} + void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid) { + struct ucall uc; + + if (get_ucall(vm, vcpuid, &uc) != UCALL_UNHANDLED) + return; + + if (uc.args[2]) /* valid_ec */ { + assert(VECTOR_IS_SYNC(uc.args[0])); + TEST_FAIL("Unexpected exception (vector:0x%lx, ec:0x%lx)", + uc.args[0], uc.args[1]); + } else { + assert(!VECTOR_IS_SYNC(uc.args[0])); + TEST_FAIL("Unexpected exception (vector:0x%lx)", + uc.args[0]); + } +} + +struct handlers { + handler_fn exception_handlers[VECTOR_NUM][ESR_EC_NUM]; +}; + +void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid) +{ + extern char vectors; + + set_reg(vm, vcpuid, ARM64_SYS_REG(VBAR_EL1), (uint64_t)&vectors); +} + +void route_exception(struct ex_regs *regs, int vector) +{ + struct handlers *handlers = (struct handlers *)exception_handlers; + bool valid_ec; + int ec = 0; + + switch (vector) { + case VECTOR_SYNC_CURRENT: + case VECTOR_SYNC_LOWER_64: + ec = (read_sysreg(esr_el1) >> ESR_EC_SHIFT) & ESR_EC_MASK; + valid_ec = true; + break; + case VECTOR_IRQ_CURRENT: + case VECTOR_IRQ_LOWER_64: + case VECTOR_FIQ_CURRENT: + case VECTOR_FIQ_LOWER_64: + case VECTOR_ERROR_CURRENT: + case VECTOR_ERROR_LOWER_64: + ec = 0; + valid_ec = false; + break; + default: + valid_ec = false; + goto unexpected_exception; + } + + if (handlers && handlers->exception_handlers[vector][ec]) + return handlers->exception_handlers[vector][ec](regs); + +unexpected_exception: + kvm_exit_unexpected_exception(vector, ec, valid_ec); +} + +void vm_init_descriptor_tables(struct kvm_vm *vm) +{ + vm->handlers = vm_vaddr_alloc(vm, sizeof(struct handlers), + vm->page_size, 0, 0); + + *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers; +} + +void vm_install_sync_handler(struct kvm_vm *vm, int vector, int ec, + void (*handler)(struct ex_regs *)) +{ + struct handlers *handlers = addr_gva2hva(vm, vm->handlers); + + assert(VECTOR_IS_SYNC(vector)); + assert(vector < VECTOR_NUM); + assert(ec < ESR_EC_NUM); + handlers->exception_handlers[vector][ec] = handler; +} + +void vm_install_exception_handler(struct kvm_vm *vm, int vector, + void (*handler)(struct ex_regs *)) +{ + struct handlers *handlers = addr_gva2hva(vm, vm->handlers); + + assert(!VECTOR_IS_SYNC(vector)); + assert(vector < VECTOR_NUM); + handlers->exception_handlers[vector][0] = handler; } diff --git a/tools/testing/selftests/kvm/lib/aarch64/ucall.c b/tools/testing/selftests/kvm/lib/aarch64/ucall.c index 2f37b90ee1a9..e0b0164e9af8 100644 --- a/tools/testing/selftests/kvm/lib/aarch64/ucall.c +++ b/tools/testing/selftests/kvm/lib/aarch64/ucall.c @@ -14,7 +14,7 @@ static bool ucall_mmio_init(struct kvm_vm *vm, vm_paddr_t gpa) if (kvm_userspace_memory_region_find(vm, gpa, gpa + 1)) return false; - virt_pg_map(vm, gpa, gpa, 0); + virt_pg_map(vm, gpa, gpa); ucall_exit_mmio_addr = (vm_vaddr_t *)gpa; sync_global_to_guest(vm, ucall_exit_mmio_addr); diff --git a/tools/testing/selftests/kvm/lib/elf.c b/tools/testing/selftests/kvm/lib/elf.c index bc75a91e00a6..eac44f5d0db0 100644 --- a/tools/testing/selftests/kvm/lib/elf.c +++ b/tools/testing/selftests/kvm/lib/elf.c @@ -111,8 +111,7 @@ static void elfhdr_get(const char *filename, Elf64_Ehdr *hdrp) * by the image and it needs to have sufficient available physical pages, to * back the virtual pages used to load the image. */ -void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename, - uint32_t data_memslot, uint32_t pgd_memslot) +void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename) { off_t offset, offset_rv; Elf64_Ehdr hdr; @@ -164,8 +163,7 @@ void kvm_vm_elf_load(struct kvm_vm *vm, const char *filename, seg_vend |= vm->page_size - 1; size_t seg_size = seg_vend - seg_vstart + 1; - vm_vaddr_t vaddr = vm_vaddr_alloc(vm, seg_size, seg_vstart, - data_memslot, pgd_memslot); + vm_vaddr_t vaddr = vm_vaddr_alloc(vm, seg_size, seg_vstart); TEST_ASSERT(vaddr == seg_vstart, "Unable to allocate " "virtual memory for segment at requested min addr,\n" " segment idx: %u\n" diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index a2b732cf96ea..5b56b57b3c20 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -365,7 +365,7 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, pages = vm_adjust_num_guest_pages(mode, pages); vm = vm_create(mode, pages, O_RDWR); - kvm_vm_elf_load(vm, program_invocation_name, 0, 0); + kvm_vm_elf_load(vm, program_invocation_name); #ifdef __x86_64__ vm_create_irqchip(vm); @@ -375,10 +375,6 @@ struct kvm_vm *vm_create_with_vcpus(enum vm_guest_mode mode, uint32_t nr_vcpus, uint32_t vcpuid = vcpuids ? vcpuids[i] : i; vm_vcpu_add_default(vm, vcpuid, guest_code); - -#ifdef __x86_64__ - vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid()); -#endif } return vm; @@ -1251,15 +1247,13 @@ static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz, * a unique set of pages, with the minimum real allocation being at least * a page. */ -vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, - uint32_t data_memslot, uint32_t pgd_memslot) +vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min) { uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0); - virt_pgd_alloc(vm, pgd_memslot); + virt_pgd_alloc(vm); vm_paddr_t paddr = vm_phy_pages_alloc(vm, pages, - KVM_UTIL_MIN_PFN * vm->page_size, - data_memslot); + KVM_UTIL_MIN_PFN * vm->page_size, 0); /* * Find an unused range of virtual page addresses of at least @@ -1271,7 +1265,7 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, for (vm_vaddr_t vaddr = vaddr_start; pages > 0; pages--, vaddr += vm->page_size, paddr += vm->page_size) { - virt_pg_map(vm, vaddr, paddr, pgd_memslot); + virt_pg_map(vm, vaddr, paddr); sparsebit_set(vm->vpages_mapped, vaddr >> vm->page_shift); @@ -1280,6 +1274,44 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, return vaddr_start; } +/* + * VM Virtual Address Allocate Pages + * + * Input Args: + * vm - Virtual Machine + * + * Output Args: None + * + * Return: + * Starting guest virtual address + * + * Allocates at least N system pages worth of bytes within the virtual address + * space of the vm. + */ +vm_vaddr_t vm_vaddr_alloc_pages(struct kvm_vm *vm, int nr_pages) +{ + return vm_vaddr_alloc(vm, nr_pages * getpagesize(), KVM_UTIL_MIN_VADDR); +} + +/* + * VM Virtual Address Allocate Page + * + * Input Args: + * vm - Virtual Machine + * + * Output Args: None + * + * Return: + * Starting guest virtual address + * + * Allocates at least one system page worth of bytes within the virtual address + * space of the vm. + */ +vm_vaddr_t vm_vaddr_alloc_page(struct kvm_vm *vm) +{ + return vm_vaddr_alloc_pages(vm, 1); +} + /* * Map a range of VM virtual address to the VM's physical address * @@ -1298,7 +1330,7 @@ vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min, * @npages starting at @vaddr to the page range starting at @paddr. */ void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - unsigned int npages, uint32_t pgd_memslot) + unsigned int npages) { size_t page_size = vm->page_size; size_t size = npages * page_size; @@ -1307,7 +1339,7 @@ void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, TEST_ASSERT(paddr + size > paddr, "Paddr overflow"); while (npages--) { - virt_pg_map(vm, vaddr, paddr, pgd_memslot); + virt_pg_map(vm, vaddr, paddr); vaddr += page_size; paddr += page_size; } @@ -2177,6 +2209,14 @@ vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min, return vm_phy_pages_alloc(vm, 1, paddr_min, memslot); } +/* Arbitrary minimum physical address used for virtual translation tables. */ +#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 + +vm_paddr_t vm_alloc_page_table(struct kvm_vm *vm) +{ + return vm_phy_page_alloc(vm, KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0); +} + /* * Address Guest Virtual to Host Virtual * @@ -2286,3 +2326,15 @@ unsigned int vm_calc_num_guest_pages(enum vm_guest_mode mode, size_t size) n = DIV_ROUND_UP(size, vm_guest_mode_params[mode].page_size); return vm_adjust_num_guest_pages(mode, n); } + +int vm_get_stats_fd(struct kvm_vm *vm) +{ + return ioctl(vm->fd, KVM_GET_STATS_FD, NULL); +} + +int vcpu_get_stats_fd(struct kvm_vm *vm, uint32_t vcpuid) +{ + struct vcpu *vcpu = vcpu_find(vm, vcpuid); + + return ioctl(vcpu->fd, KVM_GET_STATS_FD, NULL); +} diff --git a/tools/testing/selftests/kvm/lib/perf_test_util.c b/tools/testing/selftests/kvm/lib/perf_test_util.c index 7397ca299835..b488f4aefea8 100644 --- a/tools/testing/selftests/kvm/lib/perf_test_util.c +++ b/tools/testing/selftests/kvm/lib/perf_test_util.c @@ -101,7 +101,7 @@ struct kvm_vm *perf_test_create_vm(enum vm_guest_mode mode, int vcpus, guest_num_pages, 0); /* Do mapping for the demand paging memory slot */ - virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages, 0); + virt_map(vm, guest_test_virt_mem, guest_test_phys_mem, guest_num_pages); ucall_init(vm, NULL); diff --git a/tools/testing/selftests/kvm/lib/s390x/processor.c b/tools/testing/selftests/kvm/lib/s390x/processor.c index 0152f356c099..f87c7137598e 100644 --- a/tools/testing/selftests/kvm/lib/s390x/processor.c +++ b/tools/testing/selftests/kvm/lib/s390x/processor.c @@ -9,11 +9,9 @@ #include "kvm_util.h" #include "../kvm_util_internal.h" -#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 - #define PAGES_PER_REGION 4 -void virt_pgd_alloc(struct kvm_vm *vm, uint32_t memslot) +void virt_pgd_alloc(struct kvm_vm *vm) { vm_paddr_t paddr; @@ -24,7 +22,7 @@ void virt_pgd_alloc(struct kvm_vm *vm, uint32_t memslot) return; paddr = vm_phy_pages_alloc(vm, PAGES_PER_REGION, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, memslot); + KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0); memset(addr_gpa2hva(vm, paddr), 0xff, PAGES_PER_REGION * vm->page_size); vm->pgd = paddr; @@ -36,12 +34,12 @@ void virt_pgd_alloc(struct kvm_vm *vm, uint32_t memslot) * a page table (ri == 4). Returns a suitable region/segment table entry * which points to the freshly allocated pages. */ -static uint64_t virt_alloc_region(struct kvm_vm *vm, int ri, uint32_t memslot) +static uint64_t virt_alloc_region(struct kvm_vm *vm, int ri) { uint64_t taddr; taddr = vm_phy_pages_alloc(vm, ri < 4 ? PAGES_PER_REGION : 1, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, memslot); + KVM_GUEST_PAGE_TABLE_MIN_PADDR, 0); memset(addr_gpa2hva(vm, taddr), 0xff, PAGES_PER_REGION * vm->page_size); return (taddr & REGION_ENTRY_ORIGIN) @@ -49,8 +47,7 @@ static uint64_t virt_alloc_region(struct kvm_vm *vm, int ri, uint32_t memslot) | ((ri < 4 ? (PAGES_PER_REGION - 1) : 0) & REGION_ENTRY_LENGTH); } -void virt_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa, - uint32_t memslot) +void virt_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa) { int ri, idx; uint64_t *entry; @@ -77,7 +74,7 @@ void virt_pg_map(struct kvm_vm *vm, uint64_t gva, uint64_t gpa, for (ri = 1; ri <= 4; ri++) { idx = (gva >> (64 - 11 * ri)) & 0x7ffu; if (entry[idx] & REGION_ENTRY_INVALID) - entry[idx] = virt_alloc_region(vm, ri, memslot); + entry[idx] = virt_alloc_region(vm, ri); entry = addr_gpa2hva(vm, entry[idx] & REGION_ENTRY_ORIGIN); } @@ -170,7 +167,7 @@ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) vm->page_size); stack_vaddr = vm_vaddr_alloc(vm, stack_size, - DEFAULT_GUEST_STACK_VADDR_MIN, 0, 0); + DEFAULT_GUEST_STACK_VADDR_MIN); vm_vcpu_add(vm, vcpuid); diff --git a/tools/testing/selftests/kvm/lib/x86_64/apic.c b/tools/testing/selftests/kvm/lib/x86_64/apic.c new file mode 100644 index 000000000000..7168e25c194e --- /dev/null +++ b/tools/testing/selftests/kvm/lib/x86_64/apic.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * tools/testing/selftests/kvm/lib/x86_64/processor.c + * + * Copyright (C) 2021, Google LLC. + */ + +#include "apic.h" + +void apic_disable(void) +{ + wrmsr(MSR_IA32_APICBASE, + rdmsr(MSR_IA32_APICBASE) & + ~(MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD)); +} + +void xapic_enable(void) +{ + uint64_t val = rdmsr(MSR_IA32_APICBASE); + + /* Per SDM: to enable xAPIC when in x2APIC must first disable APIC */ + if (val & MSR_IA32_APICBASE_EXTD) { + apic_disable(); + wrmsr(MSR_IA32_APICBASE, + rdmsr(MSR_IA32_APICBASE) | MSR_IA32_APICBASE_ENABLE); + } else if (!(val & MSR_IA32_APICBASE_ENABLE)) { + wrmsr(MSR_IA32_APICBASE, val | MSR_IA32_APICBASE_ENABLE); + } + + /* + * Per SDM: reset value of spurious interrupt vector register has the + * APIC software enabled bit=0. It must be enabled in addition to the + * enable bit in the MSR. + */ + val = xapic_read_reg(APIC_SPIV) | APIC_SPIV_APIC_ENABLED; + xapic_write_reg(APIC_SPIV, val); +} + +void x2apic_enable(void) +{ + wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) | + MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD); + x2apic_write_reg(APIC_SPIV, + x2apic_read_reg(APIC_SPIV) | APIC_SPIV_APIC_ENABLED); +} diff --git a/tools/testing/selftests/kvm/lib/x86_64/processor.c b/tools/testing/selftests/kvm/lib/x86_64/processor.c index efe235044421..28cb881f440d 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/processor.c +++ b/tools/testing/selftests/kvm/lib/x86_64/processor.c @@ -17,13 +17,10 @@ #define DEFAULT_CODE_SELECTOR 0x8 #define DEFAULT_DATA_SELECTOR 0x10 -/* Minimum physical address used for virtual translation tables. */ -#define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000 - vm_vaddr_t exception_handlers; /* Virtual translation table structure declarations */ -struct pageMapL4Entry { +struct pageUpperEntry { uint64_t present:1; uint64_t writable:1; uint64_t user:1; @@ -33,37 +30,7 @@ struct pageMapL4Entry { uint64_t ignored_06:1; uint64_t page_size:1; uint64_t ignored_11_08:4; - uint64_t address:40; - uint64_t ignored_62_52:11; - uint64_t execute_disable:1; -}; - -struct pageDirectoryPointerEntry { - uint64_t present:1; - uint64_t writable:1; - uint64_t user:1; - uint64_t write_through:1; - uint64_t cache_disable:1; - uint64_t accessed:1; - uint64_t ignored_06:1; - uint64_t page_size:1; - uint64_t ignored_11_08:4; - uint64_t address:40; - uint64_t ignored_62_52:11; - uint64_t execute_disable:1; -}; - -struct pageDirectoryEntry { - uint64_t present:1; - uint64_t writable:1; - uint64_t user:1; - uint64_t write_through:1; - uint64_t cache_disable:1; - uint64_t accessed:1; - uint64_t ignored_06:1; - uint64_t page_size:1; - uint64_t ignored_11_08:4; - uint64_t address:40; + uint64_t pfn:40; uint64_t ignored_62_52:11; uint64_t execute_disable:1; }; @@ -79,7 +46,7 @@ struct pageTableEntry { uint64_t reserved_07:1; uint64_t global:1; uint64_t ignored_11_09:3; - uint64_t address:40; + uint64_t pfn:40; uint64_t ignored_62_52:11; uint64_t execute_disable:1; }; @@ -207,96 +174,211 @@ void sregs_dump(FILE *stream, struct kvm_sregs *sregs, } } -void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot) +void virt_pgd_alloc(struct kvm_vm *vm) { TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use " "unknown or unsupported guest mode, mode: 0x%x", vm->mode); /* If needed, create page map l4 table. */ if (!vm->pgd_created) { - vm_paddr_t paddr = vm_phy_page_alloc(vm, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot); - vm->pgd = paddr; + vm->pgd = vm_alloc_page_table(vm); vm->pgd_created = true; } } -void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, - uint32_t pgd_memslot) +static void *virt_get_pte(struct kvm_vm *vm, uint64_t pt_pfn, uint64_t vaddr, + int level) +{ + uint64_t *page_table = addr_gpa2hva(vm, pt_pfn << vm->page_shift); + int index = vaddr >> (vm->page_shift + level * 9) & 0x1ffu; + + return &page_table[index]; +} + +static struct pageUpperEntry *virt_create_upper_pte(struct kvm_vm *vm, + uint64_t pt_pfn, + uint64_t vaddr, + uint64_t paddr, + int level, + enum x86_page_size page_size) +{ + struct pageUpperEntry *pte = virt_get_pte(vm, pt_pfn, vaddr, level); + + if (!pte->present) { + pte->writable = true; + pte->present = true; + pte->page_size = (level == page_size); + if (pte->page_size) + pte->pfn = paddr >> vm->page_shift; + else + pte->pfn = vm_alloc_page_table(vm) >> vm->page_shift; + } else { + /* + * Entry already present. Assert that the caller doesn't want + * a hugepage at this level, and that there isn't a hugepage at + * this level. + */ + TEST_ASSERT(level != page_size, + "Cannot create hugepage at level: %u, vaddr: 0x%lx\n", + page_size, vaddr); + TEST_ASSERT(!pte->page_size, + "Cannot create page table at level: %u, vaddr: 0x%lx\n", + level, vaddr); + } + return pte; +} + +void __virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr, + enum x86_page_size page_size) +{ + const uint64_t pg_size = 1ull << ((page_size * 9) + 12); + struct pageUpperEntry *pml4e, *pdpe, *pde; + struct pageTableEntry *pte; + + TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, + "Unknown or unsupported guest mode, mode: 0x%x", vm->mode); + + TEST_ASSERT((vaddr % pg_size) == 0, + "Virtual address not aligned,\n" + "vaddr: 0x%lx page size: 0x%lx", vaddr, pg_size); + TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)), + "Invalid virtual address, vaddr: 0x%lx", vaddr); + TEST_ASSERT((paddr % pg_size) == 0, + "Physical address not aligned,\n" + " paddr: 0x%lx page size: 0x%lx", paddr, pg_size); + TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn, + "Physical address beyond maximum supported,\n" + " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", + paddr, vm->max_gfn, vm->page_size); + + /* + * Allocate upper level page tables, if not already present. Return + * early if a hugepage was created. + */ + pml4e = virt_create_upper_pte(vm, vm->pgd >> vm->page_shift, + vaddr, paddr, 3, page_size); + if (pml4e->page_size) + return; + + pdpe = virt_create_upper_pte(vm, pml4e->pfn, vaddr, paddr, 2, page_size); + if (pdpe->page_size) + return; + + pde = virt_create_upper_pte(vm, pdpe->pfn, vaddr, paddr, 1, page_size); + if (pde->page_size) + return; + + /* Fill in page table entry. */ + pte = virt_get_pte(vm, pde->pfn, vaddr, 0); + TEST_ASSERT(!pte->present, + "PTE already present for 4k page at vaddr: 0x%lx\n", vaddr); + pte->pfn = paddr >> vm->page_shift; + pte->writable = true; + pte->present = 1; +} + +void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr) +{ + __virt_pg_map(vm, vaddr, paddr, X86_PAGE_SIZE_4K); +} + +static struct pageTableEntry *_vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, + uint64_t vaddr) { uint16_t index[4]; - struct pageMapL4Entry *pml4e; + struct pageUpperEntry *pml4e, *pdpe, *pde; + struct pageTableEntry *pte; + struct kvm_cpuid_entry2 *entry; + struct kvm_sregs sregs; + int max_phy_addr; + /* Set the bottom 52 bits. */ + uint64_t rsvd_mask = 0x000fffffffffffff; + + entry = kvm_get_supported_cpuid_index(0x80000008, 0); + max_phy_addr = entry->eax & 0x000000ff; + /* Clear the bottom bits of the reserved mask. */ + rsvd_mask = (rsvd_mask >> max_phy_addr) << max_phy_addr; + + /* + * SDM vol 3, fig 4-11 "Formats of CR3 and Paging-Structure Entries + * with 4-Level Paging and 5-Level Paging". + * If IA32_EFER.NXE = 0 and the P flag of a paging-structure entry is 1, + * the XD flag (bit 63) is reserved. + */ + vcpu_sregs_get(vm, vcpuid, &sregs); + if ((sregs.efer & EFER_NX) == 0) { + rsvd_mask |= (1ull << 63); + } TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use " "unknown or unsupported guest mode, mode: 0x%x", vm->mode); - - TEST_ASSERT((vaddr % vm->page_size) == 0, - "Virtual address not on page boundary,\n" - " vaddr: 0x%lx vm->page_size: 0x%x", - vaddr, vm->page_size); TEST_ASSERT(sparsebit_is_set(vm->vpages_valid, (vaddr >> vm->page_shift)), "Invalid virtual address, vaddr: 0x%lx", vaddr); - TEST_ASSERT((paddr % vm->page_size) == 0, - "Physical address not on page boundary,\n" - " paddr: 0x%lx vm->page_size: 0x%x", - paddr, vm->page_size); - TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn, - "Physical address beyond beyond maximum supported,\n" - " paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x", - paddr, vm->max_gfn, vm->page_size); + /* + * Based on the mode check above there are 48 bits in the vaddr, so + * shift 16 to sign extend the last bit (bit-47), + */ + TEST_ASSERT(vaddr == (((int64_t)vaddr << 16) >> 16), + "Canonical check failed. The virtual address is invalid."); index[0] = (vaddr >> 12) & 0x1ffu; index[1] = (vaddr >> 21) & 0x1ffu; index[2] = (vaddr >> 30) & 0x1ffu; index[3] = (vaddr >> 39) & 0x1ffu; - /* Allocate page directory pointer table if not present. */ pml4e = addr_gpa2hva(vm, vm->pgd); - if (!pml4e[index[3]].present) { - pml4e[index[3]].address = vm_phy_page_alloc(vm, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot) - >> vm->page_shift; - pml4e[index[3]].writable = true; - pml4e[index[3]].present = true; - } + TEST_ASSERT(pml4e[index[3]].present, + "Expected pml4e to be present for gva: 0x%08lx", vaddr); + TEST_ASSERT((*(uint64_t*)(&pml4e[index[3]]) & + (rsvd_mask | (1ull << 7))) == 0, + "Unexpected reserved bits set."); - /* Allocate page directory table if not present. */ - struct pageDirectoryPointerEntry *pdpe; - pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size); - if (!pdpe[index[2]].present) { - pdpe[index[2]].address = vm_phy_page_alloc(vm, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot) - >> vm->page_shift; - pdpe[index[2]].writable = true; - pdpe[index[2]].present = true; - } + pdpe = addr_gpa2hva(vm, pml4e[index[3]].pfn * vm->page_size); + TEST_ASSERT(pdpe[index[2]].present, + "Expected pdpe to be present for gva: 0x%08lx", vaddr); + TEST_ASSERT(pdpe[index[2]].page_size == 0, + "Expected pdpe to map a pde not a 1-GByte page."); + TEST_ASSERT((*(uint64_t*)(&pdpe[index[2]]) & rsvd_mask) == 0, + "Unexpected reserved bits set."); - /* Allocate page table if not present. */ - struct pageDirectoryEntry *pde; - pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size); - if (!pde[index[1]].present) { - pde[index[1]].address = vm_phy_page_alloc(vm, - KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot) - >> vm->page_shift; - pde[index[1]].writable = true; - pde[index[1]].present = true; - } + pde = addr_gpa2hva(vm, pdpe[index[2]].pfn * vm->page_size); + TEST_ASSERT(pde[index[1]].present, + "Expected pde to be present for gva: 0x%08lx", vaddr); + TEST_ASSERT(pde[index[1]].page_size == 0, + "Expected pde to map a pte not a 2-MByte page."); + TEST_ASSERT((*(uint64_t*)(&pde[index[1]]) & rsvd_mask) == 0, + "Unexpected reserved bits set."); - /* Fill in page table entry. */ - struct pageTableEntry *pte; - pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size); - pte[index[0]].address = paddr >> vm->page_shift; - pte[index[0]].writable = true; - pte[index[0]].present = 1; + pte = addr_gpa2hva(vm, pde[index[1]].pfn * vm->page_size); + TEST_ASSERT(pte[index[0]].present, + "Expected pte to be present for gva: 0x%08lx", vaddr); + + return &pte[index[0]]; +} + +uint64_t vm_get_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr) +{ + struct pageTableEntry *pte = _vm_get_page_table_entry(vm, vcpuid, vaddr); + + return *(uint64_t *)pte; +} + +void vm_set_page_table_entry(struct kvm_vm *vm, int vcpuid, uint64_t vaddr, + uint64_t pte) +{ + struct pageTableEntry *new_pte = _vm_get_page_table_entry(vm, vcpuid, + vaddr); + + *(uint64_t *)new_pte = pte; } void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) { - struct pageMapL4Entry *pml4e, *pml4e_start; - struct pageDirectoryPointerEntry *pdpe, *pdpe_start; - struct pageDirectoryEntry *pde, *pde_start; + struct pageUpperEntry *pml4e, *pml4e_start; + struct pageUpperEntry *pdpe, *pdpe_start; + struct pageUpperEntry *pde, *pde_start; struct pageTableEntry *pte, *pte_start; if (!vm->pgd_created) @@ -307,8 +389,7 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) fprintf(stream, "%*s index hvaddr gpaddr " "addr w exec dirty\n", indent, ""); - pml4e_start = (struct pageMapL4Entry *) addr_gpa2hva(vm, - vm->pgd); + pml4e_start = (struct pageUpperEntry *) addr_gpa2hva(vm, vm->pgd); for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) { pml4e = &pml4e_start[n1]; if (!pml4e->present) @@ -317,11 +398,10 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) " %u\n", indent, "", pml4e - pml4e_start, pml4e, - addr_hva2gpa(vm, pml4e), (uint64_t) pml4e->address, + addr_hva2gpa(vm, pml4e), (uint64_t) pml4e->pfn, pml4e->writable, pml4e->execute_disable); - pdpe_start = addr_gpa2hva(vm, pml4e->address - * vm->page_size); + pdpe_start = addr_gpa2hva(vm, pml4e->pfn * vm->page_size); for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) { pdpe = &pdpe_start[n2]; if (!pdpe->present) @@ -331,11 +411,10 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) indent, "", pdpe - pdpe_start, pdpe, addr_hva2gpa(vm, pdpe), - (uint64_t) pdpe->address, pdpe->writable, + (uint64_t) pdpe->pfn, pdpe->writable, pdpe->execute_disable); - pde_start = addr_gpa2hva(vm, - pdpe->address * vm->page_size); + pde_start = addr_gpa2hva(vm, pdpe->pfn * vm->page_size); for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) { pde = &pde_start[n3]; if (!pde->present) @@ -344,11 +423,10 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) "0x%-12lx 0x%-10lx %u %u\n", indent, "", pde - pde_start, pde, addr_hva2gpa(vm, pde), - (uint64_t) pde->address, pde->writable, + (uint64_t) pde->pfn, pde->writable, pde->execute_disable); - pte_start = addr_gpa2hva(vm, - pde->address * vm->page_size); + pte_start = addr_gpa2hva(vm, pde->pfn * vm->page_size); for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) { pte = &pte_start[n4]; if (!pte->present) @@ -359,7 +437,7 @@ void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent) indent, "", pte - pte_start, pte, addr_hva2gpa(vm, pte), - (uint64_t) pte->address, + (uint64_t) pte->pfn, pte->writable, pte->execute_disable, pte->dirty, @@ -480,9 +558,7 @@ static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector, vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) { uint16_t index[4]; - struct pageMapL4Entry *pml4e; - struct pageDirectoryPointerEntry *pdpe; - struct pageDirectoryEntry *pde; + struct pageUpperEntry *pml4e, *pdpe, *pde; struct pageTableEntry *pte; TEST_ASSERT(vm->mode == VM_MODE_PXXV48_4K, "Attempt to use " @@ -499,43 +575,39 @@ vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva) if (!pml4e[index[3]].present) goto unmapped_gva; - pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size); + pdpe = addr_gpa2hva(vm, pml4e[index[3]].pfn * vm->page_size); if (!pdpe[index[2]].present) goto unmapped_gva; - pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size); + pde = addr_gpa2hva(vm, pdpe[index[2]].pfn * vm->page_size); if (!pde[index[1]].present) goto unmapped_gva; - pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size); + pte = addr_gpa2hva(vm, pde[index[1]].pfn * vm->page_size); if (!pte[index[0]].present) goto unmapped_gva; - return (pte[index[0]].address * vm->page_size) + (gva & 0xfffu); + return (pte[index[0]].pfn * vm->page_size) + (gva & 0xfffu); unmapped_gva: TEST_FAIL("No mapping for vm virtual address, gva: 0x%lx", gva); exit(EXIT_FAILURE); } -static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt, int gdt_memslot, - int pgd_memslot) +static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt) { if (!vm->gdt) - vm->gdt = vm_vaddr_alloc(vm, getpagesize(), - KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot); + vm->gdt = vm_vaddr_alloc_page(vm); dt->base = vm->gdt; dt->limit = getpagesize(); } static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp, - int selector, int gdt_memslot, - int pgd_memslot) + int selector) { if (!vm->tss) - vm->tss = vm_vaddr_alloc(vm, getpagesize(), - KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot); + vm->tss = vm_vaddr_alloc_page(vm); memset(segp, 0, sizeof(*segp)); segp->base = vm->tss; @@ -546,7 +618,7 @@ static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp, kvm_seg_fill_gdt_64bit(vm, segp); } -static void vcpu_setup(struct kvm_vm *vm, int vcpuid, int pgd_memslot, int gdt_memslot) +static void vcpu_setup(struct kvm_vm *vm, int vcpuid) { struct kvm_sregs sregs; @@ -555,7 +627,7 @@ static void vcpu_setup(struct kvm_vm *vm, int vcpuid, int pgd_memslot, int gdt_m sregs.idt.limit = 0; - kvm_setup_gdt(vm, &sregs.gdt, gdt_memslot, pgd_memslot); + kvm_setup_gdt(vm, &sregs.gdt); switch (vm->mode) { case VM_MODE_PXXV48_4K: @@ -567,7 +639,7 @@ static void vcpu_setup(struct kvm_vm *vm, int vcpuid, int pgd_memslot, int gdt_m kvm_seg_set_kernel_code_64bit(vm, DEFAULT_CODE_SELECTOR, &sregs.cs); kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.ds); kvm_seg_set_kernel_data_64bit(vm, DEFAULT_DATA_SELECTOR, &sregs.es); - kvm_setup_tss_64bit(vm, &sregs.tr, 0x18, gdt_memslot, pgd_memslot); + kvm_setup_tss_64bit(vm, &sregs.tr, 0x18); break; default: @@ -584,11 +656,11 @@ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) struct kvm_regs regs; vm_vaddr_t stack_vaddr; stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(), - DEFAULT_GUEST_STACK_VADDR_MIN, 0, 0); + DEFAULT_GUEST_STACK_VADDR_MIN); /* Create VCPU */ vm_vcpu_add(vm, vcpuid); - vcpu_setup(vm, vcpuid, 0, 0); + vcpu_setup(vm, vcpuid); /* Setup guest general purpose registers */ vcpu_regs_get(vm, vcpuid, ®s); @@ -600,6 +672,9 @@ void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code) /* Setup the MP state */ mp_state.mp_state = 0; vcpu_set_mp_state(vm, vcpuid, &mp_state); + + /* Setup supported CPUIDs */ + vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid()); } /* @@ -1201,7 +1276,7 @@ static void set_idt_entry(struct kvm_vm *vm, int vector, unsigned long addr, void kvm_exit_unexpected_vector(uint32_t value) { - outl(UNEXPECTED_VECTOR_PORT, value); + ucall(UCALL_UNHANDLED, 1, value); } void route_exception(struct ex_regs *regs) @@ -1222,8 +1297,8 @@ void vm_init_descriptor_tables(struct kvm_vm *vm) extern void *idt_handlers; int i; - vm->idt = vm_vaddr_alloc(vm, getpagesize(), 0x2000, 0, 0); - vm->handlers = vm_vaddr_alloc(vm, 256 * sizeof(void *), 0x2000, 0, 0); + vm->idt = vm_vaddr_alloc_page(vm); + vm->handlers = vm_vaddr_alloc_page(vm); /* Handlers have the same address in both address spaces.*/ for (i = 0; i < NUM_INTERRUPTS; i++) set_idt_entry(vm, i, (unsigned long)(&idt_handlers)[i], 0, @@ -1244,8 +1319,8 @@ void vcpu_init_descriptor_tables(struct kvm_vm *vm, uint32_t vcpuid) *(vm_vaddr_t *)addr_gva2hva(vm, (vm_vaddr_t)(&exception_handlers)) = vm->handlers; } -void vm_handle_exception(struct kvm_vm *vm, int vector, - void (*handler)(struct ex_regs *)) +void vm_install_exception_handler(struct kvm_vm *vm, int vector, + void (*handler)(struct ex_regs *)) { vm_vaddr_t *handlers = (vm_vaddr_t *)addr_gva2hva(vm, vm->handlers); @@ -1254,16 +1329,13 @@ void vm_handle_exception(struct kvm_vm *vm, int vector, void assert_on_unhandled_exception(struct kvm_vm *vm, uint32_t vcpuid) { - if (vcpu_state(vm, vcpuid)->exit_reason == KVM_EXIT_IO - && vcpu_state(vm, vcpuid)->io.port == UNEXPECTED_VECTOR_PORT - && vcpu_state(vm, vcpuid)->io.size == 4) { - /* Grab pointer to io data */ - uint32_t *data = (void *)vcpu_state(vm, vcpuid) - + vcpu_state(vm, vcpuid)->io.data_offset; + struct ucall uc; - TEST_ASSERT(false, - "Unexpected vectored event in guest (vector:0x%x)", - *data); + if (get_ucall(vm, vcpuid, &uc) == UCALL_UNHANDLED) { + uint64_t vector = uc.args[0]; + + TEST_FAIL("Unexpected vectored event in guest (vector:0x%lx)", + vector); } } diff --git a/tools/testing/selftests/kvm/lib/x86_64/svm.c b/tools/testing/selftests/kvm/lib/x86_64/svm.c index 827fe6028dd4..2ac98d70d02b 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/svm.c +++ b/tools/testing/selftests/kvm/lib/x86_64/svm.c @@ -30,17 +30,14 @@ u64 rflags; struct svm_test_data * vcpu_alloc_svm(struct kvm_vm *vm, vm_vaddr_t *p_svm_gva) { - vm_vaddr_t svm_gva = vm_vaddr_alloc(vm, getpagesize(), - 0x10000, 0, 0); + vm_vaddr_t svm_gva = vm_vaddr_alloc_page(vm); struct svm_test_data *svm = addr_gva2hva(vm, svm_gva); - svm->vmcb = (void *)vm_vaddr_alloc(vm, getpagesize(), - 0x10000, 0, 0); + svm->vmcb = (void *)vm_vaddr_alloc_page(vm); svm->vmcb_hva = addr_gva2hva(vm, (uintptr_t)svm->vmcb); svm->vmcb_gpa = addr_gva2gpa(vm, (uintptr_t)svm->vmcb); - svm->save_area = (void *)vm_vaddr_alloc(vm, getpagesize(), - 0x10000, 0, 0); + svm->save_area = (void *)vm_vaddr_alloc_page(vm); svm->save_area_hva = addr_gva2hva(vm, (uintptr_t)svm->save_area); svm->save_area_gpa = addr_gva2gpa(vm, (uintptr_t)svm->save_area); diff --git a/tools/testing/selftests/kvm/lib/x86_64/vmx.c b/tools/testing/selftests/kvm/lib/x86_64/vmx.c index 2448b30e8efa..d089d8b850b5 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86_64/vmx.c @@ -77,50 +77,48 @@ int vcpu_enable_evmcs(struct kvm_vm *vm, int vcpu_id) struct vmx_pages * vcpu_alloc_vmx(struct kvm_vm *vm, vm_vaddr_t *p_vmx_gva) { - vm_vaddr_t vmx_gva = vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vm_vaddr_t vmx_gva = vm_vaddr_alloc_page(vm); struct vmx_pages *vmx = addr_gva2hva(vm, vmx_gva); /* Setup of a region of guest memory for the vmxon region. */ - vmx->vmxon = (void *)vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vmx->vmxon = (void *)vm_vaddr_alloc_page(vm); vmx->vmxon_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmxon); vmx->vmxon_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmxon); /* Setup of a region of guest memory for a vmcs. */ - vmx->vmcs = (void *)vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vmx->vmcs = (void *)vm_vaddr_alloc_page(vm); vmx->vmcs_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmcs); vmx->vmcs_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmcs); /* Setup of a region of guest memory for the MSR bitmap. */ - vmx->msr = (void *)vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vmx->msr = (void *)vm_vaddr_alloc_page(vm); vmx->msr_hva = addr_gva2hva(vm, (uintptr_t)vmx->msr); vmx->msr_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->msr); memset(vmx->msr_hva, 0, getpagesize()); /* Setup of a region of guest memory for the shadow VMCS. */ - vmx->shadow_vmcs = (void *)vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vmx->shadow_vmcs = (void *)vm_vaddr_alloc_page(vm); vmx->shadow_vmcs_hva = addr_gva2hva(vm, (uintptr_t)vmx->shadow_vmcs); vmx->shadow_vmcs_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->shadow_vmcs); /* Setup of a region of guest memory for the VMREAD and VMWRITE bitmaps. */ - vmx->vmread = (void *)vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vmx->vmread = (void *)vm_vaddr_alloc_page(vm); vmx->vmread_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmread); vmx->vmread_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmread); memset(vmx->vmread_hva, 0, getpagesize()); - vmx->vmwrite = (void *)vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vmx->vmwrite = (void *)vm_vaddr_alloc_page(vm); vmx->vmwrite_hva = addr_gva2hva(vm, (uintptr_t)vmx->vmwrite); vmx->vmwrite_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vmwrite); memset(vmx->vmwrite_hva, 0, getpagesize()); /* Setup of a region of guest memory for the VP Assist page. */ - vmx->vp_assist = (void *)vm_vaddr_alloc(vm, getpagesize(), - 0x10000, 0, 0); + vmx->vp_assist = (void *)vm_vaddr_alloc_page(vm); vmx->vp_assist_hva = addr_gva2hva(vm, (uintptr_t)vmx->vp_assist); vmx->vp_assist_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->vp_assist); /* Setup of a region of guest memory for the enlightened VMCS. */ - vmx->enlightened_vmcs = (void *)vm_vaddr_alloc(vm, getpagesize(), - 0x10000, 0, 0); + vmx->enlightened_vmcs = (void *)vm_vaddr_alloc_page(vm); vmx->enlightened_vmcs_hva = addr_gva2hva(vm, (uintptr_t)vmx->enlightened_vmcs); vmx->enlightened_vmcs_gpa = @@ -395,7 +393,7 @@ void nested_vmx_check_supported(void) } void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm, - uint64_t nested_paddr, uint64_t paddr, uint32_t eptp_memslot) + uint64_t nested_paddr, uint64_t paddr) { uint16_t index[4]; struct eptPageTableEntry *pml4e; @@ -428,9 +426,7 @@ void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm, /* Allocate page directory pointer table if not present. */ pml4e = vmx->eptp_hva; if (!pml4e[index[3]].readable) { - pml4e[index[3]].address = vm_phy_page_alloc(vm, - KVM_EPT_PAGE_TABLE_MIN_PADDR, eptp_memslot) - >> vm->page_shift; + pml4e[index[3]].address = vm_alloc_page_table(vm) >> vm->page_shift; pml4e[index[3]].writable = true; pml4e[index[3]].readable = true; pml4e[index[3]].executable = true; @@ -440,9 +436,7 @@ void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm, struct eptPageTableEntry *pdpe; pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size); if (!pdpe[index[2]].readable) { - pdpe[index[2]].address = vm_phy_page_alloc(vm, - KVM_EPT_PAGE_TABLE_MIN_PADDR, eptp_memslot) - >> vm->page_shift; + pdpe[index[2]].address = vm_alloc_page_table(vm) >> vm->page_shift; pdpe[index[2]].writable = true; pdpe[index[2]].readable = true; pdpe[index[2]].executable = true; @@ -452,9 +446,7 @@ void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm, struct eptPageTableEntry *pde; pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size); if (!pde[index[1]].readable) { - pde[index[1]].address = vm_phy_page_alloc(vm, - KVM_EPT_PAGE_TABLE_MIN_PADDR, eptp_memslot) - >> vm->page_shift; + pde[index[1]].address = vm_alloc_page_table(vm) >> vm->page_shift; pde[index[1]].writable = true; pde[index[1]].readable = true; pde[index[1]].executable = true; @@ -494,8 +486,7 @@ void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm, * page range starting at nested_paddr to the page range starting at paddr. */ void nested_map(struct vmx_pages *vmx, struct kvm_vm *vm, - uint64_t nested_paddr, uint64_t paddr, uint64_t size, - uint32_t eptp_memslot) + uint64_t nested_paddr, uint64_t paddr, uint64_t size) { size_t page_size = vm->page_size; size_t npages = size / page_size; @@ -504,7 +495,7 @@ void nested_map(struct vmx_pages *vmx, struct kvm_vm *vm, TEST_ASSERT(paddr + size > paddr, "Paddr overflow"); while (npages--) { - nested_pg_map(vmx, vm, nested_paddr, paddr, eptp_memslot); + nested_pg_map(vmx, vm, nested_paddr, paddr); nested_paddr += page_size; paddr += page_size; } @@ -514,7 +505,7 @@ void nested_map(struct vmx_pages *vmx, struct kvm_vm *vm, * physical pages in VM. */ void nested_map_memslot(struct vmx_pages *vmx, struct kvm_vm *vm, - uint32_t memslot, uint32_t eptp_memslot) + uint32_t memslot) { sparsebit_idx_t i, last; struct userspace_mem_region *region = @@ -530,24 +521,21 @@ void nested_map_memslot(struct vmx_pages *vmx, struct kvm_vm *vm, nested_map(vmx, vm, (uint64_t)i << vm->page_shift, (uint64_t)i << vm->page_shift, - 1 << vm->page_shift, - eptp_memslot); + 1 << vm->page_shift); } } void prepare_eptp(struct vmx_pages *vmx, struct kvm_vm *vm, uint32_t eptp_memslot) { - vmx->eptp = (void *)vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + vmx->eptp = (void *)vm_vaddr_alloc_page(vm); vmx->eptp_hva = addr_gva2hva(vm, (uintptr_t)vmx->eptp); vmx->eptp_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->eptp); } -void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm, - uint32_t eptp_memslot) +void prepare_virtualize_apic_accesses(struct vmx_pages *vmx, struct kvm_vm *vm) { - vmx->apic_access = (void *)vm_vaddr_alloc(vm, getpagesize(), - 0x10000, 0, 0); + vmx->apic_access = (void *)vm_vaddr_alloc_page(vm); vmx->apic_access_hva = addr_gva2hva(vm, (uintptr_t)vmx->apic_access); vmx->apic_access_gpa = addr_gva2gpa(vm, (uintptr_t)vmx->apic_access); } diff --git a/tools/testing/selftests/kvm/memslot_perf_test.c b/tools/testing/selftests/kvm/memslot_perf_test.c index 11239652d805..d6e381e01db7 100644 --- a/tools/testing/selftests/kvm/memslot_perf_test.c +++ b/tools/testing/selftests/kvm/memslot_perf_test.c @@ -306,7 +306,7 @@ static bool prepare_vm(struct vm_data *data, int nslots, uint64_t *maxslots, guest_addr += npages * 4096; } - virt_map(data->vm, MEM_GPA, MEM_GPA, mempages, 0); + virt_map(data->vm, MEM_GPA, MEM_GPA, mempages); sync = (typeof(sync))vm_gpa2hva(data, MEM_SYNC_GPA, NULL); atomic_init(&sync->start_flag, false); diff --git a/tools/testing/selftests/kvm/set_memory_region_test.c b/tools/testing/selftests/kvm/set_memory_region_test.c index d8812f27648c..85b18bb8f762 100644 --- a/tools/testing/selftests/kvm/set_memory_region_test.c +++ b/tools/testing/selftests/kvm/set_memory_region_test.c @@ -132,7 +132,7 @@ static struct kvm_vm *spawn_vm(pthread_t *vcpu_thread, void *guest_code) gpa = vm_phy_pages_alloc(vm, 2, MEM_REGION_GPA, MEM_REGION_SLOT); TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); - virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2, 0); + virt_map(vm, MEM_REGION_GPA, MEM_REGION_GPA, 2); /* Ditto for the host mapping so that both pages can be zeroed. */ hva = addr_gpa2hva(vm, MEM_REGION_GPA); diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index fcc840088c91..b0031f2d38fd 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -73,8 +73,6 @@ static void steal_time_init(struct kvm_vm *vm) for (i = 0; i < NR_VCPUS; ++i) { int ret; - vcpu_set_cpuid(vm, i, kvm_get_supported_cpuid()); - /* ST_GPA_BASE is identity mapped */ st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); sync_global_to_guest(vm, st_gva[i]); @@ -295,7 +293,7 @@ int main(int ac, char **av) vm = vm_create_default(0, 0, guest_code); gpages = vm_calc_num_guest_pages(VM_MODE_DEFAULT, STEAL_TIME_SIZE * NR_VCPUS); vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, ST_GPA_BASE, 1, gpages, 0); - virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages, 0); + virt_map(vm, ST_GPA_BASE, ST_GPA_BASE, gpages); ucall_init(vm, NULL); /* Add the rest of the VCPUs */ diff --git a/tools/testing/selftests/kvm/x86_64/emulator_error_test.c b/tools/testing/selftests/kvm/x86_64/emulator_error_test.c new file mode 100644 index 000000000000..f070ff0224fa --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/emulator_error_test.c @@ -0,0 +1,219 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020, Google LLC. + * + * Tests for KVM_CAP_EXIT_ON_EMULATION_FAILURE capability. + */ + +#define _GNU_SOURCE /* for program_invocation_short_name */ + +#include "test_util.h" +#include "kvm_util.h" +#include "vmx.h" + +#define VCPU_ID 1 +#define PAGE_SIZE 4096 +#define MAXPHYADDR 36 + +#define MEM_REGION_GVA 0x0000123456789000 +#define MEM_REGION_GPA 0x0000000700000000 +#define MEM_REGION_SLOT 10 +#define MEM_REGION_SIZE PAGE_SIZE + +static void guest_code(void) +{ + __asm__ __volatile__("flds (%[addr])" + :: [addr]"r"(MEM_REGION_GVA)); + + GUEST_DONE(); +} + +static void run_guest(struct kvm_vm *vm) +{ + int rc; + + rc = _vcpu_run(vm, VCPU_ID); + TEST_ASSERT(rc == 0, "vcpu_run failed: %d\n", rc); +} + +/* + * Accessors to get R/M, REG, and Mod bits described in the SDM vol 2, + * figure 2-2 "Table Interpretation of ModR/M Byte (C8H)". + */ +#define GET_RM(insn_byte) (insn_byte & 0x7) +#define GET_REG(insn_byte) ((insn_byte & 0x38) >> 3) +#define GET_MOD(insn_byte) ((insn_byte & 0xc) >> 6) + +/* Ensure we are dealing with a simple 2-byte flds instruction. */ +static bool is_flds(uint8_t *insn_bytes, uint8_t insn_size) +{ + return insn_size >= 2 && + insn_bytes[0] == 0xd9 && + GET_REG(insn_bytes[1]) == 0x0 && + GET_MOD(insn_bytes[1]) == 0x0 && + /* Ensure there is no SIB byte. */ + GET_RM(insn_bytes[1]) != 0x4 && + /* Ensure there is no displacement byte. */ + GET_RM(insn_bytes[1]) != 0x5; +} + +static void process_exit_on_emulation_error(struct kvm_vm *vm) +{ + struct kvm_run *run = vcpu_state(vm, VCPU_ID); + struct kvm_regs regs; + uint8_t *insn_bytes; + uint8_t insn_size; + uint64_t flags; + + TEST_ASSERT(run->exit_reason == KVM_EXIT_INTERNAL_ERROR, + "Unexpected exit reason: %u (%s)", + run->exit_reason, + exit_reason_str(run->exit_reason)); + + TEST_ASSERT(run->emulation_failure.suberror == KVM_INTERNAL_ERROR_EMULATION, + "Unexpected suberror: %u", + run->emulation_failure.suberror); + + if (run->emulation_failure.ndata >= 1) { + flags = run->emulation_failure.flags; + if ((flags & KVM_INTERNAL_ERROR_EMULATION_FLAG_INSTRUCTION_BYTES) && + run->emulation_failure.ndata >= 3) { + insn_size = run->emulation_failure.insn_size; + insn_bytes = run->emulation_failure.insn_bytes; + + TEST_ASSERT(insn_size <= 15 && insn_size > 0, + "Unexpected instruction size: %u", + insn_size); + + TEST_ASSERT(is_flds(insn_bytes, insn_size), + "Unexpected instruction. Expected 'flds' (0xd9 /0)"); + + /* + * If is_flds() succeeded then the instruction bytes + * contained an flds instruction that is 2-bytes in + * length (ie: no prefix, no SIB, no displacement). + */ + vcpu_regs_get(vm, VCPU_ID, ®s); + regs.rip += 2; + vcpu_regs_set(vm, VCPU_ID, ®s); + } + } +} + +static void do_guest_assert(struct kvm_vm *vm, struct ucall *uc) +{ + TEST_FAIL("%s at %s:%ld", (const char *)uc->args[0], __FILE__, + uc->args[1]); +} + +static void check_for_guest_assert(struct kvm_vm *vm) +{ + struct kvm_run *run = vcpu_state(vm, VCPU_ID); + struct ucall uc; + + if (run->exit_reason == KVM_EXIT_IO && + get_ucall(vm, VCPU_ID, &uc) == UCALL_ABORT) { + do_guest_assert(vm, &uc); + } +} + +static void process_ucall_done(struct kvm_vm *vm) +{ + struct kvm_run *run = vcpu_state(vm, VCPU_ID); + struct ucall uc; + + check_for_guest_assert(vm); + + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Unexpected exit reason: %u (%s)", + run->exit_reason, + exit_reason_str(run->exit_reason)); + + TEST_ASSERT(get_ucall(vm, VCPU_ID, &uc) == UCALL_DONE, + "Unexpected ucall command: %lu, expected UCALL_DONE (%d)", + uc.cmd, UCALL_DONE); +} + +static uint64_t process_ucall(struct kvm_vm *vm) +{ + struct kvm_run *run = vcpu_state(vm, VCPU_ID); + struct ucall uc; + + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Unexpected exit reason: %u (%s)", + run->exit_reason, + exit_reason_str(run->exit_reason)); + + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_SYNC: + break; + case UCALL_ABORT: + do_guest_assert(vm, &uc); + break; + case UCALL_DONE: + process_ucall_done(vm); + break; + default: + TEST_ASSERT(false, "Unexpected ucall"); + } + + return uc.cmd; +} + +int main(int argc, char *argv[]) +{ + struct kvm_enable_cap emul_failure_cap = { + .cap = KVM_CAP_EXIT_ON_EMULATION_FAILURE, + .args[0] = 1, + }; + struct kvm_cpuid_entry2 *entry; + struct kvm_cpuid2 *cpuid; + struct kvm_vm *vm; + uint64_t gpa, pte; + uint64_t *hva; + int rc; + + /* Tell stdout not to buffer its content */ + setbuf(stdout, NULL); + + vm = vm_create_default(VCPU_ID, 0, guest_code); + + if (!kvm_check_cap(KVM_CAP_SMALLER_MAXPHYADDR)) { + printf("module parameter 'allow_smaller_maxphyaddr' is not set. Skipping test.\n"); + return 0; + } + + cpuid = kvm_get_supported_cpuid(); + + entry = kvm_get_supported_cpuid_index(0x80000008, 0); + entry->eax = (entry->eax & 0xffffff00) | MAXPHYADDR; + set_cpuid(cpuid, entry); + + vcpu_set_cpuid(vm, VCPU_ID, cpuid); + + rc = kvm_check_cap(KVM_CAP_EXIT_ON_EMULATION_FAILURE); + TEST_ASSERT(rc, "KVM_CAP_EXIT_ON_EMULATION_FAILURE is unavailable"); + vm_enable_cap(vm, &emul_failure_cap); + + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + MEM_REGION_GPA, MEM_REGION_SLOT, + MEM_REGION_SIZE / PAGE_SIZE, 0); + gpa = vm_phy_pages_alloc(vm, MEM_REGION_SIZE / PAGE_SIZE, + MEM_REGION_GPA, MEM_REGION_SLOT); + TEST_ASSERT(gpa == MEM_REGION_GPA, "Failed vm_phy_pages_alloc\n"); + virt_map(vm, MEM_REGION_GVA, MEM_REGION_GPA, 1); + hva = addr_gpa2hva(vm, MEM_REGION_GPA); + memset(hva, 0, PAGE_SIZE); + pte = vm_get_page_table_entry(vm, VCPU_ID, MEM_REGION_GVA); + vm_set_page_table_entry(vm, VCPU_ID, MEM_REGION_GVA, pte | (1ull << 36)); + + run_guest(vm); + process_exit_on_emulation_error(vm); + run_guest(vm); + + TEST_ASSERT(process_ucall(vm) == UCALL_DONE, "Expected UCALL_DONE"); + + kvm_vm_free(vm); + + return 0; +} diff --git a/tools/testing/selftests/kvm/x86_64/evmcs_test.c b/tools/testing/selftests/kvm/x86_64/evmcs_test.c index 63096cea26c6..2b46dcca86a8 100644 --- a/tools/testing/selftests/kvm/x86_64/evmcs_test.c +++ b/tools/testing/selftests/kvm/x86_64/evmcs_test.c @@ -22,15 +22,6 @@ static int ud_count; -void enable_x2apic(void) -{ - uint32_t spiv_reg = APIC_BASE_MSR + (APIC_SPIV >> 4); - - wrmsr(MSR_IA32_APICBASE, rdmsr(MSR_IA32_APICBASE) | - MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD); - wrmsr(spiv_reg, rdmsr(spiv_reg) | APIC_SPIV_APIC_ENABLED); -} - static void guest_ud_handler(struct ex_regs *regs) { ud_count++; @@ -59,7 +50,7 @@ void guest_code(struct vmx_pages *vmx_pages) #define L2_GUEST_STACK_SIZE 64 unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; - enable_x2apic(); + x2apic_enable(); GUEST_SYNC(1); GUEST_SYNC(2); @@ -121,14 +112,38 @@ void inject_nmi(struct kvm_vm *vm) vcpu_events_set(vm, VCPU_ID, &events); } +static void save_restore_vm(struct kvm_vm *vm) +{ + struct kvm_regs regs1, regs2; + struct kvm_x86_state *state; + + state = vcpu_save_state(vm, VCPU_ID); + memset(®s1, 0, sizeof(regs1)); + vcpu_regs_get(vm, VCPU_ID, ®s1); + + kvm_vm_release(vm); + + /* Restore state in a new VM. */ + kvm_vm_restart(vm, O_RDWR); + vm_vcpu_add(vm, VCPU_ID); + vcpu_set_hv_cpuid(vm, VCPU_ID); + vcpu_enable_evmcs(vm, VCPU_ID); + vcpu_load_state(vm, VCPU_ID, state); + free(state); + + memset(®s2, 0, sizeof(regs2)); + vcpu_regs_get(vm, VCPU_ID, ®s2); + TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), + "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", + (ulong) regs2.rdi, (ulong) regs2.rsi); +} + int main(int argc, char *argv[]) { vm_vaddr_t vmx_pages_gva = 0; - struct kvm_regs regs1, regs2; struct kvm_vm *vm; struct kvm_run *run; - struct kvm_x86_state *state; struct ucall uc; int stage; @@ -145,21 +160,18 @@ int main(int argc, char *argv[]) vcpu_set_hv_cpuid(vm, VCPU_ID); vcpu_enable_evmcs(vm, VCPU_ID); - run = vcpu_state(vm, VCPU_ID); - - vcpu_regs_get(vm, VCPU_ID, ®s1); - vcpu_alloc_vmx(vm, &vmx_pages_gva); vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(vm, VCPU_ID); - vm_handle_exception(vm, UD_VECTOR, guest_ud_handler); - vm_handle_exception(vm, NMI_VECTOR, guest_nmi_handler); + vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); + vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); pr_info("Running L1 which uses EVMCS to run L2\n"); for (stage = 1;; stage++) { + run = vcpu_state(vm, VCPU_ID); _vcpu_run(vm, VCPU_ID); TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, "Stage %d: unexpected exit reason: %u (%s),\n", @@ -184,32 +196,23 @@ int main(int argc, char *argv[]) uc.args[1] == stage, "Stage %d: Unexpected register values vmexit, got %lx", stage, (ulong)uc.args[1]); - state = vcpu_save_state(vm, VCPU_ID); - memset(®s1, 0, sizeof(regs1)); - vcpu_regs_get(vm, VCPU_ID, ®s1); - - kvm_vm_release(vm); - - /* Restore state in a new VM. */ - kvm_vm_restart(vm, O_RDWR); - vm_vcpu_add(vm, VCPU_ID); - vcpu_set_hv_cpuid(vm, VCPU_ID); - vcpu_enable_evmcs(vm, VCPU_ID); - vcpu_load_state(vm, VCPU_ID, state); - run = vcpu_state(vm, VCPU_ID); - free(state); - - memset(®s2, 0, sizeof(regs2)); - vcpu_regs_get(vm, VCPU_ID, ®s2); - TEST_ASSERT(!memcmp(®s1, ®s2, sizeof(regs2)), - "Unexpected register values after vcpu_load_state; rdi: %lx rsi: %lx", - (ulong) regs2.rdi, (ulong) regs2.rsi); + save_restore_vm(vm); /* Force immediate L2->L1 exit before resuming */ if (stage == 8) { pr_info("Injecting NMI into L1 before L2 had a chance to run after restore\n"); inject_nmi(vm); } + + /* + * Do KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE for a freshly + * restored VM (before the first KVM_RUN) to check that + * KVM_STATE_NESTED_EVMCS is not lost. + */ + if (stage == 9) { + pr_info("Trying extra KVM_GET_NESTED_STATE/KVM_SET_NESTED_STATE cycle\n"); + save_restore_vm(vm); + } } done: diff --git a/tools/testing/selftests/kvm/x86_64/get_cpuid_test.c b/tools/testing/selftests/kvm/x86_64/get_cpuid_test.c index 8c77537af5a1..a711f83749ea 100644 --- a/tools/testing/selftests/kvm/x86_64/get_cpuid_test.c +++ b/tools/testing/selftests/kvm/x86_64/get_cpuid_test.c @@ -145,8 +145,7 @@ static void run_vcpu(struct kvm_vm *vm, uint32_t vcpuid, int stage) struct kvm_cpuid2 *vcpu_alloc_cpuid(struct kvm_vm *vm, vm_vaddr_t *p_gva, struct kvm_cpuid2 *cpuid) { int size = sizeof(*cpuid) + cpuid->nent * sizeof(cpuid->entries[0]); - vm_vaddr_t gva = vm_vaddr_alloc(vm, size, - getpagesize(), 0, 0); + vm_vaddr_t gva = vm_vaddr_alloc(vm, size, KVM_UTIL_MIN_VADDR); struct kvm_cpuid2 *guest_cpuids = addr_gva2hva(vm, gva); memcpy(guest_cpuids, cpuid, size); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_clock.c b/tools/testing/selftests/kvm/x86_64/hyperv_clock.c index 7f1d2765572c..bab10ae787b6 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_clock.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_clock.c @@ -7,6 +7,7 @@ #include "test_util.h" #include "kvm_util.h" #include "processor.h" +#include "hyperv.h" struct ms_hyperv_tsc_page { volatile u32 tsc_sequence; @@ -15,13 +16,6 @@ struct ms_hyperv_tsc_page { volatile s64 tsc_offset; } __packed; -#define HV_X64_MSR_GUEST_OS_ID 0x40000000 -#define HV_X64_MSR_TIME_REF_COUNT 0x40000020 -#define HV_X64_MSR_REFERENCE_TSC 0x40000021 -#define HV_X64_MSR_TSC_FREQUENCY 0x40000022 -#define HV_X64_MSR_REENLIGHTENMENT_CONTROL 0x40000106 -#define HV_X64_MSR_TSC_EMULATION_CONTROL 0x40000107 - /* Simplified mul_u64_u64_shr() */ static inline u64 mul_u64_u64_shr64(u64 a, u64 b) { @@ -220,7 +214,7 @@ int main(void) vcpu_set_hv_cpuid(vm, VCPU_ID); - tsc_page_gva = vm_vaddr_alloc(vm, getpagesize(), 0x10000, 0, 0); + tsc_page_gva = vm_vaddr_alloc_page(vm); memset(addr_gpa2hva(vm, tsc_page_gva), 0x0, getpagesize()); TEST_ASSERT((addr_gva2gpa(vm, tsc_page_gva) & (getpagesize() - 1)) == 0, "TSC page has to be page aligned\n"); diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_features.c b/tools/testing/selftests/kvm/x86_64/hyperv_features.c new file mode 100644 index 000000000000..42bd658f52a8 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/hyperv_features.c @@ -0,0 +1,649 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021, Red Hat, Inc. + * + * Tests for Hyper-V features enablement + */ +#include +#include +#include + +#include "test_util.h" +#include "kvm_util.h" +#include "processor.h" +#include "hyperv.h" + +#define VCPU_ID 0 +#define LINUX_OS_ID ((u64)0x8100 << 48) + +extern unsigned char rdmsr_start; +extern unsigned char rdmsr_end; + +static u64 do_rdmsr(u32 idx) +{ + u32 lo, hi; + + asm volatile("rdmsr_start: rdmsr;" + "rdmsr_end:" + : "=a"(lo), "=c"(hi) + : "c"(idx)); + + return (((u64) hi) << 32) | lo; +} + +extern unsigned char wrmsr_start; +extern unsigned char wrmsr_end; + +static void do_wrmsr(u32 idx, u64 val) +{ + u32 lo, hi; + + lo = val; + hi = val >> 32; + + asm volatile("wrmsr_start: wrmsr;" + "wrmsr_end:" + : : "a"(lo), "c"(idx), "d"(hi)); +} + +static int nr_gp; + +static inline u64 hypercall(u64 control, vm_vaddr_t input_address, + vm_vaddr_t output_address) +{ + u64 hv_status; + + asm volatile("mov %3, %%r8\n" + "vmcall" + : "=a" (hv_status), + "+c" (control), "+d" (input_address) + : "r" (output_address) + : "cc", "memory", "r8", "r9", "r10", "r11"); + + return hv_status; +} + +static void guest_gp_handler(struct ex_regs *regs) +{ + unsigned char *rip = (unsigned char *)regs->rip; + bool r, w; + + r = rip == &rdmsr_start; + w = rip == &wrmsr_start; + GUEST_ASSERT(r || w); + + nr_gp++; + + if (r) + regs->rip = (uint64_t)&rdmsr_end; + else + regs->rip = (uint64_t)&wrmsr_end; +} + +struct msr_data { + uint32_t idx; + bool available; + bool write; + u64 write_val; +}; + +struct hcall_data { + uint64_t control; + uint64_t expect; +}; + +static void guest_msr(struct msr_data *msr) +{ + int i = 0; + + while (msr->idx) { + WRITE_ONCE(nr_gp, 0); + if (!msr->write) + do_rdmsr(msr->idx); + else + do_wrmsr(msr->idx, msr->write_val); + + if (msr->available) + GUEST_ASSERT(READ_ONCE(nr_gp) == 0); + else + GUEST_ASSERT(READ_ONCE(nr_gp) == 1); + + GUEST_SYNC(i++); + } + + GUEST_DONE(); +} + +static void guest_hcall(vm_vaddr_t pgs_gpa, struct hcall_data *hcall) +{ + int i = 0; + + wrmsr(HV_X64_MSR_GUEST_OS_ID, LINUX_OS_ID); + wrmsr(HV_X64_MSR_HYPERCALL, pgs_gpa); + + while (hcall->control) { + GUEST_ASSERT(hypercall(hcall->control, pgs_gpa, + pgs_gpa + 4096) == hcall->expect); + GUEST_SYNC(i++); + } + + GUEST_DONE(); +} + +static void hv_set_cpuid(struct kvm_vm *vm, struct kvm_cpuid2 *cpuid, + struct kvm_cpuid_entry2 *feat, + struct kvm_cpuid_entry2 *recomm, + struct kvm_cpuid_entry2 *dbg) +{ + TEST_ASSERT(set_cpuid(cpuid, feat), + "failed to set KVM_CPUID_FEATURES leaf"); + TEST_ASSERT(set_cpuid(cpuid, recomm), + "failed to set HYPERV_CPUID_ENLIGHTMENT_INFO leaf"); + TEST_ASSERT(set_cpuid(cpuid, dbg), + "failed to set HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES leaf"); + vcpu_set_cpuid(vm, VCPU_ID, cpuid); +} + +static void guest_test_msrs_access(struct kvm_vm *vm, struct msr_data *msr, + struct kvm_cpuid2 *best) +{ + struct kvm_run *run; + struct ucall uc; + int stage = 0, r; + struct kvm_cpuid_entry2 feat = { + .function = HYPERV_CPUID_FEATURES + }; + struct kvm_cpuid_entry2 recomm = { + .function = HYPERV_CPUID_ENLIGHTMENT_INFO + }; + struct kvm_cpuid_entry2 dbg = { + .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES + }; + struct kvm_enable_cap cap = {0}; + + run = vcpu_state(vm, VCPU_ID); + + while (true) { + switch (stage) { + case 0: + /* + * Only available when Hyper-V identification is set + */ + msr->idx = HV_X64_MSR_GUEST_OS_ID; + msr->write = 0; + msr->available = 0; + break; + case 1: + msr->idx = HV_X64_MSR_HYPERCALL; + msr->write = 0; + msr->available = 0; + break; + case 2: + feat.eax |= HV_MSR_HYPERCALL_AVAILABLE; + /* + * HV_X64_MSR_GUEST_OS_ID has to be written first to make + * HV_X64_MSR_HYPERCALL available. + */ + msr->idx = HV_X64_MSR_GUEST_OS_ID; + msr->write = 1; + msr->write_val = LINUX_OS_ID; + msr->available = 1; + break; + case 3: + msr->idx = HV_X64_MSR_GUEST_OS_ID; + msr->write = 0; + msr->available = 1; + break; + case 4: + msr->idx = HV_X64_MSR_HYPERCALL; + msr->write = 0; + msr->available = 1; + break; + + case 5: + msr->idx = HV_X64_MSR_VP_RUNTIME; + msr->write = 0; + msr->available = 0; + break; + case 6: + feat.eax |= HV_MSR_VP_RUNTIME_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 7: + /* Read only */ + msr->write = 1; + msr->write_val = 1; + msr->available = 0; + break; + + case 8: + msr->idx = HV_X64_MSR_TIME_REF_COUNT; + msr->write = 0; + msr->available = 0; + break; + case 9: + feat.eax |= HV_MSR_TIME_REF_COUNT_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 10: + /* Read only */ + msr->write = 1; + msr->write_val = 1; + msr->available = 0; + break; + + case 11: + msr->idx = HV_X64_MSR_VP_INDEX; + msr->write = 0; + msr->available = 0; + break; + case 12: + feat.eax |= HV_MSR_VP_INDEX_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 13: + /* Read only */ + msr->write = 1; + msr->write_val = 1; + msr->available = 0; + break; + + case 14: + msr->idx = HV_X64_MSR_RESET; + msr->write = 0; + msr->available = 0; + break; + case 15: + feat.eax |= HV_MSR_RESET_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 16: + msr->write = 1; + msr->write_val = 0; + msr->available = 1; + break; + + case 17: + msr->idx = HV_X64_MSR_REFERENCE_TSC; + msr->write = 0; + msr->available = 0; + break; + case 18: + feat.eax |= HV_MSR_REFERENCE_TSC_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 19: + msr->write = 1; + msr->write_val = 0; + msr->available = 1; + break; + + case 20: + msr->idx = HV_X64_MSR_EOM; + msr->write = 0; + msr->available = 0; + break; + case 21: + /* + * Remains unavailable even with KVM_CAP_HYPERV_SYNIC2 + * capability enabled and guest visible CPUID bit unset. + */ + cap.cap = KVM_CAP_HYPERV_SYNIC2; + vcpu_enable_cap(vm, VCPU_ID, &cap); + break; + case 22: + feat.eax |= HV_MSR_SYNIC_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 23: + msr->write = 1; + msr->write_val = 0; + msr->available = 1; + break; + + case 24: + msr->idx = HV_X64_MSR_STIMER0_CONFIG; + msr->write = 0; + msr->available = 0; + break; + case 25: + feat.eax |= HV_MSR_SYNTIMER_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 26: + msr->write = 1; + msr->write_val = 0; + msr->available = 1; + break; + case 27: + /* Direct mode test */ + msr->write = 1; + msr->write_val = 1 << 12; + msr->available = 0; + break; + case 28: + feat.edx |= HV_STIMER_DIRECT_MODE_AVAILABLE; + msr->available = 1; + break; + + case 29: + msr->idx = HV_X64_MSR_EOI; + msr->write = 0; + msr->available = 0; + break; + case 30: + feat.eax |= HV_MSR_APIC_ACCESS_AVAILABLE; + msr->write = 1; + msr->write_val = 1; + msr->available = 1; + break; + + case 31: + msr->idx = HV_X64_MSR_TSC_FREQUENCY; + msr->write = 0; + msr->available = 0; + break; + case 32: + feat.eax |= HV_ACCESS_FREQUENCY_MSRS; + msr->write = 0; + msr->available = 1; + break; + case 33: + /* Read only */ + msr->write = 1; + msr->write_val = 1; + msr->available = 0; + break; + + case 34: + msr->idx = HV_X64_MSR_REENLIGHTENMENT_CONTROL; + msr->write = 0; + msr->available = 0; + break; + case 35: + feat.eax |= HV_ACCESS_REENLIGHTENMENT; + msr->write = 0; + msr->available = 1; + break; + case 36: + msr->write = 1; + msr->write_val = 1; + msr->available = 1; + break; + case 37: + /* Can only write '0' */ + msr->idx = HV_X64_MSR_TSC_EMULATION_STATUS; + msr->write = 1; + msr->write_val = 1; + msr->available = 0; + break; + + case 38: + msr->idx = HV_X64_MSR_CRASH_P0; + msr->write = 0; + msr->available = 0; + break; + case 39: + feat.edx |= HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE; + msr->write = 0; + msr->available = 1; + break; + case 40: + msr->write = 1; + msr->write_val = 1; + msr->available = 1; + break; + + case 41: + msr->idx = HV_X64_MSR_SYNDBG_STATUS; + msr->write = 0; + msr->available = 0; + break; + case 42: + feat.edx |= HV_FEATURE_DEBUG_MSRS_AVAILABLE; + dbg.eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; + msr->write = 0; + msr->available = 1; + break; + case 43: + msr->write = 1; + msr->write_val = 0; + msr->available = 1; + break; + + case 44: + /* END */ + msr->idx = 0; + break; + } + + hv_set_cpuid(vm, best, &feat, &recomm, &dbg); + + if (msr->idx) + pr_debug("Stage %d: testing msr: 0x%x for %s\n", stage, + msr->idx, msr->write ? "write" : "read"); + else + pr_debug("Stage %d: finish\n", stage); + + r = _vcpu_run(vm, VCPU_ID); + TEST_ASSERT(!r, "vcpu_run failed: %d\n", r); + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "unexpected exit reason: %u (%s)", + run->exit_reason, exit_reason_str(run->exit_reason)); + + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_SYNC: + TEST_ASSERT(uc.args[1] == stage, + "Unexpected stage: %ld (%d expected)\n", + uc.args[1], stage); + break; + case UCALL_ABORT: + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], + __FILE__, uc.args[1]); + return; + case UCALL_DONE: + return; + } + + stage++; + } +} + +static void guest_test_hcalls_access(struct kvm_vm *vm, struct hcall_data *hcall, + void *input, void *output, struct kvm_cpuid2 *best) +{ + struct kvm_run *run; + struct ucall uc; + int stage = 0, r; + struct kvm_cpuid_entry2 feat = { + .function = HYPERV_CPUID_FEATURES, + .eax = HV_MSR_HYPERCALL_AVAILABLE + }; + struct kvm_cpuid_entry2 recomm = { + .function = HYPERV_CPUID_ENLIGHTMENT_INFO + }; + struct kvm_cpuid_entry2 dbg = { + .function = HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES + }; + + run = vcpu_state(vm, VCPU_ID); + + while (true) { + switch (stage) { + case 0: + hcall->control = 0xdeadbeef; + hcall->expect = HV_STATUS_INVALID_HYPERCALL_CODE; + break; + + case 1: + hcall->control = HVCALL_POST_MESSAGE; + hcall->expect = HV_STATUS_ACCESS_DENIED; + break; + case 2: + feat.ebx |= HV_POST_MESSAGES; + hcall->expect = HV_STATUS_INVALID_HYPERCALL_INPUT; + break; + + case 3: + hcall->control = HVCALL_SIGNAL_EVENT; + hcall->expect = HV_STATUS_ACCESS_DENIED; + break; + case 4: + feat.ebx |= HV_SIGNAL_EVENTS; + hcall->expect = HV_STATUS_INVALID_HYPERCALL_INPUT; + break; + + case 5: + hcall->control = HVCALL_RESET_DEBUG_SESSION; + hcall->expect = HV_STATUS_INVALID_HYPERCALL_CODE; + break; + case 6: + dbg.eax |= HV_X64_SYNDBG_CAP_ALLOW_KERNEL_DEBUGGING; + hcall->expect = HV_STATUS_ACCESS_DENIED; + break; + case 7: + feat.ebx |= HV_DEBUGGING; + hcall->expect = HV_STATUS_OPERATION_DENIED; + break; + + case 8: + hcall->control = HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE; + hcall->expect = HV_STATUS_ACCESS_DENIED; + break; + case 9: + recomm.eax |= HV_X64_REMOTE_TLB_FLUSH_RECOMMENDED; + hcall->expect = HV_STATUS_SUCCESS; + break; + case 10: + hcall->control = HVCALL_FLUSH_VIRTUAL_ADDRESS_SPACE_EX; + hcall->expect = HV_STATUS_ACCESS_DENIED; + break; + case 11: + recomm.eax |= HV_X64_EX_PROCESSOR_MASKS_RECOMMENDED; + hcall->expect = HV_STATUS_SUCCESS; + break; + + case 12: + hcall->control = HVCALL_SEND_IPI; + hcall->expect = HV_STATUS_ACCESS_DENIED; + break; + case 13: + recomm.eax |= HV_X64_CLUSTER_IPI_RECOMMENDED; + hcall->expect = HV_STATUS_INVALID_HYPERCALL_INPUT; + break; + case 14: + /* Nothing in 'sparse banks' -> success */ + hcall->control = HVCALL_SEND_IPI_EX; + hcall->expect = HV_STATUS_SUCCESS; + break; + + case 15: + hcall->control = HVCALL_NOTIFY_LONG_SPIN_WAIT; + hcall->expect = HV_STATUS_ACCESS_DENIED; + break; + case 16: + recomm.ebx = 0xfff; + hcall->expect = HV_STATUS_SUCCESS; + break; + + case 17: + /* END */ + hcall->control = 0; + break; + } + + hv_set_cpuid(vm, best, &feat, &recomm, &dbg); + + if (hcall->control) + pr_debug("Stage %d: testing hcall: 0x%lx\n", stage, + hcall->control); + else + pr_debug("Stage %d: finish\n", stage); + + r = _vcpu_run(vm, VCPU_ID); + TEST_ASSERT(!r, "vcpu_run failed: %d\n", r); + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "unexpected exit reason: %u (%s)", + run->exit_reason, exit_reason_str(run->exit_reason)); + + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_SYNC: + TEST_ASSERT(uc.args[1] == stage, + "Unexpected stage: %ld (%d expected)\n", + uc.args[1], stage); + break; + case UCALL_ABORT: + TEST_FAIL("%s at %s:%ld", (const char *)uc.args[0], + __FILE__, uc.args[1]); + return; + case UCALL_DONE: + return; + } + + stage++; + } +} + +int main(void) +{ + struct kvm_cpuid2 *best; + struct kvm_vm *vm; + vm_vaddr_t msr_gva, hcall_page, hcall_params; + struct kvm_enable_cap cap = { + .cap = KVM_CAP_HYPERV_ENFORCE_CPUID, + .args = {1} + }; + + /* Test MSRs */ + vm = vm_create_default(VCPU_ID, 0, guest_msr); + + msr_gva = vm_vaddr_alloc_page(vm); + memset(addr_gva2hva(vm, msr_gva), 0x0, getpagesize()); + vcpu_args_set(vm, VCPU_ID, 1, msr_gva); + vcpu_enable_cap(vm, VCPU_ID, &cap); + + vcpu_set_hv_cpuid(vm, VCPU_ID); + + best = kvm_get_supported_hv_cpuid(); + + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vm, VCPU_ID); + vm_handle_exception(vm, GP_VECTOR, guest_gp_handler); + + pr_info("Testing access to Hyper-V specific MSRs\n"); + guest_test_msrs_access(vm, addr_gva2hva(vm, msr_gva), + best); + kvm_vm_free(vm); + + /* Test hypercalls */ + vm = vm_create_default(VCPU_ID, 0, guest_hcall); + + /* Hypercall input/output */ + hcall_page = vm_vaddr_alloc_pages(vm, 2); + memset(addr_gva2hva(vm, hcall_page), 0x0, 2 * getpagesize()); + + hcall_params = vm_vaddr_alloc_page(vm); + memset(addr_gva2hva(vm, hcall_params), 0x0, getpagesize()); + + vcpu_args_set(vm, VCPU_ID, 2, addr_gva2gpa(vm, hcall_page), hcall_params); + vcpu_enable_cap(vm, VCPU_ID, &cap); + + vcpu_set_hv_cpuid(vm, VCPU_ID); + + best = kvm_get_supported_hv_cpuid(); + + pr_info("Testing access to Hyper-V hypercalls\n"); + guest_test_hcalls_access(vm, addr_gva2hva(vm, hcall_params), + addr_gva2hva(vm, hcall_page), + addr_gva2hva(vm, hcall_page) + getpagesize(), + best); + + kvm_vm_free(vm); +} diff --git a/tools/testing/selftests/kvm/x86_64/kvm_pv_test.c b/tools/testing/selftests/kvm/x86_64/kvm_pv_test.c index 732b244d6956..04ed975662c9 100644 --- a/tools/testing/selftests/kvm/x86_64/kvm_pv_test.c +++ b/tools/testing/selftests/kvm/x86_64/kvm_pv_test.c @@ -227,7 +227,7 @@ int main(void) vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(vm, VCPU_ID); - vm_handle_exception(vm, GP_VECTOR, guest_gp_handler); + vm_install_exception_handler(vm, GP_VECTOR, guest_gp_handler); enter_guest(vm); kvm_vm_free(vm); diff --git a/tools/testing/selftests/kvm/x86_64/mmu_role_test.c b/tools/testing/selftests/kvm/x86_64/mmu_role_test.c new file mode 100644 index 000000000000..523371cf8e8f --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/mmu_role_test.c @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "kvm_util.h" +#include "processor.h" + +#define VCPU_ID 1 + +#define MMIO_GPA 0x100000000ull + +static void guest_code(void) +{ + (void)READ_ONCE(*((uint64_t *)MMIO_GPA)); + (void)READ_ONCE(*((uint64_t *)MMIO_GPA)); + + GUEST_ASSERT(0); +} + +static void guest_pf_handler(struct ex_regs *regs) +{ + /* PFEC == RSVD | PRESENT (read, kernel). */ + GUEST_ASSERT(regs->error_code == 0x9); + GUEST_DONE(); +} + +static void mmu_role_test(u32 *cpuid_reg, u32 evil_cpuid_val) +{ + u32 good_cpuid_val = *cpuid_reg; + struct kvm_run *run; + struct kvm_vm *vm; + uint64_t cmd; + int r; + + /* Create VM */ + vm = vm_create_default(VCPU_ID, 0, guest_code); + run = vcpu_state(vm, VCPU_ID); + + /* Map 1gb page without a backing memlot. */ + __virt_pg_map(vm, MMIO_GPA, MMIO_GPA, X86_PAGE_SIZE_1G); + + r = _vcpu_run(vm, VCPU_ID); + + /* Guest access to the 1gb page should trigger MMIO. */ + TEST_ASSERT(r == 0, "vcpu_run failed: %d\n", r); + TEST_ASSERT(run->exit_reason == KVM_EXIT_MMIO, + "Unexpected exit reason: %u (%s), expected MMIO exit (1gb page w/o memslot)\n", + run->exit_reason, exit_reason_str(run->exit_reason)); + + TEST_ASSERT(run->mmio.len == 8, "Unexpected exit mmio size = %u", run->mmio.len); + + TEST_ASSERT(run->mmio.phys_addr == MMIO_GPA, + "Unexpected exit mmio address = 0x%llx", run->mmio.phys_addr); + + /* + * Effect the CPUID change for the guest and re-enter the guest. Its + * access should now #PF due to the PAGE_SIZE bit being reserved or + * the resulting GPA being invalid. Note, kvm_get_supported_cpuid() + * returns the struct that contains the entry being modified. Eww. + */ + *cpuid_reg = evil_cpuid_val; + vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); + + /* + * Add a dummy memslot to coerce KVM into bumping the MMIO generation. + * KVM does not "officially" support mucking with CPUID after KVM_RUN, + * and will incorrectly reuse MMIO SPTEs. Don't delete the memslot! + * KVM x86 zaps all shadow pages on memslot deletion. + */ + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, + MMIO_GPA << 1, 10, 1, 0); + + /* Set up a #PF handler to eat the RSVD #PF and signal all done! */ + vm_init_descriptor_tables(vm); + vcpu_init_descriptor_tables(vm, VCPU_ID); + vm_handle_exception(vm, PF_VECTOR, guest_pf_handler); + + r = _vcpu_run(vm, VCPU_ID); + TEST_ASSERT(r == 0, "vcpu_run failed: %d\n", r); + + cmd = get_ucall(vm, VCPU_ID, NULL); + TEST_ASSERT(cmd == UCALL_DONE, + "Unexpected guest exit, exit_reason=%s, ucall.cmd = %lu\n", + exit_reason_str(run->exit_reason), cmd); + + /* + * Restore the happy CPUID value for the next test. Yes, changes are + * indeed persistent across VM destruction. + */ + *cpuid_reg = good_cpuid_val; + + kvm_vm_free(vm); +} + +int main(int argc, char *argv[]) +{ + struct kvm_cpuid_entry2 *entry; + int opt; + + /* + * All tests are opt-in because TDP doesn't play nice with reserved #PF + * in the GVA->GPA translation. The hardware page walker doesn't let + * software change GBPAGES or MAXPHYADDR, and KVM doesn't manually walk + * the GVA on fault for performance reasons. + */ + bool do_gbpages = false; + bool do_maxphyaddr = false; + + setbuf(stdout, NULL); + + while ((opt = getopt(argc, argv, "gm")) != -1) { + switch (opt) { + case 'g': + do_gbpages = true; + break; + case 'm': + do_maxphyaddr = true; + break; + case 'h': + default: + printf("usage: %s [-g (GBPAGES)] [-m (MAXPHYADDR)]\n", argv[0]); + break; + } + } + + if (!do_gbpages && !do_maxphyaddr) { + print_skip("No sub-tests selected"); + return 0; + } + + entry = kvm_get_supported_cpuid_entry(0x80000001); + if (!(entry->edx & CPUID_GBPAGES)) { + print_skip("1gb hugepages not supported"); + return 0; + } + + if (do_gbpages) { + pr_info("Test MMIO after toggling CPUID.GBPAGES\n\n"); + mmu_role_test(&entry->edx, entry->edx & ~CPUID_GBPAGES); + } + + if (do_maxphyaddr) { + pr_info("Test MMIO after changing CPUID.MAXPHYADDR\n\n"); + entry = kvm_get_supported_cpuid_entry(0x80000008); + mmu_role_test(&entry->eax, (entry->eax & ~0xff) | 0x20); + } + + return 0; +} diff --git a/tools/testing/selftests/kvm/x86_64/set_boot_cpu_id.c b/tools/testing/selftests/kvm/x86_64/set_boot_cpu_id.c index 12c558fc8074..ae76436af0cc 100644 --- a/tools/testing/selftests/kvm/x86_64/set_boot_cpu_id.c +++ b/tools/testing/selftests/kvm/x86_64/set_boot_cpu_id.c @@ -14,16 +14,12 @@ #include "test_util.h" #include "kvm_util.h" #include "processor.h" +#include "apic.h" #define N_VCPU 2 #define VCPU_ID0 0 #define VCPU_ID1 1 -static uint32_t get_bsp_flag(void) -{ - return rdmsr(MSR_IA32_APICBASE) & MSR_IA32_APICBASE_BSP; -} - static void guest_bsp_vcpu(void *arg) { GUEST_SYNC(1); @@ -94,7 +90,7 @@ static struct kvm_vm *create_vm(void) pages = vm_adjust_num_guest_pages(VM_MODE_DEFAULT, pages); vm = vm_create(VM_MODE_DEFAULT, pages, O_RDWR); - kvm_vm_elf_load(vm, program_invocation_name, 0, 0); + kvm_vm_elf_load(vm, program_invocation_name); vm_create_irqchip(vm); return vm; @@ -106,8 +102,6 @@ static void add_x86_vcpu(struct kvm_vm *vm, uint32_t vcpuid, bool bsp_code) vm_vcpu_add_default(vm, vcpuid, guest_bsp_vcpu); else vm_vcpu_add_default(vm, vcpuid, guest_not_bsp_vcpu); - - vcpu_set_cpuid(vm, vcpuid, kvm_get_supported_cpuid()); } static void run_vm_bsp(uint32_t bsp_vcpu) diff --git a/tools/testing/selftests/kvm/x86_64/smm_test.c b/tools/testing/selftests/kvm/x86_64/smm_test.c index 613c42c5a9b8..c1f831803ad2 100644 --- a/tools/testing/selftests/kvm/x86_64/smm_test.c +++ b/tools/testing/selftests/kvm/x86_64/smm_test.c @@ -55,8 +55,8 @@ static inline void sync_with_host(uint64_t phase) void self_smi(void) { - wrmsr(APIC_BASE_MSR + (APIC_ICR >> 4), - APIC_DEST_SELF | APIC_INT_ASSERT | APIC_DM_SMI); + x2apic_write_reg(APIC_ICR, + APIC_DEST_SELF | APIC_INT_ASSERT | APIC_DM_SMI); } void guest_code(void *arg) diff --git a/tools/testing/selftests/kvm/x86_64/sync_regs_test.c b/tools/testing/selftests/kvm/x86_64/sync_regs_test.c index d672f0a473f8..fc03a150278d 100644 --- a/tools/testing/selftests/kvm/x86_64/sync_regs_test.c +++ b/tools/testing/selftests/kvm/x86_64/sync_regs_test.c @@ -24,6 +24,10 @@ #define UCALL_PIO_PORT ((uint16_t)0x1000) +struct ucall uc_none = { + .cmd = UCALL_NONE, +}; + /* * ucall is embedded here to protect against compiler reshuffling registers * before calling a function. In this test we only need to get KVM_EXIT_IO @@ -34,7 +38,8 @@ void guest_code(void) asm volatile("1: in %[port], %%al\n" "add $0x1, %%rbx\n" "jmp 1b" - : : [port] "d" (UCALL_PIO_PORT) : "rax", "rbx"); + : : [port] "d" (UCALL_PIO_PORT), "D" (&uc_none) + : "rax", "rbx"); } static void compare_regs(struct kvm_regs *left, struct kvm_regs *right) diff --git a/tools/testing/selftests/kvm/x86_64/tsc_msrs_test.c b/tools/testing/selftests/kvm/x86_64/tsc_msrs_test.c index e357d8e222d4..5a6a662f2e59 100644 --- a/tools/testing/selftests/kvm/x86_64/tsc_msrs_test.c +++ b/tools/testing/selftests/kvm/x86_64/tsc_msrs_test.c @@ -18,15 +18,6 @@ #define rounded_rdmsr(x) ROUND(rdmsr(x)) #define rounded_host_rdmsr(x) ROUND(vcpu_get_msr(vm, 0, x)) -#define GUEST_ASSERT_EQ(a, b) do { \ - __typeof(a) _a = (a); \ - __typeof(b) _b = (b); \ - if (_a != _b) \ - ucall(UCALL_ABORT, 4, \ - "Failed guest assert: " \ - #a " == " #b, __LINE__, _a, _b); \ - } while(0) - static void guest_code(void) { u64 val = 0; diff --git a/tools/testing/selftests/kvm/x86_64/userspace_msr_exit_test.c b/tools/testing/selftests/kvm/x86_64/userspace_msr_exit_test.c index 72c0d0797522..e3e20e8848d0 100644 --- a/tools/testing/selftests/kvm/x86_64/userspace_msr_exit_test.c +++ b/tools/testing/selftests/kvm/x86_64/userspace_msr_exit_test.c @@ -574,7 +574,7 @@ static void test_msr_filter_allow(void) { vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(vm, VCPU_ID); - vm_handle_exception(vm, GP_VECTOR, guest_gp_handler); + vm_install_exception_handler(vm, GP_VECTOR, guest_gp_handler); /* Process guest code userspace exits. */ run_guest_then_process_rdmsr(vm, MSR_IA32_XSS); @@ -588,12 +588,12 @@ static void test_msr_filter_allow(void) { run_guest_then_process_wrmsr(vm, MSR_NON_EXISTENT); run_guest_then_process_rdmsr(vm, MSR_NON_EXISTENT); - vm_handle_exception(vm, UD_VECTOR, guest_ud_handler); + vm_install_exception_handler(vm, UD_VECTOR, guest_ud_handler); run_guest(vm); - vm_handle_exception(vm, UD_VECTOR, NULL); + vm_install_exception_handler(vm, UD_VECTOR, NULL); if (process_ucall(vm) != UCALL_DONE) { - vm_handle_exception(vm, GP_VECTOR, guest_fep_gp_handler); + vm_install_exception_handler(vm, GP_VECTOR, guest_fep_gp_handler); /* Process emulated rdmsr and wrmsr instructions. */ run_guest_then_process_rdmsr(vm, MSR_IA32_XSS); diff --git a/tools/testing/selftests/kvm/x86_64/vmx_apic_access_test.c b/tools/testing/selftests/kvm/x86_64/vmx_apic_access_test.c index d14888b34adb..d438c4d3228a 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_apic_access_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_apic_access_test.c @@ -96,7 +96,7 @@ int main(int argc, char *argv[]) } vmx = vcpu_alloc_vmx(vm, &vmx_pages_gva); - prepare_virtualize_apic_accesses(vmx, vm, 0); + prepare_virtualize_apic_accesses(vmx, vm); vcpu_args_set(vm, VCPU_ID, 2, vmx_pages_gva, high_gpa); while (!done) { diff --git a/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c b/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c index 537de1068554..06a64980a5d2 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_dirty_log_test.c @@ -97,7 +97,7 @@ int main(int argc, char *argv[]) * Add an identity map for GVA range [0xc0000000, 0xc0002000). This * affects both L1 and L2. However... */ - virt_map(vm, GUEST_TEST_MEM, GUEST_TEST_MEM, TEST_MEM_PAGES, 0); + virt_map(vm, GUEST_TEST_MEM, GUEST_TEST_MEM, TEST_MEM_PAGES); /* * ... pages in the L2 GPA range [0xc0001000, 0xc0003000) will map to @@ -107,9 +107,9 @@ int main(int argc, char *argv[]) * meaning after the last call to virt_map. */ prepare_eptp(vmx, vm, 0); - nested_map_memslot(vmx, vm, 0, 0); - nested_map(vmx, vm, NESTED_TEST_MEM1, GUEST_TEST_MEM, 4096, 0); - nested_map(vmx, vm, NESTED_TEST_MEM2, GUEST_TEST_MEM, 4096, 0); + nested_map_memslot(vmx, vm, 0); + nested_map(vmx, vm, NESTED_TEST_MEM1, GUEST_TEST_MEM, 4096); + nested_map(vmx, vm, NESTED_TEST_MEM2, GUEST_TEST_MEM, 4096); bmap = bitmap_alloc(TEST_MEM_PAGES); host_test_mem = addr_gpa2hva(vm, GUEST_TEST_MEM); diff --git a/tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c b/tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c new file mode 100644 index 000000000000..280c01fd2412 --- /dev/null +++ b/tools/testing/selftests/kvm/x86_64/vmx_nested_tsc_scaling_test.c @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * vmx_nested_tsc_scaling_test + * + * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * This test case verifies that nested TSC scaling behaves as expected when + * both L1 and L2 are scaled using different ratios. For this test we scale + * L1 down and scale L2 up. + */ + +#include + +#include "kvm_util.h" +#include "vmx.h" +#include "kselftest.h" + + +#define VCPU_ID 0 + +/* L2 is scaled up (from L1's perspective) by this factor */ +#define L2_SCALE_FACTOR 4ULL + +#define TSC_OFFSET_L2 ((uint64_t) -33125236320908) +#define TSC_MULTIPLIER_L2 (L2_SCALE_FACTOR << 48) + +#define L2_GUEST_STACK_SIZE 64 + +enum { USLEEP, UCHECK_L1, UCHECK_L2 }; +#define GUEST_SLEEP(sec) ucall(UCALL_SYNC, 2, USLEEP, sec) +#define GUEST_CHECK(level, freq) ucall(UCALL_SYNC, 2, level, freq) + + +/* + * This function checks whether the "actual" TSC frequency of a guest matches + * its expected frequency. In order to account for delays in taking the TSC + * measurements, a difference of 1% between the actual and the expected value + * is tolerated. + */ +static void compare_tsc_freq(uint64_t actual, uint64_t expected) +{ + uint64_t tolerance, thresh_low, thresh_high; + + tolerance = expected / 100; + thresh_low = expected - tolerance; + thresh_high = expected + tolerance; + + TEST_ASSERT(thresh_low < actual, + "TSC freq is expected to be between %"PRIu64" and %"PRIu64 + " but it actually is %"PRIu64, + thresh_low, thresh_high, actual); + TEST_ASSERT(thresh_high > actual, + "TSC freq is expected to be between %"PRIu64" and %"PRIu64 + " but it actually is %"PRIu64, + thresh_low, thresh_high, actual); +} + +static void check_tsc_freq(int level) +{ + uint64_t tsc_start, tsc_end, tsc_freq; + + /* + * Reading the TSC twice with about a second's difference should give + * us an approximation of the TSC frequency from the guest's + * perspective. Now, this won't be completely accurate, but it should + * be good enough for the purposes of this test. + */ + tsc_start = rdmsr(MSR_IA32_TSC); + GUEST_SLEEP(1); + tsc_end = rdmsr(MSR_IA32_TSC); + + tsc_freq = tsc_end - tsc_start; + + GUEST_CHECK(level, tsc_freq); +} + +static void l2_guest_code(void) +{ + check_tsc_freq(UCHECK_L2); + + /* exit to L1 */ + __asm__ __volatile__("vmcall"); +} + +static void l1_guest_code(struct vmx_pages *vmx_pages) +{ + unsigned long l2_guest_stack[L2_GUEST_STACK_SIZE]; + uint32_t control; + + /* check that L1's frequency looks alright before launching L2 */ + check_tsc_freq(UCHECK_L1); + + GUEST_ASSERT(prepare_for_vmx_operation(vmx_pages)); + GUEST_ASSERT(load_vmcs(vmx_pages)); + + /* prepare the VMCS for L2 execution */ + prepare_vmcs(vmx_pages, l2_guest_code, &l2_guest_stack[L2_GUEST_STACK_SIZE]); + + /* enable TSC offsetting and TSC scaling for L2 */ + control = vmreadz(CPU_BASED_VM_EXEC_CONTROL); + control |= CPU_BASED_USE_MSR_BITMAPS | CPU_BASED_USE_TSC_OFFSETTING; + vmwrite(CPU_BASED_VM_EXEC_CONTROL, control); + + control = vmreadz(SECONDARY_VM_EXEC_CONTROL); + control |= SECONDARY_EXEC_TSC_SCALING; + vmwrite(SECONDARY_VM_EXEC_CONTROL, control); + + vmwrite(TSC_OFFSET, TSC_OFFSET_L2); + vmwrite(TSC_MULTIPLIER, TSC_MULTIPLIER_L2); + vmwrite(TSC_MULTIPLIER_HIGH, TSC_MULTIPLIER_L2 >> 32); + + /* launch L2 */ + GUEST_ASSERT(!vmlaunch()); + GUEST_ASSERT(vmreadz(VM_EXIT_REASON) == EXIT_REASON_VMCALL); + + /* check that L1's frequency still looks good */ + check_tsc_freq(UCHECK_L1); + + GUEST_DONE(); +} + +static void tsc_scaling_check_supported(void) +{ + if (!kvm_check_cap(KVM_CAP_TSC_CONTROL)) { + print_skip("TSC scaling not supported by the HW"); + exit(KSFT_SKIP); + } +} + +static void stable_tsc_check_supported(void) +{ + FILE *fp; + char buf[4]; + + fp = fopen("/sys/devices/system/clocksource/clocksource0/current_clocksource", "r"); + if (fp == NULL) + goto skip_test; + + if (fgets(buf, sizeof(buf), fp) == NULL) + goto skip_test; + + if (strncmp(buf, "tsc", sizeof(buf))) + goto skip_test; + + return; +skip_test: + print_skip("Kernel does not use TSC clocksource - assuming that host TSC is not stable"); + exit(KSFT_SKIP); +} + +int main(int argc, char *argv[]) +{ + struct kvm_vm *vm; + vm_vaddr_t vmx_pages_gva; + + uint64_t tsc_start, tsc_end; + uint64_t tsc_khz; + uint64_t l1_scale_factor; + uint64_t l0_tsc_freq = 0; + uint64_t l1_tsc_freq = 0; + uint64_t l2_tsc_freq = 0; + + nested_vmx_check_supported(); + tsc_scaling_check_supported(); + stable_tsc_check_supported(); + + /* + * We set L1's scale factor to be a random number from 2 to 10. + * Ideally we would do the same for L2's factor but that one is + * referenced by both main() and l1_guest_code() and using a global + * variable does not work. + */ + srand(time(NULL)); + l1_scale_factor = (rand() % 9) + 2; + printf("L1's scale down factor is: %"PRIu64"\n", l1_scale_factor); + printf("L2's scale up factor is: %llu\n", L2_SCALE_FACTOR); + + tsc_start = rdtsc(); + sleep(1); + tsc_end = rdtsc(); + + l0_tsc_freq = tsc_end - tsc_start; + printf("real TSC frequency is around: %"PRIu64"\n", l0_tsc_freq); + + vm = vm_create_default(VCPU_ID, 0, (void *) l1_guest_code); + vcpu_alloc_vmx(vm, &vmx_pages_gva); + vcpu_args_set(vm, VCPU_ID, 1, vmx_pages_gva); + + tsc_khz = _vcpu_ioctl(vm, VCPU_ID, KVM_GET_TSC_KHZ, NULL); + TEST_ASSERT(tsc_khz != -1, "vcpu ioctl KVM_GET_TSC_KHZ failed"); + + /* scale down L1's TSC frequency */ + vcpu_ioctl(vm, VCPU_ID, KVM_SET_TSC_KHZ, + (void *) (tsc_khz / l1_scale_factor)); + + for (;;) { + volatile struct kvm_run *run = vcpu_state(vm, VCPU_ID); + struct ucall uc; + + vcpu_run(vm, VCPU_ID); + TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, + "Got exit_reason other than KVM_EXIT_IO: %u (%s)\n", + run->exit_reason, + exit_reason_str(run->exit_reason)); + + switch (get_ucall(vm, VCPU_ID, &uc)) { + case UCALL_ABORT: + TEST_FAIL("%s", (const char *) uc.args[0]); + case UCALL_SYNC: + switch (uc.args[0]) { + case USLEEP: + sleep(uc.args[1]); + break; + case UCHECK_L1: + l1_tsc_freq = uc.args[1]; + printf("L1's TSC frequency is around: %"PRIu64 + "\n", l1_tsc_freq); + + compare_tsc_freq(l1_tsc_freq, + l0_tsc_freq / l1_scale_factor); + break; + case UCHECK_L2: + l2_tsc_freq = uc.args[1]; + printf("L2's TSC frequency is around: %"PRIu64 + "\n", l2_tsc_freq); + + compare_tsc_freq(l2_tsc_freq, + l1_tsc_freq * L2_SCALE_FACTOR); + break; + } + break; + case UCALL_DONE: + goto done; + default: + TEST_FAIL("Unknown ucall %lu", uc.cmd); + } + } + +done: + kvm_vm_free(vm); + return 0; +} diff --git a/tools/testing/selftests/kvm/x86_64/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86_64/xapic_ipi_test.c index 2f964cdc273c..afbbc40df884 100644 --- a/tools/testing/selftests/kvm/x86_64/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86_64/xapic_ipi_test.c @@ -42,8 +42,6 @@ #define HALTER_VCPU_ID 0 #define SENDER_VCPU_ID 1 -volatile uint32_t *apic_base = (volatile uint32_t *)APIC_DEFAULT_GPA; - /* * Vector for IPI from sender vCPU to halting vCPU. * Value is arbitrary and was chosen for the alternating bit pattern. Any @@ -86,45 +84,6 @@ struct thread_params { uint64_t *pipis_rcvd; /* host address of ipis_rcvd global */ }; -uint32_t read_apic_reg(uint reg) -{ - return apic_base[reg >> 2]; -} - -void write_apic_reg(uint reg, uint32_t val) -{ - apic_base[reg >> 2] = val; -} - -void disable_apic(void) -{ - wrmsr(MSR_IA32_APICBASE, - rdmsr(MSR_IA32_APICBASE) & - ~(MSR_IA32_APICBASE_ENABLE | MSR_IA32_APICBASE_EXTD)); -} - -void enable_xapic(void) -{ - uint64_t val = rdmsr(MSR_IA32_APICBASE); - - /* Per SDM: to enable xAPIC when in x2APIC must first disable APIC */ - if (val & MSR_IA32_APICBASE_EXTD) { - disable_apic(); - wrmsr(MSR_IA32_APICBASE, - rdmsr(MSR_IA32_APICBASE) | MSR_IA32_APICBASE_ENABLE); - } else if (!(val & MSR_IA32_APICBASE_ENABLE)) { - wrmsr(MSR_IA32_APICBASE, val | MSR_IA32_APICBASE_ENABLE); - } - - /* - * Per SDM: reset value of spurious interrupt vector register has the - * APIC software enabled bit=0. It must be enabled in addition to the - * enable bit in the MSR. - */ - val = read_apic_reg(APIC_SPIV) | APIC_SPIV_APIC_ENABLED; - write_apic_reg(APIC_SPIV, val); -} - void verify_apic_base_addr(void) { uint64_t msr = rdmsr(MSR_IA32_APICBASE); @@ -136,10 +95,10 @@ void verify_apic_base_addr(void) static void halter_guest_code(struct test_data_page *data) { verify_apic_base_addr(); - enable_xapic(); + xapic_enable(); - data->halter_apic_id = GET_APIC_ID_FIELD(read_apic_reg(APIC_ID)); - data->halter_lvr = read_apic_reg(APIC_LVR); + data->halter_apic_id = GET_APIC_ID_FIELD(xapic_read_reg(APIC_ID)); + data->halter_lvr = xapic_read_reg(APIC_LVR); /* * Loop forever HLTing and recording halts & wakes. Disable interrupts @@ -150,8 +109,8 @@ static void halter_guest_code(struct test_data_page *data) * TPR and PPR for diagnostic purposes in case the test fails. */ for (;;) { - data->halter_tpr = read_apic_reg(APIC_TASKPRI); - data->halter_ppr = read_apic_reg(APIC_PROCPRI); + data->halter_tpr = xapic_read_reg(APIC_TASKPRI); + data->halter_ppr = xapic_read_reg(APIC_PROCPRI); data->hlt_count++; asm volatile("sti; hlt; cli"); data->wake_count++; @@ -166,7 +125,7 @@ static void halter_guest_code(struct test_data_page *data) static void guest_ipi_handler(struct ex_regs *regs) { ipis_rcvd++; - write_apic_reg(APIC_EOI, 77); + xapic_write_reg(APIC_EOI, 77); } static void sender_guest_code(struct test_data_page *data) @@ -179,7 +138,7 @@ static void sender_guest_code(struct test_data_page *data) uint64_t tsc_start; verify_apic_base_addr(); - enable_xapic(); + xapic_enable(); /* * Init interrupt command register for sending IPIs @@ -206,8 +165,8 @@ static void sender_guest_code(struct test_data_page *data) * First IPI can be sent unconditionally because halter vCPU * starts earlier. */ - write_apic_reg(APIC_ICR2, icr2_val); - write_apic_reg(APIC_ICR, icr_val); + xapic_write_reg(APIC_ICR2, icr2_val); + xapic_write_reg(APIC_ICR, icr_val); data->ipis_sent++; /* @@ -462,13 +421,13 @@ int main(int argc, char *argv[]) vm_init_descriptor_tables(vm); vcpu_init_descriptor_tables(vm, HALTER_VCPU_ID); - vm_handle_exception(vm, IPI_VECTOR, guest_ipi_handler); + vm_install_exception_handler(vm, IPI_VECTOR, guest_ipi_handler); - virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA, 0); + virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); vm_vcpu_add_default(vm, SENDER_VCPU_ID, sender_guest_code); - test_data_page_vaddr = vm_vaddr_alloc(vm, 0x1000, 0x1000, 0, 0); + test_data_page_vaddr = vm_vaddr_alloc_page(vm); data = (struct test_data_page *)addr_gva2hva(vm, test_data_page_vaddr); memset(data, 0, sizeof(*data)); diff --git a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c index 1f4a0599683c..117bf49a3d79 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_shinfo_test.c @@ -146,7 +146,7 @@ int main(int argc, char *argv[]) /* Map a region for the shared_info page */ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, SHINFO_REGION_GPA, SHINFO_REGION_SLOT, 2, 0); - virt_map(vm, SHINFO_REGION_GVA, SHINFO_REGION_GPA, 2, 0); + virt_map(vm, SHINFO_REGION_GVA, SHINFO_REGION_GPA, 2); struct kvm_xen_hvm_config hvmc = { .flags = KVM_XEN_HVM_CONFIG_INTERCEPT_HCALL, diff --git a/tools/testing/selftests/kvm/x86_64/xen_vmcall_test.c b/tools/testing/selftests/kvm/x86_64/xen_vmcall_test.c index 8389e0bfd711..adc94452b57c 100644 --- a/tools/testing/selftests/kvm/x86_64/xen_vmcall_test.c +++ b/tools/testing/selftests/kvm/x86_64/xen_vmcall_test.c @@ -103,7 +103,7 @@ int main(int argc, char *argv[]) /* Map a region for the hypercall pages */ vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS, HCALL_REGION_GPA, HCALL_REGION_SLOT, 2, 0); - virt_map(vm, HCALL_REGION_GPA, HCALL_REGION_GPA, 2, 0); + virt_map(vm, HCALL_REGION_GPA, HCALL_REGION_GPA, 2); for (;;) { volatile struct kvm_run *run = vcpu_state(vm, VCPU_ID); diff --git a/tools/testing/selftests/seccomp/seccomp_benchmark.c b/tools/testing/selftests/seccomp/seccomp_benchmark.c index fcc806585266..6e5102a7d7c9 100644 --- a/tools/testing/selftests/seccomp/seccomp_benchmark.c +++ b/tools/testing/selftests/seccomp/seccomp_benchmark.c @@ -143,9 +143,15 @@ int main(int argc, char *argv[]) unsigned long long native, filter1, filter2, bitmap1, bitmap2; unsigned long long entry, per_filter1, per_filter2; + setbuf(stdout, NULL); + + printf("Running on:\n"); + system("uname -a"); + printf("Current BPF sysctl settings:\n"); - system("sysctl net.core.bpf_jit_enable"); - system("sysctl net.core.bpf_jit_harden"); + /* Avoid using "sysctl" which may not be installed. */ + system("grep -H . /proc/sys/net/core/bpf_jit_enable"); + system("grep -H . /proc/sys/net/core/bpf_jit_harden"); if (argc > 1) samples = strtoull(argv[1], NULL, 0); diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index e3d5c77a8612..1d64891e6492 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -235,6 +235,10 @@ struct seccomp_notif_addfd { }; #endif +#ifndef SECCOMP_ADDFD_FLAG_SEND +#define SECCOMP_ADDFD_FLAG_SEND (1UL << 1) /* Addfd and return it, atomically */ +#endif + struct seccomp_notif_addfd_small { __u64 id; char weird[4]; @@ -3959,7 +3963,7 @@ TEST(user_notification_addfd) { pid_t pid; long ret; - int status, listener, memfd, fd; + int status, listener, memfd, fd, nextfd; struct seccomp_notif_addfd addfd = {}; struct seccomp_notif_addfd_small small = {}; struct seccomp_notif_addfd_big big = {}; @@ -3968,25 +3972,34 @@ TEST(user_notification_addfd) /* 100 ms */ struct timespec delay = { .tv_nsec = 100000000 }; + /* There may be arbitrary already-open fds at test start. */ memfd = memfd_create("test", 0); ASSERT_GE(memfd, 0); + nextfd = memfd + 1; ret = prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); ASSERT_EQ(0, ret) { TH_LOG("Kernel does not support PR_SET_NO_NEW_PRIVS!"); } + /* fd: 4 */ /* Check that the basic notification machinery works */ listener = user_notif_syscall(__NR_getppid, SECCOMP_FILTER_FLAG_NEW_LISTENER); - ASSERT_GE(listener, 0); + ASSERT_EQ(listener, nextfd++); pid = fork(); ASSERT_GE(pid, 0); if (pid == 0) { + /* fds will be added and this value is expected */ if (syscall(__NR_getppid) != USER_NOTIF_MAGIC) exit(1); + + /* Atomic addfd+send is received here. Check it is a valid fd */ + if (fcntl(syscall(__NR_getppid), F_GETFD) == -1) + exit(1); + exit(syscall(__NR_getppid) != USER_NOTIF_MAGIC); } @@ -4028,14 +4041,14 @@ TEST(user_notification_addfd) /* Verify we can set an arbitrary remote fd */ fd = ioctl(listener, SECCOMP_IOCTL_NOTIF_ADDFD, &addfd); - EXPECT_GE(fd, 0); + EXPECT_EQ(fd, nextfd++); EXPECT_EQ(filecmp(getpid(), pid, memfd, fd), 0); /* Verify we can set an arbitrary remote fd with large size */ memset(&big, 0x0, sizeof(big)); big.addfd = addfd; fd = ioctl(listener, SECCOMP_IOCTL_NOTIF_ADDFD_BIG, &big); - EXPECT_GE(fd, 0); + EXPECT_EQ(fd, nextfd++); /* Verify we can set a specific remote fd */ addfd.newfd = 42; @@ -4065,6 +4078,32 @@ TEST(user_notification_addfd) ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, &req), 0); ASSERT_EQ(addfd.id, req.id); + /* Verify we can do an atomic addfd and send */ + addfd.newfd = 0; + addfd.flags = SECCOMP_ADDFD_FLAG_SEND; + fd = ioctl(listener, SECCOMP_IOCTL_NOTIF_ADDFD, &addfd); + /* + * Child has earlier "low" fds and now 42, so we expect the next + * lowest available fd to be assigned here. + */ + EXPECT_EQ(fd, nextfd++); + EXPECT_EQ(filecmp(getpid(), pid, memfd, fd), 0); + + /* + * This sets the ID of the ADD FD to the last request plus 1. The + * notification ID increments 1 per notification. + */ + addfd.id = req.id + 1; + + /* This spins until the underlying notification is generated */ + while (ioctl(listener, SECCOMP_IOCTL_NOTIF_ADDFD, &addfd) != -1 && + errno != -EINPROGRESS) + nanosleep(&delay, NULL); + + memset(&req, 0, sizeof(req)); + ASSERT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_RECV, &req), 0); + ASSERT_EQ(addfd.id, req.id); + resp.id = req.id; resp.error = 0; resp.val = USER_NOTIF_MAGIC; @@ -4125,6 +4164,10 @@ TEST(user_notification_addfd_rlimit) EXPECT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_ADDFD, &addfd), -1); EXPECT_EQ(errno, EMFILE); + addfd.flags = SECCOMP_ADDFD_FLAG_SEND; + EXPECT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_ADDFD, &addfd), -1); + EXPECT_EQ(errno, EMFILE); + addfd.newfd = 100; addfd.flags = SECCOMP_ADDFD_FLAG_SETFD; EXPECT_EQ(ioctl(listener, SECCOMP_IOCTL_NOTIF_ADDFD, &addfd), -1); diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 1c37ccd5d402..62b39149b8c8 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -63,3 +63,6 @@ config HAVE_KVM_NO_POLL config KVM_XFER_TO_GUEST_WORK bool + +config HAVE_KVM_PM_NOTIFIER + bool diff --git a/virt/kvm/binary_stats.c b/virt/kvm/binary_stats.c new file mode 100644 index 000000000000..e609d428811a --- /dev/null +++ b/virt/kvm/binary_stats.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * KVM binary statistics interface implementation + * + * Copyright 2021 Google LLC + */ + +#include +#include +#include +#include + +/** + * kvm_stats_read() - Common function to read from the binary statistics + * file descriptor. + * + * @id: identification string of the stats + * @header: stats header for a vm or a vcpu + * @desc: start address of an array of stats descriptors for a vm or a vcpu + * @stats: start address of stats data block for a vm or a vcpu + * @size_stats: the size of stats data block pointed by @stats + * @user_buffer: start address of userspace buffer + * @size: requested read size from userspace + * @offset: the start position from which the content will be read for the + * corresponding vm or vcp file descriptor + * + * The file content of a vm/vcpu file descriptor is now defined as below: + * +-------------+ + * | Header | + * +-------------+ + * | id string | + * +-------------+ + * | Descriptors | + * +-------------+ + * | Stats Data | + * +-------------+ + * Although this function allows userspace to read any amount of data (as long + * as in the limit) from any position, the typical usage would follow below + * steps: + * 1. Read header from offset 0. Get the offset of descriptors and stats data + * and some other necessary information. This is a one-time work for the + * lifecycle of the corresponding vm/vcpu stats fd. + * 2. Read id string from its offset. This is a one-time work for the lifecycle + * of the corresponding vm/vcpu stats fd. + * 3. Read descriptors from its offset and discover all the stats by parsing + * descriptors. This is a one-time work for the lifecycle of the + * corresponding vm/vcpu stats fd. + * 4. Periodically read stats data from its offset using pread. + * + * Return: the number of bytes that has been successfully read + */ +ssize_t kvm_stats_read(char *id, const struct kvm_stats_header *header, + const struct _kvm_stats_desc *desc, + void *stats, size_t size_stats, + char __user *user_buffer, size_t size, loff_t *offset) +{ + ssize_t len; + ssize_t copylen; + ssize_t remain = size; + size_t size_desc; + size_t size_header; + void *src; + loff_t pos = *offset; + char __user *dest = user_buffer; + + size_header = sizeof(*header); + size_desc = header->num_desc * sizeof(*desc); + + len = KVM_STATS_NAME_SIZE + size_header + size_desc + size_stats - pos; + len = min(len, remain); + if (len <= 0) + return 0; + remain = len; + + /* + * Copy kvm stats header. + * The header is the first block of content userspace usually read out. + * The pos is 0 and the copylen and remain would be the size of header. + * The copy of the header would be skipped if offset is larger than the + * size of header. That usually happens when userspace reads stats + * descriptors and stats data. + */ + copylen = size_header - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = (void *)header + pos; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + /* + * Copy kvm stats header id string. + * The id string is unique for every vm/vcpu, which is stored in kvm + * and kvm_vcpu structure. + * The id string is part of the stat header from the perspective of + * userspace, it is usually read out together with previous constant + * header part and could be skipped for later descriptors and stats + * data readings. + */ + copylen = header->id_offset + KVM_STATS_NAME_SIZE - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = id + pos - header->id_offset; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + /* + * Copy kvm stats descriptors. + * The descriptors copy would be skipped in the typical case that + * userspace periodically read stats data, since the pos would be + * greater than the end address of descriptors + * (header->header.desc_offset + size_desc) causing copylen <= 0. + */ + copylen = header->desc_offset + size_desc - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = (void *)desc + pos - header->desc_offset; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + /* Copy kvm stats values */ + copylen = header->data_offset + size_stats - pos; + copylen = min(copylen, remain); + if (copylen > 0) { + src = stats + pos - header->data_offset; + if (copy_to_user(dest, src, copylen)) + return -EFAULT; + remain -= copylen; + pos += copylen; + dest += copylen; + } + + *offset = pos; + return len; +} diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index b68678b52129..6866c1780cf5 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -114,7 +115,6 @@ static DEFINE_PER_CPU(struct kvm_vcpu *, kvm_running_vcpu); struct dentry *kvm_debugfs_dir; EXPORT_SYMBOL_GPL(kvm_debugfs_dir); -static int kvm_debugfs_num_entries; static const struct file_operations stat_fops_per_vm; static long kvm_vcpu_ioctl(struct file *file, unsigned int ioctl, @@ -331,7 +331,7 @@ void kvm_flush_remote_tlbs(struct kvm *kvm) */ if (!kvm_arch_flush_remote_tlb(kvm) || kvm_make_all_cpus_request(kvm, KVM_REQ_TLB_FLUSH)) - ++kvm->stat.remote_tlb_flush; + ++kvm->stat.generic.remote_tlb_flush; cmpxchg(&kvm->tlbs_dirty, dirty_count, 0); } EXPORT_SYMBOL_GPL(kvm_flush_remote_tlbs); @@ -780,6 +780,38 @@ static int kvm_init_mmu_notifier(struct kvm *kvm) #endif /* CONFIG_MMU_NOTIFIER && KVM_ARCH_WANT_MMU_NOTIFIER */ +#ifdef CONFIG_HAVE_KVM_PM_NOTIFIER +static int kvm_pm_notifier_call(struct notifier_block *bl, + unsigned long state, + void *unused) +{ + struct kvm *kvm = container_of(bl, struct kvm, pm_notifier); + + return kvm_arch_pm_notifier(kvm, state); +} + +static void kvm_init_pm_notifier(struct kvm *kvm) +{ + kvm->pm_notifier.notifier_call = kvm_pm_notifier_call; + /* Suspend KVM before we suspend ftrace, RCU, etc. */ + kvm->pm_notifier.priority = INT_MAX; + register_pm_notifier(&kvm->pm_notifier); +} + +static void kvm_destroy_pm_notifier(struct kvm *kvm) +{ + unregister_pm_notifier(&kvm->pm_notifier); +} +#else /* !CONFIG_HAVE_KVM_PM_NOTIFIER */ +static void kvm_init_pm_notifier(struct kvm *kvm) +{ +} + +static void kvm_destroy_pm_notifier(struct kvm *kvm) +{ +} +#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */ + static struct kvm_memslots *kvm_alloc_memslots(void) { int i; @@ -827,9 +859,24 @@ static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots) kvfree(slots); } +static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc) +{ + switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) { + case KVM_STATS_TYPE_INSTANT: + return 0444; + case KVM_STATS_TYPE_CUMULATIVE: + case KVM_STATS_TYPE_PEAK: + default: + return 0644; + } +} + + static void kvm_destroy_vm_debugfs(struct kvm *kvm) { int i; + int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc + + kvm_vcpu_stats_header.num_desc; if (!kvm->debugfs_dentry) return; @@ -847,7 +894,10 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) { char dir_name[ITOA_MAX_LEN * 2]; struct kvm_stat_data *stat_data; - struct kvm_stats_debugfs_item *p; + const struct _kvm_stats_desc *pdesc; + int i; + int kvm_debugfs_num_entries = kvm_vm_stats_header.num_desc + + kvm_vcpu_stats_header.num_desc; if (!debugfs_initialized()) return 0; @@ -861,15 +911,32 @@ static int kvm_create_vm_debugfs(struct kvm *kvm, int fd) if (!kvm->debugfs_stat_data) return -ENOMEM; - for (p = debugfs_entries; p->name; p++) { + for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) { + pdesc = &kvm_vm_stats_desc[i]; stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT); if (!stat_data) return -ENOMEM; stat_data->kvm = kvm; - stat_data->dbgfs_item = p; - kvm->debugfs_stat_data[p - debugfs_entries] = stat_data; - debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p), + stat_data->desc = pdesc; + stat_data->kind = KVM_STAT_VM; + kvm->debugfs_stat_data[i] = stat_data; + debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), + kvm->debugfs_dentry, stat_data, + &stat_fops_per_vm); + } + + for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) { + pdesc = &kvm_vcpu_stats_desc[i]; + stat_data = kzalloc(sizeof(*stat_data), GFP_KERNEL_ACCOUNT); + if (!stat_data) + return -ENOMEM; + + stat_data->kvm = kvm; + stat_data->desc = pdesc; + stat_data->kind = KVM_STAT_VCPU; + kvm->debugfs_stat_data[i] = stat_data; + debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), kvm->debugfs_dentry, stat_data, &stat_fops_per_vm); } @@ -909,6 +976,7 @@ static struct kvm *kvm_create_vm(unsigned long type) mutex_init(&kvm->lock); mutex_init(&kvm->irq_lock); mutex_init(&kvm->slots_lock); + mutex_init(&kvm->slots_arch_lock); INIT_LIST_HEAD(&kvm->devices); BUILD_BUG_ON(KVM_MEM_SLOTS_NUM > SHRT_MAX); @@ -963,6 +1031,7 @@ static struct kvm *kvm_create_vm(unsigned long type) mutex_unlock(&kvm_lock); preempt_notifier_inc(); + kvm_init_pm_notifier(kvm); return kvm; @@ -1010,6 +1079,7 @@ static void kvm_destroy_vm(struct kvm *kvm) int i; struct mm_struct *mm = kvm->mm; + kvm_destroy_pm_notifier(kvm); kvm_uevent_notify_change(KVM_EVENT_DESTROY_VM, kvm); kvm_destroy_vm_debugfs(kvm); kvm_arch_sync_events(kvm); @@ -1281,6 +1351,14 @@ static struct kvm_memslots *install_new_memslots(struct kvm *kvm, slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS; rcu_assign_pointer(kvm->memslots[as_id], slots); + + /* + * Acquired in kvm_set_memslot. Must be released before synchronize + * SRCU below in order to avoid deadlock with another thread + * acquiring the slots_arch_lock in an srcu critical section. + */ + mutex_unlock(&kvm->slots_arch_lock); + synchronize_srcu_expedited(&kvm->srcu); /* @@ -1307,6 +1385,18 @@ static struct kvm_memslots *install_new_memslots(struct kvm *kvm, return old_memslots; } +static size_t kvm_memslots_size(int slots) +{ + return sizeof(struct kvm_memslots) + + (sizeof(struct kvm_memory_slot) * slots); +} + +static void kvm_copy_memslots(struct kvm_memslots *to, + struct kvm_memslots *from) +{ + memcpy(to, from, kvm_memslots_size(from->used_slots)); +} + /* * Note, at a minimum, the current number of used slots must be allocated, even * when deleting a memslot, as we need a complete duplicate of the memslots for @@ -1316,19 +1406,16 @@ static struct kvm_memslots *kvm_dup_memslots(struct kvm_memslots *old, enum kvm_mr_change change) { struct kvm_memslots *slots; - size_t old_size, new_size; - - old_size = sizeof(struct kvm_memslots) + - (sizeof(struct kvm_memory_slot) * old->used_slots); + size_t new_size; if (change == KVM_MR_CREATE) - new_size = old_size + sizeof(struct kvm_memory_slot); + new_size = kvm_memslots_size(old->used_slots + 1); else - new_size = old_size; + new_size = kvm_memslots_size(old->used_slots); slots = kvzalloc(new_size, GFP_KERNEL_ACCOUNT); if (likely(slots)) - memcpy(slots, old, old_size); + kvm_copy_memslots(slots, old); return slots; } @@ -1343,9 +1430,27 @@ static int kvm_set_memslot(struct kvm *kvm, struct kvm_memslots *slots; int r; + /* + * Released in install_new_memslots. + * + * Must be held from before the current memslots are copied until + * after the new memslots are installed with rcu_assign_pointer, + * then released before the synchronize srcu in install_new_memslots. + * + * When modifying memslots outside of the slots_lock, must be held + * before reading the pointer to the current memslots until after all + * changes to those memslots are complete. + * + * These rules ensure that installing new memslots does not lose + * changes made to the previous memslots. + */ + mutex_lock(&kvm->slots_arch_lock); + slots = kvm_dup_memslots(__kvm_memslots(kvm, as_id), change); - if (!slots) + if (!slots) { + mutex_unlock(&kvm->slots_arch_lock); return -ENOMEM; + } if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { /* @@ -1356,10 +1461,9 @@ static int kvm_set_memslot(struct kvm *kvm, slot->flags |= KVM_MEMSLOT_INVALID; /* - * We can re-use the old memslots, the only difference from the - * newly installed memslots is the invalid flag, which will get - * dropped by update_memslots anyway. We'll also revert to the - * old memslots if preparing the new memory region fails. + * We can re-use the memory from the old memslots. + * It will be overwritten with a copy of the new memslots + * after reacquiring the slots_arch_lock below. */ slots = install_new_memslots(kvm, as_id, slots); @@ -1371,6 +1475,17 @@ static int kvm_set_memslot(struct kvm *kvm, * - kvm_is_visible_gfn (mmu_check_root) */ kvm_arch_flush_shadow_memslot(kvm, slot); + + /* Released in install_new_memslots. */ + mutex_lock(&kvm->slots_arch_lock); + + /* + * The arch-specific fields of the memslots could have changed + * between releasing the slots_arch_lock in + * install_new_memslots and here, so get a fresh copy of the + * slots. + */ + kvm_copy_memslots(slots, __kvm_memslots(kvm, as_id)); } r = kvm_arch_prepare_memory_region(kvm, new, mem, change); @@ -1386,8 +1501,13 @@ static int kvm_set_memslot(struct kvm *kvm, return 0; out_slots: - if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) + if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) { + slot = id_to_memslot(slots, old->id); + slot->flags &= ~KVM_MEMSLOT_INVALID; slots = install_new_memslots(kvm, as_id, slots); + } else { + mutex_unlock(&kvm->slots_arch_lock); + } kvfree(slots); return r; } @@ -2958,9 +3078,9 @@ static inline void update_halt_poll_stats(struct kvm_vcpu *vcpu, u64 poll_ns, bool waited) { if (waited) - vcpu->stat.halt_poll_fail_ns += poll_ns; + vcpu->stat.generic.halt_poll_fail_ns += poll_ns; else - vcpu->stat.halt_poll_success_ns += poll_ns; + vcpu->stat.generic.halt_poll_success_ns += poll_ns; } /* @@ -2978,16 +3098,16 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) if (vcpu->halt_poll_ns && !kvm_arch_no_poll(vcpu)) { ktime_t stop = ktime_add_ns(ktime_get(), vcpu->halt_poll_ns); - ++vcpu->stat.halt_attempted_poll; + ++vcpu->stat.generic.halt_attempted_poll; do { /* * This sets KVM_REQ_UNHALT if an interrupt * arrives. */ if (kvm_vcpu_check_block(vcpu) < 0) { - ++vcpu->stat.halt_successful_poll; + ++vcpu->stat.generic.halt_successful_poll; if (!vcpu_valid_wakeup(vcpu)) - ++vcpu->stat.halt_poll_invalid; + ++vcpu->stat.generic.halt_poll_invalid; goto out; } poll_end = cur = ktime_get(); @@ -3044,7 +3164,7 @@ bool kvm_vcpu_wake_up(struct kvm_vcpu *vcpu) waitp = kvm_arch_vcpu_get_wait(vcpu); if (rcuwait_wake_up(waitp)) { WRITE_ONCE(vcpu->ready, true); - ++vcpu->stat.halt_wakeup; + ++vcpu->stat.generic.halt_wakeup; return true; } @@ -3377,6 +3497,10 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, u32 id) vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); BUG_ON(kvm->vcpus[vcpu->vcpu_idx]); + /* Fill the stats id string for the vcpu */ + snprintf(vcpu->stats_id, sizeof(vcpu->stats_id), "kvm-%d/vcpu-%d", + task_pid_nr(current), id); + /* Now it's all set up, let userspace reach it */ kvm_get_kvm(kvm); r = create_vcpu_fd(vcpu); @@ -3426,6 +3550,44 @@ static int kvm_vcpu_ioctl_set_sigmask(struct kvm_vcpu *vcpu, sigset_t *sigset) return 0; } +static ssize_t kvm_vcpu_stats_read(struct file *file, char __user *user_buffer, + size_t size, loff_t *offset) +{ + struct kvm_vcpu *vcpu = file->private_data; + + return kvm_stats_read(vcpu->stats_id, &kvm_vcpu_stats_header, + &kvm_vcpu_stats_desc[0], &vcpu->stat, + sizeof(vcpu->stat), user_buffer, size, offset); +} + +static const struct file_operations kvm_vcpu_stats_fops = { + .read = kvm_vcpu_stats_read, + .llseek = noop_llseek, +}; + +static int kvm_vcpu_ioctl_get_stats_fd(struct kvm_vcpu *vcpu) +{ + int fd; + struct file *file; + char name[15 + ITOA_MAX_LEN + 1]; + + snprintf(name, sizeof(name), "kvm-vcpu-stats:%d", vcpu->vcpu_id); + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) + return fd; + + file = anon_inode_getfile(name, &kvm_vcpu_stats_fops, vcpu, O_RDONLY); + if (IS_ERR(file)) { + put_unused_fd(fd); + return PTR_ERR(file); + } + file->f_mode |= FMODE_PREAD; + fd_install(fd, file); + + return fd; +} + static long kvm_vcpu_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -3623,6 +3785,10 @@ static long kvm_vcpu_ioctl(struct file *filp, r = kvm_arch_vcpu_ioctl_set_fpu(vcpu, fpu); break; } + case KVM_GET_STATS_FD: { + r = kvm_vcpu_ioctl_get_stats_fd(vcpu); + break; + } default: r = kvm_arch_vcpu_ioctl(filp, ioctl, arg); } @@ -3881,6 +4047,8 @@ static long kvm_vm_ioctl_check_extension_generic(struct kvm *kvm, long arg) #else return 0; #endif + case KVM_CAP_BINARY_STATS_FD: + return 1; default: break; } @@ -3984,6 +4152,42 @@ static int kvm_vm_ioctl_enable_cap_generic(struct kvm *kvm, } } +static ssize_t kvm_vm_stats_read(struct file *file, char __user *user_buffer, + size_t size, loff_t *offset) +{ + struct kvm *kvm = file->private_data; + + return kvm_stats_read(kvm->stats_id, &kvm_vm_stats_header, + &kvm_vm_stats_desc[0], &kvm->stat, + sizeof(kvm->stat), user_buffer, size, offset); +} + +static const struct file_operations kvm_vm_stats_fops = { + .read = kvm_vm_stats_read, + .llseek = noop_llseek, +}; + +static int kvm_vm_ioctl_get_stats_fd(struct kvm *kvm) +{ + int fd; + struct file *file; + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) + return fd; + + file = anon_inode_getfile("kvm-vm-stats", + &kvm_vm_stats_fops, kvm, O_RDONLY); + if (IS_ERR(file)) { + put_unused_fd(fd); + return PTR_ERR(file); + } + file->f_mode |= FMODE_PREAD; + fd_install(fd, file); + + return fd; +} + static long kvm_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) { @@ -4166,6 +4370,9 @@ static long kvm_vm_ioctl(struct file *filp, case KVM_RESET_DIRTY_RINGS: r = kvm_vm_ioctl_reset_dirty_pages(kvm); break; + case KVM_GET_STATS_FD: + r = kvm_vm_ioctl_get_stats_fd(kvm); + break; default: r = kvm_arch_vm_ioctl(filp, ioctl, arg); } @@ -4245,6 +4452,9 @@ static int kvm_dev_ioctl_create_vm(unsigned long type) if (r < 0) goto put_kvm; + snprintf(kvm->stats_id, sizeof(kvm->stats_id), + "kvm-%d", task_pid_nr(current)); + file = anon_inode_getfile("kvm-vm", &kvm_vm_fops, kvm, O_RDWR); if (IS_ERR(file)) { put_unused_fd(r); @@ -4739,7 +4949,7 @@ static int kvm_debugfs_open(struct inode *inode, struct file *file, return -ENOENT; if (simple_attr_open(inode, file, get, - KVM_DBGFS_GET_MODE(stat_data->dbgfs_item) & 0222 + kvm_stats_debugfs_mode(stat_data->desc) & 0222 ? set : NULL, fmt)) { kvm_put_kvm(stat_data->kvm); @@ -4762,14 +4972,14 @@ static int kvm_debugfs_release(struct inode *inode, struct file *file) static int kvm_get_stat_per_vm(struct kvm *kvm, size_t offset, u64 *val) { - *val = *(ulong *)((void *)kvm + offset); + *val = *(u64 *)((void *)(&kvm->stat) + offset); return 0; } static int kvm_clear_stat_per_vm(struct kvm *kvm, size_t offset) { - *(ulong *)((void *)kvm + offset) = 0; + *(u64 *)((void *)(&kvm->stat) + offset) = 0; return 0; } @@ -4782,7 +4992,7 @@ static int kvm_get_stat_per_vcpu(struct kvm *kvm, size_t offset, u64 *val) *val = 0; kvm_for_each_vcpu(i, vcpu, kvm) - *val += *(u64 *)((void *)vcpu + offset); + *val += *(u64 *)((void *)(&vcpu->stat) + offset); return 0; } @@ -4793,7 +5003,7 @@ static int kvm_clear_stat_per_vcpu(struct kvm *kvm, size_t offset) struct kvm_vcpu *vcpu; kvm_for_each_vcpu(i, vcpu, kvm) - *(u64 *)((void *)vcpu + offset) = 0; + *(u64 *)((void *)(&vcpu->stat) + offset) = 0; return 0; } @@ -4803,14 +5013,14 @@ static int kvm_stat_data_get(void *data, u64 *val) int r = -EFAULT; struct kvm_stat_data *stat_data = (struct kvm_stat_data *)data; - switch (stat_data->dbgfs_item->kind) { + switch (stat_data->kind) { case KVM_STAT_VM: r = kvm_get_stat_per_vm(stat_data->kvm, - stat_data->dbgfs_item->offset, val); + stat_data->desc->desc.offset, val); break; case KVM_STAT_VCPU: r = kvm_get_stat_per_vcpu(stat_data->kvm, - stat_data->dbgfs_item->offset, val); + stat_data->desc->desc.offset, val); break; } @@ -4825,14 +5035,14 @@ static int kvm_stat_data_clear(void *data, u64 val) if (val) return -EINVAL; - switch (stat_data->dbgfs_item->kind) { + switch (stat_data->kind) { case KVM_STAT_VM: r = kvm_clear_stat_per_vm(stat_data->kvm, - stat_data->dbgfs_item->offset); + stat_data->desc->desc.offset); break; case KVM_STAT_VCPU: r = kvm_clear_stat_per_vcpu(stat_data->kvm, - stat_data->dbgfs_item->offset); + stat_data->desc->desc.offset); break; } @@ -4889,6 +5099,7 @@ static int vm_stat_clear(void *_offset, u64 val) } DEFINE_SIMPLE_ATTRIBUTE(vm_stat_fops, vm_stat_get, vm_stat_clear, "%llu\n"); +DEFINE_SIMPLE_ATTRIBUTE(vm_stat_readonly_fops, vm_stat_get, NULL, "%llu\n"); static int vcpu_stat_get(void *_offset, u64 *val) { @@ -4925,11 +5136,7 @@ static int vcpu_stat_clear(void *_offset, u64 val) DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_fops, vcpu_stat_get, vcpu_stat_clear, "%llu\n"); - -static const struct file_operations *stat_fops[] = { - [KVM_STAT_VCPU] = &vcpu_stat_fops, - [KVM_STAT_VM] = &vm_stat_fops, -}; +DEFINE_SIMPLE_ATTRIBUTE(vcpu_stat_readonly_fops, vcpu_stat_get, NULL, "%llu\n"); static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) { @@ -4983,15 +5190,32 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm) static void kvm_init_debug(void) { - struct kvm_stats_debugfs_item *p; + const struct file_operations *fops; + const struct _kvm_stats_desc *pdesc; + int i; kvm_debugfs_dir = debugfs_create_dir("kvm", NULL); - kvm_debugfs_num_entries = 0; - for (p = debugfs_entries; p->name; ++p, kvm_debugfs_num_entries++) { - debugfs_create_file(p->name, KVM_DBGFS_GET_MODE(p), - kvm_debugfs_dir, (void *)(long)p->offset, - stat_fops[p->kind]); + for (i = 0; i < kvm_vm_stats_header.num_desc; ++i) { + pdesc = &kvm_vm_stats_desc[i]; + if (kvm_stats_debugfs_mode(pdesc) & 0222) + fops = &vm_stat_fops; + else + fops = &vm_stat_readonly_fops; + debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), + kvm_debugfs_dir, + (void *)(long)pdesc->desc.offset, fops); + } + + for (i = 0; i < kvm_vcpu_stats_header.num_desc; ++i) { + pdesc = &kvm_vcpu_stats_desc[i]; + if (kvm_stats_debugfs_mode(pdesc) & 0222) + fops = &vcpu_stat_fops; + else + fops = &vcpu_stat_readonly_fops; + debugfs_create_file(pdesc->name, kvm_stats_debugfs_mode(pdesc), + kvm_debugfs_dir, + (void *)(long)pdesc->desc.offset, fops); } } @@ -5141,7 +5365,8 @@ int kvm_init(void *opaque, unsigned vcpu_size, unsigned vcpu_align, kmem_cache_create_usercopy("kvm_vcpu", vcpu_size, vcpu_align, SLAB_ACCOUNT, offsetof(struct kvm_vcpu, arch), - sizeof_field(struct kvm_vcpu, arch), + offsetofend(struct kvm_vcpu, stats_id) + - offsetof(struct kvm_vcpu, arch), NULL); if (!kvm_vcpu_cache) { r = -ENOMEM;