github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-06-08 06:25:52 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

MALI: rockchip: upgrade bifrost DDK to r25p1-01bet0, from r21p0-01rel0

Browse Source 
In addition, rename some configs that have been used in drivers/gpu/arm/midgard.

Change-Id: I4244ea3ac7f039f182d5e58f677d9c31417188bf
Signed-off-by: Zhen Chen <chenzhen@rock-chips.com>
This commit is contained in:
Zhen Chen 2020-08-18 18:20:15 +08:00 committed by Tao Huang
parent 83297698a1
commit 6017dfa725
161 changed files with 18688 additions and 11121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Documentation/devicetree/bindings/arm/mali-bifrost.txt
View File

@ -1,5 +1,5 @@
#
# (C) COPYRIGHT 2013-2019 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@ -19,13 +19,14 @@
#
#
* ARM Mali Midgard devices
* ARM Mali Midgard / Bifrost devices
Required properties:
- compatible : Should be mali<chip>, replacing digits with x from the back,
until malit<Major>xx, ending with arm,mali-midgard, the latter not optional.
until malit<Major>xx, and it must end with one of: "arm,malit6xx" or
"arm,mali-midgard" or "arm,mali-bifrost"
- reg : Physical base address of the device and length of the register area.
- interrupts : Contains the three IRQ lines required by T-6xx devices
- interrupt-names : Contains the names of IRQ resources in the order they were
@ -107,13 +108,22 @@ for details.
model is not found in the registered models list. If no model is specified here,
a gpu-id based model is picked if available, otherwise the default model is used.
- mali-simple-power-model: Default model used on mali
- protected-mode-switcher : Phandle to device implemented protected mode switching functionality.
Refer to Documentation/devicetree/bindings/arm/smc-protected-mode-switcher.txt for one implementation.
- idvs-group-size : Override the IDVS group size value. Tasks are sent to
cores in groups of N + 1, so i.e. 0xF means 16 tasks.
Valid values are between 0 to 0x3F (including).
- l2-size : Override L2 cache size on GPU that supports it
- l2-hash : Override L2 hash function on GPU that supports it
- arbiter_if : Phandle to the arbif platform device, used to provide KBASE with an interface
to the Arbiter. This is required when using arbitration; setting to a non-NULL
value will enable arbitration.
If arbitration is in use, then there should be no external GPU control.
When arbiter_if is in use then the following must not be:
- power_model (no IPA allowed with arbitration)
- #cooling-cells
- operating-points-v2 (no dvfs in kbase with arbitration)
- system-coherency with a value of 1 (no full coherency with arbitration)
Example for a Mali GPU with 1 clock and no regulators:

86
drivers/gpu/arm/bifrost/Kbuild
View File

@ -1,5 +1,5 @@
#
# (C) COPYRIGHT 2012-2018 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@ -20,9 +20,12 @@
#
# Driver version string which is returned to userspace via an ioctl
MALI_RELEASE_NAME ?= "r21p0-01rel0"
MALI_RELEASE_NAME ?= "r25p1-01bet0"
# Paths required for build
# make $(src) as absolute path if it isn't already, by prefixing $(srctree)
src:=$(if $(patsubst /%,,$(src)),$(srctree)/$(src),$(src))
KBASE_PATH = $(src)
KBASE_PLATFORM_PATH = $(KBASE_PATH)/platform_dummy
UMP_PATH = $(src)/../../../base
@ -34,6 +37,14 @@ MALI_UNIT_TEST ?= 0
MALI_KERNEL_TEST_API ?= 0
MALI_COVERAGE ?= 0
CONFIG_MALI_PLATFORM_NAME ?= "devicetree"
# Experimental features (corresponding -D definition should be appended to
# DEFINES below, e.g. for MALI_EXPERIMENTAL_FEATURE,
# -DMALI_EXPERIMENTAL_FEATURE=$(MALI_EXPERIMENTAL_FEATURE) should be appended)
#
# Experimental features must default to disabled, e.g.:
# MALI_EXPERIMENTAL_FEATURE ?= 0
MALI_JIT_PRESSURE_LIMIT ?= 0
MALI_INCREMENTAL_RENDERING ?= 0
# Set up our defines, which will be passed to gcc
DEFINES = \
@ -42,7 +53,9 @@ DEFINES = \
-DMALI_KERNEL_TEST_API=$(MALI_KERNEL_TEST_API) \
-DMALI_UNIT_TEST=$(MALI_UNIT_TEST) \
-DMALI_COVERAGE=$(MALI_COVERAGE) \
-DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\"
-DMALI_RELEASE_NAME=\"$(MALI_RELEASE_NAME)\" \
-DMALI_JIT_PRESSURE_LIMIT=$(MALI_JIT_PRESSURE_LIMIT) \
-DMALI_INCREMENTAL_RENDERING=$(MALI_INCREMENTAL_RENDERING)
ifeq ($(KBUILD_EXTMOD),)
# in-tree
@ -61,21 +74,16 @@ ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH)
subdir-ccflags-y += $(DEFINES) -I$(KBASE_PATH) -I$(KBASE_PLATFORM_PATH) -I$(UMP_PATH) -I$(srctree)/include/linux
SRC := \
mali_kbase_device.c \
context/mali_kbase_context.c \
debug/mali_kbase_debug_ktrace.c \
device/mali_kbase_device.c \
mali_kbase_cache_policy.c \
mali_kbase_mem.c \
mali_kbase_mem_pool_group.c \
mali_kbase_mmu.c \
mali_kbase_native_mgm.c \
mali_kbase_ctx_sched.c \
mali_kbase_jd.c \
mali_kbase_jd_debugfs.c \
mali_kbase_jm.c \
mali_kbase_gpuprops.c \
mali_kbase_js.c \
mali_kbase_js_ctx_attr.c \
mali_kbase_event.c \
mali_kbase_context.c \
mali_kbase_pm.c \
mali_kbase_config.c \
mali_kbase_vinstr.c \
@ -86,38 +94,63 @@ SRC := \
mali_kbase_hwcnt_types.c \
mali_kbase_hwcnt_virtualizer.c \
mali_kbase_softjobs.c \
mali_kbase_10969_workaround.c \
mali_kbase_hw.c \
mali_kbase_debug.c \
mali_kbase_gpu_memory_debugfs.c \
mali_kbase_mem_linux.c \
mali_kbase_core_linux.c \
mali_kbase_mem_profile_debugfs.c \
mali_kbase_mmu_mode_lpae.c \
mali_kbase_mmu_mode_aarch64.c \
mmu/mali_kbase_mmu.c \
mmu/mali_kbase_mmu_hw_direct.c \
mmu/mali_kbase_mmu_mode_lpae.c \
mmu/mali_kbase_mmu_mode_aarch64.c \
mali_kbase_disjoint_events.c \
mali_kbase_debug_mem_view.c \
mali_kbase_debug_job_fault.c \
mali_kbase_smc.c \
mali_kbase_mem_pool.c \
mali_kbase_mem_pool_debugfs.c \
mali_kbase_debugfs_helper.c \
mali_kbase_timeline.c \
mali_kbase_timeline_io.c \
mali_kbase_tlstream.c \
mali_kbase_tracepoints.c \
mali_kbase_strings.c \
mali_kbase_as_fault_debugfs.c \
mali_kbase_regs_history_debugfs.c \
thirdparty/mali_kbase_mmap.c
thirdparty/mali_kbase_mmap.c \
tl/mali_kbase_timeline.c \
tl/mali_kbase_timeline_io.c \
tl/mali_kbase_tlstream.c \
tl/mali_kbase_tracepoints.c \
gpu/mali_kbase_gpu.c
ifeq ($(MALI_USE_CSF),1)
SRC += \
debug/backend/mali_kbase_debug_ktrace_csf.c \
device/backend/mali_kbase_device_csf.c \
gpu/backend/mali_kbase_gpu_fault_csf.c \
tl/backend/mali_kbase_timeline_csf.c \
mmu/backend/mali_kbase_mmu_csf.c \
context/backend/mali_kbase_context_csf.c
else
SRC += \
mali_kbase_dummy_job_wa.c \
mali_kbase_debug_job_fault.c \
mali_kbase_event.c \
mali_kbase_jd.c \
mali_kbase_jd_debugfs.c \
mali_kbase_js.c \
mali_kbase_js_ctx_attr.c \
debug/backend/mali_kbase_debug_ktrace_jm.c \
device/backend/mali_kbase_device_jm.c \
gpu/backend/mali_kbase_gpu_fault_jm.c \
tl/backend/mali_kbase_timeline_jm.c \
mmu/backend/mali_kbase_mmu_jm.c \
context/backend/mali_kbase_context_jm.c
endif
ifeq ($(CONFIG_MALI_CINSTR_GWT),y)
SRC += mali_kbase_gwt.c
endif
ifeq ($(MALI_UNIT_TEST),1)
SRC += mali_kbase_timeline_test.c
SRC += tl/mali_kbase_timeline_test.c
endif
ifeq ($(MALI_CUSTOMER_RELEASE),0)
@ -125,7 +158,8 @@ ifeq ($(MALI_CUSTOMER_RELEASE),0)
endif
ccflags-y += -I$(KBASE_PATH)
ccflags-y += -I$(KBASE_PATH) -I$(KBASE_PATH)/debug \
-I$(KBASE_PATH)/debug/backend
# Tell the Linux build system from which .o file to create the kernel module
obj-$(CONFIG_MALI_BIFROST) += bifrost_kbase.o
@ -147,6 +181,14 @@ endif
ifeq ($(MALI_USE_CSF),1)
include $(src)/csf/Kbuild
else
# empty
endif
ifeq ($(CONFIG_MALI_ARBITER_SUPPORT),y)
include $(src)/arbiter/Kbuild
else
# empty
endif
bifrost_kbase-$(CONFIG_MALI_BIFROST_DMA_FENCE) += \

44
drivers/gpu/arm/bifrost/Kconfig
View File

@ -1,5 +1,5 @@
#
# (C) COPYRIGHT 2012-2019 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@ -49,6 +49,7 @@ config MALI_BIFROST_DVFS
config MALI_BIFROST_ENABLE_TRACE
bool "Enable kbase tracing"
depends on MALI_BIFROST
default y if MALI_BIFROST_DEBUG
default n
help
Enables tracing in kbase. Trace log available through
@ -83,6 +84,17 @@ config MALI_PLATFORM_NAME
include in the build. 'platform/$(MALI_PLATFORM_NAME)/Kbuild' must
exist.
config MALI_ARBITER_SUPPORT
bool "Enable arbiter support for Mali"
depends on MALI_BIFROST
default n
help
Enable support for the arbiter interface in the driver.
This allows an external arbiter to manage driver access
to GPU hardware in a virtualized environment
If unsure, say N.
# MALI_BIFROST_EXPERT configuration options
menuconfig MALI_BIFROST_EXPERT
@ -155,6 +167,7 @@ config MALI_BIFROST_ERROR_INJECT
config MALI_BIFROST_SYSTEM_TRACE
bool "Enable system event tracing support"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT
default y if MALI_BIFROST_DEBUG
default n
help
Choose this option to enable system trace events for each
@ -209,13 +222,17 @@ config MALI_DMA_BUF_MAP_ON_DEMAND
config MALI_DMA_BUF_LEGACY_COMPAT
bool "Enable legacy compatibility cache flush on dma-buf map"
depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND
default y
default n
help
This option enables compatibility with legacy dma-buf mapping
behavior, then the dma-buf is mapped on import, by adding cache
maintenance where MALI_DMA_BUF_MAP_ON_DEMAND would do the mapping,
including a cache flush.
This option might work-around issues related to missing cache
flushes in other drivers. This only has an effect for clients using
UK 11.18 or older. For later UK versions it is not possible.
config MALI_HW_ERRATA_1485982_NOT_AFFECTED
bool "Disable workaround for BASE_HW_ISSUE_GPU2017_1336"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT
@ -239,6 +256,14 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
tree using the property, opp-mali-errata-1485982. Otherwise the
slowest clock will be selected.
config MALI_GEM5_BUILD
bool "Enable build of Mali kernel driver for GEM5"
depends on MALI_BIFROST
default n
help
This option is to do a Mali GEM5 build.
If unsure, say N.
# Instrumentation options.
config MALI_JOB_DUMP
@ -264,5 +289,20 @@ config MALI_BIFROST_PRFCNT_SET_SECONDARY
If unsure, say N.
config MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
bool "Use secondary set of performance counters"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT && !MALI_BIFROST_PRFCNT_SET_SECONDARY && DEBUG_FS
default n
help
Select this option to make the secondary set of performance counters
available at runtime via debugfs. Kernel features that depend on an
access to the primary set of counters may become unavailable.
This feature is unsupported and unstable, and may break at any time.
Enabling this option will prevent power management from working
optimally and may cause instrumentation tools to return bogus results.
If unsure, say N.
source "drivers/gpu/arm/midgard/platform/Kconfig"
# source "drivers/gpu/arm/midgard/tests/Kconfig"

28
drivers/gpu/arm/bifrost/Mconfig
View File

@ -1,5 +1,5 @@
#
# (C) COPYRIGHT 2012-2019 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2012-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@ -39,7 +39,7 @@ config MALI_BIFROST_DVFS
config MALI_BIFROST_ENABLE_TRACE
bool "Enable kbase tracing"
depends on MALI_BIFROST
default y if MALI_BIFROST_DEBUG
default n
help
Enables tracing in kbase. Trace log available through
@ -81,6 +81,17 @@ config MALI_PLATFORM_NAME
When PLATFORM_CUSTOM is set, this needs to be set manually to
pick up the desired platform files.
config MALI_ARBITER_SUPPORT
bool "Enable arbiter support for Mali"
depends on MALI_BIFROST
default n
help
Enable support for the arbiter interface in the driver.
This allows an external arbiter to manage driver access
to GPU hardware in a virtualized environment
If unsure, say N.
# MALI_BIFROST_EXPERT configuration options
menuconfig MALI_BIFROST_EXPERT
@ -167,10 +178,11 @@ config MALI_BIFROST_ERROR_INJECT
config MALI_BIFROST_SYSTEM_TRACE
bool "Enable system event tracing support"
depends on MALI_BIFROST && MALI_BIFROST_EXPERT
default y if MALI_BIFROST_DEBUG
default n
help
Choose this option to enable system trace events for each
kbase event. This is typically used for debugging but has
kbase event. This is typically used for debugging but has
minimal overhead when not in use. Enable only if you know what
you are doing.
@ -213,7 +225,7 @@ config MALI_DMA_BUF_MAP_ON_DEMAND
config MALI_DMA_BUF_LEGACY_COMPAT
bool "Enable legacy compatibility cache flush on dma-buf map"
depends on MALI_BIFROST && !MALI_DMA_BUF_MAP_ON_DEMAND
default y
default n
help
This option enables compatibility with legacy dma-buf mapping
behavior, then the dma-buf is mapped on import, by adding cache
@ -249,6 +261,14 @@ config MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE
tree using the property, opp-mali-errata-1485982. Otherwise the
slowest clock will be selected.
config MALI_GEM5_BUILD
bool "Enable build of Mali kernel driver for GEM5"
depends on MALI_BIFROST
default n
help
This option is to do a Mali GEM5 build.
If unsure, say N.
# Instrumentation options.
# config MALI_JOB_DUMP exists in the Kernel Kconfig but is configured using CINSTR_JOB_DUMP in Mconfig.

6
drivers/gpu/arm/bifrost/platform/meson/Kbuild → drivers/gpu/arm/bifrost/arbiter/Kbuild
View File

@ -1,5 +1,5 @@
#
# (C) COPYRIGHT 2012-2017, 2019 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@ -20,5 +20,5 @@
#
mali_kbase-y += \
$(MALI_PLATFORM_DIR)/mali_kbase_config_meson.o \
$(MALI_PLATFORM_DIR)/mali_kbase_runtime_pm.o
arbiter/mali_kbase_arbif.o \
arbiter/mali_kbase_arbiter_pm.o

175
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.c Normal file
View File

@ -0,0 +1,175 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/**
* @file mali_kbase_arbif.c
* Mali arbiter interface APIs to share GPU between Virtual Machines
*/
#include <mali_kbase.h>
#include "mali_kbase_arbif.h"
#include <tl/mali_kbase_tracepoints.h>
#include <linux/of.h>
#include <linux/of_platform.h>
#include "mali_kbase_arbiter_interface.h"
static void on_gpu_stop(struct device *dev)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
KBASE_TLSTREAM_TL_EVENT_ARB_STOP_REQUESTED(kbdev, kbdev);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_STOP_EVT);
}
static void on_gpu_granted(struct device *dev)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
KBASE_TLSTREAM_TL_EVENT_ARB_GRANTED(kbdev, kbdev);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_GRANTED_EVT);
}
static void on_gpu_lost(struct device *dev)
{
struct kbase_device *kbdev = dev_get_drvdata(dev);
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_LOST_EVT);
}
int kbase_arbif_init(struct kbase_device *kbdev)
{
#ifdef CONFIG_OF
struct arbiter_if_arb_vm_ops ops;
struct arbiter_if_dev *arb_if;
struct device_node *arbiter_if_node;
struct platform_device *pdev;
int err;
dev_dbg(kbdev->dev, "%s\n", __func__);
arbiter_if_node = of_parse_phandle(kbdev->dev->of_node,
"arbiter_if", 0);
if (!arbiter_if_node) {
dev_dbg(kbdev->dev, "No arbiter_if in Device Tree\n");
/* no arbiter interface defined in device tree */
kbdev->arb.arb_dev = NULL;
kbdev->arb.arb_if = NULL;
return 0;
}
pdev = of_find_device_by_node(arbiter_if_node);
if (!pdev) {
dev_err(kbdev->dev, "Failed to find arbiter_if device\n");
return -EPROBE_DEFER;
}
if (!pdev->dev.driver || !try_module_get(pdev->dev.driver->owner)) {
dev_err(kbdev->dev, "arbiter_if driver not available\n");
return -EPROBE_DEFER;
}
kbdev->arb.arb_dev = &pdev->dev;
arb_if = platform_get_drvdata(pdev);
if (!arb_if) {
dev_err(kbdev->dev, "arbiter_if driver not ready\n");
module_put(pdev->dev.driver->owner);
return -EPROBE_DEFER;
}
kbdev->arb.arb_if = arb_if;
ops.arb_vm_gpu_stop = on_gpu_stop;
ops.arb_vm_gpu_granted = on_gpu_granted;
ops.arb_vm_gpu_lost = on_gpu_lost;
/* register kbase arbiter_if callbacks */
if (arb_if->vm_ops.vm_arb_register_dev) {
err = arb_if->vm_ops.vm_arb_register_dev(arb_if,
kbdev->dev, &ops);
if (err) {
dev_err(kbdev->dev, "Arbiter registration failed.\n");
module_put(pdev->dev.driver->owner);
return err;
}
}
#else /* CONFIG_OF */
dev_dbg(kbdev->dev, "No arbiter without Device Tree support\n");
kbdev->arb.arb_dev = NULL;
kbdev->arb.arb_if = NULL;
#endif
return 0;
}
void kbase_arbif_destroy(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_unregister_dev) {
dev_dbg(kbdev->dev, "%s\n", __func__);
arb_if->vm_ops.vm_arb_unregister_dev(kbdev->arb.arb_if);
}
kbdev->arb.arb_if = NULL;
if (kbdev->arb.arb_dev)
module_put(kbdev->arb.arb_dev->driver->owner);
kbdev->arb.arb_dev = NULL;
}
void kbase_arbif_gpu_request(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_request) {
dev_dbg(kbdev->dev, "%s\n", __func__);
arb_if->vm_ops.vm_arb_gpu_request(arb_if);
}
}
void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_stopped) {
dev_dbg(kbdev->dev, "%s\n", __func__);
KBASE_TLSTREAM_TL_EVENT_ARB_STOPPED(kbdev, kbdev);
arb_if->vm_ops.vm_arb_gpu_stopped(arb_if, gpu_required);
}
}
void kbase_arbif_gpu_active(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_active) {
dev_dbg(kbdev->dev, "%s\n", __func__);
arb_if->vm_ops.vm_arb_gpu_active(arb_if);
}
}
void kbase_arbif_gpu_idle(struct kbase_device *kbdev)
{
struct arbiter_if_dev *arb_if = kbdev->arb.arb_if;
if (arb_if && arb_if->vm_ops.vm_arb_gpu_idle) {
dev_dbg(kbdev->dev, "vm_arb_gpu_idle\n");
arb_if->vm_ops.vm_arb_gpu_idle(arb_if);
}
}

133
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbif.h Normal file
View File

@ -0,0 +1,133 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Mali arbiter interface APIs to share GPU between Virtual Machines
*/
#ifndef _MALI_KBASE_ARBIF_H_
#define _MALI_KBASE_ARBIF_H_
/**
* enum kbase_arbif_evt - Internal Arbiter event.
*
* @KBASE_VM_GPU_INITIALIZED_EVT: KBase has finished initializing
* and can be stopped
* @KBASE_VM_GPU_STOP_EVT: Stop message received from Arbiter
* @KBASE_VM_GPU_GRANTED_EVT: Grant message received from Arbiter
* @KBASE_VM_GPU_LOST_EVT: Lost message received from Arbiter
* @KBASE_VM_GPU_IDLE_EVENT: KBase has transitioned into an inactive state.
* @KBASE_VM_REF_EVENT: KBase has transitioned into an active state.
* @KBASE_VM_OS_SUSPEND_EVENT: KBase is suspending
* @KBASE_VM_OS_RESUME_EVENT: Kbase is resuming
*/
enum kbase_arbif_evt {
KBASE_VM_GPU_INITIALIZED_EVT = 1,
KBASE_VM_GPU_STOP_EVT,
KBASE_VM_GPU_GRANTED_EVT,
KBASE_VM_GPU_LOST_EVT,
KBASE_VM_GPU_IDLE_EVENT,
KBASE_VM_REF_EVENT,
KBASE_VM_OS_SUSPEND_EVENT,
KBASE_VM_OS_RESUME_EVENT,
};
/**
* kbase_arbif_init() - Initialize the arbiter interface functionality.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Initialize the arbiter interface and also determines
* if Arbiter functionality is required.
*
* Return: 0 if the Arbiter interface was successfully initialized or the
* Arbiter was not required.
*/
int kbase_arbif_init(struct kbase_device *kbdev);
/**
* kbase_arbif_destroy() - Cleanups the arbiter interface functionality.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Cleans up the arbiter interface functionality and resets the reference count
* of the arbif module used
*/
void kbase_arbif_destroy(struct kbase_device *kbdev);
/**
* kbase_arbif_gpu_request() - Send GPU request message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Sends a message to Arbiter to request GPU access.
*/
void kbase_arbif_gpu_request(struct kbase_device *kbdev);
/**
* kbase_arbif_gpu_stopped() - Send GPU stopped message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @gpu_required: true if GPU access is still required
* (Arbiter will automatically send another grant message)
*
* Sends a message to Arbiter to notify that the GPU has stopped.
* @note Once this call has been made, KBase must not attempt to access the GPU
* until the #KBASE_VM_GPU_GRANTED_EVT event has been received.
*/
void kbase_arbif_gpu_stopped(struct kbase_device *kbdev, u8 gpu_required);
/**
* kbase_arbif_gpu_active() - Send a GPU active message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Sends a message to Arbiter to report that KBase has gone active.
*/
void kbase_arbif_gpu_active(struct kbase_device *kbdev);
/**
* kbase_arbif_gpu_idle() - Send a GPU idle message to the arbiter
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Sends a message to Arbiter to report that KBase has gone idle.
*/
void kbase_arbif_gpu_idle(struct kbase_device *kbdev);
#endif /* _MALI_KBASE_ARBIF_H_ */

95
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_defs.h Normal file
View File

@ -0,0 +1,95 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Mali structures define to support arbitration feature
*/
#ifndef _MALI_KBASE_ARBITER_DEFS_H_
#define _MALI_KBASE_ARBITER_DEFS_H_
#include "mali_kbase_arbiter_pm.h"
/**
* struct kbase_arbiter_vm_state - Struct representing the state and containing the
* data of pm work
* @kbdev: Pointer to kbase device structure (must be a valid pointer)
* @vm_state_lock: The lock protecting the VM state when arbiter is used.
* This lock must also be held whenever the VM state is being
* transitioned
* @vm_state_wait: Wait queue set when GPU is granted
* @vm_state: Current state of VM
* @vm_arb_wq: Work queue for resuming or stopping work on the GPU for use
* with the Arbiter
* @vm_suspend_work: Work item for vm_arb_wq to stop current work on GPU
* @vm_resume_work: Work item for vm_arb_wq to resume current work on GPU
* @vm_arb_starting: Work queue resume in progress
* @vm_arb_stopping: Work queue suspend in progress
* @vm_arb_users_waiting: Count of users waiting for GPU
*/
struct kbase_arbiter_vm_state {
struct kbase_device *kbdev;
struct mutex vm_state_lock;
wait_queue_head_t vm_state_wait;
enum kbase_vm_state vm_state;
struct workqueue_struct *vm_arb_wq;
struct work_struct vm_suspend_work;
struct work_struct vm_resume_work;
bool vm_arb_starting;
bool vm_arb_stopping;
int vm_arb_users_waiting;
};
/**
* struct kbase_arbiter_device - Representing an instance of arbiter device,
* allocated from the probe method of Mali driver
* @arb_if: Pointer to the arbiter interface device
* @arb_dev: Pointer to the arbiter device
*/
struct kbase_arbiter_device {
struct arbiter_if_dev *arb_if;
struct device *arb_dev;
};
#endif /* _MALI_KBASE_ARBITER_DEFS_H_ */

181
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_interface.h Normal file
View File

@ -0,0 +1,181 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*
*/
/**
* @file
* Defines the Mali arbiter interface
*/
#ifndef _MALI_KBASE_ARBITER_INTERFACE_H_
#define _MALI_KBASE_ARBITER_INTERFACE_H_
/**
* @brief Mali arbiter interface version
*
* This specifies the current version of the configuration interface. Whenever
* the arbiter interface changes, so that integration effort is required, the
* version number will be increased. Each configuration must make an effort
* to check that it implements the correct version.
*
* Version history:
* 1 - Added the Mali arbiter configuration interface.
* 2 - Strip out reference code from header
* 3 - Removed DVFS utilization interface (DVFS moved to arbiter side)
*/
#define MALI_KBASE_ARBITER_INTERFACE_VERSION 3
struct arbiter_if_dev;
/**
* struct arbiter_if_arb_vm_ops - Interface to communicate messages to VM
*
* This struct contains callbacks used to deliver messages
* from the arbiter to the corresponding VM.
*
* Note that calls into these callbacks may have synchronous calls back into
* the arbiter arbiter_if_vm_arb_ops callbacks below.
* For example vm_arb_gpu_stopped() may be called as a side effect of
* arb_vm_gpu_stop() being called here.
*/
struct arbiter_if_arb_vm_ops {
/**
* arb_vm_gpu_stop() - Ask VM to stop using GPU
* @dev: The arbif kernel module device.
*
* Informs KBase to stop using the GPU as soon as possible.
* @Note: Once the driver is no longer using the GPU, a call to
* vm_arb_gpu_stopped is expected by the arbiter.
*/
void (*arb_vm_gpu_stop)(struct device *dev);
/**
* arb_vm_gpu_granted() - GPU has been granted to VM
* @dev: The arbif kernel module device.
*
* Informs KBase that the GPU can now be used by the VM.
*/
void (*arb_vm_gpu_granted)(struct device *dev);
/**
* arb_vm_gpu_lost() - VM has lost the GPU
* @dev: The arbif kernel module device.
*
* This is called if KBase takes too long to respond to the arbiter
* stop request.
* Once this is called, KBase will assume that access to the GPU
* has been lost and will fail all running jobs and reset its
* internal state.
* If successful, will respond with a vm_arb_gpu_stopped message.
*/
void (*arb_vm_gpu_lost)(struct device *dev);
};
/**
* struct arbiter_if_vm_arb_ops - Interface to communicate messages to arbiter
*
* This struct contains callbacks used to request operations
* from the VM to the arbiter
*
* Note that we must not make any synchronous calls back in to the VM
* (via arbiter_if_arb_vm_ops above) in the context of these callbacks.
*/
struct arbiter_if_vm_arb_ops {
/**
* vm_arb_register_dev() - Register VM device driver callbacks.
* @arbif_dev: The arbiter interface we are registering device callbacks
* @dev: The device structure to supply in the callbacks.
* @ops: The callbacks that the device driver supports
* (none are optional).
*/
int (*vm_arb_register_dev)(struct arbiter_if_dev *arbif_dev,
struct device *dev, struct arbiter_if_arb_vm_ops *ops);
/**
* vm_arb_unregister_dev() - Unregister VM device driver callbacks.
* @arbif_dev: The arbiter interface we are unregistering from.
*/
void (*vm_arb_unregister_dev)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_request() - Ask the arbiter interface for GPU access.
* @arbif_dev: The arbiter interface we want to issue the request.
*/
void (*vm_arb_gpu_request)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_active() - Inform arbiter that the driver has gone active
* @arbif_dev: The arbiter interface device.
*/
void (*vm_arb_gpu_active)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_idle() - Inform the arbiter that the driver has gone idle
* @arbif_dev: The arbiter interface device.
*/
void (*vm_arb_gpu_idle)(struct arbiter_if_dev *arbif_dev);
/**
* vm_arb_gpu_stopped() - Inform the arbiter that the driver has stopped
* using the GPU
* @arbif_dev: The arbiter interface device.
* @gpu_required: The GPU is still needed to do more work.
*/
void (*vm_arb_gpu_stopped)(struct arbiter_if_dev *arbif_dev,
u8 gpu_required);
};
/**
* struct arbiter_if_dev - Arbiter Interface
* @vm_ops: Callback functions for connecting KBase with
* arbiter interface device.
* @priv_data: Internal arbif data not used by KBASE.
*
* Arbiter Interface Kernel Module State used for linking KBase
* with an arbiter interface platform device
*/
struct arbiter_if_dev {
struct arbiter_if_vm_arb_ops vm_ops;
void *priv_data;
};
#endif /* _MALI_KBASE_ARBITER_INTERFACE_H_ */

645
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.c Normal file
View File

@ -0,0 +1,645 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/**
* @file mali_kbase_arbiter_pm.c
* Mali arbiter power manager state machine and APIs
*/
#include <mali_kbase.h>
#include <mali_kbase_pm.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_irq_internal.h>
#include <mali_kbase_hwcnt_context.h>
#include <mali_kbase_pm_internal.h>
#include <tl/mali_kbase_tracepoints.h>
static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev);
static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(
struct kbase_device *kbdev);
static inline const char *kbase_arbiter_pm_vm_state_str(
enum kbase_vm_state state)
{
switch (state) {
case KBASE_VM_STATE_INITIALIZING:
return "KBASE_VM_STATE_INITIALIZING";
case KBASE_VM_STATE_INITIALIZING_WITH_GPU:
return "KBASE_VM_STATE_INITIALIZING_WITH_GPU";
case KBASE_VM_STATE_SUSPENDED:
return "KBASE_VM_STATE_SUSPENDED";
case KBASE_VM_STATE_STOPPED:
return "KBASE_VM_STATE_STOPPED";
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
return "KBASE_VM_STATE_STOPPED_GPU_REQUESTED";
case KBASE_VM_STATE_STARTING:
return "KBASE_VM_STATE_STARTING";
case KBASE_VM_STATE_IDLE:
return "KBASE_VM_STATE_IDLE";
case KBASE_VM_STATE_ACTIVE:
return "KBASE_VM_STATE_ACTIVE";
case KBASE_VM_STATE_STOPPING_IDLE:
return "KBASE_VM_STATE_STOPPING_IDLE";
case KBASE_VM_STATE_STOPPING_ACTIVE:
return "KBASE_VM_STATE_STOPPING_ACTIVE";
case KBASE_VM_STATE_SUSPEND_PENDING:
return "KBASE_VM_STATE_SUSPEND_PENDING";
case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
return "KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT";
default:
KBASE_DEBUG_ASSERT(false);
return "[UnknownState]";
}
}
static inline const char *kbase_arbiter_pm_vm_event_str(
enum kbase_arbif_evt evt)
{
switch (evt) {
case KBASE_VM_GPU_INITIALIZED_EVT:
return "KBASE_VM_GPU_INITIALIZED_EVT";
case KBASE_VM_GPU_STOP_EVT:
return "KBASE_VM_GPU_STOP_EVT";
case KBASE_VM_GPU_GRANTED_EVT:
return "KBASE_VM_GPU_GRANTED_EVT";
case KBASE_VM_GPU_LOST_EVT:
return "KBASE_VM_GPU_LOST_EVT";
case KBASE_VM_OS_SUSPEND_EVENT:
return "KBASE_VM_OS_SUSPEND_EVENT";
case KBASE_VM_OS_RESUME_EVENT:
return "KBASE_VM_OS_RESUME_EVENT";
case KBASE_VM_GPU_IDLE_EVENT:
return "KBASE_VM_GPU_IDLE_EVENT";
case KBASE_VM_REF_EVENT:
return "KBASE_VM_REF_EVENT";
default:
KBASE_DEBUG_ASSERT(false);
return "[UnknownEvent]";
}
}
static void kbase_arbiter_pm_vm_set_state(struct kbase_device *kbdev,
enum kbase_vm_state new_state)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
dev_dbg(kbdev->dev, "VM set_state %s -> %s",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state),
kbase_arbiter_pm_vm_state_str(new_state));
lockdep_assert_held(&arb_vm_state->vm_state_lock);
arb_vm_state->vm_state = new_state;
wake_up(&arb_vm_state->vm_state_wait);
}
static void kbase_arbiter_pm_suspend_wq(struct work_struct *data)
{
struct kbase_arbiter_vm_state *arb_vm_state = container_of(data,
struct kbase_arbiter_vm_state,
vm_suspend_work);
struct kbase_device *kbdev = arb_vm_state->kbdev;
mutex_lock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">%s\n", __func__);
if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE ||
arb_vm_state->vm_state ==
KBASE_VM_STATE_STOPPING_ACTIVE ||
arb_vm_state->vm_state ==
KBASE_VM_STATE_SUSPEND_PENDING) {
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">kbase_pm_driver_suspend\n");
kbase_pm_driver_suspend(kbdev);
dev_dbg(kbdev->dev, "<kbase_pm_driver_suspend\n");
mutex_lock(&arb_vm_state->vm_state_lock);
}
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, "<%s\n", __func__);
}
static void kbase_arbiter_pm_resume_wq(struct work_struct *data)
{
struct kbase_arbiter_vm_state *arb_vm_state = container_of(data,
struct kbase_arbiter_vm_state,
vm_resume_work);
struct kbase_device *kbdev = arb_vm_state->kbdev;
mutex_lock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">%s\n", __func__);
arb_vm_state->vm_arb_starting = true;
if (arb_vm_state->vm_state == KBASE_VM_STATE_STARTING) {
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, ">kbase_pm_driver_resume\n");
kbase_pm_driver_resume(kbdev, true);
dev_dbg(kbdev->dev, "<kbase_pm_driver_resume\n");
mutex_lock(&arb_vm_state->vm_state_lock);
} else if (arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_ACTIVE) {
kbase_arbiter_pm_vm_stopped(kbdev);
}
arb_vm_state->vm_arb_starting = false;
mutex_unlock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, "<%s\n", __func__);
}
int kbase_arbiter_pm_early_init(struct kbase_device *kbdev)
{
int err;
struct kbase_arbiter_vm_state *arb_vm_state = NULL;
arb_vm_state = kmalloc(sizeof(struct kbase_arbiter_vm_state),
GFP_KERNEL);
if (arb_vm_state == NULL)
return -ENOMEM;
arb_vm_state->kbdev = kbdev;
arb_vm_state->vm_state = KBASE_VM_STATE_INITIALIZING;
mutex_init(&arb_vm_state->vm_state_lock);
init_waitqueue_head(&arb_vm_state->vm_state_wait);
arb_vm_state->vm_arb_wq = alloc_ordered_workqueue("kbase_vm_arb_wq",
WQ_HIGHPRI);
if (!arb_vm_state->vm_arb_wq) {
dev_err(kbdev->dev, "Failed to allocate vm_arb workqueue\n");
return -ENOMEM;
}
INIT_WORK(&arb_vm_state->vm_suspend_work, kbase_arbiter_pm_suspend_wq);
INIT_WORK(&arb_vm_state->vm_resume_work, kbase_arbiter_pm_resume_wq);
arb_vm_state->vm_arb_starting = false;
arb_vm_state->vm_arb_users_waiting = 0;
kbdev->pm.arb_vm_state = arb_vm_state;
err = kbase_arbif_init(kbdev);
if (err) {
goto arbif_init_fail;
}
if (kbdev->arb.arb_if) {
kbase_arbif_gpu_request(kbdev);
dev_dbg(kbdev->dev, "Waiting for initial GPU assignment...\n");
wait_event(arb_vm_state->vm_state_wait,
arb_vm_state->vm_state ==
KBASE_VM_STATE_INITIALIZING_WITH_GPU);
dev_dbg(kbdev->dev,
"Waiting for initial GPU assignment - done\n");
}
return 0;
arbif_init_fail:
destroy_workqueue(arb_vm_state->vm_arb_wq);
kfree(arb_vm_state);
kbdev->pm.arb_vm_state = NULL;
return err;
}
void kbase_arbiter_pm_early_term(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
mutex_lock(&arb_vm_state->vm_state_lock);
if (arb_vm_state->vm_state > KBASE_VM_STATE_STOPPED_GPU_REQUESTED)
kbase_arbif_gpu_stopped(kbdev, false);
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_arbif_destroy(kbdev);
destroy_workqueue(arb_vm_state->vm_arb_wq);
arb_vm_state->vm_arb_wq = NULL;
kfree(kbdev->pm.arb_vm_state);
kbdev->pm.arb_vm_state = NULL;
}
void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
mutex_lock(&arb_vm_state->vm_state_lock);
if (!kbdev->arb.arb_if ||
arb_vm_state->vm_state >
KBASE_VM_STATE_STOPPED_GPU_REQUESTED)
kbase_release_interrupts(kbdev);
mutex_unlock(&arb_vm_state->vm_state_lock);
}
void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev)
{
bool request_gpu = false;
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
if (arb_vm_state->vm_arb_users_waiting > 0 &&
arb_vm_state->vm_state == KBASE_VM_STATE_STOPPING_IDLE)
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
dev_dbg(kbdev->dev, "%s %s\n", __func__,
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
kbase_release_interrupts(kbdev);
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STOPPING_ACTIVE:
request_gpu = true;
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
break;
case KBASE_VM_STATE_STOPPING_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STOPPED);
break;
case KBASE_VM_STATE_SUSPEND_PENDING:
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
break;
default:
dev_warn(kbdev->dev, "unexpected pm_stop VM state %u",
arb_vm_state->vm_state);
break;
}
kbase_arbif_gpu_stopped(kbdev, request_gpu);
}
static void kbase_arbiter_pm_vm_gpu_start(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_INITIALIZING:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_INITIALIZING_WITH_GPU);
break;
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_STARTING);
kbase_install_interrupts(kbdev);
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_resume_work);
break;
case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
kbase_arbif_gpu_stopped(kbdev, false);
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
break;
default:
dev_warn(kbdev->dev,
"GPU_GRANTED when not expected - state %s\n",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
break;
}
}
static void kbase_arbiter_pm_vm_gpu_stop(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
if (arb_vm_state->vm_state == KBASE_VM_STATE_INITIALIZING_WITH_GPU) {
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
}
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_IDLE);
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_suspend_work);
break;
case KBASE_VM_STATE_ACTIVE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_suspend_work);
break;
case KBASE_VM_STATE_STARTING:
dev_dbg(kbdev->dev, "Got GPU_STOP event while STARTING.");
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
if (arb_vm_state->vm_arb_starting)
queue_work(arb_vm_state->vm_arb_wq,
&arb_vm_state->vm_suspend_work);
break;
case KBASE_VM_STATE_SUSPEND_PENDING:
/* Suspend finishes with a stop so nothing else to do */
break;
default:
dev_warn(kbdev->dev, "GPU_STOP when not expected - state %s\n",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
break;
}
}
static void kbase_gpu_lost(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STARTING:
case KBASE_VM_STATE_ACTIVE:
case KBASE_VM_STATE_IDLE:
dev_warn(kbdev->dev, "GPU lost in state %s",
kbase_arbiter_pm_vm_state_str(arb_vm_state->vm_state));
kbase_arbiter_pm_vm_gpu_stop(kbdev);
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_pm_handle_gpu_lost(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
break;
case KBASE_VM_STATE_STOPPING_IDLE:
case KBASE_VM_STATE_STOPPING_ACTIVE:
case KBASE_VM_STATE_SUSPEND_PENDING:
dev_info(kbdev->dev, "GPU lost while stopping");
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_pm_handle_gpu_lost(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
break;
case KBASE_VM_STATE_SUSPENDED:
case KBASE_VM_STATE_STOPPED:
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
dev_info(kbdev->dev, "GPU lost while already stopped");
break;
case KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT:
dev_info(kbdev->dev, "GPU lost while waiting to suspend");
kbase_arbiter_pm_vm_set_state(kbdev, KBASE_VM_STATE_SUSPENDED);
break;
default:
break;
}
}
static inline bool kbase_arbiter_pm_vm_os_suspend_ready_state(
struct kbase_device *kbdev)
{
switch (kbdev->pm.arb_vm_state->vm_state) {
case KBASE_VM_STATE_SUSPENDED:
case KBASE_VM_STATE_STOPPED:
case KBASE_VM_STATE_IDLE:
case KBASE_VM_STATE_ACTIVE:
return true;
default:
return false;
}
}
static void kbase_arbiter_pm_vm_os_prepare_suspend(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
enum kbase_vm_state prev_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
if (kbdev->arb.arb_if) {
if (kbdev->pm.arb_vm_state->vm_state ==
KBASE_VM_STATE_SUSPENDED)
return;
}
/* Block suspend OS function until we are in a stable state
* with vm_state_lock
*/
while (!kbase_arbiter_pm_vm_os_suspend_ready_state(kbdev)) {
prev_state = arb_vm_state->vm_state;
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STOPPING_ACTIVE:
case KBASE_VM_STATE_STOPPING_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_PENDING);
break;
case KBASE_VM_STATE_STOPPED_GPU_REQUESTED:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT);
break;
case KBASE_VM_STATE_STARTING:
if (!arb_vm_state->vm_arb_starting) {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_PENDING);
kbase_arbiter_pm_vm_stopped(kbdev);
}
break;
default:
break;
}
mutex_unlock(&arb_vm_state->vm_state_lock);
wait_event(arb_vm_state->vm_state_wait,
arb_vm_state->vm_state != prev_state);
mutex_lock(&arb_vm_state->vm_state_lock);
}
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STOPPED:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPENDED);
break;
case KBASE_VM_STATE_IDLE:
case KBASE_VM_STATE_ACTIVE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_SUSPEND_PENDING);
mutex_unlock(&arb_vm_state->vm_state_lock);
/* Ensure resume has completed fully before starting suspend */
flush_work(&arb_vm_state->vm_resume_work);
kbase_pm_driver_suspend(kbdev);
mutex_lock(&arb_vm_state->vm_state_lock);
break;
case KBASE_VM_STATE_SUSPENDED:
break;
default:
KBASE_DEBUG_ASSERT_MSG(false, "Unexpected state to suspend");
break;
}
}
static void kbase_arbiter_pm_vm_os_resume(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
KBASE_DEBUG_ASSERT_MSG(arb_vm_state->vm_state ==
KBASE_VM_STATE_SUSPENDED,
"Unexpected state to resume");
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
kbase_arbif_gpu_request(kbdev);
/* Release lock and block resume OS function until we have
* asynchronously received the GRANT message from the Arbiter and
* fully resumed
*/
mutex_unlock(&arb_vm_state->vm_state_lock);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
flush_work(&arb_vm_state->vm_resume_work);
mutex_lock(&arb_vm_state->vm_state_lock);
}
void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev,
enum kbase_arbif_evt evt)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
if (!kbdev->arb.arb_if)
return;
mutex_lock(&arb_vm_state->vm_state_lock);
dev_dbg(kbdev->dev, "%s %s\n", __func__,
kbase_arbiter_pm_vm_event_str(evt));
switch (evt) {
case KBASE_VM_GPU_GRANTED_EVT:
kbase_arbiter_pm_vm_gpu_start(kbdev);
break;
case KBASE_VM_GPU_STOP_EVT:
kbase_arbiter_pm_vm_gpu_stop(kbdev);
break;
case KBASE_VM_GPU_LOST_EVT:
dev_info(kbdev->dev, "KBASE_ARBIF_GPU_LOST_EVT!");
kbase_gpu_lost(kbdev);
break;
case KBASE_VM_OS_SUSPEND_EVENT:
kbase_arbiter_pm_vm_os_prepare_suspend(kbdev);
break;
case KBASE_VM_OS_RESUME_EVENT:
kbase_arbiter_pm_vm_os_resume(kbdev);
break;
case KBASE_VM_GPU_IDLE_EVENT:
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_ACTIVE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_IDLE);
kbase_arbif_gpu_idle(kbdev);
break;
default:
break;
}
break;
case KBASE_VM_REF_EVENT:
switch (arb_vm_state->vm_state) {
case KBASE_VM_STATE_STARTING:
KBASE_TLSTREAM_TL_EVENT_ARB_STARTED(kbdev, kbdev);
/* FALL THROUGH */
case KBASE_VM_STATE_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_ACTIVE);
kbase_arbif_gpu_active(kbdev);
break;
case KBASE_VM_STATE_STOPPING_IDLE:
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
break;
default:
break;
}
break;
case KBASE_VM_GPU_INITIALIZED_EVT:
lockdep_assert_held(&kbdev->pm.lock);
if (kbdev->pm.active_count > 0) {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_ACTIVE);
kbase_arbif_gpu_active(kbdev);
} else {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_IDLE);
kbase_arbif_gpu_idle(kbdev);
}
break;
default:
dev_alert(kbdev->dev, "Got Unknown Event!");
break;
}
mutex_unlock(&arb_vm_state->vm_state_lock);
}
static void kbase_arbiter_pm_vm_wait_gpu_assignment(struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
dev_dbg(kbdev->dev, "Waiting for GPU assignment...\n");
wait_event(arb_vm_state->vm_state_wait,
arb_vm_state->vm_state == KBASE_VM_STATE_IDLE ||
arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE);
dev_dbg(kbdev->dev, "Waiting for GPU assignment - done\n");
}
static inline bool kbase_arbiter_pm_vm_gpu_assigned_lockheld(
struct kbase_device *kbdev)
{
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
lockdep_assert_held(&arb_vm_state->vm_state_lock);
return (arb_vm_state->vm_state == KBASE_VM_STATE_IDLE ||
arb_vm_state->vm_state == KBASE_VM_STATE_ACTIVE);
}
int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
enum kbase_pm_suspend_handler suspend_handler)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
struct kbase_arbiter_vm_state *arb_vm_state = kbdev->pm.arb_vm_state;
if (kbdev->arb.arb_if) {
mutex_lock(&arb_vm_state->vm_state_lock);
while (!kbase_arbiter_pm_vm_gpu_assigned_lockheld(kbdev)) {
/* Update VM state since we have GPU work to do */
if (arb_vm_state->vm_state ==
KBASE_VM_STATE_STOPPING_IDLE)
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPING_ACTIVE);
else if (arb_vm_state->vm_state ==
KBASE_VM_STATE_STOPPED) {
kbase_arbiter_pm_vm_set_state(kbdev,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED);
kbase_arbif_gpu_request(kbdev);
} else if (arb_vm_state->vm_state ==
KBASE_VM_STATE_INITIALIZING_WITH_GPU)
break;
if (suspend_handler !=
KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE) {
if (suspend_handler ==
KBASE_PM_SUSPEND_HANDLER_VM_GPU_GRANTED
||
kbdev->pm.active_count > 0)
break;
mutex_unlock(&arb_vm_state->vm_state_lock);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
return 1;
}
if (arb_vm_state->vm_state ==
KBASE_VM_STATE_INITIALIZING_WITH_GPU)
break;
/* Need to synchronously wait for GPU assignment */
arb_vm_state->vm_arb_users_waiting++;
mutex_unlock(&arb_vm_state->vm_state_lock);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_arbiter_pm_vm_wait_gpu_assignment(kbdev);
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
mutex_lock(&arb_vm_state->vm_state_lock);
arb_vm_state->vm_arb_users_waiting--;
}
mutex_unlock(&arb_vm_state->vm_state_lock);
}
return 0;
}

159
drivers/gpu/arm/bifrost/arbiter/mali_kbase_arbiter_pm.h Normal file
View File

@ -0,0 +1,159 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU license.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/**
* @file
* Mali arbiter power manager state machine and APIs
*/
#ifndef _MALI_KBASE_ARBITER_PM_H_
#define _MALI_KBASE_ARBITER_PM_H_
#include "mali_kbase_arbif.h"
/**
* enum kbase_vm_state - Current PM Arbitration state.
*
* @KBASE_VM_STATE_INITIALIZING: Special state before arbiter is initialized.
* @KBASE_VM_STATE_INITIALIZING_WITH_GPU: Initialization after GPU
* has been granted.
* @KBASE_VM_STATE_SUSPENDED: KBase is suspended by OS and GPU is not assigned.
* @KBASE_VM_STATE_STOPPED: GPU is not assigned to KBase and is not required.
* @KBASE_VM_STATE_STOPPED_GPU_REQUESTED: GPU is not assigned to KBase
* but a request has been made.
* @KBASE_VM_STATE_STARTING: GPU is assigned and KBase is getting ready to run.
* @KBASE_VM_STATE_IDLE: GPU is assigned but KBase has no work to do
* @KBASE_VM_STATE_ACTIVE: GPU is assigned and KBase is busy using it
* @KBASE_VM_STATE_SUSPEND_PENDING: OS is going into suspend mode.
* @KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT: OS is going into suspend mode but GPU
* has already been requested.
* In this situation we must wait for
* the Arbiter to send a GRANTED message
* and respond immediately with
* a STOPPED message before entering
* the suspend mode.
* @KBASE_VM_STATE_STOPPING_IDLE: Arbiter has sent a stopped message and there
* is currently no work to do on the GPU.
* @KBASE_VM_STATE_STOPPING_ACTIVE: Arbiter has sent a stopped message when
* KBase has work to do.
*/
enum kbase_vm_state {
KBASE_VM_STATE_INITIALIZING,
KBASE_VM_STATE_INITIALIZING_WITH_GPU,
KBASE_VM_STATE_SUSPENDED,
KBASE_VM_STATE_STOPPED,
KBASE_VM_STATE_STOPPED_GPU_REQUESTED,
KBASE_VM_STATE_STARTING,
KBASE_VM_STATE_IDLE,
KBASE_VM_STATE_ACTIVE,
KBASE_VM_STATE_SUSPEND_PENDING,
KBASE_VM_STATE_SUSPEND_WAIT_FOR_GRANT,
KBASE_VM_STATE_STOPPING_IDLE,
KBASE_VM_STATE_STOPPING_ACTIVE
};
/**
* kbase_arbiter_pm_early_init() - Initialize arbiter for VM Paravirtualized use
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Initialize the arbiter and other required resources during the runtime
* and request the GPU for the VM for the first time.
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
int kbase_arbiter_pm_early_init(struct kbase_device *kbdev);
/**
* kbase_arbiter_pm_early_term() - Shutdown arbiter and free resources.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Clean up all the resources
*/
void kbase_arbiter_pm_early_term(struct kbase_device *kbdev);
/**
* kbase_arbiter_pm_release_interrupts() - Release the GPU interrupts
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Releases interrupts if needed (GPU is available) otherwise does nothing
*/
void kbase_arbiter_pm_release_interrupts(struct kbase_device *kbdev);
/**
* kbase_arbiter_pm_vm_event() - Dispatch VM event to the state machine
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* The state machine function. Receives events and transitions states
* according the event received and the current state
*/
void kbase_arbiter_pm_vm_event(struct kbase_device *kbdev,
enum kbase_arbif_evt event);
/**
* kbase_arbiter_pm_ctx_active_handle_suspend() - Handle suspend operation for
* arbitration mode
* @kbdev: The kbase device structure for the device (must be a valid pointer)
* @suspend_handler: The handler code for how to handle a suspend
* that might occur
*
* This function handles a suspend event from the driver,
* communicating with the arbiter and waiting synchronously for the GPU
* to be granted again depending on the VM state.
*
* Return: 0 if success, 1 if failure due to system suspending/suspended
*/
int kbase_arbiter_pm_ctx_active_handle_suspend(struct kbase_device *kbdev,
enum kbase_pm_suspend_handler suspend_handler);
/**
* kbase_arbiter_pm_vm_stopped() - Handle stop event for the VM
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* This function handles a stop event for the VM.
* It will update the VM state and forward the stop event to the driver.
*/
void kbase_arbiter_pm_vm_stopped(struct kbase_device *kbdev);
#endif /*_MALI_KBASE_ARBITER_PM_H_ */

18
drivers/gpu/arm/bifrost/backend/gpu/Kbuild
View File

@ -1,5 +1,5 @@
#
# (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved.
# (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
#
# This program is free software and is provided to you under the terms of the
# GNU General Public License version 2 as published by the Free Software
@ -22,16 +22,10 @@
BACKEND += \
backend/gpu/mali_kbase_cache_policy_backend.c \
backend/gpu/mali_kbase_device_hw.c \
backend/gpu/mali_kbase_gpu.c \
backend/gpu/mali_kbase_gpuprops_backend.c \
backend/gpu/mali_kbase_debug_job_fault_backend.c \
backend/gpu/mali_kbase_irq_linux.c \
backend/gpu/mali_kbase_instr_backend.c \
backend/gpu/mali_kbase_jm_as.c \
backend/gpu/mali_kbase_jm_hw.c \
backend/gpu/mali_kbase_jm_rb.c \
backend/gpu/mali_kbase_js_backend.c \
backend/gpu/mali_kbase_mmu_hw_direct.c \
backend/gpu/mali_kbase_pm_backend.c \
backend/gpu/mali_kbase_pm_driver.c \
backend/gpu/mali_kbase_pm_metrics.c \
@ -42,6 +36,16 @@ BACKEND += \
backend/gpu/mali_kbase_time.c \
backend/gpu/mali_kbase_l2_mmu_config.c
ifeq ($(MALI_USE_CSF),1)
# empty
else
BACKEND += \
backend/gpu/mali_kbase_jm_as.c \
backend/gpu/mali_kbase_debug_job_fault_backend.c \
backend/gpu/mali_kbase_jm_hw.c \
backend/gpu/mali_kbase_jm_rb.c
endif
ifeq ($(MALI_CUSTOMER_RELEASE),0)
BACKEND += \
backend/gpu/mali_kbase_pm_always_on_demand.c

4
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_debug_job_fault_backend.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2012-2015,2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2012-2015,2018-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -70,6 +70,8 @@ static int mmu_reg_snapshot[] = {
static int as_reg_snapshot[] = {
AS_TRANSTAB_LO,
AS_TRANSTAB_HI,
AS_TRANSCFG_LO,
AS_TRANSCFG_HI,
AS_MEMATTR_LO,
AS_MEMATTR_HI,
AS_FAULTSTATUS,

4
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_devfreq.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -21,7 +21,7 @@
*/
#include <mali_kbase.h>
#include <mali_kbase_tracepoints.h>
#include <tl/mali_kbase_tracepoints.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <linux/of.h>

77
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_device_hw.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2016, 2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -25,15 +25,15 @@
*
*/
#include <mali_kbase.h>
#include <gpu/mali_kbase_gpu_fault.h>
#include <backend/gpu/mali_kbase_instr_internal.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <backend/gpu/mali_kbase_mmu_hw_direct.h>
#include <mali_kbase_reset_gpu.h>
#include <mmu/mali_kbase_mmu.h>
#if !defined(CONFIG_MALI_BIFROST_NO_MALI)
#ifdef CONFIG_DEBUG_FS
int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size)
@ -143,8 +143,8 @@ void kbase_io_history_dump(struct kbase_device *kbdev)
&h->buf[(h->count - iters + i) % h->size];
char const access = (io->addr & 1) ? 'w' : 'r';
dev_err(kbdev->dev, "%6i: %c: reg 0x%p val %08x\n", i, access,
(void *)(io->addr & ~0x1), io->value);
dev_err(kbdev->dev, "%6i: %c: reg 0x%016lx val %08x\n", i,
access, (unsigned long)(io->addr & ~0x1), io->value);
}
spin_unlock_irqrestore(&h->lock, flags);
@ -190,6 +190,15 @@ u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset)
}
KBASE_EXPORT_TEST_API(kbase_reg_read);
bool kbase_is_gpu_lost(struct kbase_device *kbdev)
{
u32 val;
val = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
return val == 0;
}
#endif /* !defined(CONFIG_MALI_BIFROST_NO_MALI) */
/**
@ -203,23 +212,19 @@ KBASE_EXPORT_TEST_API(kbase_reg_read);
*/
static void kbase_report_gpu_fault(struct kbase_device *kbdev, int multiple)
{
u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
u32 status = kbase_reg_read(kbdev,
GPU_CONTROL_REG(GPU_FAULTSTATUS));
u32 status = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS));
u64 address = (u64) kbase_reg_read(kbdev,
GPU_CONTROL_REG(GPU_FAULTADDRESS_HI)) << 32;
address |= kbase_reg_read(kbdev,
GPU_CONTROL_REG(GPU_FAULTADDRESS_LO));
if ((gpu_id & GPU_ID2_PRODUCT_MODEL) != GPU_ID2_PRODUCT_TULX) {
dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx",
status,
kbase_exception_name(kbdev, status & 0xFF),
address);
if (multiple)
dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n");
}
dev_warn(kbdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx",
status,
kbase_gpu_exception_name(status & 0xFF),
address);
if (multiple)
dev_warn(kbdev->dev, "There were multiple GPU faults - some have not been reported\n");
}
static bool kbase_gpu_fault_interrupt(struct kbase_device *kbdev, int multiple)
@ -249,7 +254,7 @@ void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev)
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK),
irq_mask | CLEAN_CACHES_COMPLETED);
KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_CLEAN_INV_CACHES);
@ -284,7 +289,7 @@ static void kbase_clean_caches_done(struct kbase_device *kbdev)
if (kbdev->cache_clean_queued) {
kbdev->cache_clean_queued = false;
KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_CLEAN_INV_CACHES);
} else {
@ -299,25 +304,45 @@ static void kbase_clean_caches_done(struct kbase_device *kbdev)
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev)
static inline bool get_cache_clean_flag(struct kbase_device *kbdev)
{
bool cache_clean_in_progress;
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
while (kbdev->cache_clean_in_progress) {
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
cache_clean_in_progress = kbdev->cache_clean_in_progress;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return cache_clean_in_progress;
}
void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev)
{
while (get_cache_clean_flag(kbdev)) {
wait_event_interruptible(kbdev->cache_clean_wait,
!kbdev->cache_clean_in_progress);
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev,
unsigned int wait_timeout_ms)
{
long remaining = msecs_to_jiffies(wait_timeout_ms);
while (remaining && get_cache_clean_flag(kbdev)) {
remaining = wait_event_timeout(kbdev->cache_clean_wait,
!kbdev->cache_clean_in_progress,
remaining);
}
return (remaining ? 0 : -ETIMEDOUT);
}
void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
{
bool clear_gpu_fault = false;
KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, NULL, 0u, val);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ, NULL, val);
if (val & GPU_FAULT)
clear_gpu_fault = kbase_gpu_fault_interrupt(kbdev,
val & MULTIPLE_GPU_FAULTS);
@ -328,7 +353,7 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
if (val & PRFCNT_SAMPLE_COMPLETED)
kbase_instr_hwcnt_sample_done(kbdev);
KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u, val);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, val);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), val);
/* kbase_pm_check_transitions (called by kbase_pm_power_changed) must
@ -358,5 +383,5 @@ void kbase_gpu_interrupt(struct kbase_device *kbdev, u32 val)
}
KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, NULL, 0u, val);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_IRQ_DONE, NULL, val);
}

30
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_device_internal.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014,2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014,2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -50,6 +50,18 @@ void kbase_reg_write(struct kbase_device *kbdev, u32 offset, u32 value);
*/
u32 kbase_reg_read(struct kbase_device *kbdev, u32 offset);
/**
* kbase_is_gpu_lost() - Has the GPU been lost.
* @kbdev: Kbase device pointer
*
* This function will return true if the GPU has been lost.
* When this happens register reads will be zero. A zero GPU_ID is
* invalid so this is used to detect GPU_LOST
*
* Return: True if GPU LOST
*/
bool kbase_is_gpu_lost(struct kbase_device *kbdev);
/**
* kbase_gpu_start_cache_clean - Start a cache clean
* @kbdev: Kbase device
@ -76,6 +88,22 @@ void kbase_gpu_start_cache_clean_nolock(struct kbase_device *kbdev);
*/
void kbase_gpu_wait_cache_clean(struct kbase_device *kbdev);
/**
* kbase_gpu_wait_cache_clean_timeout - Wait for certain time for cache
* cleaning to finish
* @kbdev: Kbase device
* @wait_timeout_ms: Time, in milli seconds, to wait for cache clean to complete.
*
* This function will take hwaccess_lock, and may sleep. This is supposed to be
* called from paths (like GPU reset) where an indefinite wait for the completion
* of cache clean operation can cause deadlock, as the operation may never
* complete.
*
* Return: 0 if successful or a negative error code on failure.
*/
int kbase_gpu_wait_cache_clean_timeout(struct kbase_device *kbdev,
unsigned int wait_timeout_ms);
/**
* kbase_gpu_cache_clean_wait_complete - Called after the cache cleaning is
* finished. Would also be called after

162
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpu.c
View File

@ -1,162 +0,0 @@
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Register-based HW access backend APIs
*/
#include <mali_kbase.h>
#include <mali_kbase_hwaccess_backend.h>
#include <mali_kbase_reset_gpu.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
#include <backend/gpu/mali_kbase_js_internal.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
int kbase_backend_early_init(struct kbase_device *kbdev)
{
int err;
err = kbasep_platform_device_init(kbdev);
if (err)
return err;
err = kbase_pm_runtime_init(kbdev);
if (err)
goto fail_runtime_pm;
/* Ensure we can access the GPU registers */
kbase_pm_register_access_enable(kbdev);
/* Find out GPU properties based on the GPU feature registers */
kbase_gpuprops_set(kbdev);
/* We're done accessing the GPU registers for now. */
kbase_pm_register_access_disable(kbdev);
err = kbase_install_interrupts(kbdev);
if (err)
goto fail_interrupts;
return 0;
fail_interrupts:
kbase_pm_runtime_term(kbdev);
fail_runtime_pm:
kbasep_platform_device_term(kbdev);
return err;
}
void kbase_backend_early_term(struct kbase_device *kbdev)
{
kbase_release_interrupts(kbdev);
kbase_pm_runtime_term(kbdev);
kbasep_platform_device_term(kbdev);
}
int kbase_backend_late_init(struct kbase_device *kbdev)
{
int err;
err = kbase_hwaccess_pm_init(kbdev);
if (err)
return err;
err = kbase_reset_gpu_init(kbdev);
if (err)
goto fail_reset_gpu_init;
err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT);
if (err)
goto fail_pm_powerup;
err = kbase_backend_timer_init(kbdev);
if (err)
goto fail_timer;
#ifdef CONFIG_MALI_BIFROST_DEBUG
#ifndef CONFIG_MALI_BIFROST_NO_MALI
if (kbasep_common_test_interrupt_handlers(kbdev) != 0) {
dev_err(kbdev->dev, "Interrupt assigment check failed.\n");
err = -EINVAL;
goto fail_interrupt_test;
}
#endif /* !CONFIG_MALI_BIFROST_NO_MALI */
#endif /* CONFIG_MALI_BIFROST_DEBUG */
err = kbase_job_slot_init(kbdev);
if (err)
goto fail_job_slot;
/* Do the initialisation of devfreq.
* Devfreq needs backend_timer_init() for completion of its
* initialisation and it also needs to catch the first callback
* occurence of the runtime_suspend event for maintaining state
* coherence with the backend power management, hence needs to be
* placed before the kbase_pm_context_idle().
*/
err = kbase_backend_devfreq_init(kbdev);
if (err)
goto fail_devfreq_init;
/* Idle the GPU and/or cores, if the policy wants it to */
kbase_pm_context_idle(kbdev);
/* Update gpuprops with L2_FEATURES if applicable */
kbase_gpuprops_update_l2_features(kbdev);
init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait);
return 0;
fail_devfreq_init:
kbase_job_slot_term(kbdev);
fail_job_slot:
#ifdef CONFIG_MALI_BIFROST_DEBUG
#ifndef CONFIG_MALI_BIFROST_NO_MALI
fail_interrupt_test:
#endif /* !CONFIG_MALI_BIFROST_NO_MALI */
#endif /* CONFIG_MALI_BIFROST_DEBUG */
kbase_backend_timer_term(kbdev);
fail_timer:
kbase_hwaccess_pm_halt(kbdev);
fail_pm_powerup:
kbase_reset_gpu_term(kbdev);
fail_reset_gpu_init:
kbase_hwaccess_pm_term(kbdev);
return err;
}
void kbase_backend_late_term(struct kbase_device *kbdev)
{
kbase_backend_devfreq_term(kbdev);
kbase_job_slot_halt(kbdev);
kbase_job_slot_term(kbdev);
kbase_backend_timer_term(kbdev);
kbase_hwaccess_pm_halt(kbdev);
kbase_reset_gpu_term(kbdev);
kbase_hwaccess_pm_term(kbdev);
}

87
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_gpuprops_backend.c
View File

@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -20,8 +21,6 @@
*
*/
/*
* Base kernel property query backend APIs
*/
@ -31,79 +30,93 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <mali_kbase_hwaccess_gpuprops.h>
void kbase_backend_gpuprops_get(struct kbase_device *kbdev,
int kbase_backend_gpuprops_get(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
int i;
struct kbase_gpuprops_regdump registers;
/* Fill regdump with the content of the relevant registers */
regdump->gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
registers.gpu_id = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_ID));
regdump->l2_features = kbase_reg_read(kbdev,
registers.l2_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_FEATURES));
regdump->core_features = kbase_reg_read(kbdev,
registers.core_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(CORE_FEATURES));
regdump->tiler_features = kbase_reg_read(kbdev,
registers.tiler_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_FEATURES));
regdump->mem_features = kbase_reg_read(kbdev,
registers.mem_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MEM_FEATURES));
regdump->mmu_features = kbase_reg_read(kbdev,
registers.mmu_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(MMU_FEATURES));
regdump->as_present = kbase_reg_read(kbdev,
registers.as_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(AS_PRESENT));
regdump->js_present = kbase_reg_read(kbdev,
registers.js_present = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_PRESENT));
for (i = 0; i < GPU_MAX_JOB_SLOTS; i++)
regdump->js_features[i] = kbase_reg_read(kbdev,
registers.js_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JS_FEATURES_REG(i)));
for (i = 0; i < BASE_GPU_NUM_TEXTURE_FEATURES_REGISTERS; i++)
regdump->texture_features[i] = kbase_reg_read(kbdev,
registers.texture_features[i] = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TEXTURE_FEATURES_REG(i)));
regdump->thread_max_threads = kbase_reg_read(kbdev,
registers.thread_max_threads = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_THREADS));
regdump->thread_max_workgroup_size = kbase_reg_read(kbdev,
registers.thread_max_workgroup_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_WORKGROUP_SIZE));
regdump->thread_max_barrier_size = kbase_reg_read(kbdev,
registers.thread_max_barrier_size = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_MAX_BARRIER_SIZE));
regdump->thread_features = kbase_reg_read(kbdev,
registers.thread_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_FEATURES));
regdump->thread_tls_alloc = kbase_reg_read(kbdev,
registers.thread_tls_alloc = kbase_reg_read(kbdev,
GPU_CONTROL_REG(THREAD_TLS_ALLOC));
regdump->shader_present_lo = kbase_reg_read(kbdev,
registers.shader_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_LO));
regdump->shader_present_hi = kbase_reg_read(kbdev,
registers.shader_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_PRESENT_HI));
regdump->tiler_present_lo = kbase_reg_read(kbdev,
registers.tiler_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_LO));
regdump->tiler_present_hi = kbase_reg_read(kbdev,
registers.tiler_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_PRESENT_HI));
regdump->l2_present_lo = kbase_reg_read(kbdev,
registers.l2_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_LO));
regdump->l2_present_hi = kbase_reg_read(kbdev,
registers.l2_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_PRESENT_HI));
regdump->stack_present_lo = kbase_reg_read(kbdev,
registers.stack_present_lo = kbase_reg_read(kbdev,
GPU_CONTROL_REG(STACK_PRESENT_LO));
regdump->stack_present_hi = kbase_reg_read(kbdev,
registers.stack_present_hi = kbase_reg_read(kbdev,
GPU_CONTROL_REG(STACK_PRESENT_HI));
if (!kbase_is_gpu_lost(kbdev)) {
*regdump = registers;
return 0;
} else
return -EIO;
}
void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
int kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_COHERENCY_REG)) {
u32 coherency_features;
/* Ensure we can access the GPU registers */
kbase_pm_register_access_enable(kbdev);
regdump->coherency_features = kbase_reg_read(kbdev,
coherency_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(COHERENCY_FEATURES));
if (kbase_is_gpu_lost(kbdev))
return -EIO;
regdump->coherency_features = coherency_features;
/* We're done accessing the GPU registers for now. */
kbase_pm_register_access_disable(kbdev);
} else {
@ -112,14 +125,22 @@ void kbase_backend_gpuprops_get_features(struct kbase_device *kbdev,
COHERENCY_FEATURE_BIT(COHERENCY_NONE) |
COHERENCY_FEATURE_BIT(COHERENCY_ACE_LITE);
}
return 0;
}
void kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
int kbase_backend_gpuprops_get_l2_features(struct kbase_device *kbdev,
struct kbase_gpuprops_regdump *regdump)
{
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) {
regdump->l2_features = kbase_reg_read(kbdev,
u32 l2_features = kbase_reg_read(kbdev,
GPU_CONTROL_REG(L2_FEATURES));
}
}
if (kbase_is_gpu_lost(kbdev))
return -EIO;
regdump->l2_features = l2_features;
}
return 0;
}

42
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_backend.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -27,11 +27,12 @@
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase_hwaccess_instr.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <backend/gpu/mali_kbase_instr_internal.h>
int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
struct kbase_context *kctx,
struct kbase_instr_hwcnt_enable *enable)
@ -70,7 +71,11 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
/* Configure */
prfcnt_config = kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT;
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
if (kbdev->hwcnt.backend.use_secondary_override)
#else
if (enable->use_secondary)
#endif
prfcnt_config |= 1 << PRFCNT_CONFIG_SETSELECT_SHIFT;
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
@ -80,16 +85,17 @@ int kbase_instr_hwcnt_enable_internal(struct kbase_device *kbdev,
enable->dump_buffer & 0xFFFFFFFF);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
enable->dump_buffer >> 32);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN),
enable->jm_bm);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN),
enable->shader_bm);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN),
enable->mmu_l2_bm);
/* Due to PRLAM-8186 we need to disable the Tiler before we enable the
* HW counter dump. */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
enable->tiler_bm);
enable->tiler_bm);
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
prfcnt_config | PRFCNT_CONFIG_MODE_MANUAL);
@ -198,6 +204,7 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
*/
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_DUMPING;
/* Reconfigure the dump address */
kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
kbdev->hwcnt.addr & 0xFFFFFFFF);
@ -205,8 +212,9 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
kbdev->hwcnt.addr >> 32);
/* Start dumping */
KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL, NULL,
kbdev->hwcnt.addr, 0);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_SAMPLE, NULL,
kbdev->hwcnt.addr);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_PRFCNT_SAMPLE);
@ -216,6 +224,8 @@ int kbase_instr_hwcnt_request_dump(struct kbase_context *kctx)
unlock:
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
return err;
}
KBASE_EXPORT_SYMBOL(kbase_instr_hwcnt_request_dump);
@ -277,6 +287,7 @@ void kbasep_cache_clean_worker(struct work_struct *data)
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
}
void kbase_instr_hwcnt_sample_done(struct kbase_device *kbdev)
{
unsigned long flags;
@ -348,7 +359,7 @@ int kbase_instr_hwcnt_clear(struct kbase_context *kctx)
goto out;
/* Clear the counters */
KBASE_TRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_PRFCNT_CLEAR, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_PRFCNT_CLEAR);
@ -369,8 +380,14 @@ int kbase_instr_backend_init(struct kbase_device *kbdev)
init_waitqueue_head(&kbdev->hwcnt.backend.wait);
INIT_WORK(&kbdev->hwcnt.backend.cache_clean_work,
kbasep_cache_clean_worker);
kbdev->hwcnt.backend.triggered = 0;
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
kbdev->hwcnt.backend.use_secondary_override = false;
#endif
kbdev->hwcnt.backend.cache_clean_wq =
alloc_workqueue("Mali cache cleaning workqueue", 0, 1);
if (NULL == kbdev->hwcnt.backend.cache_clean_wq)
@ -383,3 +400,12 @@ void kbase_instr_backend_term(struct kbase_device *kbdev)
{
destroy_workqueue(kbdev->hwcnt.backend.cache_clean_wq);
}
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
void kbase_instr_backend_debugfs_init(struct kbase_device *kbdev)
{
debugfs_create_bool("hwcnt_use_secondary", S_IRUGO | S_IWUSR,
kbdev->mali_debugfs_directory,
&kbdev->hwcnt.backend.use_secondary_override);
}
#endif

5
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_instr_defs.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014, 2016, 2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014, 2016, 2018, 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -47,6 +47,9 @@ enum kbase_instr_state {
struct kbase_instr_backend {
wait_queue_head_t wait;
int triggered;
#ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS
bool use_secondary_override;
#endif
enum kbase_instr_state state;
struct workqueue_struct *cache_clean_wq;

35
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_irq_linux.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2016,2018-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -79,8 +79,6 @@ static irqreturn_t kbase_job_irq_handler(int irq, void *data)
return IRQ_HANDLED;
}
KBASE_EXPORT_TEST_API(kbase_job_irq_handler);
static irqreturn_t kbase_mmu_irq_handler(int irq, void *data)
{
unsigned long flags;
@ -153,8 +151,6 @@ static irqreturn_t kbase_gpu_irq_handler(int irq, void *data)
return IRQ_HANDLED;
}
KBASE_EXPORT_TEST_API(kbase_gpu_irq_handler);
static irq_handler_t kbase_handler_table[] = {
[JOB_IRQ_TAG] = kbase_job_irq_handler,
[MMU_IRQ_TAG] = kbase_mmu_irq_handler,
@ -166,6 +162,35 @@ static irq_handler_t kbase_handler_table[] = {
#define MMU_IRQ_HANDLER MMU_IRQ_TAG
#define GPU_IRQ_HANDLER GPU_IRQ_TAG
/**
* kbase_gpu_irq_test_handler - Variant (for test) of kbase_gpu_irq_handler()
* @irq: IRQ number
* @data: Data associated with this IRQ (i.e. kbdev)
* @val: Value of the GPU_CONTROL_REG(GPU_IRQ_STATUS)
*
* Handle the GPU device interrupt source requests reflected in the
* given source bit-pattern. The test code caller is responsible for
* undertaking the required device power maintenace.
*
* Return: IRQ_HANDLED if the requests are from the GPU device,
* IRQ_NONE otherwise
*/
irqreturn_t kbase_gpu_irq_test_handler(int irq, void *data, u32 val)
{
struct kbase_device *kbdev = kbase_untag(data);
if (!val)
return IRQ_NONE;
dev_dbg(kbdev->dev, "%s: irq %d irqstatus 0x%x\n", __func__, irq, val);
kbase_gpu_interrupt(kbdev, val);
return IRQ_HANDLED;
}
KBASE_EXPORT_TEST_API(kbase_gpu_irq_test_handler);
/**
* kbase_set_custom_irq_handler - Set a custom IRQ handler
* @kbdev: Device for which the handler is to be registered

7
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_as.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -148,8 +148,7 @@ int kbase_backend_find_and_release_free_address_space(
*/
if (as_kctx && !kbase_ctx_flag(as_kctx, KCTX_PRIVILEGED) &&
atomic_read(&as_kctx->refcount) == 1) {
if (!kbasep_js_runpool_retain_ctx_nolock(kbdev,
as_kctx)) {
if (!kbase_ctx_sched_inc_refcount_nolock(as_kctx)) {
WARN(1, "Failed to retain active context\n");
spin_unlock_irqrestore(&kbdev->hwaccess_lock,
@ -236,7 +235,7 @@ bool kbase_backend_use_ctx(struct kbase_device *kbdev,
if (kbase_ctx_flag(kctx, KCTX_PRIVILEGED)) {
/* We need to retain it to keep the corresponding address space
*/
kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
kbase_ctx_sched_retain_ctx_refcount(kctx);
}
return true;

7
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_defs.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2016, 2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2016, 2018-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -62,9 +62,6 @@ struct slot_rb {
/**
* struct kbase_backend_data - GPU backend specific data for HW access layer
* @slot_rb: Slot ringbuffers
* @rmu_workaround_flag: When PRLAM-8987 is present, this flag determines
* whether slots 0/1 or slot 2 are currently being
* pulled from
* @scheduling_timer: The timer tick used for rescheduling jobs
* @timer_running: Is the timer running? The runpool_mutex must be
* held whilst modifying this.
@ -83,8 +80,6 @@ struct slot_rb {
struct kbase_backend_data {
struct slot_rb slot_rb[BASE_JM_MAX_NR_SLOTS];
bool rmu_workaround_flag;
struct hrtimer scheduling_timer;
bool timer_running;

251
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_hw.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -26,8 +26,9 @@
#include <mali_kbase.h>
#include <mali_kbase_config.h>
#include <mali_midg_regmap.h>
#include <mali_kbase_tracepoints.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <tl/mali_kbase_tracepoints.h>
#include <mali_linux_trace.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_reset_gpu.h>
@ -37,17 +38,8 @@
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
#define beenthere(kctx, f, a...) \
dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev);
static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
struct kbase_context *kctx)
{
return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT));
}
static u64 kbase_job_write_affinity(struct kbase_device *kbdev,
base_jd_core_req core_req,
int js)
@ -87,6 +79,17 @@ static u64 kbase_job_write_affinity(struct kbase_device *kbdev,
kbdev->pm.debug_core_mask[js];
}
if (unlikely(!affinity)) {
#ifdef CONFIG_MALI_BIFROST_DEBUG
u64 shaders_ready =
kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
WARN_ON(!(shaders_ready & kbdev->pm.backend.shaders_avail));
#endif
affinity = kbdev->pm.backend.shaders_avail;
}
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO),
affinity & 0xFFFFFFFF);
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI),
@ -95,13 +98,90 @@ static u64 kbase_job_write_affinity(struct kbase_device *kbdev,
return affinity;
}
/**
* select_job_chain() - Select which job chain to submit to the GPU
* @katom: Pointer to the atom about to be submitted to the GPU
*
* Selects one of the fragment job chains attached to the special atom at the
* end of a renderpass, or returns the address of the single job chain attached
* to any other type of atom.
*
* Which job chain is selected depends upon whether the tiling phase of the
* renderpass completed normally or was soft-stopped because it used too
* much memory. It also depends upon whether one of the fragment job chains
* has already been run as part of the same renderpass.
*
* Return: GPU virtual address of the selected job chain
*/
static u64 select_job_chain(struct kbase_jd_atom *katom)
{
struct kbase_context *const kctx = katom->kctx;
u64 jc = katom->jc;
struct kbase_jd_renderpass *rp;
lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
if (!(katom->core_req & BASE_JD_REQ_END_RENDERPASS))
return jc;
compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <=
ARRAY_SIZE(kctx->jctx.renderpasses),
"Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[katom->renderpass_id];
/* We can read a subset of renderpass state without holding
* higher-level locks (but not end_katom, for example).
* If the end-of-renderpass atom is running with as-yet indeterminate
* OOM state then assume that the start atom was not soft-stopped.
*/
switch (rp->state) {
case KBASE_JD_RP_OOM:
/* Tiling ran out of memory.
* Start of incremental rendering, used once.
*/
jc = katom->jc_fragment.norm_read_forced_write;
break;
case KBASE_JD_RP_START:
case KBASE_JD_RP_PEND_OOM:
/* Tiling completed successfully first time.
* Single-iteration rendering, used once.
*/
jc = katom->jc_fragment.norm_read_norm_write;
break;
case KBASE_JD_RP_RETRY_OOM:
/* Tiling ran out of memory again.
* Continuation of incremental rendering, used as
* many times as required.
*/
jc = katom->jc_fragment.forced_read_forced_write;
break;
case KBASE_JD_RP_RETRY:
case KBASE_JD_RP_RETRY_PEND_OOM:
/* Tiling completed successfully this time.
* End of incremental rendering, used once.
*/
jc = katom->jc_fragment.forced_read_norm_write;
break;
default:
WARN_ON(1);
break;
}
dev_dbg(kctx->kbdev->dev,
"Selected job chain 0x%llx for end atom %p in state %d\n",
jc, (void *)katom, (int)rp->state);
katom->jc = jc;
return jc;
}
void kbase_job_hw_submit(struct kbase_device *kbdev,
struct kbase_jd_atom *katom,
int js)
{
struct kbase_context *kctx;
u32 cfg;
u64 jc_head = katom->jc;
u64 const jc_head = select_job_chain(katom);
u64 affinity;
KBASE_DEBUG_ASSERT(kbdev);
@ -112,6 +192,9 @@ void kbase_job_hw_submit(struct kbase_device *kbdev,
/* Command register must be available */
KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
dev_dbg(kctx->kbdev->dev, "Write JS_HEAD_NEXT 0x%llx for atom %p\n",
jc_head, (void *)katom);
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO),
jc_head & 0xFFFFFFFF);
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI),
@ -142,8 +225,8 @@ void kbase_job_hw_submit(struct kbase_device *kbdev,
cfg |= JS_CONFIG_THREAD_PRI(8);
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) &&
(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED))
if ((katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED) ||
(katom->core_req & BASE_JD_REQ_END_RENDERPASS))
cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK;
if (kbase_hw_has_feature(kbdev,
@ -175,7 +258,7 @@ void kbase_job_hw_submit(struct kbase_device *kbdev,
dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx",
katom, kctx, js, jc_head);
KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
(u32)affinity);
KBASE_TLSTREAM_AUX_EVENT_JOB_SLOT(kbdev, kctx,
@ -207,6 +290,10 @@ void kbase_job_hw_submit(struct kbase_device *kbdev,
kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
}
#endif
trace_sysgraph_gpu(SGR_SUBMIT, kctx->id,
kbase_jd_atom_id(kctx, katom), js);
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
JS_COMMAND_START);
}
@ -274,7 +361,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done)
KBASE_DEBUG_ASSERT(kbdev);
KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ, NULL, NULL, 0, done);
end_timestamp = ktime_get();
@ -459,7 +546,7 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done)
*/
kbasep_try_reset_gpu_early_locked(kbdev);
}
KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
KBASE_KTRACE_ADD_JM(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
}
void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
@ -468,7 +555,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
base_jd_core_req core_reqs,
struct kbase_jd_atom *target_katom)
{
#if KBASE_TRACE_ENABLE
#if KBASE_KTRACE_ENABLE
u32 status_reg_before;
u64 job_in_head_before;
u32 status_reg_after;
@ -496,7 +583,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
/* We are about to issue a soft stop, so mark the atom as having
* been soft stopped */
target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED;
target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED;
/* Mark the point where we issue the soft-stop command */
KBASE_TLSTREAM_TL_EVENT_ATOM_SOFTSTOP_ISSUE(kbdev, target_katom);
@ -524,7 +611,7 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action);
#if KBASE_TRACE_ENABLE
#if KBASE_KTRACE_ENABLE
status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS));
if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
struct kbase_jd_atom *head;
@ -534,36 +621,28 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
head_kctx = head->kctx;
if (status_reg_before == BASE_JD_EVENT_ACTIVE)
KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx,
head, job_in_head_before, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, job_in_head_before, js);
else
KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
0, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
switch (action) {
case JS_COMMAND_SOFT_STOP:
KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx,
head, head->jc, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_SOFT_STOP_0:
KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx,
head, head->jc, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_SOFT_STOP_1:
KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx,
head, head->jc, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_HARD_STOP:
KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx,
head, head->jc, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_HARD_STOP_0:
KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx,
head, head->jc, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, js);
break;
case JS_COMMAND_HARD_STOP_1:
KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx,
head, head->jc, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js);
break;
default:
BUG();
@ -571,36 +650,28 @@ void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
}
} else {
if (status_reg_before == BASE_JD_EVENT_ACTIVE)
KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
job_in_head_before, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, job_in_head_before, js);
else
KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
0, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
switch (action) {
case JS_COMMAND_SOFT_STOP:
KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0,
js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js);
break;
case JS_COMMAND_SOFT_STOP_0:
KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL,
0, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js);
break;
case JS_COMMAND_SOFT_STOP_1:
KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL,
0, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js);
break;
case JS_COMMAND_HARD_STOP:
KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0,
js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js);
break;
case JS_COMMAND_HARD_STOP_0:
KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL,
0, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js);
break;
case JS_COMMAND_HARD_STOP_1:
KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL,
0, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js);
break;
default:
BUG();
@ -660,6 +731,70 @@ void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
}
}
static int softstop_start_rp_nolock(
struct kbase_context *kctx, struct kbase_va_region *reg)
{
struct kbase_device *const kbdev = kctx->kbdev;
struct kbase_jd_atom *katom;
struct kbase_jd_renderpass *rp;
lockdep_assert_held(&kbdev->hwaccess_lock);
katom = kbase_gpu_inspect(kbdev, 1, 0);
if (!katom) {
dev_dbg(kctx->kbdev->dev, "No atom on job slot\n");
return -ESRCH;
}
if (!(katom->core_req & BASE_JD_REQ_START_RENDERPASS)) {
dev_dbg(kctx->kbdev->dev,
"Atom %p on job slot is not start RP\n", (void *)katom);
return -EPERM;
}
compiletime_assert((1ull << (sizeof(katom->renderpass_id) * 8)) <=
ARRAY_SIZE(kctx->jctx.renderpasses),
"Should check invalid access to renderpasses");
rp = &kctx->jctx.renderpasses[katom->renderpass_id];
if (WARN_ON(rp->state != KBASE_JD_RP_START &&
rp->state != KBASE_JD_RP_RETRY))
return -EINVAL;
dev_dbg(kctx->kbdev->dev, "OOM in state %d with region %p\n",
(int)rp->state, (void *)reg);
if (WARN_ON(katom != rp->start_katom))
return -EINVAL;
dev_dbg(kctx->kbdev->dev, "Adding region %p to list %p\n",
(void *)reg, (void *)&rp->oom_reg_list);
list_move_tail(&reg->link, &rp->oom_reg_list);
dev_dbg(kctx->kbdev->dev, "Added region to list\n");
rp->state = (rp->state == KBASE_JD_RP_START ?
KBASE_JD_RP_PEND_OOM : KBASE_JD_RP_RETRY_PEND_OOM);
kbase_job_slot_softstop(kbdev, 1, katom);
return 0;
}
int kbase_job_slot_softstop_start_rp(struct kbase_context *const kctx,
struct kbase_va_region *const reg)
{
struct kbase_device *const kbdev = kctx->kbdev;
int err;
unsigned long flags;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
err = softstop_start_rp_nolock(kctx, reg);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return err;
}
void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx)
{
struct kbase_device *kbdev = kctx->kbdev;
@ -749,6 +884,9 @@ KBASE_EXPORT_TEST_API(kbase_job_slot_term);
void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
struct kbase_jd_atom *target_katom, u32 sw_flags)
{
dev_dbg(kbdev->dev, "Soft-stop atom %p with flags 0x%x (s:%d)\n",
target_katom, sw_flags, js);
KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom,
JS_COMMAND_SOFT_STOP | sw_flags);
@ -907,7 +1045,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
KBASE_RESET_GPU_SILENT)
silent = true;
KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD_JM(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
/* Disable GPU hardware counters.
* This call will block until counters are disabled.
@ -1035,7 +1173,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
dev_err(kbdev->dev, "Reset complete");
/* Try submitting some jobs to restart processing */
KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD_JM(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0);
kbase_js_sched_all(kbdev);
/* Process any pending slot updates */
@ -1050,7 +1188,7 @@ static void kbasep_reset_timeout_worker(struct work_struct *data)
kbase_hwcnt_context_enable(kbdev->hwcnt_gpu_ctx);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD_JM(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
}
static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
@ -1281,4 +1419,3 @@ void kbase_reset_gpu_term(struct kbase_device *kbdev)
{
destroy_workqueue(kbdev->hwaccess.backend.reset_workq);
}

10
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_internal.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2016, 2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -34,6 +34,7 @@
#include <linux/atomic.h>
#include <backend/gpu/mali_kbase_jm_rb.h>
#include <backend/gpu/mali_kbase_device_internal.h>
/**
* kbase_job_submit_nolock() - Submit a job to a certain job-slot
@ -70,6 +71,13 @@ static inline char *kbasep_make_job_slot_string(int js, char *js_string,
}
#endif
static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
struct kbase_context *kctx)
{
return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT));
}
/**
* kbase_job_hw_submit() - Submit a job to the GPU
* @kbdev: Device pointer

77
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_jm_rb.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -26,12 +26,12 @@
*/
#include <mali_kbase.h>
#include <gpu/mali_kbase_gpu_fault.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_jm.h>
#include <mali_kbase_js.h>
#include <mali_kbase_tracepoints.h>
#include <tl/mali_kbase_tracepoints.h>
#include <mali_kbase_hwcnt_context.h>
#include <mali_kbase_10969_workaround.h>
#include <mali_kbase_reset_gpu.h>
#include <backend/gpu/mali_kbase_cache_policy_backend.h>
#include <backend/gpu/mali_kbase_device_internal.h>
@ -299,9 +299,6 @@ static void kbase_gpu_release_atom(struct kbase_device *kbdev,
case KBASE_ATOM_GPU_RB_READY:
/* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
/* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
break;
@ -617,11 +614,15 @@ static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev,
return -EAGAIN;
}
if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) ||
kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
if (kbase_pm_get_ready_cores(kbdev,
KBASE_PM_CORE_L2) ||
kbase_pm_get_trans_cores(kbdev,
KBASE_PM_CORE_L2) ||
kbase_is_gpu_lost(kbdev)) {
/*
* The L2 is still powered, wait for all the users to
* finish with it before doing the actual reset.
* The L2 is still powered, wait for all
* the users to finish with it before doing
* the actual reset.
*/
return -EAGAIN;
}
@ -809,7 +810,11 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
lockdep_assert_held(&kbdev->hwaccess_lock);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbase_reset_gpu_is_active(kbdev) || kbase_is_gpu_lost(kbdev))
#else
if (kbase_reset_gpu_is_active(kbdev))
#endif
return;
for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
@ -834,8 +839,7 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
break;
case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
if (katom[idx]->atom_flags &
KBASE_KATOM_FLAG_X_DEP_BLOCKED)
if (kbase_js_atom_blocked_on_x_dep(katom[idx]))
break;
katom[idx]->gpu_rb_state =
@ -930,12 +934,6 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
if (!cores_ready)
break;
katom[idx]->gpu_rb_state =
KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
/* ***TRANSITION TO HIGHER STATE*** */
/* fallthrough */
case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
katom[idx]->gpu_rb_state =
KBASE_ATOM_GPU_RB_READY;
@ -967,11 +965,11 @@ void kbase_backend_slot_update(struct kbase_device *kbdev)
other_slots_busy(kbdev, js))
break;
if ((kbdev->serialize_jobs &
KBASE_SERIALIZE_RESET) &&
kbase_reset_gpu_is_active(kbdev))
#ifdef CONFIG_MALI_GEM5_BUILD
if (!kbasep_jm_is_js_free(kbdev, js,
katom[idx]->kctx))
break;
#endif
/* Check if this job needs the cycle counter
* enabled before submission */
if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
@ -1015,6 +1013,8 @@ void kbase_backend_run_atom(struct kbase_device *kbdev,
struct kbase_jd_atom *katom)
{
lockdep_assert_held(&kbdev->hwaccess_lock);
dev_dbg(kbdev->dev, "Backend running atom %p\n", (void *)katom);
kbase_gpu_enqueue_atom(kbdev, katom);
kbase_backend_slot_update(kbdev);
}
@ -1073,6 +1073,10 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
struct kbase_context *kctx = katom->kctx;
dev_dbg(kbdev->dev,
"Atom %p completed on hw with code 0x%x and job_tail 0x%llx (s:%d)\n",
(void *)katom, completion_code, job_tail, js);
lockdep_assert_held(&kbdev->hwaccess_lock);
/*
@ -1125,12 +1129,11 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
if (!kbase_ctx_flag(katom->kctx, KCTX_DYING))
dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)",
js, completion_code,
kbase_exception_name
(kbdev,
kbase_gpu_exception_name(
completion_code));
#if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
KBASE_TRACE_DUMP(kbdev);
#if KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR != 0
KBASE_KTRACE_DUMP(kbdev);
#endif
kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
@ -1184,23 +1187,22 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
}
}
KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
js, completion_code);
KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc, js, completion_code);
if (job_tail != 0 && job_tail != katom->jc) {
bool was_updated = (job_tail != katom->jc);
/* Some of the job has been executed */
dev_dbg(kbdev->dev,
"Update job chain address of atom %p to resume from 0x%llx\n",
(void *)katom, job_tail);
/* Some of the job has been executed, so we update the job chain
* address to where we should resume from */
katom->jc = job_tail;
if (was_updated)
KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
katom, job_tail, js);
KBASE_KTRACE_ADD_JM_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
katom, job_tail, js);
}
/* Only update the event code for jobs that weren't cancelled */
if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
katom->event_code = (base_jd_event_code)completion_code;
katom->event_code = (enum base_jd_event_code)completion_code;
/* Complete the job, and start new ones
*
@ -1250,8 +1252,9 @@ void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
katom = kbase_jm_complete(kbdev, katom, end_timestamp);
if (katom) {
/* Cross-slot dependency has now become runnable. Try to submit
* it. */
dev_dbg(kbdev->dev,
"Cross-slot dependency %p has become runnable.\n",
(void *)katom);
/* Check if there are lower priority jobs to soft stop */
kbase_job_slot_ctx_priority_check_locked(kctx, katom);

6
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_js_backend.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -301,8 +301,7 @@ void kbase_backend_ctx_count_changed(struct kbase_device *kbdev)
HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
HRTIMER_MODE_REL);
KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u,
0u);
KBASE_KTRACE_ADD_JM(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u);
}
}
@ -313,7 +312,6 @@ int kbase_backend_timer_init(struct kbase_device *kbdev)
hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC,
HRTIMER_MODE_REL);
backend->scheduling_timer.function = timer_callback;
backend->timer_running = false;
return 0;

18
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.c
View File

@ -1,7 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -21,7 +21,6 @@
*
*/
#include <mali_kbase.h>
#include <mali_kbase_bits.h>
#include <mali_kbase_config_defaults.h>
@ -76,7 +75,7 @@ static const struct l2_mmu_config_limit limits[] = {
{KBASE_3BIT_AID_32, GENMASK(17, 15), 15} },
};
void kbase_set_mmu_quirks(struct kbase_device *kbdev)
int kbase_set_mmu_quirks(struct kbase_device *kbdev)
{
/* All older GPUs had 2 bits for both fields, this is a default */
struct l2_mmu_config_limit limit = {
@ -101,18 +100,23 @@ void kbase_set_mmu_quirks(struct kbase_device *kbdev)
mmu_config = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG));
if (kbase_is_gpu_lost(kbdev))
return -EIO;
mmu_config &= ~(limit.read.mask | limit.write.mask);
/* Can't use FIELD_PREP() macro here as the mask isn't constant */
mmu_config |= (limit.read.value << limit.read.shift) |
(limit.write.value << limit.write.shift);
(limit.write.value << limit.write.shift);
kbdev->hw_quirks_mmu = mmu_config;
if (kbdev->system_coherency == COHERENCY_ACE) {
/* Allow memory configuration disparity to be ignored,
* we optimize the use of shared memory and thus we
* expect some disparity in the memory configuration.
*/
* we optimize the use of shared memory and thus we
* expect some disparity in the memory configuration.
*/
kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY;
}
return 0;
}

15
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_l2_mmu_config.h
View File

@ -20,13 +20,22 @@
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
*/
#ifndef _KBASE_L2_MMU_CONFIG_H_
@ -38,7 +47,9 @@
*
* Use this function to initialise the hw_quirks_mmu field, for instance to set
* the MAX_READS and MAX_WRITES to sane defaults for each GPU.
*
* Return: Zero for succeess or a Linux error code
*/
void kbase_set_mmu_quirks(struct kbase_device *kbdev);
int kbase_set_mmu_quirks(struct kbase_device *kbdev);
#endif /* _KBASE_L2_MMU_CONFIG_H */

62
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_mmu_hw_direct.h
View File

@ -1,62 +0,0 @@
/*
*
* (C) COPYRIGHT 2014-2015, 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Interface file for the direct implementation for MMU hardware access
*
* Direct MMU hardware interface
*
* This module provides the interface(s) that are required by the direct
* register access implementation of the MMU hardware interface
*/
#ifndef _KBASE_MMU_HW_DIRECT_H_
#define _KBASE_MMU_HW_DIRECT_H_
#include <mali_kbase_defs.h>
/**
* kbase_mmu_interrupt - Process an MMU interrupt.
*
* Process the MMU interrupt that was reported by the &kbase_device.
*
* @kbdev: Pointer to the kbase device for which the interrupt happened.
* @irq_stat: Value of the MMU_IRQ_STATUS register.
*/
void kbase_mmu_interrupt(struct kbase_device *kbdev, u32 irq_stat);
/**
* kbase_mmu_bus_fault_interrupt - Process a bus fault interrupt.
*
* Process the bus fault interrupt that was reported for a particular GPU
* address space.
*
* @kbdev: Pointer to the kbase device for which bus fault was reported.
* @status: Value of the GPU_FAULTSTATUS register.
* @as_nr: GPU address space for which the bus fault occurred.
*
* Return: zero if the operation was successful, non-zero otherwise.
*/
int kbase_mmu_bus_fault_interrupt(struct kbase_device *kbdev,
u32 status, u32 as_nr);
#endif /* _KBASE_MMU_HW_DIRECT_H_ */

127
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_backend.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -26,16 +26,18 @@
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_pm.h>
#include <mali_kbase_hwaccess_jm.h>
#include <mali_kbase_hwcnt_context.h>
#include <backend/gpu/mali_kbase_js_internal.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
#include <mali_kbase_hwcnt_context.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_devfreq.h>
#include <mali_kbase_dummy_job_wa.h>
#include <mali_kbase_irq_internal.h>
static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data);
static void kbase_pm_hwcnt_disable_worker(struct work_struct *data);
@ -138,6 +140,9 @@ int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
kbdev->pm.backend.ca_cores_enabled = ~0ull;
kbdev->pm.backend.gpu_powered = false;
kbdev->pm.suspending = false;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbdev->pm.gpu_lost = false;
#endif
#ifdef CONFIG_MALI_BIFROST_DEBUG
kbdev->pm.backend.driver_ready_for_irqs = false;
#endif /* CONFIG_MALI_BIFROST_DEBUG */
@ -244,7 +249,6 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
pm.backend.gpu_poweroff_wait_work);
struct kbase_pm_device_data *pm = &kbdev->pm;
struct kbase_pm_backend_data *backend = &pm->backend;
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
unsigned long flags;
/* Wait for power transitions to complete. We do this with no locks held
@ -252,8 +256,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
*/
kbase_pm_wait_for_desired_state(kbdev);
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
kbase_pm_lock(kbdev);
if (!backend->poweron_required) {
unsigned long flags;
@ -271,11 +274,9 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
* process. Interrupts are disabled so no more faults
* should be generated at this point.
*/
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_pm_unlock(kbdev);
kbase_flush_mmu_wqs(kbdev);
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
kbase_pm_lock(kbdev);
/* Turn off clock now that fault have been handled. We
* dropped locks so poweron_required may have changed -
@ -301,8 +302,7 @@ static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
}
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_pm_unlock(kbdev);
wake_up(&kbdev->pm.backend.poweroff_wait);
}
@ -517,14 +517,12 @@ KBASE_EXPORT_TEST_API(kbase_pm_wait_for_poweroff_complete);
int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
unsigned int flags)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
unsigned long irq_flags;
int ret;
KBASE_DEBUG_ASSERT(kbdev != NULL);
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
kbase_pm_lock(kbdev);
/* A suspend won't happen during startup/insmod */
KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
@ -533,8 +531,7 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
* them. */
ret = kbase_pm_init_hw(kbdev, flags);
if (ret) {
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_pm_unlock(kbdev);
return ret;
}
@ -564,8 +561,7 @@ int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
/* Turn on the GPU and any cores needed by the policy */
kbase_pm_do_poweron(kbdev, false);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_pm_unlock(kbdev);
return 0;
}
@ -615,7 +611,7 @@ void kbase_pm_power_changed(struct kbase_device *kbdev)
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_pm_update_state(kbdev);
kbase_backend_slot_update(kbdev);
kbase_backend_slot_update(kbdev);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
@ -627,6 +623,11 @@ void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev,
lockdep_assert_held(&kbdev->hwaccess_lock);
lockdep_assert_held(&kbdev->pm.lock);
if (kbase_dummy_job_wa_enabled(kbdev)) {
dev_warn(kbdev->dev, "Change of core mask not supported for slot 0 as dummy job WA is enabled");
new_core_mask_js0 = kbdev->pm.debug_core_mask[0];
}
kbdev->pm.debug_core_mask[0] = new_core_mask_js0;
kbdev->pm.debug_core_mask[1] = new_core_mask_js1;
kbdev->pm.debug_core_mask[2] = new_core_mask_js2;
@ -648,20 +649,16 @@ void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev)
void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
/* Force power off the GPU and all cores (regardless of policy), only
* after the PM active count reaches zero (otherwise, we risk turning it
* off prematurely) */
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
kbase_pm_lock(kbdev);
kbase_pm_do_poweroff(kbdev);
kbase_backend_timer_suspend(kbdev);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_pm_unlock(kbdev);
kbase_pm_wait_for_poweroff_complete(kbdev);
@ -671,16 +668,80 @@ void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
kbase_pm_lock(kbdev);
kbdev->pm.suspending = false;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbdev->pm.gpu_lost = false;
#endif
kbase_pm_do_poweron(kbdev, true);
kbase_backend_timer_resume(kbdev);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_pm_unlock(kbdev);
}
#ifdef CONFIG_MALI_ARBITER_SUPPORT
void kbase_pm_handle_gpu_lost(struct kbase_device *kbdev)
{
unsigned long flags;
struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
ktime_t end_timestamp = ktime_get();
/* Full GPU reset will have been done by hypervisor, so cancel */
atomic_set(&kbdev->hwaccess.backend.reset_gpu,
KBASE_RESET_GPU_NOT_PENDING);
hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
/* GPU is no longer mapped to VM. So no interrupts will be received
* and Mali registers have been replaced by dummy RAM
*/
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
spin_lock(&kbdev->mmu_mask_change);
kbdev->irq_reset_flush = true;
spin_unlock(&kbdev->mmu_mask_change);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
kbase_synchronize_irqs(kbdev);
kbase_flush_mmu_wqs(kbdev);
kbdev->irq_reset_flush = false;
/* Clear all jobs running on the GPU */
mutex_lock(&kbdev->pm.lock);
kbdev->pm.gpu_lost = true;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->protected_mode = false;
if (!kbdev->pm.backend.protected_entry_transition_override)
kbase_backend_reset(kbdev, &end_timestamp);
kbase_pm_metrics_update(kbdev, NULL);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
/* Cancel any pending HWC dumps */
spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
kbdev->hwcnt.backend.triggered = 1;
wake_up(&kbdev->hwcnt.backend.wait);
spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
/* Wait for all threads keeping GPU active to complete */
mutex_unlock(&kbdev->pm.lock);
wait_event(kbdev->pm.zero_active_count_wait,
kbdev->pm.active_count == 0);
mutex_lock(&kbdev->pm.lock);
/* Update state to GPU off */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbdev->pm.backend.shaders_desired = false;
kbdev->pm.backend.l2_desired = false;
backend->l2_state = KBASE_L2_OFF;
backend->shaders_state = KBASE_SHADERS_OFF_CORESTACK_OFF;
kbdev->pm.backend.gpu_powered = false;
backend->poweroff_wait_in_progress = false;
KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0);
wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
kbase_gpu_cache_clean_wait_complete(kbdev);
backend->poweroff_wait_in_progress = false;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
wake_up(&kbdev->pm.backend.poweroff_wait);
mutex_unlock(&kbdev->pm.lock);
}
#endif /* CONFIG_MALI_ARBITER_SUPPORT */

8
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_ca.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2013-2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2013-2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -30,6 +30,7 @@
#ifdef MALI_BIFROST_NO_MALI
#include <backend/gpu/mali_kbase_model_dummy.h>
#endif
#include <mali_kbase_dummy_job_wa.h>
int kbase_pm_ca_init(struct kbase_device *kbdev)
{
@ -64,6 +65,11 @@ void kbase_devfreq_set_core_mask(struct kbase_device *kbdev, u64 core_mask)
goto unlock;
}
if (kbase_dummy_job_wa_enabled(kbdev)) {
dev_err(kbdev->dev, "Dynamic core scaling not supported as dummy job WA is enabled");
goto unlock;
}
pm_backend->ca_cores_enabled = core_mask;
kbase_pm_update_state(kbdev);

17
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_defs.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -301,9 +301,17 @@ union kbase_pm_policy_data {
* @l2_always_on: If true, disable powering down of l2 cache.
* @shaders_state: The current state of the shader state machine.
* @shaders_avail: This is updated by the state machine when it is in a state
* where it can handle changes to the core availability. This
* is internal to the shader state machine and should *not* be
* modified elsewhere.
* where it can write to the SHADER_PWRON or PWROFF registers
* to have the same set of available cores as specified by
* @shaders_desired_mask. So it would eventually have the same
* value as @shaders_desired_mask and would precisely indicate
* the cores that are currently available. This is internal to
* shader state machine and should *not* be modified elsewhere.
* @shaders_desired_mask: This is updated by the state machine when it is in
* a state where it can handle changes to the core
* availability (either by DVFS or sysfs). This is
* internal to the shader state machine and should
* *not* be modified elsewhere.
* @shaders_desired: True if the PM active count or power policy requires the
* shader cores to be on. This is used as an input to the
* shader power state machine. The current state of the
@ -401,6 +409,7 @@ struct kbase_pm_backend_data {
enum kbase_l2_core_state l2_state;
enum kbase_shader_core_state shaders_state;
u64 shaders_avail;
u64 shaders_desired_mask;
bool l2_desired;
bool l2_always_on;
bool shaders_desired;

296
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_driver.c
View File

@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -20,16 +21,14 @@
*
*/
/*
* Base kernel Power Management hardware control
*/
#include <mali_kbase.h>
#include <mali_kbase_config_defaults.h>
#include <mali_midg_regmap.h>
#include <mali_kbase_tracepoints.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <tl/mali_kbase_tracepoints.h>
#include <mali_kbase_pm.h>
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_smc.h>
@ -42,6 +41,10 @@
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_l2_mmu_config.h>
#include <mali_kbase_dummy_job_wa.h>
#ifdef CONFIG_MALI_ARBITER_SUPPORT
#include <arbiter/mali_kbase_arbiter_pm.h>
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
#include <linux/of.h>
@ -280,16 +283,13 @@ static void kbase_pm_invoke(struct kbase_device *kbdev,
if (action == ACTION_PWRON)
switch (core_type) {
case KBASE_PM_CORE_SHADER:
KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u,
lo);
KBASE_KTRACE_ADD(kbdev, PM_PWRON, NULL, cores);
break;
case KBASE_PM_CORE_TILER:
KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL,
NULL, 0u, lo);
KBASE_KTRACE_ADD(kbdev, PM_PWRON_TILER, NULL, cores);
break;
case KBASE_PM_CORE_L2:
KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL,
0u, lo);
KBASE_KTRACE_ADD(kbdev, PM_PWRON_L2, NULL, cores);
break;
default:
break;
@ -297,16 +297,13 @@ static void kbase_pm_invoke(struct kbase_device *kbdev,
else if (action == ACTION_PWROFF)
switch (core_type) {
case KBASE_PM_CORE_SHADER:
KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL,
0u, lo);
KBASE_KTRACE_ADD(kbdev, PM_PWROFF, NULL, cores);
break;
case KBASE_PM_CORE_TILER:
KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL,
NULL, 0u, lo);
KBASE_KTRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, cores);
break;
case KBASE_PM_CORE_L2:
KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL,
0u, lo);
KBASE_KTRACE_ADD(kbdev, PM_PWROFF_L2, NULL, cores);
/* disable snoops before L2 is turned off */
kbase_pm_cache_snoop_disable(kbdev);
break;
@ -315,11 +312,18 @@ static void kbase_pm_invoke(struct kbase_device *kbdev,
}
}
if (lo != 0)
kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo);
if (hi != 0)
kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi);
if (kbase_dummy_job_wa_enabled(kbdev) &&
action == ACTION_PWRON &&
core_type == KBASE_PM_CORE_SHADER &&
!(kbdev->dummy_job_wa.flags &
KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER)) {
kbase_dummy_job_wa_execute(kbdev, cores);
} else {
if (lo != 0)
kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo);
if (hi != 0)
kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi);
}
}
/**
@ -436,16 +440,13 @@ u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
switch (type) {
case KBASE_PM_CORE_SHADER:
KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u,
(u32) result);
KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED, NULL, result);
break;
case KBASE_PM_CORE_TILER:
KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u,
(u32) result);
KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, result);
break;
case KBASE_PM_CORE_L2:
KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u,
(u32) result);
KBASE_KTRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, result);
break;
default:
break;
@ -535,7 +536,7 @@ static const char *kbase_l2_core_state_to_string(enum kbase_l2_core_state state)
return strings[state];
}
static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
static int kbase_pm_l2_update_state(struct kbase_device *kbdev)
{
struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
u64 l2_present = kbdev->gpu_props.props.raw_props.l2_present;
@ -555,6 +556,13 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
u64 tiler_ready = kbase_pm_get_ready_cores(kbdev,
KBASE_PM_CORE_TILER);
/*
* kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
* are vulnerable to corruption if gpu is lost
*/
if (kbase_is_gpu_lost(kbdev))
return -EIO;
/* mask off ready from trans in case transitions finished
* between the register reads
*/
@ -592,10 +600,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
case KBASE_L2_PEND_ON:
if (!l2_trans && l2_ready == l2_present && !tiler_trans
&& tiler_ready == tiler_present) {
KBASE_TRACE_ADD(kbdev,
PM_CORES_CHANGE_AVAILABLE_TILER,
NULL, NULL, 0u,
(u32)tiler_ready);
KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, tiler_ready);
/*
* Ensure snoops are enabled after L2 is powered
* up. Note that kbase keeps track of the snoop
@ -765,8 +770,7 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
kbase_gpu_start_cache_clean_nolock(
kbdev);
KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
NULL, NULL, 0u, 0u);
KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, 0u);
backend->l2_state = KBASE_L2_PEND_OFF;
break;
@ -811,8 +815,6 @@ static u64 kbase_pm_l2_update_state(struct kbase_device *kbdev)
&kbdev->pm.backend.gpu_poweroff_wait_work);
}
if (backend->l2_state == KBASE_L2_ON)
return l2_present;
return 0;
}
@ -889,13 +891,14 @@ static const char *kbase_shader_core_state_to_string(
return strings[state];
}
static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
static int kbase_pm_shaders_update_state(struct kbase_device *kbdev)
{
struct kbase_pm_backend_data *backend = &kbdev->pm.backend;
struct kbasep_pm_tick_timer_state *stt =
&kbdev->pm.backend.shader_tick_timer;
enum kbase_shader_core_state prev_state;
u64 stacks_avail = 0;
int err = 0;
lockdep_assert_held(&kbdev->hwaccess_lock);
@ -917,6 +920,15 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
stacks_ready = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_STACK);
}
/*
* kbase_pm_get_ready_cores and kbase_pm_get_trans_cores
* are vulnerable to corruption if gpu is lost
*/
if (kbase_is_gpu_lost(kbdev)) {
err = -EIO;
break;
}
/* mask off ready from trans in case transitions finished
* between the register reads
*/
@ -931,7 +943,8 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
* except at certain points where we can handle it,
* i.e. off and SHADERS_ON_CORESTACK_ON.
*/
backend->shaders_avail = kbase_pm_ca_get_core_mask(kbdev);
backend->shaders_desired_mask =
kbase_pm_ca_get_core_mask(kbdev);
backend->pm_shaders_core_mask = 0;
if (backend->shaders_desired &&
@ -958,6 +971,8 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
case KBASE_SHADERS_OFF_CORESTACK_PEND_ON:
if (!stacks_trans && stacks_ready == stacks_avail) {
backend->shaders_avail =
backend->shaders_desired_mask;
kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
backend->shaders_avail, ACTION_PWRON);
@ -967,9 +982,7 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
case KBASE_SHADERS_PEND_ON_CORESTACK_ON:
if (!shaders_trans && shaders_ready == backend->shaders_avail) {
KBASE_TRACE_ADD(kbdev,
PM_CORES_CHANGE_AVAILABLE,
NULL, NULL, 0u, (u32)shaders_ready);
KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, shaders_ready);
backend->pm_shaders_core_mask = shaders_ready;
backend->hwcnt_desired = true;
if (backend->hwcnt_disabled) {
@ -983,11 +996,12 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
break;
case KBASE_SHADERS_ON_CORESTACK_ON:
backend->shaders_avail = kbase_pm_ca_get_core_mask(kbdev);
backend->shaders_desired_mask =
kbase_pm_ca_get_core_mask(kbdev);
/* If shaders to change state, trigger a counter dump */
if (!backend->shaders_desired ||
(backend->shaders_avail != shaders_ready)) {
(backend->shaders_desired_mask != shaders_ready)) {
backend->hwcnt_desired = false;
if (!backend->hwcnt_disabled)
kbase_pm_trigger_hwcnt_disable(kbdev);
@ -997,7 +1011,7 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
break;
case KBASE_SHADERS_ON_CORESTACK_ON_RECHECK:
backend->shaders_avail =
backend->shaders_desired_mask =
kbase_pm_ca_get_core_mask(kbdev);
if (!backend->hwcnt_disabled) {
@ -1005,6 +1019,10 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
;
} else if (!backend->shaders_desired) {
if (kbdev->pm.backend.protected_transition_override ||
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbase_pm_is_suspending(kbdev) ||
kbase_pm_is_gpu_lost(kbdev) ||
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
!stt->configured_ticks ||
WARN_ON(stt->cancel_queued)) {
backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
@ -1031,19 +1049,20 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
backend->shaders_state = KBASE_SHADERS_WAIT_OFF_CORESTACK_ON;
}
} else if (backend->shaders_avail & ~shaders_ready) {
} else if (backend->shaders_desired_mask & ~shaders_ready) {
/* set cores ready but not available to
* meet KBASE_SHADERS_PEND_ON_CORESTACK_ON
* check pass
*/
backend->shaders_avail |= shaders_ready;
backend->shaders_avail =
(backend->shaders_desired_mask | shaders_ready);
kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
backend->shaders_avail & ~shaders_ready,
ACTION_PWRON);
backend->shaders_state =
KBASE_SHADERS_PEND_ON_CORESTACK_ON;
} else if (shaders_ready & ~backend->shaders_avail) {
} else if (shaders_ready & ~backend->shaders_desired_mask) {
backend->shaders_state =
KBASE_SHADERS_WAIT_GPU_IDLE;
} else {
@ -1063,6 +1082,11 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
backend->shaders_state = KBASE_SHADERS_ON_CORESTACK_ON_RECHECK;
} else if (stt->remaining_ticks == 0) {
backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
#ifdef CONFIG_MALI_ARBITER_SUPPORT
} else if (kbase_pm_is_suspending(kbdev) ||
kbase_pm_is_gpu_lost(kbdev)) {
backend->shaders_state = KBASE_SHADERS_WAIT_FINISHED_CORESTACK_ON;
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
}
break;
@ -1104,21 +1128,24 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
* meet KBASE_SHADERS_PEND_ON_CORESTACK_ON
* check pass
*/
backend->shaders_avail &= shaders_ready;
/* shaders_desired_mask shall be a subset of
* shaders_ready
*/
WARN_ON(backend->shaders_desired_mask & ~shaders_ready);
WARN_ON(!(backend->shaders_desired_mask & shaders_ready));
backend->shaders_avail =
backend->shaders_desired_mask;
kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
shaders_ready & ~backend->shaders_avail, ACTION_PWROFF);
backend->shaders_state = KBASE_SHADERS_PEND_ON_CORESTACK_ON;
KBASE_TRACE_ADD(kbdev,
PM_CORES_CHANGE_AVAILABLE,
NULL, NULL, 0u,
(u32)(shaders_ready & ~backend->shaders_avail));
KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, (shaders_ready & ~backend->shaders_avail));
} else {
kbase_pm_invoke(kbdev, KBASE_PM_CORE_SHADER,
shaders_ready, ACTION_PWROFF);
KBASE_TRACE_ADD(kbdev,
PM_CORES_CHANGE_AVAILABLE,
NULL, NULL, 0u, 0u);
KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u);
backend->shaders_state = KBASE_SHADERS_PEND_OFF_CORESTACK_ON;
}
@ -1167,6 +1194,8 @@ static void kbase_pm_shaders_update_state(struct kbase_device *kbdev)
backend->shaders_state));
} while (backend->shaders_state != prev_state);
return err;
}
static bool kbase_pm_is_in_desired_state_nolock(struct kbase_device *kbdev)
@ -1259,24 +1288,29 @@ void kbase_pm_update_state(struct kbase_device *kbdev)
if (!kbdev->pm.backend.gpu_powered)
return; /* Do nothing if the GPU is off */
kbase_pm_l2_update_state(kbdev);
kbase_pm_shaders_update_state(kbdev);
if (kbase_pm_l2_update_state(kbdev))
return;
if (kbase_pm_shaders_update_state(kbdev))
return;
/* If the shaders just turned off, re-invoke the L2 state machine, in
* case it was waiting for the shaders to turn off before powering down
* the L2.
*/
if (prev_shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF &&
kbdev->pm.backend.shaders_state == KBASE_SHADERS_OFF_CORESTACK_OFF)
kbase_pm_l2_update_state(kbdev);
kbdev->pm.backend.shaders_state ==
KBASE_SHADERS_OFF_CORESTACK_OFF) {
if (kbase_pm_l2_update_state(kbdev))
return;
}
if (kbase_pm_is_in_desired_state_nolock(kbdev)) {
KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL,
true, kbdev->pm.backend.shaders_avail);
KBASE_KTRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, kbdev->pm.backend.shaders_avail);
kbase_pm_trace_power_state(kbdev);
KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, 0);
wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
}
}
@ -1555,7 +1589,7 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
kbdev->poweroff_pending = false;
KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
KBASE_KTRACE_ADD(kbdev, PM_GPU_ON, NULL, 0u);
if (is_resume && kbdev->pm.backend.callback_power_resume) {
kbdev->pm.backend.callback_power_resume(kbdev);
@ -1580,6 +1614,15 @@ void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&kbdev->mmu_hw_mutex);
if (kbdev->dummy_job_wa.flags &
KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) {
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
kbase_dummy_job_wa_execute(kbdev,
kbase_pm_get_present_cores(kbdev,
KBASE_PM_CORE_SHADER));
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
}
/* Enable the interrupts */
kbase_pm_enable_interrupts(kbdev);
@ -1609,7 +1652,7 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev)
return true;
}
KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
KBASE_KTRACE_ADD(kbdev, PM_GPU_OFF, NULL, 0u);
/* Disable interrupts. This also clears any outstanding interrupts */
kbase_pm_disable_interrupts(kbdev);
@ -1630,6 +1673,9 @@ bool kbase_pm_clock_off(struct kbase_device *kbdev)
/* The GPU power may be turned off from this point */
kbdev->pm.backend.gpu_powered = false;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
#ifdef CONFIG_MALI_ARBITER_SUPPORT
kbase_arbiter_pm_vm_event(kbdev, KBASE_VM_GPU_IDLE_EVENT);
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
if (kbdev->pm.backend.callback_power_off)
kbdev->pm.backend.callback_power_off(kbdev);
@ -1683,10 +1729,11 @@ static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
return HRTIMER_NORESTART;
}
static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
static int kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
{
kbdev->hw_quirks_jm = kbase_reg_read(kbdev,
u32 hw_quirks_jm = kbase_reg_read(kbdev,
GPU_CONTROL_REG(JM_CONFIG));
if (GPU_ID2_MODEL_MATCH_VALUE(prod_id) == GPU_ID2_PRODUCT_TMIX) {
/* Only for tMIx */
u32 coherency_features;
@ -1699,11 +1746,17 @@ static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
*/
if (coherency_features ==
COHERENCY_FEATURE_BIT(COHERENCY_ACE)) {
kbdev->hw_quirks_jm |= (COHERENCY_ACE_LITE |
hw_quirks_jm |= (COHERENCY_ACE_LITE |
COHERENCY_ACE) <<
JM_FORCE_COHERENCY_FEATURES_SHIFT;
}
}
if (kbase_is_gpu_lost(kbdev))
return -EIO;
kbdev->hw_quirks_jm = hw_quirks_jm;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_IDVS_GROUP_SIZE)) {
int default_idvs_group_size = 0xF;
u32 tmp;
@ -1712,53 +1765,71 @@ static void kbase_set_jm_quirks(struct kbase_device *kbdev, const u32 prod_id)
"idvs-group-size", &tmp))
tmp = default_idvs_group_size;
if (tmp > JM_MAX_IDVS_GROUP_SIZE) {
if (tmp > IDVS_GROUP_MAX_SIZE) {
dev_err(kbdev->dev,
"idvs-group-size of %d is too large. Maximum value is %d",
tmp, JM_MAX_IDVS_GROUP_SIZE);
tmp, IDVS_GROUP_MAX_SIZE);
tmp = default_idvs_group_size;
}
kbdev->hw_quirks_jm |= tmp << JM_IDVS_GROUP_SIZE_SHIFT;
kbdev->hw_quirks_jm |= tmp << IDVS_GROUP_SIZE_SHIFT;
}
#define MANUAL_POWER_CONTROL ((u32)(1 << 8))
if (corestack_driver_control)
kbdev->hw_quirks_jm |= MANUAL_POWER_CONTROL;
return 0;
}
static void kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id)
static int kbase_set_sc_quirks(struct kbase_device *kbdev, const u32 prod_id)
{
kbdev->hw_quirks_sc = kbase_reg_read(kbdev,
u32 hw_quirks_sc = kbase_reg_read(kbdev,
GPU_CONTROL_REG(SHADER_CONFIG));
if (kbase_is_gpu_lost(kbdev))
return -EIO;
if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */
kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */
kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_2968_TTRX_3162))
kbdev->hw_quirks_sc |= SC_VAR_ALGORITHM;
hw_quirks_sc |= SC_VAR_ALGORITHM;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_TLS_HASHING))
kbdev->hw_quirks_sc |= SC_TLS_HASH_ENABLE;
hw_quirks_sc |= SC_TLS_HASH_ENABLE;
kbdev->hw_quirks_sc = hw_quirks_sc;
return 0;
}
static void kbase_set_tiler_quirks(struct kbase_device *kbdev)
static int kbase_set_tiler_quirks(struct kbase_device *kbdev)
{
kbdev->hw_quirks_tiler = kbase_reg_read(kbdev,
u32 hw_quirks_tiler = kbase_reg_read(kbdev,
GPU_CONTROL_REG(TILER_CONFIG));
if (kbase_is_gpu_lost(kbdev))
return -EIO;
/* Set tiler clock gate override if required */
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
kbdev->hw_quirks_tiler = hw_quirks_tiler;
return 0;
}
static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
static int kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
{
struct device_node *np = kbdev->dev->of_node;
const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
GPU_ID_VERSION_PRODUCT_ID_SHIFT;
int error = 0;
kbdev->hw_quirks_jm = 0;
kbdev->hw_quirks_sc = 0;
@ -1771,7 +1842,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_jm = [0x%x] in Devicetree\n",
kbdev->hw_quirks_jm);
} else {
kbase_set_jm_quirks(kbdev, prod_id);
error = kbase_set_jm_quirks(kbdev, prod_id);
if (error)
return error;
}
if (!of_property_read_u32(np, "quirks_sc",
@ -1780,7 +1853,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_sc = [0x%x] in Devicetree\n",
kbdev->hw_quirks_sc);
} else {
kbase_set_sc_quirks(kbdev, prod_id);
error = kbase_set_sc_quirks(kbdev, prod_id);
if (error)
return error;
}
if (!of_property_read_u32(np, "quirks_tiler",
@ -1789,7 +1864,9 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_tiler = [0x%x] in Devicetree\n",
kbdev->hw_quirks_tiler);
} else {
kbase_set_tiler_quirks(kbdev);
error = kbase_set_tiler_quirks(kbdev);
if (error)
return error;
}
if (!of_property_read_u32(np, "quirks_mmu",
@ -1798,8 +1875,10 @@ static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
"Found quirks_mmu = [0x%x] in Devicetree\n",
kbdev->hw_quirks_mmu);
} else {
kbase_set_mmu_quirks(kbdev);
error = kbase_set_mmu_quirks(kbdev);
}
return error;
}
static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
@ -1861,7 +1940,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
struct kbasep_reset_timeout_data rtdata;
int ret;
KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, 0);
KBASE_TLSTREAM_JD_GPU_SOFT_RESET(kbdev, kbdev);
@ -1902,8 +1981,9 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
/* No interrupt has been received - check if the RAWSTAT register says
* the reset has completed */
if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) &
RESET_COMPLETED) {
if ((kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT)) &
RESET_COMPLETED)
|| kbase_is_gpu_lost(kbdev)) {
/* The interrupt is set in the RAWSTAT; this suggests that the
* interrupts are not getting to the CPU */
dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
@ -1916,7 +1996,7 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
* reset */
dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
RESET_TIMEOUT);
KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
KBASE_KTRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, 0);
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_HARD_RESET);
@ -1944,29 +2024,20 @@ static int kbase_pm_do_reset(struct kbase_device *kbdev)
return -EINVAL;
}
static int kbasep_protected_mode_enable(struct protected_mode_device *pdev)
int kbase_pm_protected_mode_enable(struct kbase_device *const kbdev)
{
struct kbase_device *kbdev = pdev->data;
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
GPU_COMMAND_SET_PROTECTED_MODE);
return 0;
}
static int kbasep_protected_mode_disable(struct protected_mode_device *pdev)
int kbase_pm_protected_mode_disable(struct kbase_device *const kbdev)
{
struct kbase_device *kbdev = pdev->data;
lockdep_assert_held(&kbdev->pm.lock);
return kbase_pm_do_reset(kbdev);
}
struct protected_mode_ops kbase_native_protected_ops = {
.protected_mode_enable = kbasep_protected_mode_enable,
.protected_mode_disable = kbasep_protected_mode_disable
};
int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
{
unsigned long irq_flags;
@ -1994,16 +2065,12 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
/* The cores should be made unavailable due to the reset */
spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
if (kbdev->pm.backend.shaders_state != KBASE_SHADERS_OFF_CORESTACK_OFF)
KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
NULL, 0u, (u32)0u);
KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, 0u);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, irq_flags);
/* Soft reset the GPU */
if (kbdev->protected_mode_support)
err = kbdev->protected_ops->protected_mode_disable(
kbdev->protected_dev);
else
err = kbase_pm_do_reset(kbdev);
err = kbdev->protected_ops->protected_mode_disable(
kbdev->protected_dev);
spin_lock_irqsave(&kbdev->hwaccess_lock, irq_flags);
kbdev->protected_mode = false;
@ -2012,19 +2079,18 @@ int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
if (err)
goto exit;
if (flags & PM_HW_ISSUES_DETECT)
kbase_pm_hw_issues_detect(kbdev);
if (flags & PM_HW_ISSUES_DETECT) {
err = kbase_pm_hw_issues_detect(kbdev);
if (err)
goto exit;
}
kbase_pm_hw_issues_apply(kbdev);
kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency);
/* Sanity check protected mode was left after reset */
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) {
u32 gpu_status = kbase_reg_read(kbdev,
GPU_CONTROL_REG(GPU_STATUS));
WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE);
}
WARN_ON(kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS)) &
GPU_STATUS_PROTECTED_MODE_ACTIVE);
/* If cycle counter was in use re-enable it, enable_irqs will only be
* false when called from kbase_pm_powerup */

27
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_internal.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -682,4 +682,29 @@ extern bool corestack_driver_control;
* Return: true if l2 need to power on
*/
bool kbase_pm_is_l2_desired(struct kbase_device *kbdev);
/**
* kbase_pm_lock - Lock all necessary mutexes to perform PM actions
*
* @kbdev: Device pointer
*
* This function locks correct mutexes independent of GPU architecture.
*/
static inline void kbase_pm_lock(struct kbase_device *kbdev)
{
mutex_lock(&kbdev->js_data.runpool_mutex);
mutex_lock(&kbdev->pm.lock);
}
/**
* kbase_pm_unlock - Unlock mutexes locked by kbase_pm_lock
*
* @kbdev: Device pointer
*/
static inline void kbase_pm_unlock(struct kbase_device *kbdev)
{
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&kbdev->js_data.runpool_mutex);
}
#endif /* _KBASE_BACKEND_PM_INTERNAL_H_ */

6
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_metrics.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -31,6 +31,7 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <backend/gpu/mali_kbase_jm_rb.h>
#include <backend/gpu/mali_kbase_pm_defs.h>
#include <mali_linux_trace.h>
/* When VSync is being hit aim for utilisation between 70-90% */
#define KBASE_PM_VSYNC_MIN_UTILISATION 70
@ -284,8 +285,11 @@ static void kbase_pm_metrics_active_calc(struct kbase_device *kbdev)
active_cl_ctx[device_nr] = 1;
} else {
kbdev->pm.backend.metrics.active_gl_ctx[js] = 1;
trace_sysgraph(SGR_ACTIVE, 0, js);
}
kbdev->pm.backend.metrics.gpu_active = true;
} else {
trace_sysgraph(SGR_INACTIVE, 0, js);
}
}
}

24
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_policy.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -25,7 +25,7 @@
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase_pm.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
@ -102,6 +102,8 @@ void kbase_pm_update_active(struct kbase_device *kbdev)
* when there are contexts active */
KBASE_DEBUG_ASSERT(pm->active_count == 0);
pm->backend.poweron_required = false;
/* Request power off */
if (pm->backend.gpu_powered) {
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
@ -157,8 +159,7 @@ void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
shaders_desired = kbdev->pm.backend.pm_current_policy->shaders_needed(kbdev);
if (kbdev->pm.backend.shaders_desired != shaders_desired) {
KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
(u32)kbdev->pm.backend.shaders_desired);
KBASE_KTRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, kbdev->pm.backend.shaders_desired);
kbdev->pm.backend.shaders_desired = shaders_desired;
kbase_pm_update_state(kbdev);
@ -199,22 +200,20 @@ KBASE_EXPORT_TEST_API(kbase_pm_get_policy);
void kbase_pm_set_policy(struct kbase_device *kbdev,
const struct kbase_pm_policy *new_policy)
{
struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
const struct kbase_pm_policy *old_policy;
unsigned long flags;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(new_policy != NULL);
KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
KBASE_KTRACE_ADD(kbdev, PM_SET_POLICY, NULL, new_policy->id);
/* During a policy change we pretend the GPU is active */
/* A suspend won't happen here, because we're in a syscall from a
* userspace thread */
kbase_pm_context_active(kbdev);
mutex_lock(&js_devdata->runpool_mutex);
mutex_lock(&kbdev->pm.lock);
kbase_pm_lock(kbdev);
/* Remove the policy to prevent IRQ handlers from working on it */
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
@ -222,13 +221,11 @@ void kbase_pm_set_policy(struct kbase_device *kbdev,
kbdev->pm.backend.pm_current_policy = NULL;
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
old_policy->id);
KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, old_policy->id);
if (old_policy->term)
old_policy->term(kbdev);
KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
new_policy->id);
KBASE_KTRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, new_policy->id);
if (new_policy->init)
new_policy->init(kbdev);
@ -242,8 +239,7 @@ void kbase_pm_set_policy(struct kbase_device *kbdev,
kbase_pm_update_active(kbdev);
kbase_pm_update_cores_state(kbdev);
mutex_unlock(&kbdev->pm.lock);
mutex_unlock(&js_devdata->runpool_mutex);
kbase_pm_unlock(kbdev);
/* Now the policy change is finished, we release our fake context active
* reference */

2
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_pm_shader_states.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2018-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software

10
drivers/gpu/arm/bifrost/backend/gpu/mali_kbase_time.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2016,2018-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2016,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -26,7 +26,7 @@
#include <backend/gpu/mali_kbase_pm_internal.h>
void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
u64 *system_time, struct timespec *ts)
u64 *system_time, struct timespec64 *ts)
{
u32 hi1, hi2;
@ -60,7 +60,11 @@ void kbase_backend_get_gpu_time(struct kbase_device *kbdev, u64 *cycle_counter,
/* Record the CPU's idea of current time */
if (ts != NULL)
getrawmonotonic(ts);
#if (KERNEL_VERSION(4, 17, 0) > LINUX_VERSION_CODE)
*ts = ktime_to_timespec64(ktime_get_raw());
#else
ktime_get_raw_ts64(ts);
#endif
kbase_pm_release_gpu_cycle_counter(kbdev);
}

76
drivers/gpu/arm/bifrost/build.bp
View File

@ -1,13 +1,16 @@
/*
* Copyright:
* ----------------------------------------------------------------------------
* This confidential and proprietary software may be used only as authorized
* by a licensing agreement from ARM Limited.
* (C) COPYRIGHT 2017-2019 ARM Limited, ALL RIGHTS RESERVED
* The entire notice above must be reproduced on all authorized copies and
* copies may only be made to the extent permitted by a licensing agreement
* from ARM Limited.
* ----------------------------------------------------------------------------
*
* (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
/* Kernel-side tests may include mali_kbase's headers. Therefore any config
@ -34,9 +37,6 @@ bob_defaults {
buslog: {
kbuild_options: ["CONFIG_MALI_BUSLOG=y"],
},
cinstr_job_dump: {
kbuild_options: ["CONFIG_MALI_JOB_DUMP=y"],
},
cinstr_vector_dump: {
kbuild_options: ["CONFIG_MALI_VECTOR_DUMP=y"],
},
@ -46,6 +46,9 @@ bob_defaults {
mali_gator_support: {
kbuild_options: ["CONFIG_MALI_BIFROST_GATOR_SUPPORT=y"],
},
mali_midgard_enable_trace: {
kbuild_options: ["CONFIG_MALI_BIFROST_ENABLE_TRACE=y"],
},
mali_system_trace: {
kbuild_options: ["CONFIG_MALI_BIFROST_SYSTEM_TRACE=y"],
},
@ -61,6 +64,12 @@ bob_defaults {
mali_dma_buf_legacy_compat: {
kbuild_options: ["CONFIG_MALI_DMA_BUF_LEGACY_COMPAT=y"],
},
mali_arbiter_support: {
kbuild_options: ["CONFIG_MALI_ARBITER_SUPPORT=y"],
},
mali_gem5_build: {
kbuild_options: ["CONFIG_MALI_GEM5_BUILD=y"],
},
kbuild_options: [
"MALI_UNIT_TEST={{.unit_test_code}}",
"MALI_CUSTOMER_RELEASE={{.release}}",
@ -79,6 +88,8 @@ bob_kernel_module {
"backend/gpu/*.c",
"backend/gpu/*.h",
"backend/gpu/Kbuild",
"context/*.c",
"context/*.h",
"ipa/*.c",
"ipa/*.h",
"ipa/Kbuild",
@ -87,6 +98,16 @@ bob_kernel_module {
"platform/*/*.h",
"platform/*/Kbuild",
"thirdparty/*.c",
"debug/*.c",
"debug/*.h",
"device/*.c",
"device/*.h",
"gpu/*.c",
"gpu/*.h",
"tl/*.c",
"tl/*.h",
"mmu/*.c",
"mmu/*.h",
],
kbuild_options: [
"CONFIG_MALI_KUTF=n",
@ -111,6 +132,9 @@ bob_kernel_module {
cinstr_secondary_hwc: {
kbuild_options: ["CONFIG_MALI_BIFROST_PRFCNT_SET_SECONDARY=y"],
},
cinstr_secondary_hwc_via_debug_fs: {
kbuild_options: ["CONFIG_MALI_PRFCNT_SET_SECONDARY_VIA_DEBUG_FS=y"],
},
mali_2mb_alloc: {
kbuild_options: ["CONFIG_MALI_2MB_ALLOC=y"],
},
@ -120,11 +144,39 @@ bob_kernel_module {
mali_hw_errata_1485982_use_clock_alternative: {
kbuild_options: ["CONFIG_MALI_HW_ERRATA_1485982_USE_CLOCK_ALTERNATIVE=y"],
},
gpu_has_job_manager: {
srcs: [
"context/backend/*_jm.c",
"debug/backend/*_jm.c",
"debug/backend/*_jm.h",
"device/backend/*_jm.c",
"gpu/backend/*_jm.c",
"gpu/backend/*_jm.h",
"jm/*.h",
"tl/backend/*_jm.c",
"mmu/backend/*_jm.c",
],
},
gpu_has_csf: {
srcs: [
"context/backend/*_csf.c",
"csf/*.c",
"csf/*.h",
"csf/Kbuild",
"debug/backend/*_csf.c",
"debug/backend/*_csf.h",
"device/backend/*_csf.c",
"gpu/backend/*_csf.c",
"gpu/backend/*_csf.h",
"tl/backend/*_csf.c",
"mmu/backend/*_csf.c",
],
},
mali_arbiter_support: {
srcs: [
"arbiter/*.c",
"arbiter/*.h",
"arbiter/Kbuild",
],
},
defaults: ["mali_kbase_shared_config_defaults"],

213
drivers/gpu/arm/bifrost/context/backend/mali_kbase_context_jm.c Normal file
View File

@ -0,0 +1,213 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel context APIs for Job Manager GPUs
*/
#include <context/mali_kbase_context_internal.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase.h>
#include <mali_kbase_ctx_sched.h>
#include <mali_kbase_dma_fence.h>
#include <mali_kbase_mem_linux.h>
#include <mali_kbase_mem_pool_group.h>
#include <mmu/mali_kbase_mmu.h>
#include <tl/mali_kbase_timeline.h>
#include <tl/mali_kbase_tracepoints.h>
#ifdef CONFIG_DEBUG_FS
#include <mali_kbase_debug_mem_view.h>
#include <mali_kbase_mem_pool_debugfs.h>
void kbase_context_debugfs_init(struct kbase_context *const kctx)
{
kbase_debug_mem_view_init(kctx);
kbase_mem_pool_debugfs_init(kctx->kctx_dentry, kctx);
kbase_jit_debugfs_init(kctx);
kbasep_jd_debugfs_ctx_init(kctx);
kbase_debug_job_fault_context_init(kctx);
}
KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init);
void kbase_context_debugfs_term(struct kbase_context *const kctx)
{
debugfs_remove_recursive(kctx->kctx_dentry);
kbase_debug_job_fault_context_term(kctx);
}
KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term);
#else
void kbase_context_debugfs_init(struct kbase_context *const kctx)
{
CSTD_UNUSED(kctx);
}
KBASE_EXPORT_SYMBOL(kbase_context_debugfs_init);
void kbase_context_debugfs_term(struct kbase_context *const kctx)
{
CSTD_UNUSED(kctx);
}
KBASE_EXPORT_SYMBOL(kbase_context_debugfs_term);
#endif /* CONFIG_DEBUG_FS */
static int kbase_context_kbase_timer_setup(struct kbase_context *kctx)
{
kbase_timer_setup(&kctx->soft_job_timeout,
kbasep_soft_job_timeout_worker);
return 0;
}
static int kbase_context_submit_check(struct kbase_context *kctx)
{
struct kbasep_js_kctx_info *js_kctx_info = &kctx->jctx.sched_info;
unsigned long irq_flags = 0;
base_context_create_flags const flags = kctx->create_flags;
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags);
/* Translate the flags */
if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0)
kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED);
spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
return 0;
}
static const struct kbase_context_init context_init[] = {
{kbase_context_common_init, kbase_context_common_term, NULL},
{kbase_context_mem_pool_group_init, kbase_context_mem_pool_group_term,
"Memory pool goup initialization failed"},
{kbase_mem_evictable_init, kbase_mem_evictable_deinit,
"Memory evictable initialization failed"},
{kbasep_js_kctx_init, kbasep_js_kctx_term,
"JS kctx initialization failed"},
{kbase_jd_init, kbase_jd_exit,
"JD initialization failed"},
{kbase_event_init, kbase_event_cleanup,
"Event initialization failed"},
{kbase_dma_fence_init, kbase_dma_fence_term,
"DMA fence initialization failed"},
{kbase_context_mmu_init, kbase_context_mmu_term,
"MMU initialization failed"},
{kbase_context_mem_alloc_page, kbase_context_mem_pool_free,
"Memory alloc page failed"},
{kbase_region_tracker_init, kbase_region_tracker_term,
"Region tracker initialization failed"},
{kbase_sticky_resource_init, kbase_context_sticky_resource_term,
"Sticky resource initialization failed"},
{kbase_jit_init, kbase_jit_term,
"JIT initialization failed"},
{kbase_context_kbase_timer_setup, NULL, NULL},
{kbase_context_submit_check, NULL, NULL},
};
static void kbase_context_term_partial(
struct kbase_context *kctx,
unsigned int i)
{
while (i-- > 0) {
if (context_init[i].term)
context_init[i].term(kctx);
}
}
struct kbase_context *kbase_create_context(struct kbase_device *kbdev,
bool is_compat,
base_context_create_flags const flags,
unsigned long const api_version,
struct file *const filp)
{
struct kbase_context *kctx;
unsigned int i = 0;
if (WARN_ON(!kbdev))
return NULL;
/* Validate flags */
if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS)))
return NULL;
/* zero-inited as lot of code assume it's zero'ed out on create */
kctx = vzalloc(sizeof(*kctx));
if (WARN_ON(!kctx))
return NULL;
kctx->kbdev = kbdev;
kctx->api_version = api_version;
kctx->filp = filp;
kctx->create_flags = flags;
if (is_compat)
kbase_ctx_flag_set(kctx, KCTX_COMPAT);
#if defined(CONFIG_64BIT)
else
kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA);
#endif /* !defined(CONFIG_64BIT) */
for (i = 0; i < ARRAY_SIZE(context_init); i++) {
int err = context_init[i].init(kctx);
if (err) {
dev_err(kbdev->dev, "%s error = %d\n",
context_init[i].err_mes, err);
kbase_context_term_partial(kctx, i);
return NULL;
}
}
return kctx;
}
KBASE_EXPORT_SYMBOL(kbase_create_context);
void kbase_destroy_context(struct kbase_context *kctx)
{
struct kbase_device *kbdev;
if (WARN_ON(!kctx))
return;
kbdev = kctx->kbdev;
if (WARN_ON(!kbdev))
return;
/* Ensure the core is powered up for the destroy process
* A suspend won't happen here, because we're in a syscall
* from a userspace thread.
*/
kbase_pm_context_active(kbdev);
kbase_mem_pool_group_mark_dying(&kctx->mem_pools);
kbase_jd_zap_context(kctx);
flush_workqueue(kctx->jctx.job_done_wq);
kbase_context_term_partial(kctx, ARRAY_SIZE(context_init));
kbase_pm_context_idle(kbdev);
}
KBASE_EXPORT_SYMBOL(kbase_destroy_context);

210
drivers/gpu/arm/bifrost/context/mali_kbase_context.c Normal file
View File

@ -0,0 +1,210 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel context APIs
*/
#include <mali_kbase.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase_mem_linux.h>
#include <mali_kbase_dma_fence.h>
#include <mali_kbase_ctx_sched.h>
#include <mali_kbase_mem_pool_group.h>
#include <tl/mali_kbase_tracepoints.h>
#include <tl/mali_kbase_timeline.h>
#include <mmu/mali_kbase_mmu.h>
#include <context/mali_kbase_context_internal.h>
int kbase_context_common_init(struct kbase_context *kctx)
{
const unsigned long cookies_mask = KBASE_COOKIE_MASK;
/* creating a context is considered a disjoint event */
kbase_disjoint_event(kctx->kbdev);
kctx->as_nr = KBASEP_AS_NR_INVALID;
atomic_set(&kctx->refcount, 0);
spin_lock_init(&kctx->mm_update_lock);
kctx->process_mm = NULL;
atomic_set(&kctx->nonmapped_pages, 0);
atomic_set(&kctx->permanent_mapped_pages, 0);
kctx->tgid = current->tgid;
kctx->pid = current->pid;
atomic_set(&kctx->used_pages, 0);
mutex_init(&kctx->reg_lock);
spin_lock_init(&kctx->mem_partials_lock);
INIT_LIST_HEAD(&kctx->mem_partials);
spin_lock_init(&kctx->waiting_soft_jobs_lock);
INIT_LIST_HEAD(&kctx->waiting_soft_jobs);
init_waitqueue_head(&kctx->event_queue);
atomic_set(&kctx->event_count, 0);
atomic_set(&kctx->event_closed, false);
bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG);
#ifdef CONFIG_GPU_TRACEPOINTS
atomic_set(&kctx->jctx.work_id, 0);
#endif
kctx->id = atomic_add_return(1, &(kctx->kbdev->ctx_num)) - 1;
mutex_init(&kctx->legacy_hwcnt_lock);
mutex_lock(&kctx->kbdev->kctx_list_lock);
list_add(&kctx->kctx_list_link, &kctx->kbdev->kctx_list);
KBASE_TLSTREAM_TL_KBASE_NEW_CTX(kctx->kbdev, kctx->id,
kctx->kbdev->gpu_props.props.raw_props.gpu_id);
KBASE_TLSTREAM_TL_NEW_CTX(kctx->kbdev, kctx, kctx->id,
(u32)(kctx->tgid));
mutex_unlock(&kctx->kbdev->kctx_list_lock);
return 0;
}
void kbase_context_common_term(struct kbase_context *kctx)
{
unsigned long flags;
int pages;
mutex_lock(&kctx->kbdev->mmu_hw_mutex);
spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
kbase_ctx_sched_remove_ctx(kctx);
spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
mutex_unlock(&kctx->kbdev->mmu_hw_mutex);
pages = atomic_read(&kctx->used_pages);
if (pages != 0)
dev_warn(kctx->kbdev->dev,
"%s: %d pages in use!\n", __func__, pages);
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
mutex_lock(&kctx->kbdev->kctx_list_lock);
KBASE_TLSTREAM_TL_KBASE_DEL_CTX(kctx->kbdev, kctx->id);
KBASE_TLSTREAM_TL_DEL_CTX(kctx->kbdev, kctx);
list_del(&kctx->kctx_list_link);
mutex_unlock(&kctx->kbdev->kctx_list_lock);
KBASE_KTRACE_ADD(kctx->kbdev, CORE_CTX_DESTROY, kctx, 0u);
/* Flush the timeline stream, so the user can see the termination
* tracepoints being fired.
* The "if" statement below is for optimization. It is safe to call
* kbase_timeline_streams_flush when timeline is disabled.
*/
if (atomic_read(&kctx->kbdev->timeline_flags) != 0)
kbase_timeline_streams_flush(kctx->kbdev->timeline);
vfree(kctx);
}
int kbase_context_mem_pool_group_init(struct kbase_context *kctx)
{
return kbase_mem_pool_group_init(&kctx->mem_pools,
kctx->kbdev,
&kctx->kbdev->mem_pool_defaults,
&kctx->kbdev->mem_pools);
}
void kbase_context_mem_pool_group_term(struct kbase_context *kctx)
{
kbase_mem_pool_group_term(&kctx->mem_pools);
}
int kbase_context_mmu_init(struct kbase_context *kctx)
{
kbase_mmu_init(kctx->kbdev,
&kctx->mmu, kctx,
base_context_mmu_group_id_get(kctx->create_flags));
return 0;
}
void kbase_context_mmu_term(struct kbase_context *kctx)
{
kbase_mmu_term(kctx->kbdev, &kctx->mmu);
}
int kbase_context_mem_alloc_page(struct kbase_context *kctx)
{
struct page *p;
p = kbase_mem_alloc_page(&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK]);
if (!p)
return -ENOMEM;
kctx->aliasing_sink_page = as_tagged(page_to_phys(p));
return 0;
}
void kbase_context_mem_pool_free(struct kbase_context *kctx)
{
/* drop the aliasing sink page now that it can't be mapped anymore */
kbase_mem_pool_free(
&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK],
as_page(kctx->aliasing_sink_page),
false);
}
void kbase_context_sticky_resource_term(struct kbase_context *kctx)
{
unsigned long pending_regions_to_clean;
kbase_gpu_vm_lock(kctx);
kbase_sticky_resource_term(kctx);
/* free pending region setups */
pending_regions_to_clean = KBASE_COOKIE_MASK;
bitmap_andnot(&pending_regions_to_clean, &pending_regions_to_clean,
kctx->cookies, BITS_PER_LONG);
while (pending_regions_to_clean) {
unsigned int cookie = find_first_bit(&pending_regions_to_clean,
BITS_PER_LONG);
if (!WARN_ON(!kctx->pending_regions[cookie])) {
dev_dbg(kctx->kbdev->dev, "Freeing pending unmapped region\n");
kbase_mem_phy_alloc_put(
kctx->pending_regions[cookie]->cpu_alloc);
kbase_mem_phy_alloc_put(
kctx->pending_regions[cookie]->gpu_alloc);
kfree(kctx->pending_regions[cookie]);
kctx->pending_regions[cookie] = NULL;
}
bitmap_clear(&pending_regions_to_clean, cookie, 1);
}
kbase_gpu_vm_unlock(kctx);
}

34
drivers/gpu/arm/bifrost/mali_kbase_context.h → drivers/gpu/arm/bifrost/context/mali_kbase_context.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2017, 2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -18,6 +18,16 @@
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2011-2017, 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
*/
#ifndef _KBASE_CONTEXT_H_
@ -25,6 +35,28 @@
#include <linux/atomic.h>
/**
* kbase_context_debugfs_init - Initialize the kctx platform
* specific debugfs
*
* @kctx: kbase context
*
* This initializes some debugfs interfaces specific to the platform the source
* is compiled for.
*/
void kbase_context_debugfs_init(struct kbase_context *const kctx);
/**
* kbase_context_debugfs_term - Terminate the kctx platform
* specific debugfs
*
* @kctx: kbase context
*
* This terminates some debugfs interfaces specific to the platform the source
* is compiled for.
*/
void kbase_context_debugfs_term(struct kbase_context *const kctx);
/**
* kbase_create_context() - Create a kernel base context.
*

60
drivers/gpu/arm/bifrost/context/mali_kbase_context_internal.h Normal file
View File

@ -0,0 +1,60 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*/
#include <mali_kbase.h>
typedef int kbase_context_init_method(struct kbase_context *kctx);
typedef void kbase_context_term_method(struct kbase_context *kctx);
/**
* struct kbase_context_init - Device init/term methods.
* @init: Function pointer to a initialise method.
* @term: Function pointer to a terminate method.
* @err_mes: Error message to be printed when init method fails.
*/
struct kbase_context_init {
kbase_context_init_method *init;
kbase_context_term_method *term;
char *err_mes;
};
int kbase_context_common_init(struct kbase_context *kctx);
void kbase_context_common_term(struct kbase_context *kctx);
int kbase_context_mem_pool_group_init(struct kbase_context *kctx);
void kbase_context_mem_pool_group_term(struct kbase_context *kctx);
int kbase_context_mmu_init(struct kbase_context *kctx);
void kbase_context_mmu_term(struct kbase_context *kctx);
int kbase_context_mem_alloc_page(struct kbase_context *kctx);
void kbase_context_mem_pool_free(struct kbase_context *kctx);
void kbase_context_sticky_resource_term(struct kbase_context *kctx);

170
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_codes_jm.h Normal file
View File

@ -0,0 +1,170 @@
/*
*
* (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE *****
* ***** DO NOT INCLUDE DIRECTLY *****
* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL *****
*/
/*
* The purpose of this header file is just to contain a list of trace code
* identifiers
*
* IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THAT
* DESCRIBED IN mali_kbase_debug_ktrace_codes.h
*/
#if 0 /* Dummy section to avoid breaking formatting */
int dummy_array[] = {
#endif
/*
* Job Slot management events
*/
/* info_val==irq rawstat at start */
KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ),
/* info_val==jobs processed */
KBASE_KTRACE_CODE_MAKE_CODE(JM_IRQ_END),
/* In the following:
*
* - ctx is set if a corresponding job found (NULL otherwise, e.g. some
* soft-stop cases)
* - uatom==kernel-side mapped uatom address (for correlation with
* user-side)
*/
/* info_val==exit code; gpu_addr==chain gpuaddr */
KBASE_KTRACE_CODE_MAKE_CODE(JM_JOB_DONE),
/* gpu_addr==JS_HEAD_NEXT written, info_val==lower 32 bits of
* affinity
*/
KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT),
/* gpu_addr is as follows:
* - If JS_STATUS active after soft-stop, val==gpu addr written to
* JS_HEAD on submit
* - otherwise gpu_addr==0
*/
KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP),
KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_0),
KBASE_KTRACE_CODE_MAKE_CODE(JM_SOFTSTOP_1),
/* gpu_addr==JS_HEAD read */
KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP),
/* gpu_addr==JS_HEAD read */
KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_0),
/* gpu_addr==JS_HEAD read */
KBASE_KTRACE_CODE_MAKE_CODE(JM_HARDSTOP_1),
/* gpu_addr==JS_TAIL read */
KBASE_KTRACE_CODE_MAKE_CODE(JM_UPDATE_HEAD),
/* gpu_addr is as follows:
* - If JS_STATUS active before soft-stop, val==JS_HEAD
* - otherwise gpu_addr==0
*/
/* gpu_addr==JS_HEAD read */
KBASE_KTRACE_CODE_MAKE_CODE(JM_CHECK_HEAD),
KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS),
KBASE_KTRACE_CODE_MAKE_CODE(JM_FLUSH_WORKQS_DONE),
/* info_val == is_scheduled */
KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_NON_SCHEDULED),
/* info_val == is_scheduled */
KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_SCHEDULED),
KBASE_KTRACE_CODE_MAKE_CODE(JM_ZAP_DONE),
/* info_val == nr jobs submitted */
KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_SOFT_OR_HARD_STOP),
/* gpu_addr==JS_HEAD_NEXT last written */
KBASE_KTRACE_CODE_MAKE_CODE(JM_SLOT_EVICT),
KBASE_KTRACE_CODE_MAKE_CODE(JM_SUBMIT_AFTER_RESET),
KBASE_KTRACE_CODE_MAKE_CODE(JM_BEGIN_RESET_WORKER),
KBASE_KTRACE_CODE_MAKE_CODE(JM_END_RESET_WORKER),
/*
* Job dispatch events
*/
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE),
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER),
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_WORKER_END),
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JD_DONE_TRY_RUN_NEXT_JOB),
/* gpu_addr==0, info_val==0, uatom==0 */
KBASE_KTRACE_CODE_MAKE_CODE(JD_ZAP_CONTEXT),
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL),
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JD_CANCEL_WORKER),
/*
* Scheduler Core events
*/
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JS_ADD_JOB),
/* gpu_addr==last value written/would be written to JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JS_REMOVE_JOB),
KBASE_KTRACE_CODE_MAKE_CODE(JS_TRY_SCHEDULE_HEAD_CTX),
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_TRY_RUN_NEXT_JOB),
/* gpu_addr==value to write into JS_HEAD */
KBASE_KTRACE_CODE_MAKE_CODE(JS_JOB_DONE_RETRY_NEEDED),
KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_SUBMIT_TO_BLOCKED),
/* info_val == lower 32 bits of affinity */
KBASE_KTRACE_CODE_MAKE_CODE(JS_AFFINITY_CURRENT),
/* info_val == lower 32 bits of affinity */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_CORES_FAILED),
/* info_val == lower 32 bits of affinity */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_INUSE_FAILED),
/* info_val == lower 32 bits of rechecked affinity */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED),
/* info_val == lower 32 bits of rechecked affinity */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED),
/* info_val == lower 32 bits of affinity */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CORE_REF_AFFINITY_WOULD_VIOLATE),
/* info_val == the ctx attribute now on ctx */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_CTX),
/* info_val == the ctx attribute now on runpool */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_ON_RUNPOOL),
/* info_val == the ctx attribute now off ctx */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_CTX),
/* info_val == the ctx attribute now off runpool */
KBASE_KTRACE_CODE_MAKE_CODE(JS_CTX_ATTR_NOW_OFF_RUNPOOL),
/*
* Scheduler Policy events
*/
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_INIT_CTX),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TERM_CTX),
/* info_val == whether it was evicted */
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TRY_EVICT_CTX),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_FOREACH_CTX_JOBS),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_CTX),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_HEAD_CTX),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_ADD_CTX),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_RUNPOOL_REMOVE_CTX),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_DEQUEUE_JOB_IRQ),
/* gpu_addr==JS_HEAD to write if the job were run */
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_ENQUEUE_JOB),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_START),
KBASE_KTRACE_CODE_MAKE_CODE(JS_POLICY_TIMER_END),
#if 0 /* Dummy section to avoid breaking formatting */
};
#endif
/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */

75
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_defs_jm.h Normal file
View File

@ -0,0 +1,75 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_DEBUG_KTRACE_DEFS_JM_H_
#define _KBASE_DEBUG_KTRACE_DEFS_JM_H_
/**
* DOC: KTrace version history, JM variant
* 1.0:
* - Original version (implicit, header did not carry version information)
* 2.0:
* - Introduced version information into the header
* - some changes of parameter names in header
* - trace now uses all 64-bits of info_val
* - Non-JM specific parts moved to using info_val instead of refcount/gpu_addr
*/
#define KBASE_KTRACE_VERSION_MAJOR 2
#define KBASE_KTRACE_VERSION_MINOR 0
/* indicates if the trace message has a valid refcount member */
#define KBASE_KTRACE_FLAG_JM_REFCOUNT (((kbase_ktrace_flag_t)1) << 0)
/* indicates if the trace message has a valid jobslot member */
#define KBASE_KTRACE_FLAG_JM_JOBSLOT (((kbase_ktrace_flag_t)1) << 1)
/* indicates if the trace message has valid atom related info. */
#define KBASE_KTRACE_FLAG_JM_ATOM (((kbase_ktrace_flag_t)1) << 2)
/**
* struct kbase_ktrace_backend - backend specific part of a trace message
*
* @atom_udata: Copy of the user data sent for the atom in base_jd_submit.
* Only valid if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags
* @gpu_addr: GPU address, usually of the job-chain represented by an atom.
* @atom_number: id of the atom for which trace message was added. Only valid
* if KBASE_KTRACE_FLAG_JM_ATOM is set in @flags
* @code: Identifies the event, refer to enum kbase_ktrace_code.
* @flags: indicates information about the trace message itself. Used
* during dumping of the message.
* @jobslot: job-slot for which trace message was added, valid only for
* job-slot management events.
* @refcount: reference count for the context, valid for certain events
* related to scheduler core and policy.
*/
struct kbase_ktrace_backend {
/* Place 64 and 32-bit members together */
u64 atom_udata[2]; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */
u64 gpu_addr;
int atom_number; /* Only valid for KBASE_KTRACE_FLAG_JM_ATOM */
/* Pack smaller members together */
kbase_ktrace_code_t code;
kbase_ktrace_flag_t flags;
u8 jobslot;
u8 refcount;
};
#endif /* _KBASE_DEBUG_KTRACE_DEFS_JM_H_ */

113
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.c Normal file
View File

@ -0,0 +1,113 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
#include "debug/mali_kbase_debug_ktrace_internal.h"
#include "debug/backend/mali_kbase_debug_ktrace_jm.h"
#if KBASE_KTRACE_TARGET_RBUF
void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written)
{
*written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0),
"katom,gpu_addr,jobslot,refcount"), 0);
}
void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg,
char *buffer, int sz, s32 *written)
{
/* katom */
if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_ATOM)
*written += MAX(snprintf(buffer + *written,
MAX(sz - *written, 0),
"atom %d (ud: 0x%llx 0x%llx)",
trace_msg->backend.atom_number,
trace_msg->backend.atom_udata[0],
trace_msg->backend.atom_udata[1]), 0);
/* gpu_addr */
if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_BACKEND)
*written += MAX(snprintf(buffer + *written,
MAX(sz - *written, 0),
",%.8llx,", trace_msg->backend.gpu_addr), 0);
else
*written += MAX(snprintf(buffer + *written,
MAX(sz - *written, 0),
",,"), 0);
/* jobslot */
if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_JOBSLOT)
*written += MAX(snprintf(buffer + *written,
MAX(sz - *written, 0),
"%d", trace_msg->backend.jobslot), 0);
*written += MAX(snprintf(buffer + *written, MAX(sz - *written, 0),
","), 0);
/* refcount */
if (trace_msg->backend.flags & KBASE_KTRACE_FLAG_JM_REFCOUNT)
*written += MAX(snprintf(buffer + *written,
MAX(sz - *written, 0),
"%d", trace_msg->backend.refcount), 0);
}
void kbasep_ktrace_add_jm(struct kbase_device *kbdev,
enum kbase_ktrace_code code, struct kbase_context *kctx,
struct kbase_jd_atom *katom, u64 gpu_addr,
kbase_ktrace_flag_t flags, int refcount, int jobslot,
u64 info_val)
{
unsigned long irqflags;
struct kbase_ktrace_msg *trace_msg;
spin_lock_irqsave(&kbdev->ktrace.lock, irqflags);
/* Reserve and update indices */
trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace);
/* Fill the common part of the message (including backend.flags) */
kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags,
info_val);
/* Indicate to the common code that backend-specific parts will be
* valid
*/
trace_msg->backend.flags |= KBASE_KTRACE_FLAG_BACKEND;
/* Fill the JM-specific parts of the message */
if (katom) {
trace_msg->backend.flags |= KBASE_KTRACE_FLAG_JM_ATOM;
trace_msg->backend.atom_number = kbase_jd_atom_id(katom->kctx, katom);
trace_msg->backend.atom_udata[0] = katom->udata.blob[0];
trace_msg->backend.atom_udata[1] = katom->udata.blob[1];
}
trace_msg->backend.gpu_addr = gpu_addr;
trace_msg->backend.jobslot = jobslot;
/* Clamp refcount */
trace_msg->backend.refcount = MIN((unsigned int)refcount, 0xFF);
/* Done */
spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags);
}
#endif /* KBASE_KTRACE_TARGET_RBUF */

362
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_ktrace_jm.h Normal file
View File

@ -0,0 +1,362 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_DEBUG_KTRACE_JM_H_
#define _KBASE_DEBUG_KTRACE_JM_H_
/*
* KTrace target for internal ringbuffer
*/
#if KBASE_KTRACE_TARGET_RBUF
/**
* kbasep_ktrace_add_jm - internal function to add trace about Job Management
* @kbdev: kbase device
* @code: trace code
* @kctx: kbase context, or NULL if no context
* @katom: kbase atom, or NULL if no atom
* @gpu_addr: GPU address, usually related to @katom
* @flags: flags about the message
* @refcount: reference count information to add to the trace
* @jobslot: jobslot information to add to the trace
* @info_val: generic information about @code to add to the trace
*
* PRIVATE: do not use directly. Use KBASE_KTRACE_ADD_JM() instead.
*/
void kbasep_ktrace_add_jm(struct kbase_device *kbdev,
enum kbase_ktrace_code code, struct kbase_context *kctx,
struct kbase_jd_atom *katom, u64 gpu_addr,
kbase_ktrace_flag_t flags, int refcount, int jobslot,
u64 info_val);
#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
jobslot) \
kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, 0)
#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \
jobslot, info_val) \
kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
gpu_addr, KBASE_KTRACE_FLAG_JM_JOBSLOT, 0, jobslot, \
info_val)
#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \
refcount) \
kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, 0)
#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
gpu_addr, refcount, info_val) \
kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
gpu_addr, KBASE_KTRACE_FLAG_JM_REFCOUNT, refcount, 0, \
info_val)
#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \
kbasep_ktrace_add_jm(kbdev, KBASE_KTRACE_CODE(code), kctx, katom, \
gpu_addr, 0, 0, 0, info_val)
#else /* KBASE_KTRACE_TARGET_RBUF */
#define KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
jobslot) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(jobslot);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \
jobslot, info_val) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(jobslot);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \
refcount) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(refcount);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
gpu_addr, refcount, info_val) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \
info_val)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#endif /* KBASE_KTRACE_TARGET_RBUF */
/*
* KTrace target for Linux's ftrace
*/
#if KBASE_KTRACE_TARGET_FTRACE
#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
jobslot) \
trace_mali_##code(jobslot, 0)
#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \
gpu_addr, jobslot, info_val) \
trace_mali_##code(jobslot, info_val)
#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \
gpu_addr, refcount) \
trace_mali_##code(refcount, 0)
#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
gpu_addr, refcount, info_val) \
trace_mali_##code(refcount, info_val)
#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \
info_val) \
trace_mali_##code(gpu_addr, info_val)
#else /* KBASE_KTRACE_TARGET_FTRACE */
#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
jobslot) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(jobslot);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, \
gpu_addr, jobslot, info_val) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(jobslot);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, \
gpu_addr, refcount) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(refcount);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
gpu_addr, refcount, info_val) \
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#define KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, \
info_val)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(kctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#endif /* KBASE_KTRACE_TARGET_FTRACE */
/*
* Master set of macros to route KTrace to any of the targets
*/
/**
* KBASE_KTRACE_ADD_JM_SLOT - Add trace values about a job-slot
* @kbdev: kbase device
* @code: trace code
* @kctx: kbase context, or NULL if no context
* @katom: kbase atom, or NULL if no atom
* @gpu_addr: GPU address, usually related to @katom
* @jobslot: jobslot information to add to the trace
*
* Note: Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when
* KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
* to this macro must:
* a) be static or static inline, and
* b) just return 0 and have no other statements present in the body.
*/
#define KBASE_KTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, gpu_addr, \
jobslot) \
do { \
/* capture values that could come from non-pure function calls */ \
u64 __gpu_addr = gpu_addr; \
int __jobslot = jobslot; \
KBASE_KTRACE_RBUF_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \
KBASE_KTRACE_FTRACE_ADD_JM_SLOT(kbdev, code, kctx, katom, __gpu_addr, __jobslot); \
} while (0)
/**
* KBASE_KTRACE_ADD_JM_SLOT_INFO - Add trace values about a job-slot, with info
* @kbdev: kbase device
* @code: trace code
* @kctx: kbase context, or NULL if no context
* @katom: kbase atom, or NULL if no atom
* @gpu_addr: GPU address, usually related to @katom
* @jobslot: jobslot information to add to the trace
* @info_val: generic information about @code to add to the trace
*
* Note: Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when
* KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
* to this macro must:
* a) be static or static inline, and
* b) just return 0 and have no other statements present in the body.
*/
#define KBASE_KTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, gpu_addr, \
jobslot, info_val) \
do { \
/* capture values that could come from non-pure function calls */ \
u64 __gpu_addr = gpu_addr; \
int __jobslot = jobslot; \
u64 __info_val = info_val; \
KBASE_KTRACE_RBUF_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \
KBASE_KTRACE_FTRACE_ADD_JM_SLOT_INFO(kbdev, code, kctx, katom, __gpu_addr, __jobslot, __info_val); \
} while (0)
/**
* KBASE_KTRACE_ADD_JM_REFCOUNT - Add trace values about a kctx refcount
* @kbdev: kbase device
* @code: trace code
* @kctx: kbase context, or NULL if no context
* @katom: kbase atom, or NULL if no atom
* @gpu_addr: GPU address, usually related to @katom
* @refcount: reference count information to add to the trace
*
* Note: Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when
* KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
* to this macro must:
* a) be static or static inline, and
* b) just return 0 and have no other statements present in the body.
*/
#define KBASE_KTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, gpu_addr, \
refcount) \
do { \
/* capture values that could come from non-pure function calls */ \
u64 __gpu_addr = gpu_addr; \
int __refcount = refcount; \
KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \
KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount); \
} while (0)
/**
* KBASE_KTRACE_ADD_JM_REFCOUNT_INFO - Add trace values about a kctx refcount,
* and info
* @kbdev: kbase device
* @code: trace code
* @kctx: kbase context, or NULL if no context
* @katom: kbase atom, or NULL if no atom
* @gpu_addr: GPU address, usually related to @katom
* @refcount: reference count information to add to the trace
* @info_val: generic information about @code to add to the trace
*
* Note: Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when
* KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
* to this macro must:
* a) be static or static inline, and
* b) just return 0 and have no other statements present in the body.
*/
#define KBASE_KTRACE_ADD_JM_REFCOUNT_INFO(kbdev, code, kctx, katom, \
gpu_addr, refcount, info_val) \
do { \
/* capture values that could come from non-pure function calls */ \
u64 __gpu_addr = gpu_addr; \
int __refcount = refcount; \
u64 __info_val = info_val; \
KBASE_KTRACE_RBUF_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \
KBASE_KTRACE_FTRACE_ADD_JM_REFCOUNT(kbdev, code, kctx, katom, __gpu_addr, __refcount, __info_val); \
} while (0)
/**
* KBASE_KTRACE_ADD_JM - Add trace values (no slot or refcount)
* @kbdev: kbase device
* @code: trace code
* @kctx: kbase context, or NULL if no context
* @katom: kbase atom, or NULL if no atom
* @gpu_addr: GPU address, usually related to @katom
* @info_val: generic information about @code to add to the trace
*
* Note: Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when
* KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
* to this macro must:
* a) be static or static inline, and
* b) just return 0 and have no other statements present in the body.
*/
#define KBASE_KTRACE_ADD_JM(kbdev, code, kctx, katom, gpu_addr, info_val) \
do { \
/* capture values that could come from non-pure function calls */ \
u64 __gpu_addr = gpu_addr; \
u64 __info_val = info_val; \
KBASE_KTRACE_RBUF_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \
KBASE_KTRACE_FTRACE_ADD_JM(kbdev, code, kctx, katom, __gpu_addr, __info_val); \
} while (0)
#endif /* _KBASE_DEBUG_KTRACE_JM_H_ */

150
drivers/gpu/arm/bifrost/debug/backend/mali_kbase_debug_linux_ktrace_jm.h Normal file
View File

@ -0,0 +1,150 @@
/*
*
* (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* NOTE: This must **only** be included through mali_linux_trace.h,
* otherwise it will fail to setup tracepoints correctly
*/
#if !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)
#define _KBASE_DEBUG_LINUX_KTRACE_JM_H_
DECLARE_EVENT_CLASS(mali_jm_slot_template,
TP_PROTO(int jobslot, u64 info_val),
TP_ARGS(jobslot, info_val),
TP_STRUCT__entry(
__field(unsigned int, jobslot)
__field(u64, info_val)
),
TP_fast_assign(
__entry->jobslot = jobslot;
__entry->info_val = info_val;
),
TP_printk("jobslot=%u info=0x%llx", __entry->jobslot, __entry->info_val)
);
#define DEFINE_MALI_JM_SLOT_EVENT(name) \
DEFINE_EVENT(mali_jm_slot_template, mali_##name, \
TP_PROTO(int jobslot, u64 info_val), \
TP_ARGS(jobslot, info_val))
DEFINE_MALI_JM_SLOT_EVENT(JM_SUBMIT);
DEFINE_MALI_JM_SLOT_EVENT(JM_JOB_DONE);
DEFINE_MALI_JM_SLOT_EVENT(JM_UPDATE_HEAD);
DEFINE_MALI_JM_SLOT_EVENT(JM_CHECK_HEAD);
DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP);
DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_0);
DEFINE_MALI_JM_SLOT_EVENT(JM_SOFTSTOP_1);
DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP);
DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_0);
DEFINE_MALI_JM_SLOT_EVENT(JM_HARDSTOP_1);
DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_SOFT_OR_HARD_STOP);
DEFINE_MALI_JM_SLOT_EVENT(JM_SLOT_EVICT);
DEFINE_MALI_JM_SLOT_EVENT(JM_BEGIN_RESET_WORKER);
DEFINE_MALI_JM_SLOT_EVENT(JM_END_RESET_WORKER);
DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_ON_RECHECK_FAILED);
DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_SUBMIT_TO_BLOCKED);
DEFINE_MALI_JM_SLOT_EVENT(JS_AFFINITY_CURRENT);
DEFINE_MALI_JM_SLOT_EVENT(JD_DONE_TRY_RUN_NEXT_JOB);
DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_CORES_FAILED);
DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REGISTER_INUSE_FAILED);
DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_REQUEST_ON_RECHECK_FAILED);
DEFINE_MALI_JM_SLOT_EVENT(JS_CORE_REF_AFFINITY_WOULD_VIOLATE);
DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_TRY_RUN_NEXT_JOB);
DEFINE_MALI_JM_SLOT_EVENT(JS_JOB_DONE_RETRY_NEEDED);
DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB);
DEFINE_MALI_JM_SLOT_EVENT(JS_POLICY_DEQUEUE_JOB_IRQ);
#undef DEFINE_MALI_JM_SLOT_EVENT
DECLARE_EVENT_CLASS(mali_jm_refcount_template,
TP_PROTO(int refcount, u64 info_val),
TP_ARGS(refcount, info_val),
TP_STRUCT__entry(
__field(unsigned int, refcount)
__field(u64, info_val)
),
TP_fast_assign(
__entry->refcount = refcount;
__entry->info_val = info_val;
),
TP_printk("refcount=%u info=0x%llx", __entry->refcount, __entry->info_val)
);
#define DEFINE_MALI_JM_REFCOUNT_EVENT(name) \
DEFINE_EVENT(mali_jm_refcount_template, mali_##name, \
TP_PROTO(int refcount, u64 info_val), \
TP_ARGS(refcount, info_val))
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_ADD_JOB);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_REMOVE_JOB);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_TRY_SCHEDULE_HEAD_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_INIT_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TERM_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_ENQUEUE_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_DEQUEUE_HEAD_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_TRY_EVICT_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_ADD_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_RUNPOOL_REMOVE_CTX);
DEFINE_MALI_JM_REFCOUNT_EVENT(JS_POLICY_FOREACH_CTX_JOBS);
#undef DEFINE_MALI_JM_REFCOUNT_EVENT
DECLARE_EVENT_CLASS(mali_jm_add_template,
TP_PROTO(u64 gpu_addr, u64 info_val),
TP_ARGS(gpu_addr, info_val),
TP_STRUCT__entry(
__field(u64, gpu_addr)
__field(u64, info_val)
),
TP_fast_assign(
__entry->gpu_addr = gpu_addr;
__entry->info_val = info_val;
),
TP_printk("gpu_addr=0x%llx info=0x%llx", __entry->gpu_addr, __entry->info_val)
);
#define DEFINE_MALI_JM_ADD_EVENT(name) \
DEFINE_EVENT(mali_jm_add_template, mali_##name, \
TP_PROTO(u64 gpu_addr, u64 info_val), \
TP_ARGS(gpu_addr, info_val))
DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER);
DEFINE_MALI_JM_ADD_EVENT(JD_DONE_WORKER_END);
DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL_WORKER);
DEFINE_MALI_JM_ADD_EVENT(JD_DONE);
DEFINE_MALI_JM_ADD_EVENT(JD_CANCEL);
DEFINE_MALI_JM_ADD_EVENT(JD_ZAP_CONTEXT);
DEFINE_MALI_JM_ADD_EVENT(JM_IRQ);
DEFINE_MALI_JM_ADD_EVENT(JM_IRQ_END);
DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS);
DEFINE_MALI_JM_ADD_EVENT(JM_FLUSH_WORKQS_DONE);
DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_NON_SCHEDULED);
DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_SCHEDULED);
DEFINE_MALI_JM_ADD_EVENT(JM_ZAP_DONE);
DEFINE_MALI_JM_ADD_EVENT(JM_SUBMIT_AFTER_RESET);
DEFINE_MALI_JM_ADD_EVENT(JM_JOB_COMPLETE);
DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_RUNPOOL);
DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_RUNPOOL);
DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_ON_CTX);
DEFINE_MALI_JM_ADD_EVENT(JS_CTX_ATTR_NOW_OFF_CTX);
DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_END);
DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_TIMER_START);
DEFINE_MALI_JM_ADD_EVENT(JS_POLICY_ENQUEUE_JOB);
#undef DEFINE_MALI_JM_ADD_EVENT
#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_JM_H_) || defined(TRACE_HEADER_MULTI_READ)*/

342
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.c Normal file
View File

@ -0,0 +1,342 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
#include "debug/mali_kbase_debug_ktrace_internal.h"
int kbase_ktrace_init(struct kbase_device *kbdev)
{
#if KBASE_KTRACE_TARGET_RBUF
struct kbase_ktrace_msg *rbuf;
/* See also documentation of enum kbase_ktrace_code */
compiletime_assert(sizeof(kbase_ktrace_code_t) == sizeof(unsigned long long) ||
KBASE_KTRACE_CODE_COUNT <= (1ull << (sizeof(kbase_ktrace_code_t) * BITS_PER_BYTE)),
"kbase_ktrace_code_t not wide enough for KBASE_KTRACE_CODE_COUNT");
rbuf = kmalloc_array(KBASE_KTRACE_SIZE, sizeof(*rbuf), GFP_KERNEL);
if (!rbuf)
return -EINVAL;
kbdev->ktrace.rbuf = rbuf;
spin_lock_init(&kbdev->ktrace.lock);
#endif /* KBASE_KTRACE_TARGET_RBUF */
return 0;
}
void kbase_ktrace_term(struct kbase_device *kbdev)
{
#if KBASE_KTRACE_TARGET_RBUF
kfree(kbdev->ktrace.rbuf);
#endif /* KBASE_KTRACE_TARGET_RBUF */
}
void kbase_ktrace_hook_wrapper(void *param)
{
struct kbase_device *kbdev = (struct kbase_device *)param;
KBASE_KTRACE_DUMP(kbdev);
}
#if KBASE_KTRACE_TARGET_RBUF
static const char * const kbasep_ktrace_code_string[] = {
/*
* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
* THIS MUST BE USED AT THE START OF THE ARRAY
*/
#define KBASE_KTRACE_CODE_MAKE_CODE(X) # X
#include "debug/mali_kbase_debug_ktrace_codes.h"
#undef KBASE_KTRACE_CODE_MAKE_CODE
};
static void kbasep_ktrace_format_header(char *buffer, int sz, s32 written)
{
written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
"secs,thread_id,cpu,code,kctx,"), 0);
kbasep_ktrace_backend_format_header(buffer, sz, &written);
written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
",info_val,ktrace_version=%u.%u",
KBASE_KTRACE_VERSION_MAJOR,
KBASE_KTRACE_VERSION_MINOR), 0);
buffer[sz - 1] = 0;
}
static void kbasep_ktrace_format_msg(struct kbase_ktrace_msg *trace_msg,
char *buffer, int sz)
{
s32 written = 0;
/* Initial part of message:
*
* secs,thread_id,cpu,code,kctx,
*/
written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
"%d.%.6d,%d,%d,%s,%p,",
(int)trace_msg->timestamp.tv_sec,
(int)(trace_msg->timestamp.tv_nsec / 1000),
trace_msg->thread_id, trace_msg->cpu,
kbasep_ktrace_code_string[trace_msg->backend.code],
trace_msg->kctx), 0);
/* Backend parts */
kbasep_ktrace_backend_format_msg(trace_msg, buffer, sz,
&written);
/* Rest of message:
*
* ,info_val
*
* Note that the last column is empty, it's simply to hold the ktrace
* version in the header
*/
written += MAX(snprintf(buffer + written, MAX(sz - written, 0),
",0x%.16llx",
(unsigned long long)trace_msg->info_val), 0);
buffer[sz - 1] = 0;
}
static void kbasep_ktrace_dump_msg(struct kbase_device *kbdev,
struct kbase_ktrace_msg *trace_msg)
{
char buffer[KTRACE_DUMP_MESSAGE_SIZE];
lockdep_assert_held(&kbdev->ktrace.lock);
kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer));
dev_dbg(kbdev->dev, "%s", buffer);
}
struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace)
{
struct kbase_ktrace_msg *trace_msg;
lockdep_assert_held(&ktrace->lock);
trace_msg = &ktrace->rbuf[ktrace->next_in];
/* Update the ringbuffer indices */
ktrace->next_in = (ktrace->next_in + 1) & KBASE_KTRACE_MASK;
if (ktrace->next_in == ktrace->first_out)
ktrace->first_out = (ktrace->first_out + 1) & KBASE_KTRACE_MASK;
return trace_msg;
}
void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace,
struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code,
struct kbase_context *kctx, kbase_ktrace_flag_t flags,
u64 info_val)
{
lockdep_assert_held(&ktrace->lock);
trace_msg->thread_id = task_pid_nr(current);
trace_msg->cpu = task_cpu(current);
ktime_get_real_ts64(&trace_msg->timestamp);
trace_msg->kctx = kctx;
trace_msg->info_val = info_val;
trace_msg->backend.code = code;
trace_msg->backend.flags = flags;
}
void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code,
struct kbase_context *kctx, kbase_ktrace_flag_t flags,
u64 info_val)
{
unsigned long irqflags;
struct kbase_ktrace_msg *trace_msg;
spin_lock_irqsave(&kbdev->ktrace.lock, irqflags);
/* Reserve and update indices */
trace_msg = kbasep_ktrace_reserve(&kbdev->ktrace);
/* Fill the common part of the message (including backend.flags) */
kbasep_ktrace_msg_init(&kbdev->ktrace, trace_msg, code, kctx, flags,
info_val);
/* Done */
spin_unlock_irqrestore(&kbdev->ktrace.lock, irqflags);
}
static void kbasep_ktrace_clear_locked(struct kbase_device *kbdev)
{
lockdep_assert_held(&kbdev->ktrace.lock);
kbdev->ktrace.first_out = kbdev->ktrace.next_in;
}
void kbasep_ktrace_clear(struct kbase_device *kbdev)
{
unsigned long flags;
spin_lock_irqsave(&kbdev->ktrace.lock, flags);
kbasep_ktrace_clear_locked(kbdev);
spin_unlock_irqrestore(&kbdev->ktrace.lock, flags);
}
void kbasep_ktrace_dump(struct kbase_device *kbdev)
{
unsigned long flags;
u32 start;
u32 end;
char buffer[KTRACE_DUMP_MESSAGE_SIZE] = "Dumping trace:\n";
kbasep_ktrace_format_header(buffer, sizeof(buffer), strlen(buffer));
dev_dbg(kbdev->dev, "%s", buffer);
spin_lock_irqsave(&kbdev->ktrace.lock, flags);
start = kbdev->ktrace.first_out;
end = kbdev->ktrace.next_in;
while (start != end) {
struct kbase_ktrace_msg *trace_msg = &kbdev->ktrace.rbuf[start];
kbasep_ktrace_dump_msg(kbdev, trace_msg);
start = (start + 1) & KBASE_KTRACE_MASK;
}
dev_dbg(kbdev->dev, "TRACE_END");
kbasep_ktrace_clear_locked(kbdev);
spin_unlock_irqrestore(&kbdev->ktrace.lock, flags);
}
#ifdef CONFIG_DEBUG_FS
struct trace_seq_state {
struct kbase_ktrace_msg trace_buf[KBASE_KTRACE_SIZE];
u32 start;
u32 end;
};
static void *kbasep_ktrace_seq_start(struct seq_file *s, loff_t *pos)
{
struct trace_seq_state *state = s->private;
int i;
if (*pos == 0)
/* See Documentation/filesystems/seq_file.txt */
return SEQ_START_TOKEN;
if (*pos > KBASE_KTRACE_SIZE)
return NULL;
i = state->start + *pos;
if ((state->end >= state->start && i >= state->end) ||
i >= state->end + KBASE_KTRACE_SIZE)
return NULL;
i &= KBASE_KTRACE_MASK;
return &state->trace_buf[i];
}
static void kbasep_ktrace_seq_stop(struct seq_file *s, void *data)
{
}
static void *kbasep_ktrace_seq_next(struct seq_file *s, void *data, loff_t *pos)
{
struct trace_seq_state *state = s->private;
int i;
if (data != SEQ_START_TOKEN)
(*pos)++;
i = (state->start + *pos) & KBASE_KTRACE_MASK;
if (i == state->end)
return NULL;
return &state->trace_buf[i];
}
static int kbasep_ktrace_seq_show(struct seq_file *s, void *data)
{
struct kbase_ktrace_msg *trace_msg = data;
char buffer[KTRACE_DUMP_MESSAGE_SIZE];
/* If this is the start, print a header */
if (data == SEQ_START_TOKEN)
kbasep_ktrace_format_header(buffer, sizeof(buffer), 0);
else
kbasep_ktrace_format_msg(trace_msg, buffer, sizeof(buffer));
seq_printf(s, "%s\n", buffer);
return 0;
}
static const struct seq_operations kbasep_ktrace_seq_ops = {
.start = kbasep_ktrace_seq_start,
.next = kbasep_ktrace_seq_next,
.stop = kbasep_ktrace_seq_stop,
.show = kbasep_ktrace_seq_show,
};
static int kbasep_ktrace_debugfs_open(struct inode *inode, struct file *file)
{
struct kbase_device *kbdev = inode->i_private;
unsigned long flags;
struct trace_seq_state *state;
state = __seq_open_private(file, &kbasep_ktrace_seq_ops,
sizeof(*state));
if (!state)
return -ENOMEM;
spin_lock_irqsave(&kbdev->ktrace.lock, flags);
state->start = kbdev->ktrace.first_out;
state->end = kbdev->ktrace.next_in;
memcpy(state->trace_buf, kbdev->ktrace.rbuf, sizeof(state->trace_buf));
spin_unlock_irqrestore(&kbdev->ktrace.lock, flags);
return 0;
}
static const struct file_operations kbasep_ktrace_debugfs_fops = {
.owner = THIS_MODULE,
.open = kbasep_ktrace_debugfs_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release_private,
};
void kbase_ktrace_debugfs_init(struct kbase_device *kbdev)
{
debugfs_create_file("mali_trace", 0444,
kbdev->mali_debugfs_directory, kbdev,
&kbasep_ktrace_debugfs_fops);
}
#endif /* CONFIG_DEBUG_FS */
#else /* KBASE_KTRACE_TARGET_RBUF */
#ifdef CONFIG_DEBUG_FS
void kbase_ktrace_debugfs_init(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
}
#endif /* CONFIG_DEBUG_FS */
#endif /* KBASE_KTRACE_TARGET_RBUF */

219
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace.h Normal file
View File

@ -0,0 +1,219 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* DOC: Kbase's own trace, 'KTrace'
*
* Low overhead trace specific to kbase, aimed at:
* - common use-cases for tracing kbase specific functionality to do with
* running work on the GPU
* - easy 1-line addition of new types of trace
*
* KTrace can be recorded in one or more of the following targets:
* - KBASE_KTRACE_TARGET_RBUF: low overhead ringbuffer protected by an
* irq-spinlock, output available via dev_dbg() and debugfs file
* - KBASE_KTRACE_TARGET_FTRACE: ftrace based tracepoints under 'mali' events
*/
#ifndef _KBASE_DEBUG_KTRACE_H_
#define _KBASE_DEBUG_KTRACE_H_
#include "debug/backend/mali_kbase_debug_ktrace_jm.h"
/**
* kbase_ktrace_init - initialize kbase ktrace.
* @kbdev: kbase device
*/
int kbase_ktrace_init(struct kbase_device *kbdev);
/**
* kbase_ktrace_term - terminate kbase ktrace.
* @kbdev: kbase device
*/
void kbase_ktrace_term(struct kbase_device *kbdev);
/**
* kbase_ktrace_hook_wrapper - wrapper so that dumping ktrace can be done via a
* callback.
* @param: kbase device, cast to void pointer
*/
void kbase_ktrace_hook_wrapper(void *param);
#ifdef CONFIG_DEBUG_FS
/**
* kbase_ktrace_debugfs_init - initialize kbase ktrace for debugfs usage, if
* the selected targets support it.
* @kbdev: kbase device
*
* There is no matching 'term' call, debugfs_remove_recursive() is sufficient.
*/
void kbase_ktrace_debugfs_init(struct kbase_device *kbdev);
#endif /* CONFIG_DEBUG_FS */
/*
* KTrace target for internal ringbuffer
*/
#if KBASE_KTRACE_TARGET_RBUF
/**
* kbasep_ktrace_add - internal function to add trace to the ringbuffer.
* @kbdev: kbase device
* @code: ktrace code
* @kctx: kbase context, or NULL if no context
* @flags: flags about the message
* @info_val: generic information about @code to add to the trace
*
* PRIVATE: do not use directly. Use KBASE_KTRACE_ADD() instead.
*/
void kbasep_ktrace_add(struct kbase_device *kbdev, enum kbase_ktrace_code code,
struct kbase_context *kctx, kbase_ktrace_flag_t flags,
u64 info_val);
/**
* kbasep_ktrace_clear - clear the trace ringbuffer
* @kbdev: kbase device
*
* PRIVATE: do not use directly. Use KBASE_KTRACE_CLEAR() instead.
*/
void kbasep_ktrace_clear(struct kbase_device *kbdev);
/**
* kbasep_ktrace_dump - dump ktrace ringbuffer to dev_dbg(), then clear it
* @kbdev: kbase device
*
* PRIVATE: do not use directly. Use KBASE_KTRACE_DUMP() instead.
*/
void kbasep_ktrace_dump(struct kbase_device *kbdev);
#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \
kbasep_ktrace_add(kbdev, KBASE_KTRACE_CODE(code), kctx, 0, \
info_val) \
#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \
kbasep_ktrace_clear(kbdev)
#define KBASE_KTRACE_RBUF_DUMP(kbdev) \
kbasep_ktrace_dump(kbdev)
#else /* KBASE_KTRACE_TARGET_RBUF */
#define KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, info_val) \
do { \
CSTD_UNUSED(kbdev); \
CSTD_NOP(code); \
CSTD_UNUSED(kctx); \
CSTD_UNUSED(info_val); \
CSTD_NOP(0); \
} while (0)
#define KBASE_KTRACE_RBUF_CLEAR(kbdev) \
do { \
CSTD_UNUSED(kbdev); \
CSTD_NOP(0); \
} while (0)
#define KBASE_KTRACE_RBUF_DUMP(kbdev) \
do { \
CSTD_UNUSED(kbdev); \
CSTD_NOP(0); \
} while (0)
#endif /* KBASE_KTRACE_TARGET_RBUF */
/*
* KTrace target for Linux's ftrace
*/
#if KBASE_KTRACE_TARGET_FTRACE
#include "mali_linux_trace.h"
#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \
trace_mali_##code(info_val)
#else /* KBASE_KTRACE_TARGET_FTRACE */
#define KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, info_val) \
do { \
CSTD_UNUSED(kbdev); \
CSTD_NOP(code); \
CSTD_UNUSED(kctx); \
CSTD_UNUSED(info_val); \
CSTD_NOP(0); \
} while (0)
#endif /* KBASE_KTRACE_TARGET_FTRACE */
/* No 'clear' implementation for ftrace yet */
#define KBASE_KTRACE_FTRACE_CLEAR(kbdev) \
do { \
CSTD_UNUSED(kbdev); \
CSTD_NOP(0); \
} while (0)
/* No 'dump' implementation for ftrace yet */
#define KBASE_KTRACE_FTRACE_DUMP(kbdev) \
do { \
CSTD_UNUSED(kbdev); \
CSTD_NOP(0); \
} while (0)
/*
* Master set of macros to route KTrace to any of the targets
*/
/**
* KBASE_KTRACE_ADD - Add trace values
* @kbdev: kbase device
* @code: trace code
* @kctx: kbase context, or NULL if no context
* @info_val: generic information about @code to add to the trace
*
* Note: Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when
* KBASE_KTRACE_ENABLE == 0 any functions called to get the parameters supplied
* to this macro must:
* a) be static or static inline, and
* b) just return 0 and have no other statements present in the body.
*/
#define KBASE_KTRACE_ADD(kbdev, code, kctx, info_val) \
do { \
/* capture values that could come from non-pure function calls */ \
u64 __info_val = info_val; \
KBASE_KTRACE_RBUF_ADD(kbdev, code, kctx, __info_val); \
KBASE_KTRACE_FTRACE_ADD(kbdev, code, kctx, __info_val); \
} while (0)
/**
* KBASE_KTRACE_CLEAR - Clear the trace, if applicable to the target(s)
* @kbdev: kbase device
*/
#define KBASE_KTRACE_CLEAR(kbdev) \
do { \
KBASE_KTRACE_RBUF_CLEAR(kbdev); \
KBASE_KTRACE_FTRACE_CLEAR(kbdev); \
} while (0)
/**
* KBASE_KTRACE_DUMP - Dump the trace, if applicable to the target(s)
* @kbdev: kbase device
*/
#define KBASE_KTRACE_DUMP(kbdev) \
do { \
KBASE_KTRACE_RBUF_DUMP(kbdev); \
KBASE_KTRACE_FTRACE_DUMP(kbdev); \
} while (0)
#endif /* _KBASE_DEBUG_KTRACE_H_ */

158
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_codes.h Normal file
View File

@ -0,0 +1,158 @@
/*
*
* (C) COPYRIGHT 2011-2015,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* ***** IMPORTANT: THIS IS NOT A NORMAL HEADER FILE *****
* ***** DO NOT INCLUDE DIRECTLY *****
* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL *****
*/
/*
* The purpose of this header file is just to contain a list of trace code
* identifiers
*
* Each identifier is wrapped in a macro, so that its string form and enum form
* can be created
*
* Each macro is separated with a comma, to allow insertion into an array
* initializer or enum definition block.
*
* This allows automatic creation of an enum and a corresponding array of
* strings
*
* Before #including, the includer MUST #define KBASE_KTRACE_CODE_MAKE_CODE.
* After #including, the includer MUST #under KBASE_KTRACE_CODE_MAKE_CODE.
*
* e.g.:
* #define KBASE_KTRACE_CODE( X ) KBASE_KTRACE_CODE_ ## X
* typedef enum
* {
* #define KBASE_KTRACE_CODE_MAKE_CODE( X ) KBASE_KTRACE_CODE( X )
* #include "mali_kbase_debug_ktrace_codes.h"
* #undef KBASE_KTRACE_CODE_MAKE_CODE
* } kbase_ktrace_code;
*
* IMPORTANT: THIS FILE MUST NOT BE USED FOR ANY OTHER PURPOSE OTHER THAN THE ABOVE
*
*
* The use of the macro here is:
* - KBASE_KTRACE_CODE_MAKE_CODE( X )
*
* Which produces:
* - For an enum, KBASE_KTRACE_CODE_X
* - For a string, "X"
*
*
* For example:
* - KBASE_KTRACE_CODE_MAKE_CODE( JM_JOB_COMPLETE ) expands to:
* - KBASE_KTRACE_CODE_JM_JOB_COMPLETE for the enum
* - "JM_JOB_COMPLETE" for the string
* - To use it to trace an event, do:
* - KBASE_KTRACE_ADD( kbdev, JM_JOB_COMPLETE, subcode, kctx, uatom, val );
*/
#if 0 /* Dummy section to avoid breaking formatting */
int dummy_array[] = {
#endif
/*
* Core events
*/
/* no info_val */
KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_DESTROY),
/* no info_val */
KBASE_KTRACE_CODE_MAKE_CODE(CORE_CTX_HWINSTR_TERM),
/* info_val == GPU_IRQ_STATUS register */
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ),
/* info_val == bits cleared */
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_CLEAR),
/* info_val == GPU_IRQ_STATUS register */
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_IRQ_DONE),
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_SOFT_RESET),
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_HARD_RESET),
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_CLEAR),
/* info_val == dump address */
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_PRFCNT_SAMPLE),
KBASE_KTRACE_CODE_MAKE_CODE(CORE_GPU_CLEAN_INV_CACHES),
/*
* Power Management Events
*/
KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERING_UP),
KBASE_KTRACE_CODE_MAKE_CODE(PM_JOB_SUBMIT_AFTER_POWERED_UP),
KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON),
KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_TILER),
KBASE_KTRACE_CODE_MAKE_CODE(PM_PWRON_L2),
KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF),
KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_TILER),
KBASE_KTRACE_CODE_MAKE_CODE(PM_PWROFF_L2),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_TILER),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_POWERED_L2),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_DESIRED_TILER),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_CHANGE_AVAILABLE_TILER),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CORES_AVAILABLE_TILER),
KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED),
KBASE_KTRACE_CODE_MAKE_CODE(PM_DESIRED_REACHED_TILER),
KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_SHADER_NEEDED),
KBASE_KTRACE_CODE_MAKE_CODE(PM_RELEASE_CHANGE_TILER_NEEDED),
KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_SHADER_NEEDED),
KBASE_KTRACE_CODE_MAKE_CODE(PM_REQUEST_CHANGE_TILER_NEEDED),
KBASE_KTRACE_CODE_MAKE_CODE(PM_WAKE_WAITERS),
/* info_val == kbdev->pm.active_count*/
KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_ACTIVE),
/* info_val == kbdev->pm.active_count*/
KBASE_KTRACE_CODE_MAKE_CODE(PM_CONTEXT_IDLE),
KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_ON),
KBASE_KTRACE_CODE_MAKE_CODE(PM_GPU_OFF),
/* info_val == policy number, or -1 for "Already changing" */
KBASE_KTRACE_CODE_MAKE_CODE(PM_SET_POLICY),
KBASE_KTRACE_CODE_MAKE_CODE(PM_CA_SET_POLICY),
/* info_val == policy number */
KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_INIT),
/* info_val == policy number */
KBASE_KTRACE_CODE_MAKE_CODE(PM_CURRENT_POLICY_TERM),
/*
* Context Scheduler events
*/
/* info_val == kctx->refcount */
KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RETAIN_CTX_NOLOCK),
/* info_val == kctx->refcount */
KBASE_KTRACE_CODE_MAKE_CODE(SCHED_RELEASE_CTX),
#include "debug/backend/mali_kbase_debug_ktrace_codes_jm.h"
/*
* Unused code just to make it easier to not have a comma at the end.
* All other codes MUST come before this
*/
KBASE_KTRACE_CODE_MAKE_CODE(DUMMY)
#if 0 /* Dummy section to avoid breaking formatting */
};
#endif
/* ***** THE LACK OF HEADER GUARDS IS INTENTIONAL ***** */

152
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_defs.h Normal file
View File

@ -0,0 +1,152 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_DEBUG_KTRACE_DEFS_H_
#define _KBASE_DEBUG_KTRACE_DEFS_H_
/* Enable SW tracing when set */
#if defined(CONFIG_MALI_BIFROST_ENABLE_TRACE) || defined(CONFIG_MALI_BIFROST_SYSTEM_TRACE)
#define KBASE_KTRACE_ENABLE 1
#endif
#ifndef KBASE_KTRACE_ENABLE
#ifdef CONFIG_MALI_BIFROST_DEBUG
#define KBASE_KTRACE_ENABLE 1
#else /* CONFIG_MALI_BIFROST_DEBUG */
#define KBASE_KTRACE_ENABLE 0
#endif /* CONFIG_MALI_BIFROST_DEBUG */
#endif /* KBASE_KTRACE_ENABLE */
/* Select targets for recording of trace:
*
*/
#if KBASE_KTRACE_ENABLE
#ifdef CONFIG_MALI_BIFROST_SYSTEM_TRACE
#define KBASE_KTRACE_TARGET_FTRACE 1
#else /* CONFIG_MALI_BIFROST_SYSTEM_TRACE */
#define KBASE_KTRACE_TARGET_FTRACE 0
#endif /* CONFIG_MALI_BIFROST_SYSTEM_TRACE */
#ifdef CONFIG_MALI_BIFROST_ENABLE_TRACE
#define KBASE_KTRACE_TARGET_RBUF 1
#else /* CONFIG_MALI_BIFROST_ENABLE_TRACE*/
#define KBASE_KTRACE_TARGET_RBUF 0
#endif /* CONFIG_MALI_BIFROST_ENABLE_TRACE */
#else /* KBASE_KTRACE_ENABLE */
#define KBASE_KTRACE_TARGET_FTRACE 0
#define KBASE_KTRACE_TARGET_RBUF 0
#endif /* KBASE_KTRACE_ENABLE */
/*
* NOTE: KBASE_KTRACE_VERSION_MAJOR, KBASE_KTRACE_VERSION_MINOR are kept in
* the backend, since updates can be made to one backend in a way that doesn't
* affect the other.
*
* However, modifying the common part could require both backend versions to be
* updated.
*/
#if KBASE_KTRACE_TARGET_RBUF
typedef u8 kbase_ktrace_flag_t;
typedef u8 kbase_ktrace_code_t;
/*
* struct kbase_ktrace_backend - backend specific part of a trace message
*
* At the very least, this must contain a kbase_ktrace_code_t 'code' member and
* a kbase_ktrace_flag_t 'flags' member
*/
struct kbase_ktrace_backend;
#include "debug/backend/mali_kbase_debug_ktrace_defs_jm.h"
/* Indicates if the trace message has backend related info.
*
* If not set, consider the &kbase_ktrace_backend part of a &kbase_ktrace_msg
* as uninitialized, apart from the mandatory parts:
* - code
* - flags
*/
#define KBASE_KTRACE_FLAG_BACKEND (((kbase_ktrace_flag_t)1) << 7)
#define KBASE_KTRACE_SHIFT 8 /* 256 entries */
#define KBASE_KTRACE_SIZE (1 << KBASE_KTRACE_SHIFT)
#define KBASE_KTRACE_MASK ((1 << KBASE_KTRACE_SHIFT)-1)
#define KBASE_KTRACE_CODE(X) KBASE_KTRACE_CODE_ ## X
/* Note: compiletime_assert() about this against kbase_ktrace_code_t is in
* kbase_ktrace_init()
*/
enum kbase_ktrace_code {
/*
* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
* THIS MUST BE USED AT THE START OF THE ENUM
*/
#define KBASE_KTRACE_CODE_MAKE_CODE(X) KBASE_KTRACE_CODE(X)
#include <debug/mali_kbase_debug_ktrace_codes.h>
#undef KBASE_KTRACE_CODE_MAKE_CODE
/* Comma on its own, to extend the list */
,
/* Must be the last in the enum */
KBASE_KTRACE_CODE_COUNT
};
/**
* struct kbase_ktrace - object representing a trace message added to trace
* buffer trace_rbuf in &kbase_device
* @timestamp: CPU timestamp at which the trace message was added.
* @thread_id: id of the thread in the context of which trace message was
* added.
* @cpu: indicates which CPU the @thread_id was scheduled on when the
* trace message was added.
* @kctx: Pointer to the kbase context for which the trace message was
* added. Will be NULL for certain trace messages associated with
* the &kbase_device itself, such as power management events.
* Will point to the appropriate context corresponding to
* backend-specific events.
* @info_val: value specific to the type of event being traced. Refer to the
* specific code in enum kbase_ktrace_code
* @backend: backend-specific trace information. All backends must implement
* a minimum common set of members
*/
struct kbase_ktrace_msg {
struct timespec64 timestamp;
u32 thread_id;
u32 cpu;
void *kctx;
u64 info_val;
struct kbase_ktrace_backend backend;
};
struct kbase_ktrace {
spinlock_t lock;
u16 first_out;
u16 next_in;
struct kbase_ktrace_msg *rbuf;
};
#endif /* KBASE_KTRACE_TARGET_RBUF */
#endif /* _KBASE_DEBUG_KTRACE_DEFS_H_ */

89
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_ktrace_internal.h Normal file
View File

@ -0,0 +1,89 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_DEBUG_KTRACE_INTERNAL_H_
#define _KBASE_DEBUG_KTRACE_INTERNAL_H_
#if KBASE_KTRACE_TARGET_RBUF
#define KTRACE_DUMP_MESSAGE_SIZE 256
/**
* kbasep_ktrace_backend_format_header - format the backend part of the header
* @buffer: buffer to write to
* @sz: size of @buffer in bytes
* @written: pointer to storage for updating bytes written so far to @buffer
*
* The backend must format only the non-common backend specific parts of the
* header. It must format them as though they were standalone. The caller will
* handle adding any delimiters around this.
*/
void kbasep_ktrace_backend_format_header(char *buffer, int sz, s32 *written);
/**
* kbasep_ktrace_backend_format_msg - format the backend part of the message
* @trace_msg: ktrace message
* @buffer: buffer to write to
* @sz: size of @buffer in bytes
* @written: pointer to storage for updating bytes written so far to @buffer
*
* The backend must format only the non-common backend specific parts of the
* message. It must format them as though they were standalone. The caller will
* handle adding any delimiters around this.
*
* A caller may have the flags member of @trace_msg with
* %KBASE_KTRACE_FLAG_BACKEND clear. The backend must handle that setting
* appropriately.
*/
void kbasep_ktrace_backend_format_msg(struct kbase_ktrace_msg *trace_msg,
char *buffer, int sz, s32 *written);
/**
* kbasep_ktrace_reserve - internal function to reserve space for a ktrace
* message
* @ktrace: kbase device's ktrace
*
* This may also empty the oldest entry in the ringbuffer to make space.
*/
struct kbase_ktrace_msg *kbasep_ktrace_reserve(struct kbase_ktrace *ktrace);
/**
* kbasep_ktrace_msg_init - internal function to initialize just the common
* part of a ktrace message
* @ktrace: kbase device's ktrace
* @trace_msg: ktrace message to initialize
* @code: ktrace code
* @kctx: kbase context, or NULL if no context
* @flags: flags about the message
* @info_val: generic information about @code to add to the trace
*
* The common part includes the mandatory parts of the backend part
*/
void kbasep_ktrace_msg_init(struct kbase_ktrace *ktrace,
struct kbase_ktrace_msg *trace_msg, enum kbase_ktrace_code code,
struct kbase_context *kctx, kbase_ktrace_flag_t flags,
u64 info_val);
#endif /* KBASE_KTRACE_TARGET_RBUF */
#endif /* _KBASE_DEBUG_KTRACE_INTERNAL_H_ */

99
drivers/gpu/arm/bifrost/debug/mali_kbase_debug_linux_ktrace.h Normal file
View File

@ -0,0 +1,99 @@
/*
*
* (C) COPYRIGHT 2014,2018,2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* NOTE: This must **only** be included through mali_linux_trace.h,
* otherwise it will fail to setup tracepoints correctly
*/
#if !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ)
#define _KBASE_DEBUG_LINUX_KTRACE_H_
#if KBASE_KTRACE_TARGET_FTRACE
DECLARE_EVENT_CLASS(mali_add_template,
TP_PROTO(u64 info_val),
TP_ARGS(info_val),
TP_STRUCT__entry(
__field(u64, info_val)
),
TP_fast_assign(
__entry->info_val = info_val;
),
TP_printk("info=0x%llx", __entry->info_val)
);
#define DEFINE_MALI_ADD_EVENT(name) \
DEFINE_EVENT(mali_add_template, mali_##name, \
TP_PROTO(u64 info_val), \
TP_ARGS(info_val))
DEFINE_MALI_ADD_EVENT(CORE_CTX_DESTROY);
DEFINE_MALI_ADD_EVENT(CORE_CTX_HWINSTR_TERM);
DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ);
DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_CLEAR);
DEFINE_MALI_ADD_EVENT(CORE_GPU_IRQ_DONE);
DEFINE_MALI_ADD_EVENT(CORE_GPU_SOFT_RESET);
DEFINE_MALI_ADD_EVENT(CORE_GPU_HARD_RESET);
DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_SAMPLE);
DEFINE_MALI_ADD_EVENT(CORE_GPU_PRFCNT_CLEAR);
DEFINE_MALI_ADD_EVENT(CORE_GPU_CLEAN_INV_CACHES);
DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_DESIRED);
DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERING_UP);
DEFINE_MALI_ADD_EVENT(PM_JOB_SUBMIT_AFTER_POWERED_UP);
DEFINE_MALI_ADD_EVENT(PM_PWRON);
DEFINE_MALI_ADD_EVENT(PM_PWRON_TILER);
DEFINE_MALI_ADD_EVENT(PM_PWRON_L2);
DEFINE_MALI_ADD_EVENT(PM_PWROFF);
DEFINE_MALI_ADD_EVENT(PM_PWROFF_TILER);
DEFINE_MALI_ADD_EVENT(PM_PWROFF_L2);
DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED);
DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_TILER);
DEFINE_MALI_ADD_EVENT(PM_CORES_POWERED_L2);
DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED);
DEFINE_MALI_ADD_EVENT(PM_DESIRED_REACHED_TILER);
DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_SHADER_NEEDED);
DEFINE_MALI_ADD_EVENT(PM_REQUEST_CHANGE_TILER_NEEDED);
DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_SHADER_NEEDED);
DEFINE_MALI_ADD_EVENT(PM_RELEASE_CHANGE_TILER_NEEDED);
DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE);
DEFINE_MALI_ADD_EVENT(PM_CORES_AVAILABLE_TILER);
DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE);
DEFINE_MALI_ADD_EVENT(PM_CORES_CHANGE_AVAILABLE_TILER);
DEFINE_MALI_ADD_EVENT(PM_GPU_ON);
DEFINE_MALI_ADD_EVENT(PM_GPU_OFF);
DEFINE_MALI_ADD_EVENT(PM_SET_POLICY);
DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_INIT);
DEFINE_MALI_ADD_EVENT(PM_CURRENT_POLICY_TERM);
DEFINE_MALI_ADD_EVENT(PM_CA_SET_POLICY);
DEFINE_MALI_ADD_EVENT(PM_CONTEXT_ACTIVE);
DEFINE_MALI_ADD_EVENT(PM_CONTEXT_IDLE);
DEFINE_MALI_ADD_EVENT(PM_WAKE_WAITERS);
DEFINE_MALI_ADD_EVENT(SCHED_RETAIN_CTX_NOLOCK);
DEFINE_MALI_ADD_EVENT(SCHED_RELEASE_CTX);
#undef DEFINE_MALI_ADD_EVENT
#include "mali_kbase_debug_linux_ktrace_jm.h"
#endif /* KBASE_KTRACE_TARGET_FTRACE */
#endif /* !defined(_KBASE_DEBUG_LINUX_KTRACE_H_) || defined(TRACE_HEADER_MULTI_READ) */

260
drivers/gpu/arm/bifrost/device/backend/mali_kbase_device_jm.c Normal file
View File

@ -0,0 +1,260 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include "../mali_kbase_device_internal.h"
#include "../mali_kbase_device.h"
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_hwaccess_backend.h>
#include <mali_kbase_ctx_sched.h>
#include <mali_kbase_reset_gpu.h>
#ifdef CONFIG_MALI_BIFROST_NO_MALI
#include <mali_kbase_model_linux.h>
#endif
#ifdef CONFIG_MALI_ARBITER_SUPPORT
#include <arbiter/mali_kbase_arbiter_pm.h>
#endif
#include <mali_kbase.h>
#include <backend/gpu/mali_kbase_irq_internal.h>
#include <backend/gpu/mali_kbase_jm_internal.h>
#include <backend/gpu/mali_kbase_js_internal.h>
#include <backend/gpu/mali_kbase_pm_internal.h>
#include <mali_kbase_dummy_job_wa.h>
/**
* kbase_backend_late_init - Perform any backend-specific initialization.
* @kbdev: Device pointer
*
* Return: 0 on success, or an error code on failure.
*/
static int kbase_backend_late_init(struct kbase_device *kbdev)
{
int err;
err = kbase_hwaccess_pm_init(kbdev);
if (err)
return err;
err = kbase_reset_gpu_init(kbdev);
if (err)
goto fail_reset_gpu_init;
err = kbase_hwaccess_pm_powerup(kbdev, PM_HW_ISSUES_DETECT);
if (err)
goto fail_pm_powerup;
err = kbase_backend_timer_init(kbdev);
if (err)
goto fail_timer;
#ifdef CONFIG_MALI_BIFROST_DEBUG
#ifndef CONFIG_MALI_BIFROST_NO_MALI
if (kbasep_common_test_interrupt_handlers(kbdev) != 0) {
dev_err(kbdev->dev, "Interrupt assignment check failed.\n");
err = -EINVAL;
goto fail_interrupt_test;
}
#endif /* !CONFIG_MALI_BIFROST_NO_MALI */
#endif /* CONFIG_MALI_BIFROST_DEBUG */
err = kbase_job_slot_init(kbdev);
if (err)
goto fail_job_slot;
/* Do the initialisation of devfreq.
* Devfreq needs backend_timer_init() for completion of its
* initialisation and it also needs to catch the first callback
* occurrence of the runtime_suspend event for maintaining state
* coherence with the backend power management, hence needs to be
* placed before the kbase_pm_context_idle().
*/
err = kbase_backend_devfreq_init(kbdev);
if (err)
goto fail_devfreq_init;
/* Idle the GPU and/or cores, if the policy wants it to */
kbase_pm_context_idle(kbdev);
/* Update gpuprops with L2_FEATURES if applicable */
err = kbase_gpuprops_update_l2_features(kbdev);
if (err)
goto fail_update_l2_features;
init_waitqueue_head(&kbdev->hwaccess.backend.reset_wait);
return 0;
fail_update_l2_features:
fail_devfreq_init:
kbase_job_slot_term(kbdev);
fail_job_slot:
#ifdef CONFIG_MALI_BIFROST_DEBUG
#ifndef CONFIG_MALI_BIFROST_NO_MALI
fail_interrupt_test:
#endif /* !CONFIG_MALI_BIFROST_NO_MALI */
#endif /* CONFIG_MALI_BIFROST_DEBUG */
kbase_backend_timer_term(kbdev);
fail_timer:
kbase_hwaccess_pm_halt(kbdev);
fail_pm_powerup:
kbase_reset_gpu_term(kbdev);
fail_reset_gpu_init:
kbase_hwaccess_pm_term(kbdev);
return err;
}
/**
* kbase_backend_late_term - Perform any backend-specific termination.
* @kbdev: Device pointer
*/
static void kbase_backend_late_term(struct kbase_device *kbdev)
{
kbase_backend_devfreq_term(kbdev);
kbase_job_slot_halt(kbdev);
kbase_job_slot_term(kbdev);
kbase_backend_timer_term(kbdev);
kbase_hwaccess_pm_halt(kbdev);
kbase_reset_gpu_term(kbdev);
kbase_hwaccess_pm_term(kbdev);
}
static const struct kbase_device_init dev_init[] = {
#ifdef CONFIG_MALI_BIFROST_NO_MALI
{kbase_gpu_device_create, kbase_gpu_device_destroy,
"Dummy model initialization failed"},
#else
{assign_irqs, NULL,
"IRQ search failed"},
{registers_map, registers_unmap,
"Register map failed"},
#endif
{kbase_device_io_history_init, kbase_device_io_history_term,
"Register access history initialization failed"},
{kbase_device_pm_init, kbase_device_pm_term,
"Power management initialization failed"},
{kbase_device_early_init, kbase_device_early_term,
"Early device initialization failed"},
{kbase_device_populate_max_freq, NULL,
"Populating max frequency failed"},
{kbase_device_misc_init, kbase_device_misc_term,
"Miscellaneous device initialization failed"},
{kbase_ctx_sched_init, kbase_ctx_sched_term,
"Context scheduler initialization failed"},
{kbase_mem_init, kbase_mem_term,
"Memory subsystem initialization failed"},
{kbase_device_coherency_init, NULL,
"Device coherency init failed"},
{kbase_protected_mode_init, kbase_protected_mode_term,
"Protected mode subsystem initialization failed"},
{kbase_device_list_init, kbase_device_list_term,
"Device list setup failed"},
{kbasep_js_devdata_init, kbasep_js_devdata_term,
"Job JS devdata initialization failed"},
{kbase_device_timeline_init, kbase_device_timeline_term,
"Timeline stream initialization failed"},
{kbase_device_hwcnt_backend_gpu_init,
kbase_device_hwcnt_backend_gpu_term,
"GPU hwcnt backend creation failed"},
{kbase_device_hwcnt_context_init, kbase_device_hwcnt_context_term,
"GPU hwcnt context initialization failed"},
{kbase_device_hwcnt_virtualizer_init,
kbase_device_hwcnt_virtualizer_term,
"GPU hwcnt virtualizer initialization failed"},
{kbase_device_vinstr_init, kbase_device_vinstr_term,
"Virtual instrumentation initialization failed"},
{kbase_backend_late_init, kbase_backend_late_term,
"Late backend initialization failed"},
#ifdef MALI_KBASE_BUILD
{kbase_debug_job_fault_dev_init, kbase_debug_job_fault_dev_term,
"Job fault debug initialization failed"},
{kbase_device_debugfs_init, kbase_device_debugfs_term,
"DebugFS initialization failed"},
/* Sysfs init needs to happen before registering the device with
* misc_register(), otherwise it causes a race condition between
* registering the device and a uevent event being generated for
* userspace, causing udev rules to run which might expect certain
* sysfs attributes present. As a result of the race condition
* we avoid, some Mali sysfs entries may have appeared to udev
* to not exist.
* For more information, see
* https://www.kernel.org/doc/Documentation/driver-model/device.txt, the
* paragraph that starts with "Word of warning", currently the
* second-last paragraph.
*/
{kbase_sysfs_init, kbase_sysfs_term, "SysFS group creation failed"},
{kbase_device_misc_register, kbase_device_misc_deregister,
"Misc device registration failed"},
#ifdef CONFIG_MALI_BUSLOG
{buslog_init, buslog_term, "Bus log client registration failed"},
#endif
{kbase_gpuprops_populate_user_buffer, kbase_gpuprops_free_user_buffer,
"GPU property population failed"},
#endif
{kbase_dummy_job_wa_load, kbase_dummy_job_wa_cleanup,
"Dummy job workaround load failed"},
};
static void kbase_device_term_partial(struct kbase_device *kbdev,
unsigned int i)
{
while (i-- > 0) {
if (dev_init[i].term)
dev_init[i].term(kbdev);
}
}
void kbase_device_term(struct kbase_device *kbdev)
{
kbase_device_term_partial(kbdev, ARRAY_SIZE(dev_init));
kbasep_js_devdata_halt(kbdev);
kbase_mem_halt(kbdev);
}
int kbase_device_init(struct kbase_device *kbdev)
{
int err = 0;
unsigned int i = 0;
dev_info(kbdev->dev, "Kernel DDK version %s", MALI_RELEASE_NAME);
kbase_device_id_init(kbdev);
kbase_disjoint_init(kbdev);
for (i = 0; i < ARRAY_SIZE(dev_init); i++) {
err = dev_init[i].init(kbdev);
if (err) {
dev_err(kbdev->dev, "%s error = %d\n",
dev_init[i].err_mes, err);
kbase_device_term_partial(kbdev, i);
break;
}
}
return err;
}

429
drivers/gpu/arm/bifrost/device/mali_kbase_device.c Normal file
View File

@ -0,0 +1,429 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel device APIs
*/
#include <linux/debugfs.h>
#include <linux/dma-mapping.h>
#include <linux/seq_file.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of_platform.h>
#include <linux/types.h>
#include <mali_kbase.h>
#include <mali_kbase_defs.h>
#include <mali_kbase_hwaccess_instr.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_config_defaults.h>
#include <tl/mali_kbase_timeline.h>
#include "mali_kbase_vinstr.h"
#include "mali_kbase_hwcnt_context.h"
#include "mali_kbase_hwcnt_virtualizer.h"
#include "mali_kbase_device.h"
#include "mali_kbase_device_internal.h"
#include "backend/gpu/mali_kbase_pm_internal.h"
#include "backend/gpu/mali_kbase_irq_internal.h"
#ifdef CONFIG_MALI_ARBITER_SUPPORT
#include "arbiter/mali_kbase_arbiter_pm.h"
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
/* NOTE: Magic - 0x45435254 (TRCE in ASCII).
* Supports tracing feature provided in the base module.
* Please keep it in sync with the value of base module.
*/
#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254
/* Number of register accesses for the buffer that we allocate during
* initialization time. The buffer size can be changed later via debugfs.
*/
#define KBASEP_DEFAULT_REGISTER_HISTORY_SIZE ((u16)512)
static DEFINE_MUTEX(kbase_dev_list_lock);
static LIST_HEAD(kbase_dev_list);
static int kbase_dev_nr;
struct kbase_device *kbase_device_alloc(void)
{
return kzalloc(sizeof(struct kbase_device), GFP_KERNEL);
}
static int kbase_device_as_init(struct kbase_device *kbdev, int i)
{
kbdev->as[i].number = i;
kbdev->as[i].bf_data.addr = 0ULL;
kbdev->as[i].pf_data.addr = 0ULL;
kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i);
if (!kbdev->as[i].pf_wq)
return -EINVAL;
INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker);
INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker);
return 0;
}
static void kbase_device_as_term(struct kbase_device *kbdev, int i)
{
destroy_workqueue(kbdev->as[i].pf_wq);
}
static int kbase_device_all_as_init(struct kbase_device *kbdev)
{
int i, err;
for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
err = kbase_device_as_init(kbdev, i);
if (err)
goto free_workqs;
}
return 0;
free_workqs:
for (; i > 0; i--)
kbase_device_as_term(kbdev, i);
return err;
}
static void kbase_device_all_as_term(struct kbase_device *kbdev)
{
int i;
for (i = 0; i < kbdev->nr_hw_address_spaces; i++)
kbase_device_as_term(kbdev, i);
}
int kbase_device_misc_init(struct kbase_device * const kbdev)
{
int err;
#ifdef CONFIG_ARM64
struct device_node *np = NULL;
#endif /* CONFIG_ARM64 */
spin_lock_init(&kbdev->mmu_mask_change);
mutex_init(&kbdev->mmu_hw_mutex);
#ifdef CONFIG_ARM64
kbdev->cci_snoop_enabled = false;
np = kbdev->dev->of_node;
if (np != NULL) {
if (of_property_read_u32(np, "snoop_enable_smc",
&kbdev->snoop_enable_smc))
kbdev->snoop_enable_smc = 0;
if (of_property_read_u32(np, "snoop_disable_smc",
&kbdev->snoop_disable_smc))
kbdev->snoop_disable_smc = 0;
/* Either both or none of the calls should be provided. */
if (!((kbdev->snoop_disable_smc == 0
&& kbdev->snoop_enable_smc == 0)
|| (kbdev->snoop_disable_smc != 0
&& kbdev->snoop_enable_smc != 0))) {
WARN_ON(1);
err = -EINVAL;
goto fail;
}
}
#endif /* CONFIG_ARM64 */
/* Get the list of workarounds for issues on the current HW
* (identified by the GPU_ID register)
*/
err = kbase_hw_set_issues_mask(kbdev);
if (err)
goto fail;
/* Set the list of features available on the current HW
* (identified by the GPU_ID register)
*/
kbase_hw_set_features_mask(kbdev);
err = kbase_gpuprops_set_features(kbdev);
if (err)
goto fail;
/* On Linux 4.0+, dma coherency is determined from device tree */
#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops);
#endif
/* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our
* device structure was created by device-tree
*/
if (!kbdev->dev->dma_mask)
kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask;
err = dma_set_mask(kbdev->dev,
DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits));
if (err)
goto dma_set_mask_failed;
err = dma_set_coherent_mask(kbdev->dev,
DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits));
if (err)
goto dma_set_mask_failed;
kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces;
err = kbase_device_all_as_init(kbdev);
if (err)
goto as_init_failed;
spin_lock_init(&kbdev->hwcnt.lock);
err = kbase_ktrace_init(kbdev);
if (err)
goto term_as;
init_waitqueue_head(&kbdev->cache_clean_wait);
kbase_debug_assert_register_hook(&kbase_ktrace_hook_wrapper, kbdev);
atomic_set(&kbdev->ctx_num, 0);
err = kbase_instr_backend_init(kbdev);
if (err)
goto term_trace;
kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD;
kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
kbdev->mmu_mode = kbase_mmu_mode_get_aarch64();
else
kbdev->mmu_mode = kbase_mmu_mode_get_lpae();
mutex_init(&kbdev->kctx_list_lock);
INIT_LIST_HEAD(&kbdev->kctx_list);
spin_lock_init(&kbdev->hwaccess_lock);
return 0;
term_trace:
kbase_ktrace_term(kbdev);
term_as:
kbase_device_all_as_term(kbdev);
as_init_failed:
dma_set_mask_failed:
fail:
return err;
}
void kbase_device_misc_term(struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(kbdev);
WARN_ON(!list_empty(&kbdev->kctx_list));
#if KBASE_KTRACE_ENABLE
kbase_debug_assert_register_hook(NULL, NULL);
#endif
kbase_instr_backend_term(kbdev);
kbase_ktrace_term(kbdev);
kbase_device_all_as_term(kbdev);
}
void kbase_device_free(struct kbase_device *kbdev)
{
kfree(kbdev);
}
void kbase_device_id_init(struct kbase_device *kbdev)
{
scnprintf(kbdev->devname, DEVNAME_SIZE, "%s%d", kbase_drv_name,
kbase_dev_nr);
kbdev->id = kbase_dev_nr;
}
void kbase_increment_device_id(void)
{
kbase_dev_nr++;
}
int kbase_device_hwcnt_backend_gpu_init(struct kbase_device *kbdev)
{
return kbase_hwcnt_backend_gpu_create(kbdev, &kbdev->hwcnt_gpu_iface);
}
void kbase_device_hwcnt_backend_gpu_term(struct kbase_device *kbdev)
{
kbase_hwcnt_backend_gpu_destroy(&kbdev->hwcnt_gpu_iface);
}
int kbase_device_hwcnt_context_init(struct kbase_device *kbdev)
{
return kbase_hwcnt_context_init(&kbdev->hwcnt_gpu_iface,
&kbdev->hwcnt_gpu_ctx);
}
void kbase_device_hwcnt_context_term(struct kbase_device *kbdev)
{
kbase_hwcnt_context_term(kbdev->hwcnt_gpu_ctx);
}
int kbase_device_hwcnt_virtualizer_init(struct kbase_device *kbdev)
{
return kbase_hwcnt_virtualizer_init(kbdev->hwcnt_gpu_ctx,
KBASE_HWCNT_GPU_VIRTUALIZER_DUMP_THRESHOLD_NS,
&kbdev->hwcnt_gpu_virt);
}
void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev)
{
kbase_hwcnt_virtualizer_term(kbdev->hwcnt_gpu_virt);
}
int kbase_device_timeline_init(struct kbase_device *kbdev)
{
atomic_set(&kbdev->timeline_flags, 0);
return kbase_timeline_init(&kbdev->timeline, &kbdev->timeline_flags);
}
void kbase_device_timeline_term(struct kbase_device *kbdev)
{
kbase_timeline_term(kbdev->timeline);
}
int kbase_device_vinstr_init(struct kbase_device *kbdev)
{
return kbase_vinstr_init(kbdev->hwcnt_gpu_virt, &kbdev->vinstr_ctx);
}
void kbase_device_vinstr_term(struct kbase_device *kbdev)
{
kbase_vinstr_term(kbdev->vinstr_ctx);
}
int kbase_device_io_history_init(struct kbase_device *kbdev)
{
return kbase_io_history_init(&kbdev->io_history,
KBASEP_DEFAULT_REGISTER_HISTORY_SIZE);
}
void kbase_device_io_history_term(struct kbase_device *kbdev)
{
kbase_io_history_term(&kbdev->io_history);
}
int kbase_device_misc_register(struct kbase_device *kbdev)
{
return misc_register(&kbdev->mdev);
}
void kbase_device_misc_deregister(struct kbase_device *kbdev)
{
misc_deregister(&kbdev->mdev);
}
int kbase_device_list_init(struct kbase_device *kbdev)
{
const struct list_head *dev_list;
dev_list = kbase_device_get_list();
list_add(&kbdev->entry, &kbase_dev_list);
kbase_device_put_list(dev_list);
return 0;
}
void kbase_device_list_term(struct kbase_device *kbdev)
{
const struct list_head *dev_list;
dev_list = kbase_device_get_list();
list_del(&kbdev->entry);
kbase_device_put_list(dev_list);
}
const struct list_head *kbase_device_get_list(void)
{
mutex_lock(&kbase_dev_list_lock);
return &kbase_dev_list;
}
KBASE_EXPORT_TEST_API(kbase_device_get_list);
void kbase_device_put_list(const struct list_head *dev_list)
{
mutex_unlock(&kbase_dev_list_lock);
}
KBASE_EXPORT_TEST_API(kbase_device_put_list);
int kbase_device_early_init(struct kbase_device *kbdev)
{
int err;
err = kbasep_platform_device_init(kbdev);
if (err)
return err;
err = kbase_pm_runtime_init(kbdev);
if (err)
goto fail_runtime_pm;
/* Ensure we can access the GPU registers */
kbase_pm_register_access_enable(kbdev);
/* Find out GPU properties based on the GPU feature registers */
kbase_gpuprops_set(kbdev);
/* We're done accessing the GPU registers for now. */
kbase_pm_register_access_disable(kbdev);
err = kbase_install_interrupts(kbdev);
if (err)
goto fail_interrupts;
return 0;
fail_interrupts:
kbase_pm_runtime_term(kbdev);
fail_runtime_pm:
kbasep_platform_device_term(kbdev);
return err;
}
void kbase_device_early_term(struct kbase_device *kbdev)
{
#ifdef CONFIG_MALI_ARBITER_SUPPORT
if (kbdev->arb.arb_if)
kbase_arbiter_pm_release_interrupts(kbdev);
else
kbase_release_interrupts(kbdev);
#else
kbase_release_interrupts(kbdev);
#endif /* CONFIG_MALI_ARBITER_SUPPORT */
kbase_pm_runtime_term(kbdev);
kbasep_platform_device_term(kbdev);
}

71
drivers/gpu/arm/bifrost/device/mali_kbase_device.h Normal file
View File

@ -0,0 +1,71 @@
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
/**
* kbase_device_get_list - get device list.
*
* Get access to device list.
*
* Return: Pointer to the linked list head.
*/
const struct list_head *kbase_device_get_list(void);
/**
* kbase_device_put_list - put device list.
*
* @dev_list: head of linked list containing device list.
*
* Put access to the device list.
*/
void kbase_device_put_list(const struct list_head *dev_list);
/**
* Kbase_increment_device_id - increment device id.
*
* Used to increment device id on successful initialization of the device.
*/
void kbase_increment_device_id(void);
/**
* kbase_device_init - Device initialisation.
*
* This is called from device probe to initialise various other
* components needed.
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Return: 0 on success and non-zero value on failure.
*/
int kbase_device_init(struct kbase_device *kbdev);
/**
* kbase_device_term - Device termination.
*
* This is called from device remove to terminate various components that
* were initialised during kbase_device_init.
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
*/
void kbase_device_term(struct kbase_device *kbdev);

78
drivers/gpu/arm/bifrost/device/mali_kbase_device_internal.h Normal file
View File

@ -0,0 +1,78 @@
/*
*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
typedef int kbase_device_init_method(struct kbase_device *kbdev);
typedef void kbase_device_term_method(struct kbase_device *kbdev);
/**
* struct kbase_device_init - Device init/term methods.
* @init: Function pointer to a initialise method.
* @term: Function pointer to a terminate method.
* @err_mes: Error message to be printed when init method fails.
*/
struct kbase_device_init {
kbase_device_init_method *init;
kbase_device_term_method *term;
char *err_mes;
};
int kbase_device_vinstr_init(struct kbase_device *kbdev);
void kbase_device_vinstr_term(struct kbase_device *kbdev);
int kbase_device_timeline_init(struct kbase_device *kbdev);
void kbase_device_timeline_term(struct kbase_device *kbdev);
int kbase_device_hwcnt_backend_gpu_init(struct kbase_device *kbdev);
void kbase_device_hwcnt_backend_gpu_term(struct kbase_device *kbdev);
int kbase_device_hwcnt_context_init(struct kbase_device *kbdev);
void kbase_device_hwcnt_context_term(struct kbase_device *kbdev);
int kbase_device_hwcnt_virtualizer_init(struct kbase_device *kbdev);
void kbase_device_hwcnt_virtualizer_term(struct kbase_device *kbdev);
int kbase_device_list_init(struct kbase_device *kbdev);
void kbase_device_list_term(struct kbase_device *kbdev);
int kbase_device_io_history_init(struct kbase_device *kbdev);
void kbase_device_io_history_term(struct kbase_device *kbdev);
int kbase_device_misc_register(struct kbase_device *kbdev);
void kbase_device_misc_deregister(struct kbase_device *kbdev);
void kbase_device_id_init(struct kbase_device *kbdev);
/**
* kbase_device_early_init - Perform any device-specific initialization.
* @kbdev: Device pointer
*
* Return: 0 on success, or an error code on failure.
*/
int kbase_device_early_init(struct kbase_device *kbdev);
/**
* kbase_device_early_term - Perform any device-specific termination.
* @kbdev: Device pointer
*/
void kbase_device_early_term(struct kbase_device *kbdev);

181
drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_fault_jm.c Normal file
View File

@ -0,0 +1,181 @@
/*
*
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <mali_kbase.h>
#include "../mali_kbase_gpu_fault.h"
const char *kbase_gpu_exception_name(u32 const exception_code)
{
const char *e;
switch (exception_code) {
/* Non-Fault Status code */
case 0x00:
e = "NOT_STARTED/IDLE/OK";
break;
case 0x01:
e = "DONE";
break;
case 0x02:
e = "INTERRUPTED";
break;
case 0x03:
e = "STOPPED";
break;
case 0x04:
e = "TERMINATED";
break;
case 0x08:
e = "ACTIVE";
break;
/* Job exceptions */
case 0x40:
e = "JOB_CONFIG_FAULT";
break;
case 0x41:
e = "JOB_POWER_FAULT";
break;
case 0x42:
e = "JOB_READ_FAULT";
break;
case 0x43:
e = "JOB_WRITE_FAULT";
break;
case 0x44:
e = "JOB_AFFINITY_FAULT";
break;
case 0x48:
e = "JOB_BUS_FAULT";
break;
case 0x50:
e = "INSTR_INVALID_PC";
break;
case 0x51:
e = "INSTR_INVALID_ENC";
break;
case 0x52:
e = "INSTR_TYPE_MISMATCH";
break;
case 0x53:
e = "INSTR_OPERAND_FAULT";
break;
case 0x54:
e = "INSTR_TLS_FAULT";
break;
case 0x55:
e = "INSTR_BARRIER_FAULT";
break;
case 0x56:
e = "INSTR_ALIGN_FAULT";
break;
case 0x58:
e = "DATA_INVALID_FAULT";
break;
case 0x59:
e = "TILE_RANGE_FAULT";
break;
case 0x5A:
e = "ADDR_RANGE_FAULT";
break;
case 0x60:
e = "OUT_OF_MEMORY";
break;
/* GPU exceptions */
case 0x80:
e = "DELAYED_BUS_FAULT";
break;
case 0x88:
e = "SHAREABILITY_FAULT";
break;
/* MMU exceptions */
case 0xC0:
case 0xC1:
case 0xC2:
case 0xC3:
case 0xC4:
case 0xC5:
case 0xC6:
case 0xC7:
e = "TRANSLATION_FAULT";
break;
case 0xC8:
e = "PERMISSION_FAULT";
break;
case 0xC9:
case 0xCA:
case 0xCB:
case 0xCC:
case 0xCD:
case 0xCE:
case 0xCF:
e = "PERMISSION_FAULT";
break;
case 0xD0:
case 0xD1:
case 0xD2:
case 0xD3:
case 0xD4:
case 0xD5:
case 0xD6:
case 0xD7:
e = "TRANSTAB_BUS_FAULT";
break;
case 0xD8:
e = "ACCESS_FLAG";
break;
case 0xD9:
case 0xDA:
case 0xDB:
case 0xDC:
case 0xDD:
case 0xDE:
case 0xDF:
e = "ACCESS_FLAG";
break;
case 0xE0:
case 0xE1:
case 0xE2:
case 0xE3:
case 0xE4:
case 0xE5:
case 0xE6:
case 0xE7:
e = "ADDRESS_SIZE_FAULT";
break;
case 0xE8:
case 0xE9:
case 0xEA:
case 0xEB:
case 0xEC:
case 0xED:
case 0xEE:
case 0xEF:
e = "MEMORY_ATTRIBUTES_FAULT";
break;
default:
e = "UNKNOWN";
break;
};
return e;
}

68
drivers/gpu/arm/bifrost/mali_midg_regmap_jm.h → drivers/gpu/arm/bifrost/gpu/backend/mali_kbase_gpu_regmap_jm.h
View File

@ -20,15 +20,69 @@
*
*/
#ifndef _MIDG_REGMAP_JM_H_
#define _MIDG_REGMAP_JM_H_
#ifndef _KBASE_GPU_REGMAP_JM_H_
#define _KBASE_GPU_REGMAP_JM_H_
/* Set to implementation defined, outer caching */
#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull
/* Set to write back memory, outer caching */
#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull
/* Set to inner non-cacheable, outer-non-cacheable
* Setting defined by the alloc bits is ignored, but set to a valid encoding:
* - no-alloc on read
* - no alloc on write
*/
#define AS_MEMATTR_AARCH64_NON_CACHEABLE 0x4Cull
/* Symbols for default MEMATTR to use
* Default is - HW implementation defined caching
*/
#define AS_MEMATTR_INDEX_DEFAULT 0
#define AS_MEMATTR_INDEX_DEFAULT_ACE 3
/* HW implementation defined caching */
#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0
/* Force cache on */
#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1
/* Write-alloc */
#define AS_MEMATTR_INDEX_WRITE_ALLOC 2
/* Outer coherent, inner implementation defined policy */
#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3
/* Outer coherent, write alloc inner */
#define AS_MEMATTR_INDEX_OUTER_WA 4
/* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */
#define AS_MEMATTR_INDEX_NON_CACHEABLE 5
/* GPU control registers */
#define CORE_FEATURES 0x008 /* (RO) Shader Core Features */
#define JS_PRESENT 0x01C /* (RO) Job slots present */
#define LATEST_FLUSH 0x038 /* (RO) Flush ID of latest clean-and-invalidate operation */
#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */
#define LATEST_FLUSH 0x038 /* (RO) Flush ID of latest
* clean-and-invalidate operation
*/
#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory
* region base address, low word
*/
#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory
* region base address, high word
*/
#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter
* configuration
*/
#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable
* flags for Job Manager
*/
#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable
* flags for shader cores
*/
#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable
* flags for tiler
*/
#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable
* flags for MMU/L2 cache
*/
#define JS0_FEATURES 0x0C0 /* (RO) Features of job slot 0 */
#define JS1_FEATURES 0x0C4 /* (RO) Features of job slot 1 */
@ -140,7 +194,7 @@
* The values are separated to avoid dependency of userspace and kernel code.
*/
/* Group of values representing the job status insead a particular fault */
/* Group of values representing the job status instead of a particular fault */
#define JS_STATUS_NO_EXCEPTION_BASE 0x00
#define JS_STATUS_INTERRUPTED (JS_STATUS_NO_EXCEPTION_BASE + 0x02) /* 0x02 means INTERRUPTED */
#define JS_STATUS_STOPPED (JS_STATUS_NO_EXCEPTION_BASE + 0x03) /* 0x03 means STOPPED */
@ -192,8 +246,6 @@
#define JM_JOB_THROTTLE_LIMIT_SHIFT (3)
#define JM_MAX_JOB_THROTTLE_LIMIT (0x3F)
#define JM_FORCE_COHERENCY_FEATURES_SHIFT (2)
#define JM_IDVS_GROUP_SIZE_SHIFT (16)
#define JM_MAX_IDVS_GROUP_SIZE (0x3F)
/* GPU_COMMAND values */
#define GPU_COMMAND_NOP 0x00 /* No operation, nothing happens */
@ -207,4 +259,4 @@
#define GPU_COMMAND_CLEAN_INV_CACHES 0x08 /* Clean and invalidate all caches */
#define GPU_COMMAND_SET_PROTECTED_MODE 0x09 /* Places the GPU in protected mode */
#endif /* _MIDG_REGMAP_JM_H_ */
#endif /* _KBASE_GPU_REGMAP_JM_H_ */

43
drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_platform.h → drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2017, 2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -20,27 +20,22 @@
*
*/
/**
* Power management configuration
*
* Attached value: pointer to @ref kbase_pm_callback_conf
* Default value: See @ref kbase_pm_callback_conf
*/
#define POWER_MANAGEMENT_CALLBACKS (&pm_callbacks)
#include <mali_kbase.h>
#include <mali_kbase_defs.h>
/**
* Platform specific configuration functions
*
* Attached value: pointer to @ref kbase_platform_funcs_conf
* Default value: See @ref kbase_platform_funcs_conf
*/
#define PLATFORM_FUNCS (NULL)
extern struct kbase_pm_callback_conf pm_callbacks;
/**
* Autosuspend delay
*
* The delay time (in milliseconds) to be used for autosuspend
*/
#define AUTO_SUSPEND_DELAY (100)
const char *kbase_gpu_access_type_name(u32 fault_status)
{
switch (AS_FAULTSTATUS_ACCESS_TYPE_GET(fault_status)) {
case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
return "ATOMIC";
case AS_FAULTSTATUS_ACCESS_TYPE_READ:
return "READ";
case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
return "WRITE";
case AS_FAULTSTATUS_ACCESS_TYPE_EX:
return "EXECUTE";
default:
WARN_ON(1);
return NULL;
}
}

26
drivers/gpu/arm/bifrost/platform/meson/mali_kbase_config_meson.c → drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2015, 2017, 2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -20,22 +20,12 @@
*
*/
#include <mali_kbase_config.h>
#ifndef _KBASE_GPU_H_
#define _KBASE_GPU_H_
static struct kbase_platform_config dummy_platform_config;
#include "mali_kbase_gpu_regmap.h"
#include "mali_kbase_gpu_fault.h"
#include "mali_kbase_gpu_coherency.h"
#include "mali_kbase_gpu_id.h"
struct kbase_platform_config *kbase_get_platform_config(void)
{
return &dummy_platform_config;
}
#ifndef CONFIG_OF
int kbase_platform_register(void)
{
return 0;
}
void kbase_platform_unregister(void)
{
}
#endif
#endif /* _KBASE_GPU_H_ */

8
drivers/gpu/arm/bifrost/mali_midg_coherency.h → drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_coherency.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2015 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -20,12 +20,12 @@
*
*/
#ifndef _MIDG_COHERENCY_H_
#define _MIDG_COHERENCY_H_
#ifndef _KBASE_GPU_COHERENCY_H_
#define _KBASE_GPU_COHERENCY_H_
#define COHERENCY_ACE_LITE 0
#define COHERENCY_ACE 1
#define COHERENCY_NONE 31
#define COHERENCY_FEATURE_BIT(x) (1 << (x))
#endif /* _MIDG_COHERENCY_H_ */
#endif /* _KBASE_GPU_COHERENCY_H_ */

59
drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_fault.h Normal file
View File

@ -0,0 +1,59 @@
/*
*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_GPU_FAULT_H_
#define _KBASE_GPU_FAULT_H_
/** Returns the name associated with a Mali exception code
*
* @exception_code: exception code
*
* This function is called from the interrupt handler when a GPU fault occurs.
*
* Return: name associated with the exception code
*/
const char *kbase_gpu_exception_name(u32 exception_code);
/** Returns the name associated with a Mali fatal exception code
*
* @fatal_exception_code: fatal exception code
*
* This function is called from the interrupt handler when a GPU fatal
* exception occurs.
*
* Return: name associated with the fatal exception code
*/
const char *kbase_gpu_fatal_exception_name(u32 const fatal_exception_code);
/**
* kbase_gpu_access_type_name - Convert MMU_AS_CONTROL.FAULTSTATUS.ACCESS_TYPE
* into string.
* @fault_status: value of FAULTSTATUS register.
*
* After MMU fault, this function can be used to get readable information about
* access_type of the MMU fault.
*
* Return: String of the access type.
*/
const char *kbase_gpu_access_type_name(u32 fault_status);
#endif /* _KBASE_GPU_FAULT_H_ */

11
drivers/gpu/arm/bifrost/mali_kbase_gpu_id.h → drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_id.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2015-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2015-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -19,6 +19,7 @@
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_GPU_ID_H_
#define _KBASE_GPU_ID_H_
@ -97,12 +98,14 @@
#define GPU_ID2_PRODUCT_TNAX GPU_ID2_MODEL_MAKE(9, 1)
#define GPU_ID2_PRODUCT_TBEX GPU_ID2_MODEL_MAKE(9, 2)
#define GPU_ID2_PRODUCT_LBEX GPU_ID2_MODEL_MAKE(9, 4)
#define GPU_ID2_PRODUCT_TULX GPU_ID2_MODEL_MAKE(10, 0)
#define GPU_ID2_PRODUCT_TDUX GPU_ID2_MODEL_MAKE(10, 1)
#define GPU_ID2_PRODUCT_TODX GPU_ID2_MODEL_MAKE(10, 2)
#define GPU_ID2_PRODUCT_TIDX GPU_ID2_MODEL_MAKE(10, 3)
#define GPU_ID2_PRODUCT_TGRX GPU_ID2_MODEL_MAKE(10, 3)
#define GPU_ID2_PRODUCT_TVAX GPU_ID2_MODEL_MAKE(10, 4)
#define GPU_ID2_PRODUCT_LODX GPU_ID2_MODEL_MAKE(10, 5)
#define GPU_ID2_PRODUCT_LODX GPU_ID2_MODEL_MAKE(10, 7)
#define GPU_ID2_PRODUCT_TTUX GPU_ID2_MODEL_MAKE(11, 2)
#define GPU_ID2_PRODUCT_LTUX GPU_ID2_MODEL_MAKE(11, 3)
#define GPU_ID2_PRODUCT_TE2X GPU_ID2_MODEL_MAKE(11, 1)
/* Helper macro to create a GPU_ID assuming valid values for id, major,
minor, status */

91
drivers/gpu/arm/bifrost/mali_midg_regmap.h → drivers/gpu/arm/bifrost/gpu/mali_kbase_gpu_regmap.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -20,12 +20,12 @@
*
*/
#ifndef _MIDG_REGMAP_H_
#define _MIDG_REGMAP_H_
#ifndef _KBASE_GPU_REGMAP_H_
#define _KBASE_GPU_REGMAP_H_
#include "mali_midg_coherency.h"
#include "mali_kbase_gpu_coherency.h"
#include "mali_kbase_gpu_id.h"
#include "mali_midg_regmap_jm.h"
#include "backend/mali_kbase_gpu_regmap_jm.h"
/* Begin Register Offsets */
/* GPU control registers */
@ -54,18 +54,15 @@
#define L2_CONFIG 0x048 /* (RW) Level 2 cache configuration */
#define GROUPS_L2_COHERENT (1 << 0) /* Cores groups are l2 coherent */
#define SUPER_L2_COHERENT (1 << 1) /* Shader cores within a core
* supergroup are l2 coherent
*/
#define PWR_KEY 0x050 /* (WO) Power manager key register */
#define PWR_OVERRIDE0 0x054 /* (RW) Power manager override settings */
#define PWR_OVERRIDE1 0x058 /* (RW) Power manager override settings */
#define PRFCNT_BASE_LO 0x060 /* (RW) Performance counter memory region base address, low word */
#define PRFCNT_BASE_HI 0x064 /* (RW) Performance counter memory region base address, high word */
#define PRFCNT_CONFIG 0x068 /* (RW) Performance counter configuration */
#define PRFCNT_JM_EN 0x06C /* (RW) Performance counter enable flags for Job Manager */
#define PRFCNT_SHADER_EN 0x070 /* (RW) Performance counter enable flags for shader cores */
#define PRFCNT_TILER_EN 0x074 /* (RW) Performance counter enable flags for tiler */
#define PRFCNT_MMU_L2_EN 0x07C /* (RW) Performance counter enable flags for MMU/L2 cache */
#define CYCLE_COUNT_LO 0x090 /* (RO) Cycle counter, low word */
#define CYCLE_COUNT_HI 0x094 /* (RO) Cycle counter, high word */
#define TIMESTAMP_LO 0x098 /* (RO) Global time stamp counter, low word */
@ -235,9 +232,17 @@
#define PRFCNT_SAMPLE_COMPLETED (1 << 16) /* Set when a performance count sample has completed. */
#define CLEAN_CACHES_COMPLETED (1 << 17) /* Set when a cache clean operation has completed. */
#define GPU_IRQ_REG_ALL (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \
/* Include POWER_CHANGED_SINGLE in debug builds for use in irq latency test.
*/
#define GPU_IRQ_REG_COMMON (GPU_FAULT | MULTIPLE_GPU_FAULTS | RESET_COMPLETED \
| POWER_CHANGED_ALL | PRFCNT_SAMPLE_COMPLETED)
#ifdef CONFIG_MALI_BIFROST_DEBUG
#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON | POWER_CHANGED_SINGLE)
#else /* CONFIG_MALI_BIFROST_DEBUG */
#define GPU_IRQ_REG_ALL (GPU_IRQ_REG_COMMON)
#endif /* CONFIG_MALI_BIFROST_DEBUG */
/*
* MMU_IRQ_RAWSTAT register values. Values are valid also for
* MMU_IRQ_CLEAR, MMU_IRQ_MASK, MMU_IRQ_STATUS registers.
@ -281,11 +286,25 @@
#define AS_FAULTSTATUS_EXCEPTION_CODE_ADDRESS_SIZE_FAULT (0x4<<3)
#define AS_FAULTSTATUS_EXCEPTION_CODE_MEMORY_ATTRIBUTES_FAULT (0x5<<3)
#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3<<8)
#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0<<8)
#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1<<8)
#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2<<8)
#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3<<8)
#define AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT 0
#define AS_FAULTSTATUS_EXCEPTION_TYPE_MASK (0xFF << AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT)
#define AS_FAULTSTATUS_EXCEPTION_TYPE_GET(reg_val) \
(((reg_val)&AS_FAULTSTATUS_EXCEPTION_TYPE_MASK) >> AS_FAULTSTATUS_EXCEPTION_TYPE_SHIFT)
#define AS_FAULTSTATUS_ACCESS_TYPE_SHIFT 8
#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << AS_FAULTSTATUS_ACCESS_TYPE_SHIFT)
#define AS_FAULTSTATUS_ACCESS_TYPE_GET(reg_val) \
(((reg_val)&AS_FAULTSTATUS_ACCESS_TYPE_MASK) >> AS_FAULTSTATUS_ACCESS_TYPE_SHIFT)
#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0)
#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1)
#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2)
#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3)
#define AS_FAULTSTATUS_SOURCE_ID_SHIFT 16
#define AS_FAULTSTATUS_SOURCE_ID_MASK (0xFFFF << AS_FAULTSTATUS_SOURCE_ID_SHIFT)
#define AS_FAULTSTATUS_SOURCE_ID_GET(reg_val) \
(((reg_val)&AS_FAULTSTATUS_SOURCE_ID_MASK) >> AS_FAULTSTATUS_SOURCE_ID_SHIFT)
/*
* Begin MMU TRANSCFG register values
@ -353,17 +372,6 @@
/* Inner write-alloc cache setup, no outer caching */
#define AS_MEMATTR_WRITE_ALLOC 0x8Dull
/* Set to implementation defined, outer caching */
#define AS_MEMATTR_AARCH64_OUTER_IMPL_DEF 0x88ull
/* Set to write back memory, outer caching */
#define AS_MEMATTR_AARCH64_OUTER_WA 0x8Dull
/* Set to inner non-cacheable, outer-non-cacheable
* Setting defined by the alloc bits is ignored, but set to a valid encoding:
* - no-alloc on read
* - no alloc on write
*/
#define AS_MEMATTR_AARCH64_NON_CACHEABLE 0x4Cull
/* Use GPU implementation-defined caching policy. */
#define AS_MEMATTR_LPAE_IMPL_DEF_CACHE_POLICY 0x48ull
/* The attribute set to force all resources to be cached. */
@ -380,24 +388,6 @@
*/
#define AS_MEMATTR_LPAE_NON_CACHEABLE_RESERVED
/* Symbols for default MEMATTR to use
* Default is - HW implementation defined caching */
#define AS_MEMATTR_INDEX_DEFAULT 0
#define AS_MEMATTR_INDEX_DEFAULT_ACE 3
/* HW implementation defined caching */
#define AS_MEMATTR_INDEX_IMPL_DEF_CACHE_POLICY 0
/* Force cache on */
#define AS_MEMATTR_INDEX_FORCE_TO_CACHE_ALL 1
/* Write-alloc */
#define AS_MEMATTR_INDEX_WRITE_ALLOC 2
/* Outer coherent, inner implementation defined policy */
#define AS_MEMATTR_INDEX_OUTER_IMPL_DEF 3
/* Outer coherent, write alloc inner */
#define AS_MEMATTR_INDEX_OUTER_WA 4
/* Normal memory, inner non-cacheable, outer non-cacheable (ARMv8 mode only) */
#define AS_MEMATTR_INDEX_NON_CACHEABLE 5
/* L2_MMU_CONFIG register */
#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT (23)
#define L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY (0x1 << L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY_SHIFT)
@ -440,5 +430,8 @@
#define L2_CONFIG_HASH_MASK (0xFFul << L2_CONFIG_HASH_SHIFT)
/* End L2_CONFIG register */
/* IDVS_GROUP register */
#define IDVS_GROUP_SIZE_SHIFT (16)
#define IDVS_GROUP_MAX_SIZE (0x3F)
#endif /* _MIDG_REGMAP_H_ */
#endif /* _KBASE_GPU_REGMAP_H_ */

1001
drivers/gpu/arm/bifrost/jm/mali_base_jm_kernel.h Normal file
View File

File diff suppressed because it is too large Load Diff

818
drivers/gpu/arm/bifrost/jm/mali_kbase_jm_defs.h Normal file
View File

@ -0,0 +1,818 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Definitions (types, defines, etcs) specific to Job Manager Kbase.
* They are placed here to allow the hierarchy of header files to work.
*/
#ifndef _KBASE_JM_DEFS_H_
#define _KBASE_JM_DEFS_H_
#include "mali_kbase_js_defs.h"
/* Dump Job slot trace on error (only active if KBASE_KTRACE_ENABLE != 0) */
#define KBASE_KTRACE_DUMP_ON_JOB_SLOT_ERROR 1
/*
* Number of milliseconds before resetting the GPU when a job cannot be "zapped"
* from the hardware. Note that the time is actually
* ZAP_TIMEOUT+SOFT_STOP_RESET_TIMEOUT between the context zap starting and
* the GPU actually being reset to give other contexts time for their jobs
* to be soft-stopped and removed from the hardware before resetting.
*/
#define ZAP_TIMEOUT 1000
/*
* Prevent soft-stops from occurring in scheduling situations
*
* This is not due to HW issues, but when scheduling is desired to be more
* predictable.
*
* Therefore, soft stop may still be disabled due to HW issues.
*
* Soft stop will still be used for non-scheduling purposes e.g. when
* terminating a context.
*
* if not in use, define this value to 0 instead of being undefined.
*/
#define KBASE_DISABLE_SCHEDULING_SOFT_STOPS 0
/*
* Prevent hard-stops from occurring in scheduling situations
*
* This is not due to HW issues, but when scheduling is desired to be more
* predictable.
*
* Hard stop will still be used for non-scheduling purposes e.g. when
* terminating a context.
*
* if not in use, define this value to 0 instead of being undefined.
*/
#define KBASE_DISABLE_SCHEDULING_HARD_STOPS 0
/* Atom has been previously soft-stopped */
#define KBASE_KATOM_FLAG_BEEN_SOFT_STOPPED (1<<1)
/* Atom has been previously retried to execute */
#define KBASE_KATOM_FLAGS_RERUN (1<<2)
/* Atom submitted with JOB_CHAIN_FLAG bit set in JS_CONFIG_NEXT register, helps
* to disambiguate short-running job chains during soft/hard stopping of jobs
*/
#define KBASE_KATOM_FLAGS_JOBCHAIN (1<<3)
/* Atom has been previously hard-stopped. */
#define KBASE_KATOM_FLAG_BEEN_HARD_STOPPED (1<<4)
/* Atom has caused us to enter disjoint state */
#define KBASE_KATOM_FLAG_IN_DISJOINT (1<<5)
/* Atom blocked on cross-slot dependency */
#define KBASE_KATOM_FLAG_X_DEP_BLOCKED (1<<7)
/* Atom has fail dependency on cross-slot dependency */
#define KBASE_KATOM_FLAG_FAIL_BLOCKER (1<<8)
/* Atom is currently in the list of atoms blocked on cross-slot dependencies */
#define KBASE_KATOM_FLAG_JSCTX_IN_X_DEP_LIST (1<<9)
/* Atom is currently holding a context reference */
#define KBASE_KATOM_FLAG_HOLDING_CTX_REF (1<<10)
/* Atom requires GPU to be in protected mode */
#define KBASE_KATOM_FLAG_PROTECTED (1<<11)
/* Atom has been stored in runnable_tree */
#define KBASE_KATOM_FLAG_JSCTX_IN_TREE (1<<12)
/* Atom is waiting for L2 caches to power up in order to enter protected mode */
#define KBASE_KATOM_FLAG_HOLDING_L2_REF_PROT (1<<13)
/* SW related flags about types of JS_COMMAND action
* NOTE: These must be masked off by JS_COMMAND_MASK
*/
/* This command causes a disjoint event */
#define JS_COMMAND_SW_CAUSES_DISJOINT 0x100
/* Bitmask of all SW related flags */
#define JS_COMMAND_SW_BITS (JS_COMMAND_SW_CAUSES_DISJOINT)
#if (JS_COMMAND_SW_BITS & JS_COMMAND_MASK)
#error "JS_COMMAND_SW_BITS not masked off by JS_COMMAND_MASK." \
"Must update JS_COMMAND_SW_<..> bitmasks"
#endif
/* Soft-stop command that causes a Disjoint event. This of course isn't
* entirely masked off by JS_COMMAND_MASK
*/
#define JS_COMMAND_SOFT_STOP_WITH_SW_DISJOINT \
(JS_COMMAND_SW_CAUSES_DISJOINT | JS_COMMAND_SOFT_STOP)
#define KBASEP_ATOM_ID_INVALID BASE_JD_ATOM_COUNT
/* Serialize atoms within a slot (ie only one atom per job slot) */
#define KBASE_SERIALIZE_INTRA_SLOT (1 << 0)
/* Serialize atoms between slots (ie only one job slot running at any time) */
#define KBASE_SERIALIZE_INTER_SLOT (1 << 1)
/* Reset the GPU after each atom completion */
#define KBASE_SERIALIZE_RESET (1 << 2)
#ifdef CONFIG_DEBUG_FS
/**
* struct base_job_fault_event - keeps track of the atom which faulted or which
* completed after the faulty atom but before the
* debug data for faulty atom was dumped.
*
* @event_code: event code for the atom, should != BASE_JD_EVENT_DONE for
* the atom which faulted.
* @katom: pointer to the atom for which job fault occurred or which
* completed after the faulty atom.
* @job_fault_work: work item, queued only for the faulty atom, which waits for
* the dumping to get completed and then does the bottom half
* of job done for the atoms which followed the faulty atom.
* @head: List head used to store the atom in the global list of
* faulty atoms or context specific list of atoms which got
* completed during the dump.
* @reg_offset: offset of the register to be dumped next, only applicable
* for the faulty atom.
*/
struct base_job_fault_event {
u32 event_code;
struct kbase_jd_atom *katom;
struct work_struct job_fault_work;
struct list_head head;
int reg_offset;
};
#endif
/**
* struct kbase_jd_atom_dependency - Contains the dependency info for an atom.
* @atom: pointer to the dependee atom.
* @dep_type: type of dependency on the dependee @atom, i.e. order or data
* dependency. BASE_JD_DEP_TYPE_INVALID indicates no dependency.
*/
struct kbase_jd_atom_dependency {
struct kbase_jd_atom *atom;
u8 dep_type;
};
/**
* kbase_jd_katom_dep_atom - Retrieves a read-only reference to the
* dependee atom.
* @dep: pointer to the dependency info structure.
*
* Return: readonly reference to dependee atom.
*/
static inline const struct kbase_jd_atom *
kbase_jd_katom_dep_atom(const struct kbase_jd_atom_dependency *dep)
{
LOCAL_ASSERT(dep != NULL);
return (const struct kbase_jd_atom *)(dep->atom);
}
/**
* kbase_jd_katom_dep_type - Retrieves the dependency type info
*
* @dep: pointer to the dependency info structure.
*
* Return: the type of dependency there is on the dependee atom.
*/
static inline u8 kbase_jd_katom_dep_type(
const struct kbase_jd_atom_dependency *dep)
{
LOCAL_ASSERT(dep != NULL);
return dep->dep_type;
}
/**
* kbase_jd_katom_dep_set - sets up the dependency info structure
* as per the values passed.
* @const_dep: pointer to the dependency info structure to be setup.
* @a: pointer to the dependee atom.
* @type: type of dependency there is on the dependee atom.
*/
static inline void kbase_jd_katom_dep_set(
const struct kbase_jd_atom_dependency *const_dep,
struct kbase_jd_atom *a, u8 type)
{
struct kbase_jd_atom_dependency *dep;
LOCAL_ASSERT(const_dep != NULL);
dep = (struct kbase_jd_atom_dependency *)const_dep;
dep->atom = a;
dep->dep_type = type;
}
/**
* kbase_jd_katom_dep_clear - resets the dependency info structure
*
* @const_dep: pointer to the dependency info structure to be setup.
*/
static inline void kbase_jd_katom_dep_clear(
const struct kbase_jd_atom_dependency *const_dep)
{
struct kbase_jd_atom_dependency *dep;
LOCAL_ASSERT(const_dep != NULL);
dep = (struct kbase_jd_atom_dependency *)const_dep;
dep->atom = NULL;
dep->dep_type = BASE_JD_DEP_TYPE_INVALID;
}
/**
* enum kbase_atom_gpu_rb_state - The state of an atom, pertinent after it
* becomes runnable, with respect to job slot
* ringbuffer/fifo.
* @KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB: Atom not currently present in slot fifo,
* which implies that either atom has not become
* runnable due to dependency or has completed
* the execution on GPU.
* @KBASE_ATOM_GPU_RB_WAITING_BLOCKED: Atom has been added to slot fifo but is
* blocked due to cross slot dependency,
* can't be submitted to GPU.
* @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV: Atom has been added to slot
* fifo but is waiting for the completion of
* previously added atoms in current & other
* slots, as their protected mode requirements
* do not match with the current atom.
* @KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION: Atom is in slot fifo
* and is waiting for completion of protected
* mode transition, needed before the atom is
* submitted to GPU.
* @KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE: Atom is in slot fifo but is
* waiting for the cores, which are needed to
* execute the job chain represented by the atom,
* to become available
* @KBASE_ATOM_GPU_RB_READY: Atom is in slot fifo and can be submitted to
* GPU.
* @KBASE_ATOM_GPU_RB_SUBMITTED: Atom is in slot fifo and has been submitted
* to GPU.
* @KBASE_ATOM_GPU_RB_RETURN_TO_JS: Atom must be returned to JS due to some
* failure, but only after the previously added
* atoms in fifo have completed or have also
* been returned to JS.
*/
enum kbase_atom_gpu_rb_state {
KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB,
KBASE_ATOM_GPU_RB_WAITING_BLOCKED,
KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV,
KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION,
KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE,
KBASE_ATOM_GPU_RB_READY,
KBASE_ATOM_GPU_RB_SUBMITTED,
KBASE_ATOM_GPU_RB_RETURN_TO_JS = -1
};
/**
* enum kbase_atom_enter_protected_state - The state of an atom with respect to
* the preparation for GPU's entry into protected mode,
* becomes pertinent only after atom's state with respect
* to slot ringbuffer is
* KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
* @KBASE_ATOM_ENTER_PROTECTED_CHECK: Starting state. Check if there are any
* atoms currently submitted to GPU and protected mode
* transition is not already in progress.
* @KBASE_ATOM_ENTER_PROTECTED_HWCNT: Wait for hardware counter context to
* become disabled before entry into protected mode.
* @KBASE_ATOM_ENTER_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
* preparation for the coherency change. L2 shall be
* powered down and GPU shall come out of fully
* coherent mode before entering protected mode.
* @KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY: Prepare coherency change;
* for BASE_HW_ISSUE_TGOX_R1_1234 also request L2 power on
* so that coherency register contains correct value when
* GPU enters protected mode.
* @KBASE_ATOM_ENTER_PROTECTED_FINISHED: End state; for
* BASE_HW_ISSUE_TGOX_R1_1234 check
* that L2 is powered up and switch GPU to protected mode.
*/
enum kbase_atom_enter_protected_state {
/*
* NOTE: The integer value of this must match
* KBASE_ATOM_EXIT_PROTECTED_CHECK.
*/
KBASE_ATOM_ENTER_PROTECTED_CHECK = 0,
KBASE_ATOM_ENTER_PROTECTED_HWCNT,
KBASE_ATOM_ENTER_PROTECTED_IDLE_L2,
KBASE_ATOM_ENTER_PROTECTED_SET_COHERENCY,
KBASE_ATOM_ENTER_PROTECTED_FINISHED,
};
/**
* enum kbase_atom_exit_protected_state - The state of an atom with respect to
* the preparation for GPU's exit from protected mode,
* becomes pertinent only after atom's state with respect
* to slot ngbuffer is
* KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
* @KBASE_ATOM_EXIT_PROTECTED_CHECK: Starting state. Check if there are any
* atoms currently submitted to GPU and protected mode
* transition is not already in progress.
* @KBASE_ATOM_EXIT_PROTECTED_IDLE_L2: Wait for the L2 to become idle in
* preparation for the reset, as exiting protected mode
* requires a reset.
* @KBASE_ATOM_EXIT_PROTECTED_RESET: Issue the reset to trigger exit from
* protected mode
* @KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT: End state, Wait for the reset to
* complete
*/
enum kbase_atom_exit_protected_state {
/*
* NOTE: The integer value of this must match
* KBASE_ATOM_ENTER_PROTECTED_CHECK.
*/
KBASE_ATOM_EXIT_PROTECTED_CHECK = 0,
KBASE_ATOM_EXIT_PROTECTED_IDLE_L2,
KBASE_ATOM_EXIT_PROTECTED_RESET,
KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT,
};
/**
* struct kbase_ext_res - Contains the info for external resources referred
* by an atom, which have been mapped on GPU side.
* @gpu_address: Start address of the memory region allocated for
* the resource from GPU virtual address space.
* @alloc: pointer to physical pages tracking object, set on
* mapping the external resource on GPU side.
*/
struct kbase_ext_res {
u64 gpu_address;
struct kbase_mem_phy_alloc *alloc;
};
/**
* struct kbase_jd_atom - object representing the atom, containing the complete
* state and attributes of an atom.
* @work: work item for the bottom half processing of the atom,
* by JD or JS, after it got executed on GPU or the
* input fence got signaled
* @start_timestamp: time at which the atom was submitted to the GPU, by
* updating the JS_HEAD_NEXTn register.
* @udata: copy of the user data sent for the atom in
* base_jd_submit.
* @kctx: Pointer to the base context with which the atom is
* associated.
* @dep_head: Array of 2 list heads, pointing to the two list of
* atoms
* which are blocked due to dependency on this atom.
* @dep_item: Array of 2 list heads, used to store the atom in the
* list of other atoms depending on the same dependee
* atom.
* @dep: Array containing the dependency info for the 2 atoms
* on which the atom depends upon.
* @jd_item: List head used during job dispatch job_done
* processing - as dependencies may not be entirely
* resolved at this point,
* we need to use a separate list head.
* @in_jd_list: flag set to true if atom's @jd_item is currently on
* a list, prevents atom being processed twice.
* @jit_ids: Zero-terminated array of IDs of just-in-time memory
* allocations written to by the atom. When the atom
* completes, the value stored at the
* &struct_base_jit_alloc_info.heap_info_gpu_addr of
* each allocation is read in order to enforce an
* overall physical memory usage limit.
* @nr_extres: number of external resources referenced by the atom.
* @extres: pointer to the location containing info about
* @nr_extres external resources referenced by the atom.
* @device_nr: indicates the coregroup with which the atom is
* associated, when
* BASE_JD_REQ_SPECIFIC_COHERENT_GROUP specified.
* @jc: GPU address of the job-chain.
* @softjob_data: Copy of data read from the user space buffer that @jc
* points to.
* @fence: Stores either an input or output sync fence,
* depending on soft-job type
* @sync_waiter: Pointer to the sync fence waiter structure passed to
* the callback function on signaling of the input
* fence.
* @dma_fence: object containing pointers to both input & output
* fences and other related members used for explicit
* sync through soft jobs and for the implicit
* synchronization required on access to external
* resources.
* @event_code: Event code for the job chain represented by the atom,
* both HW and low-level SW events are represented by
* event codes.
* @core_req: bitmask of BASE_JD_REQ_* flags specifying either
* Hw or Sw requirements for the job chain represented
* by the atom.
* @ticks: Number of scheduling ticks for which atom has been
* running on the GPU.
* @sched_priority: Priority of the atom for Job scheduling, as per the
* KBASE_JS_ATOM_SCHED_PRIO_*.
* @completed: Wait queue to wait upon for the completion of atom.
* @status: Indicates at high level at what stage the atom is in,
* as per KBASE_JD_ATOM_STATE_*, that whether it is not
* in use or its queued in JD or given to JS or
* submitted to Hw or it completed the execution on Hw.
* @work_id: used for GPU tracepoints, its a snapshot of the
* 'work_id' counter in kbase_jd_context which is
* incremented on every call to base_jd_submit.
* @slot_nr: Job slot chosen for the atom.
* @atom_flags: bitmask of KBASE_KATOM_FLAG* flags capturing the
* excat low level state of the atom.
* @gpu_rb_state: bitmnask of KBASE_ATOM_GPU_RB_* flags, precisely
* tracking atom's state after it has entered
* Job scheduler on becoming runnable. Atom
* could be blocked due to cross slot dependency
* or waiting for the shader cores to become available
* or waiting for protected mode transitions to
* complete.
* @need_cache_flush_cores_retained: flag indicating that manual flush of GPU
* cache is needed for the atom and the shader cores
* used for atom have been kept on.
* @blocked: flag indicating that atom's resubmission to GPU is
* blocked till the work item is scheduled to return the
* atom to JS.
* @pre_dep: Pointer to atom that this atom has same-slot
* dependency on
* @post_dep: Pointer to atom that has same-slot dependency on
* this atom
* @x_pre_dep: Pointer to atom that this atom has cross-slot
* dependency on
* @x_post_dep: Pointer to atom that has cross-slot dependency on
* this atom
* @flush_id: The GPU's flush count recorded at the time of
* submission,
* used for the cache flush optimization
* @fault_event: Info for dumping the debug data on Job fault.
* @queue: List head used for 4 different purposes :
* Adds atom to the list of dma-buf fence waiting atoms.
* Adds atom to the list of atoms blocked due to cross
* slot dependency.
* Adds atom to the list of softjob atoms for which JIT
* allocation has been deferred
* Adds atom to the list of softjob atoms waiting for
* the signaling of fence.
* @jit_node: Used to keep track of all JIT free/alloc jobs in
* submission order
* @jit_blocked: Flag indicating that JIT allocation requested through
* softjob atom will be reattempted after the impending
* free of other active JIT allocations.
* @will_fail_event_code: If non-zero, this indicates that the atom will fail
* with the set event_code when the atom is processed.
* Used for special handling of atoms, which have a data
* dependency on the failed atoms.
* @protected_state: State of the atom, as per
* KBASE_ATOM_(ENTER|EXIT)_PROTECTED_*,
* when transitioning into or out of protected mode.
* Atom will be either entering or exiting the
* protected mode.
* @runnable_tree_node: The node added to context's job slot specific rb tree
* when the atom becomes runnable.
* @age: Age of atom relative to other atoms in the context,
* is snapshot of the age_count counter in kbase
* context.
*/
struct kbase_jd_atom {
struct work_struct work;
ktime_t start_timestamp;
struct base_jd_udata udata;
struct kbase_context *kctx;
struct list_head dep_head[2];
struct list_head dep_item[2];
const struct kbase_jd_atom_dependency dep[2];
struct list_head jd_item;
bool in_jd_list;
#if MALI_JIT_PRESSURE_LIMIT
u8 jit_ids[2];
#endif /* MALI_JIT_PRESSURE_LIMIT */
u16 nr_extres;
struct kbase_ext_res *extres;
u32 device_nr;
u64 jc;
void *softjob_data;
#if defined(CONFIG_SYNC)
struct sync_fence *fence;
struct sync_fence_waiter sync_waiter;
#endif /* CONFIG_SYNC */
#if defined(CONFIG_MALI_BIFROST_DMA_FENCE) || defined(CONFIG_SYNC_FILE)
struct {
/* Use the functions/API defined in mali_kbase_fence.h to
* when working with this sub struct
*/
#if defined(CONFIG_SYNC_FILE)
/* Input fence */
#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
struct fence *fence_in;
#else
struct dma_fence *fence_in;
#endif
#endif
/* This points to the dma-buf output fence for this atom. If
* this is NULL then there is no fence for this atom and the
* following fields related to dma_fence may have invalid data.
*
* The context and seqno fields contain the details for this
* fence.
*
* This fence is signaled when the katom is completed,
* regardless of the event_code of the katom (signal also on
* failure).
*/
#if (KERNEL_VERSION(4, 10, 0) > LINUX_VERSION_CODE)
struct fence *fence;
#else
struct dma_fence *fence;
#endif
/* The dma-buf fence context number for this atom. A unique
* context number is allocated to each katom in the context on
* context creation.
*/
unsigned int context;
/* The dma-buf fence sequence number for this atom. This is
* increased every time this katom uses dma-buf fence.
*/
atomic_t seqno;
/* This contains a list of all callbacks set up to wait on
* other fences. This atom must be held back from JS until all
* these callbacks have been called and dep_count have reached
* 0. The initial value of dep_count must be equal to the
* number of callbacks on this list.
*
* This list is protected by jctx.lock. Callbacks are added to
* this list when the atom is built and the wait are set up.
* All the callbacks then stay on the list until all callbacks
* have been called and the atom is queued, or cancelled, and
* then all callbacks are taken off the list and freed.
*/
struct list_head callbacks;
/* Atomic counter of number of outstandind dma-buf fence
* dependencies for this atom. When dep_count reaches 0 the
* atom may be queued.
*
* The special value "-1" may only be set after the count
* reaches 0, while holding jctx.lock. This indicates that the
* atom has been handled, either queued in JS or cancelled.
*
* If anyone but the dma-fence worker sets this to -1 they must
* ensure that any potentially queued worker must have
* completed before allowing the atom to be marked as unused.
* This can be done by flushing the fence work queue:
* kctx->dma_fence.wq.
*/
atomic_t dep_count;
} dma_fence;
#endif /* CONFIG_MALI_BIFROST_DMA_FENCE || CONFIG_SYNC_FILE */
/* Note: refer to kbasep_js_atom_retained_state, which will take a copy
* of some of the following members
*/
enum base_jd_event_code event_code;
base_jd_core_req core_req;
u8 jobslot;
u8 renderpass_id;
struct base_jd_fragment jc_fragment;
u32 ticks;
int sched_priority;
wait_queue_head_t completed;
enum kbase_jd_atom_state status;
#ifdef CONFIG_GPU_TRACEPOINTS
int work_id;
#endif
int slot_nr;
u32 atom_flags;
int retry_count;
enum kbase_atom_gpu_rb_state gpu_rb_state;
bool need_cache_flush_cores_retained;
atomic_t blocked;
struct kbase_jd_atom *pre_dep;
struct kbase_jd_atom *post_dep;
struct kbase_jd_atom *x_pre_dep;
struct kbase_jd_atom *x_post_dep;
u32 flush_id;
#ifdef CONFIG_DEBUG_FS
struct base_job_fault_event fault_event;
#endif
struct list_head queue;
struct list_head jit_node;
bool jit_blocked;
enum base_jd_event_code will_fail_event_code;
union {
enum kbase_atom_enter_protected_state enter;
enum kbase_atom_exit_protected_state exit;
} protected_state;
struct rb_node runnable_tree_node;
u32 age;
};
static inline bool kbase_jd_katom_is_protected(
const struct kbase_jd_atom *katom)
{
return (bool)(katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED);
}
/*
* Theory of operations:
*
* Atom objects are statically allocated within the context structure.
*
* Each atom is the head of two lists, one for the "left" set of dependencies,
* one for the "right" set.
*/
#define KBASE_JD_DEP_QUEUE_SIZE 256
/**
* enum kbase_jd_renderpass_state - State of a renderpass
* @KBASE_JD_RP_COMPLETE: Unused or completed renderpass. Can only transition to
* START.
* @KBASE_JD_RP_START: Renderpass making a first attempt at tiling.
* Can transition to PEND_OOM or COMPLETE.
* @KBASE_JD_RP_PEND_OOM: Renderpass whose first attempt at tiling used too much
* memory and has a soft-stop pending. Can transition to
* OOM or COMPLETE.
* @KBASE_JD_RP_OOM: Renderpass whose first attempt at tiling used too much
* memory and therefore switched to incremental
* rendering. The fragment job chain is forced to run.
* Can only transition to RETRY.
* @KBASE_JD_RP_RETRY: Renderpass making a second or subsequent attempt at
* tiling. Can transition to RETRY_PEND_OOM or COMPLETE.
* @KBASE_JD_RP_RETRY_PEND_OOM: Renderpass whose second or subsequent attempt at
* tiling used too much memory again and has a
* soft-stop pending. Can transition to RETRY_OOM
* or COMPLETE.
* @KBASE_JD_RP_RETRY_OOM: Renderpass whose second or subsequent attempt at
* tiling used too much memory again. The fragment job
* chain is forced to run. Can only transition to RETRY.
*
* A state machine is used to control incremental rendering.
*/
enum kbase_jd_renderpass_state {
KBASE_JD_RP_COMPLETE, /* COMPLETE => START */
KBASE_JD_RP_START, /* START => PEND_OOM or COMPLETE */
KBASE_JD_RP_PEND_OOM, /* PEND_OOM => OOM or COMPLETE */
KBASE_JD_RP_OOM, /* OOM => RETRY */
KBASE_JD_RP_RETRY, /* RETRY => RETRY_PEND_OOM or
* COMPLETE
*/
KBASE_JD_RP_RETRY_PEND_OOM, /* RETRY_PEND_OOM => RETRY_OOM or
* COMPLETE
*/
KBASE_JD_RP_RETRY_OOM, /* RETRY_OOM => RETRY */
};
/**
* struct kbase_jd_renderpass - Data for a renderpass
* @state: Current state of the renderpass. If KBASE_JD_RP_COMPLETE then
* all other members are invalid.
* Both the job dispatcher context and hwaccess_lock must be
* locked to modify this so that it can be read with either
* (or both) locked.
* @start_katom: Address of the atom that is the start of a renderpass.
* Both the job dispatcher context and hwaccess_lock must be
* locked to modify this so that it can be read with either
* (or both) locked.
* @end_katom: Address of the atom that is the end of a renderpass, or NULL
* if that atom hasn't been added to the job scheduler yet.
* The job dispatcher context and hwaccess_lock must be
* locked to modify this so that it can be read with either
* (or both) locked.
* @oom_reg_list: A list of region structures which triggered out-of-memory.
* The hwaccess_lock must be locked to access this.
*
* Atoms tagged with BASE_JD_REQ_START_RENDERPASS or BASE_JD_REQ_END_RENDERPASS
* are associated with an object of this type, which is created and maintained
* by kbase to keep track of each renderpass.
*/
struct kbase_jd_renderpass {
enum kbase_jd_renderpass_state state;
struct kbase_jd_atom *start_katom;
struct kbase_jd_atom *end_katom;
struct list_head oom_reg_list;
};
/**
* struct kbase_jd_context - per context object encapsulating all the
* Job dispatcher related state.
* @lock: lock to serialize the updates made to the
* Job dispatcher state and kbase_jd_atom objects.
* @sched_info: Structure encapsulating all the Job scheduling
* info.
* @atoms: Array of the objects representing atoms,
* containing the complete state and attributes
* of an atom.
* @renderpasses: Array of renderpass state for incremental
* rendering, indexed by user-specified renderpass
* ID.
* @job_nr: Tracks the number of atoms being processed by the
* kbase. This includes atoms that are not tracked by
* scheduler: 'not ready to run' & 'dependency-only'
* jobs.
* @zero_jobs_wait: Waitq that reflects whether there are no jobs
* (including SW-only dependency jobs). This is set
* when no jobs are present on the ctx, and clear
* when there are jobs.
* This must be updated atomically with @job_nr.
* note: Job Dispatcher knows about more jobs than
* the Job Scheduler as it is unaware of jobs that
* are blocked on dependencies and SW-only dependency
* jobs. This waitq can be waited upon to find out
* when the context jobs are all done/cancelled
* (including those that might've been blocked
* on dependencies) - and so, whether it can be
* terminated. However, it should only be terminated
* once it is not present in the run-pool.
* Since the waitq is only set under @lock,
* the waiter should also briefly obtain and drop
* @lock to guarantee that the setter has completed
* its work on the kbase_context
* @job_done_wq: Workqueue to which the per atom work item is
* queued for bottom half processing when the
* atom completes
* execution on GPU or the input fence get signaled.
* @tb_lock: Lock to serialize the write access made to @tb to
* to store the register access trace messages.
* @tb: Pointer to the Userspace accessible buffer storing
* the trace messages for register read/write
* accesses made by the Kbase. The buffer is filled
* in circular fashion.
* @tb_wrap_offset: Offset to the end location in the trace buffer,
* the write pointer is moved to the beginning on
* reaching this offset.
* @work_id: atomic variable used for GPU tracepoints,
* incremented on every call to base_jd_submit.
* @jit_atoms_head: A list of the just-in-time memory soft-jobs, both
* allocate & free, in submission order, protected
* by kbase_jd_context.lock.
* @jit_pending_alloc: A list of just-in-time memory allocation
* soft-jobs which will be reattempted after the
* impending free of other active allocations.
*/
struct kbase_jd_context {
struct mutex lock;
struct kbasep_js_kctx_info sched_info;
struct kbase_jd_atom atoms[BASE_JD_ATOM_COUNT];
struct kbase_jd_renderpass renderpasses[BASE_JD_RP_COUNT];
struct workqueue_struct *job_done_wq;
wait_queue_head_t zero_jobs_wait;
spinlock_t tb_lock;
u32 *tb;
u32 job_nr;
size_t tb_wrap_offset;
#ifdef CONFIG_GPU_TRACEPOINTS
atomic_t work_id;
#endif
struct list_head jit_atoms_head;
struct list_head jit_pending_alloc;
};
/**
* struct jsctx_queue - JS context atom queue
* @runnable_tree: Root of RB-tree containing currently runnable atoms on this
* job slot.
* @x_dep_head: Head item of the linked list of atoms blocked on cross-slot
* dependencies. Atoms on this list will be moved to the
* runnable_tree when the blocking atom completes.
*
* hwaccess_lock must be held when accessing this structure.
*/
struct jsctx_queue {
struct rb_root runnable_tree;
struct list_head x_dep_head;
};
#endif /* _KBASE_JM_DEFS_H_ */

136
drivers/gpu/arm/bifrost/jm/mali_kbase_jm_ioctl.h Normal file
View File

@ -0,0 +1,136 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_JM_IOCTL_H_
#define _KBASE_JM_IOCTL_H_
#include <asm-generic/ioctl.h>
#include <linux/types.h>
/*
* 11.1:
* - Add BASE_MEM_TILER_ALIGN_TOP under base_mem_alloc_flags
* 11.2:
* - KBASE_MEM_QUERY_FLAGS can return KBASE_REG_PF_GROW and KBASE_REG_PROTECTED,
* which some user-side clients prior to 11.2 might fault if they received
* them
* 11.3:
* - New ioctls KBASE_IOCTL_STICKY_RESOURCE_MAP and
* KBASE_IOCTL_STICKY_RESOURCE_UNMAP
* 11.4:
* - New ioctl KBASE_IOCTL_MEM_FIND_GPU_START_AND_OFFSET
* 11.5:
* - New ioctl: KBASE_IOCTL_MEM_JIT_INIT (old ioctl renamed to _OLD)
* 11.6:
* - Added flags field to base_jit_alloc_info structure, which can be used to
* specify pseudo chunked tiler alignment for JIT allocations.
* 11.7:
* - Removed UMP support
* 11.8:
* - Added BASE_MEM_UNCACHED_GPU under base_mem_alloc_flags
* 11.9:
* - Added BASE_MEM_PERMANENT_KERNEL_MAPPING and BASE_MEM_FLAGS_KERNEL_ONLY
* under base_mem_alloc_flags
* 11.10:
* - Enabled the use of nr_extres field of base_jd_atom_v2 structure for
* JIT_ALLOC and JIT_FREE type softjobs to enable multiple JIT allocations
* with one softjob.
* 11.11:
* - Added BASE_MEM_GPU_VA_SAME_4GB_PAGE under base_mem_alloc_flags
* 11.12:
* - Removed ioctl: KBASE_IOCTL_GET_PROFILING_CONTROLS
* 11.13:
* - New ioctl: KBASE_IOCTL_MEM_EXEC_INIT
* 11.14:
* - Add BASE_MEM_GROUP_ID_MASK, base_mem_group_id_get, base_mem_group_id_set
* under base_mem_alloc_flags
* 11.15:
* - Added BASEP_CONTEXT_MMU_GROUP_ID_MASK under base_context_create_flags.
* - Require KBASE_IOCTL_SET_FLAGS before BASE_MEM_MAP_TRACKING_HANDLE can be
* passed to mmap().
* 11.16:
* - Extended ioctl KBASE_IOCTL_MEM_SYNC to accept imported dma-buf.
* - Modified (backwards compatible) ioctl KBASE_IOCTL_MEM_IMPORT behavior for
* dma-buf. Now, buffers are mapped on GPU when first imported, no longer
* requiring external resource or sticky resource tracking. UNLESS,
* CONFIG_MALI_DMA_BUF_MAP_ON_DEMAND is enabled.
* 11.17:
* - Added BASE_JD_REQ_JOB_SLOT.
* - Reused padding field in base_jd_atom_v2 to pass job slot number.
* - New ioctl: KBASE_IOCTL_GET_CPU_GPU_TIMEINFO
* 11.18:
* - Added BASE_MEM_IMPORT_SYNC_ON_MAP_UNMAP under base_mem_alloc_flags
* 11.19:
* - Extended base_jd_atom_v2 to allow a renderpass ID to be specified.
* 11.20:
* - Added new phys_pages member to kbase_ioctl_mem_jit_init for
* KBASE_IOCTL_MEM_JIT_INIT, previous variants of this renamed to use _10_2
* (replacing '_OLD') and _11_5 suffixes
* - Replaced compat_core_req (deprecated in 10.3) with jit_id[2] in
* base_jd_atom_v2. It must currently be initialized to zero.
* - Added heap_info_gpu_addr to base_jit_alloc_info, and
* BASE_JIT_ALLOC_HEAP_INFO_IS_SIZE allowable in base_jit_alloc_info's
* flags member. Previous variants of this structure are kept and given _10_2
* and _11_5 suffixes.
* - The above changes are checked for safe values in usual builds
* 11.21:
* - v2.0 of mali_trace debugfs file, which now versions the file separately
*/
#define BASE_UK_VERSION_MAJOR 11
#define BASE_UK_VERSION_MINOR 21
/**
* struct kbase_ioctl_job_submit - Submit jobs/atoms to the kernel
*
* @addr: Memory address of an array of struct base_jd_atom_v2
* @nr_atoms: Number of entries in the array
* @stride: sizeof(struct base_jd_atom_v2)
*/
struct kbase_ioctl_job_submit {
__u64 addr;
__u32 nr_atoms;
__u32 stride;
};
#define KBASE_IOCTL_JOB_SUBMIT \
_IOW(KBASE_IOCTL_TYPE, 2, struct kbase_ioctl_job_submit)
#define KBASE_IOCTL_POST_TERM \
_IO(KBASE_IOCTL_TYPE, 4)
/**
* struct kbase_ioctl_soft_event_update - Update the status of a soft-event
* @event: GPU address of the event which has been updated
* @new_status: The new status to set
* @flags: Flags for future expansion
*/
struct kbase_ioctl_soft_event_update {
__u64 event;
__u32 new_status;
__u32 flags;
};
#define KBASE_IOCTL_SOFT_EVENT_UPDATE \
_IOW(KBASE_IOCTL_TYPE, 28, struct kbase_ioctl_soft_event_update)
#endif /* _KBASE_JM_IOCTL_H_ */

892
drivers/gpu/arm/bifrost/jm/mali_kbase_jm_js.h Normal file
View File

@ -0,0 +1,892 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Job Scheduler Interface.
* These interfaces are Internal to KBase.
*/
#ifndef _KBASE_JM_JS_H_
#define _KBASE_JM_JS_H_
#include "mali_kbase_js_ctx_attr.h"
/**
* kbasep_js_devdata_init - Initialize the Job Scheduler
*
* The struct kbasep_js_device_data sub-structure of kbdev must be zero
* initialized before passing to the kbasep_js_devdata_init() function. This is
* to give efficient error path code.
*/
int kbasep_js_devdata_init(struct kbase_device * const kbdev);
/**
* kbasep_js_devdata_halt - Halt the Job Scheduler.
*
* It is safe to call this on kbdev even if it the kbasep_js_device_data
* sub-structure was never initialized/failed initialization, to give efficient
* error-path code.
*
* For this to work, the struct kbasep_js_device_data sub-structure of kbdev
* must be zero initialized before passing to the kbasep_js_devdata_init()
* function. This is to give efficient error path code.
*
* It is a programming error to call this whilst there are still kbase_context
* structures registered with this scheduler.
*
*/
void kbasep_js_devdata_halt(struct kbase_device *kbdev);
/**
* kbasep_js_devdata_term - Terminate the Job Scheduler
*
* It is safe to call this on kbdev even if it the kbasep_js_device_data
* sub-structure was never initialized/failed initialization, to give efficient
* error-path code.
*
* For this to work, the struct kbasep_js_device_data sub-structure of kbdev
* must be zero initialized before passing to the kbasep_js_devdata_init()
* function. This is to give efficient error path code.
*
* It is a programming error to call this whilst there are still kbase_context
* structures registered with this scheduler.
*/
void kbasep_js_devdata_term(struct kbase_device *kbdev);
/**
* kbasep_js_kctx_init - Initialize the Scheduling Component of a
* struct kbase_context on the Job Scheduler.
*
* This effectively registers a struct kbase_context with a Job Scheduler.
*
* It does not register any jobs owned by the struct kbase_context with
* the scheduler. Those must be separately registered by kbasep_js_add_job().
*
* The struct kbase_context must be zero initialized before passing to the
* kbase_js_init() function. This is to give efficient error path code.
*/
int kbasep_js_kctx_init(struct kbase_context *const kctx);
/**
* kbasep_js_kctx_term - Terminate the Scheduling Component of a
* struct kbase_context on the Job Scheduler
*
* This effectively de-registers a struct kbase_context from its Job Scheduler
*
* It is safe to call this on a struct kbase_context that has never had or
* failed initialization of its jctx.sched_info member, to give efficient
* error-path code.
*
* For this to work, the struct kbase_context must be zero intitialized before
* passing to the kbase_js_init() function.
*
* It is a Programming Error to call this whilst there are still jobs
* registered with this context.
*/
void kbasep_js_kctx_term(struct kbase_context *kctx);
/**
* kbasep_js_add_job - Add a job chain to the Job Scheduler,
* and take necessary actions to
* schedule the context/run the job.
*
* This atomically does the following:
* * Update the numbers of jobs information
* * Add the job to the run pool if necessary (part of init_job)
*
* Once this is done, then an appropriate action is taken:
* * If the ctx is scheduled, it attempts to start the next job (which might be
* this added job)
* * Otherwise, and if this is the first job on the context, it enqueues it on
* the Policy Queue
*
* The Policy's Queue can be updated by this in the following ways:
* * In the above case that this is the first job on the context
* * If the context is high priority and the context is not scheduled, then it
* could cause the Policy to schedule out a low-priority context, allowing
* this context to be scheduled in.
*
* If the context is already scheduled on the RunPool, then adding a job to it
* is guaranteed not to update the Policy Queue. And so, the caller is
* guaranteed to not need to try scheduling a context from the Run Pool - it
* can safely assert that the result is false.
*
* It is a programming error to have more than U32_MAX jobs in flight at a time.
*
* The following locking conditions are made on the caller:
* * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
* * it must not hold hwaccess_lock (as this will be obtained internally)
* * it must not hold kbasep_js_device_data::runpool_mutex (as this will be
* obtained internally)
* * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used
* internally).
*
* Return: true indicates that the Policy Queue was updated, and so the
* caller will need to try scheduling a context onto the Run Pool,
* false indicates that no updates were made to the Policy Queue,
* so no further action is required from the caller. This is always returned
* when the context is currently scheduled.
*/
bool kbasep_js_add_job(struct kbase_context *kctx, struct kbase_jd_atom *atom);
/**
* kbasep_js_remove_job - Remove a job chain from the Job Scheduler,
* except for its 'retained state'.
*
* Completely removing a job requires several calls:
* * kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of
* the atom
* * kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler
* * kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the
* remaining state held as part of the job having been run.
*
* In the common case of atoms completing normally, this set of actions is more
* optimal for spinlock purposes than having kbasep_js_remove_job() handle all
* of the actions.
*
* In the case of canceling atoms, it is easier to call
* kbasep_js_remove_cancelled_job(), which handles all the necessary actions.
*
* It is a programming error to call this when:
* * a atom is not a job belonging to kctx.
* * a atom has already been removed from the Job Scheduler.
* * a atom is still in the runpool
*
* Do not use this for removing jobs being killed by kbase_jd_cancel() - use
* kbasep_js_remove_cancelled_job() instead.
*
* The following locking conditions are made on the caller:
* * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
*
*/
void kbasep_js_remove_job(struct kbase_device *kbdev,
struct kbase_context *kctx, struct kbase_jd_atom *atom);
/**
* kbasep_js_remove_cancelled_job - Completely remove a job chain from the
* Job Scheduler, in the case
* where the job chain was cancelled.
*
* This is a variant of kbasep_js_remove_job() that takes care of removing all
* of the retained state too. This is generally useful for cancelled atoms,
* which need not be handled in an optimal way.
*
* It is a programming error to call this when:
* * a atom is not a job belonging to kctx.
* * a atom has already been removed from the Job Scheduler.
* * a atom is still in the runpool:
* * it is not being killed with kbasep_jd_cancel()
*
* The following locking conditions are made on the caller:
* * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
* * it must not hold the hwaccess_lock, (as this will be obtained
* internally)
* * it must not hold kbasep_js_device_data::runpool_mutex (as this could be
* obtained internally)
*
* Return: true indicates that ctx attributes have changed and the caller
* should call kbase_js_sched_all() to try to run more jobs and
* false otherwise.
*/
bool kbasep_js_remove_cancelled_job(struct kbase_device *kbdev,
struct kbase_context *kctx,
struct kbase_jd_atom *katom);
/**
* kbasep_js_runpool_requeue_or_kill_ctx - Handling the requeuing/killing of a
* context that was evicted from the
* policy queue or runpool.
*
* This should be used whenever handing off a context that has been evicted
* from the policy queue or the runpool:
* * If the context is not dying and has jobs, it gets re-added to the policy
* queue
* * Otherwise, it is not added
*
* In addition, if the context is dying the jobs are killed asynchronously.
*
* In all cases, the Power Manager active reference is released
* (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true.
* has_pm_ref must be set to false whenever the context was not previously in
* the runpool and does not hold a Power Manager active refcount. Note that
* contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an
* active refcount even though they weren't in the runpool.
*
* The following locking conditions are made on the caller:
* * it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
* * it must not hold kbasep_jd_device_data::queue_mutex (as this will be
* obtained internally)
*/
void kbasep_js_runpool_requeue_or_kill_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx, bool has_pm_ref);
/**
* kbasep_js_runpool_release_ctx - Release a refcount of a context being busy,
* allowing it to be scheduled out.
*
* When the refcount reaches zero and the context might be scheduled out
* (depending on whether the Scheduling Policy has deemed it so, or if it has
* run out of jobs).
*
* If the context does get scheduled out, then The following actions will be
* taken as part of deschduling a context:
* For the context being descheduled:
* * If the context is in the processing of dying (all the jobs are being
* removed from it), then descheduling also kills off any jobs remaining in the
* context.
* * If the context is not dying, and any jobs remain after descheduling the
* context then it is re-enqueued to the Policy's Queue.
* * Otherwise, the context is still known to the scheduler, but remains absent
* from the Policy Queue until a job is next added to it.
* * In all descheduling cases, the Power Manager active reference (obtained
* during kbasep_js_try_schedule_head_ctx()) is released
* (kbase_pm_context_idle()).
*
* Whilst the context is being descheduled, this also handles actions that
* cause more atoms to be run:
* * Attempt submitting atoms when the Context Attributes on the Runpool have
* changed. This is because the context being scheduled out could mean that
* there are more opportunities to run atoms.
* * Attempt submitting to a slot that was previously blocked due to affinity
* restrictions. This is usually only necessary when releasing a context
* happens as part of completing a previous job, but is harmless nonetheless.
* * Attempt scheduling in a new context (if one is available), and if
* necessary, running a job from that new context.
*
* Unlike retaining a context in the runpool, this function cannot be called
* from IRQ context.
*
* It is a programming error to call this on a kctx that is not currently
* scheduled, or that already has a zero refcount.
*
* The following locking conditions are made on the caller:
* * it must not hold the hwaccess_lock, because it will be used internally.
* * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
* * it must not hold kbasep_js_device_data::runpool_mutex (as this will be
* obtained internally)
* * it must not hold the kbase_device::mmu_hw_mutex (as this will be
* obtained internally)
* * it must not hold kbasep_jd_device_data::queue_mutex (as this will be
* obtained internally)
*
*/
void kbasep_js_runpool_release_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx);
/**
* kbasep_js_runpool_release_ctx_and_katom_retained_state - Variant of
* kbasep_js_runpool_release_ctx() that handles additional
* actions from completing an atom.
*
* This is usually called as part of completing an atom and releasing the
* refcount on the context held by the atom.
*
* Therefore, the extra actions carried out are part of handling actions queued
* on a completed atom, namely:
* * Releasing the atom's context attributes
* * Retrying the submission on a particular slot, because we couldn't submit
* on that slot from an IRQ handler.
*
* The locking conditions of this function are the same as those for
* kbasep_js_runpool_release_ctx()
*/
void kbasep_js_runpool_release_ctx_and_katom_retained_state(
struct kbase_device *kbdev,
struct kbase_context *kctx,
struct kbasep_js_atom_retained_state *katom_retained_state);
/**
* kbasep_js_runpool_release_ctx_nolock - Variant of
* kbase_js_runpool_release_ctx() that assumes that
* kbasep_js_device_data::runpool_mutex and
* kbasep_js_kctx_info::ctx::jsctx_mutex are held by the caller, and does not
* attempt to schedule new contexts.
*/
void kbasep_js_runpool_release_ctx_nolock(struct kbase_device *kbdev,
struct kbase_context *kctx);
/**
* kbasep_js_schedule_privileged_ctx - Schedule in a privileged context
*
* This schedules a context in regardless of the context priority.
* If the runpool is full, a context will be forced out of the runpool and the
* function will wait for the new context to be scheduled in.
* The context will be kept scheduled in (and the corresponding address space
* reserved) until kbasep_js_release_privileged_ctx is called).
*
* The following locking conditions are made on the caller:
* * it must not hold the hwaccess_lock, because it will be used internally.
* * it must not hold kbasep_js_device_data::runpool_mutex (as this will be
* obtained internally)
* * it must not hold the kbase_device::mmu_hw_mutex (as this will be
* obtained internally)
* * it must not hold kbasep_jd_device_data::queue_mutex (again, it's used
* internally).
* * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will
* be used internally.
*
*/
void kbasep_js_schedule_privileged_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx);
/**
* kbasep_js_release_privileged_ctx - Release a privileged context,
* allowing it to be scheduled out.
*
* See kbasep_js_runpool_release_ctx for potential side effects.
*
* The following locking conditions are made on the caller:
* * it must not hold the hwaccess_lock, because it will be used internally.
* * it must not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
* * it must not hold kbasep_js_device_data::runpool_mutex (as this will be
* obtained internally)
* * it must not hold the kbase_device::mmu_hw_mutex (as this will be
* obtained internally)
*
*/
void kbasep_js_release_privileged_ctx(struct kbase_device *kbdev,
struct kbase_context *kctx);
/**
* kbase_js_try_run_jobs - Try to submit the next job on each slot
*
* The following locks may be used:
* * kbasep_js_device_data::runpool_mutex
* * hwaccess_lock
*/
void kbase_js_try_run_jobs(struct kbase_device *kbdev);
/**
* kbasep_js_suspend - Suspend the job scheduler during a Power Management
* Suspend event.
*
* Causes all contexts to be removed from the runpool, and prevents any
* contexts from (re)entering the runpool.
*
* This does not handle suspending the one privileged context: the caller must
* instead do this by by suspending the GPU HW Counter Instrumentation.
*
* This will eventually cause all Power Management active references held by
* contexts on the runpool to be released, without running any more atoms.
*
* The caller must then wait for all Power Management active refcount to become
* zero before completing the suspend.
*
* The emptying mechanism may take some time to complete, since it can wait for
* jobs to complete naturally instead of forcing them to end quickly. However,
* this is bounded by the Job Scheduler's Job Timeouts. Hence, this
* function is guaranteed to complete in a finite time.
*/
void kbasep_js_suspend(struct kbase_device *kbdev);
/**
* kbasep_js_resume - Resume the Job Scheduler after a Power Management
* Resume event.
*
* This restores the actions from kbasep_js_suspend():
* * Schedules contexts back into the runpool
* * Resumes running atoms on the GPU
*/
void kbasep_js_resume(struct kbase_device *kbdev);
/**
* kbase_js_dep_resolved_submit - Submit an atom to the job scheduler.
*
* @kctx: Context pointer
* @atom: Pointer to the atom to submit
*
* The atom is enqueued on the context's ringbuffer. The caller must have
* ensured that all dependencies can be represented in the ringbuffer.
*
* Caller must hold jctx->lock
*
* Return: true if the context requires to be enqueued, otherwise false.
*/
bool kbase_js_dep_resolved_submit(struct kbase_context *kctx,
struct kbase_jd_atom *katom);
/**
* jsctx_ll_flush_to_rb() - Pushes atoms from the linked list to ringbuffer.
* @kctx: Context Pointer
* @prio: Priority (specifies the queue together with js).
* @js: Job slot (specifies the queue together with prio).
*
* Pushes all possible atoms from the linked list to the ringbuffer.
* Number of atoms are limited to free space in the ringbuffer and
* number of available atoms in the linked list.
*
*/
void jsctx_ll_flush_to_rb(struct kbase_context *kctx, int prio, int js);
/**
* kbase_js_pull - Pull an atom from a context in the job scheduler for
* execution.
*
* @kctx: Context to pull from
* @js: Job slot to pull from
*
* The atom will not be removed from the ringbuffer at this stage.
*
* The HW access lock must be held when calling this function.
*
* Return: a pointer to an atom, or NULL if there are no atoms for this
* slot that can be currently run.
*/
struct kbase_jd_atom *kbase_js_pull(struct kbase_context *kctx, int js);
/**
* kbase_js_unpull - Return an atom to the job scheduler ringbuffer.
*
* @kctx: Context pointer
* @atom: Pointer to the atom to unpull
*
* An atom is 'unpulled' if execution is stopped but intended to be returned to
* later. The most common reason for this is that the atom has been
* soft-stopped. Another reason is if an end-of-renderpass atom completed
* but will need to be run again as part of the same renderpass.
*
* Note that if multiple atoms are to be 'unpulled', they must be returned in
* the reverse order to which they were originally pulled. It is a programming
* error to return atoms in any other order.
*
* The HW access lock must be held when calling this function.
*
*/
void kbase_js_unpull(struct kbase_context *kctx, struct kbase_jd_atom *katom);
/**
* kbase_js_complete_atom_wq - Complete an atom from jd_done_worker(),
* removing it from the job
* scheduler ringbuffer.
* @kctx: Context pointer
* @katom: Pointer to the atom to complete
*
* If the atom failed then all dependee atoms marked for failure propagation
* will also fail.
*
* Return: true if the context is now idle (no jobs pulled) false otherwise.
*/
bool kbase_js_complete_atom_wq(struct kbase_context *kctx,
struct kbase_jd_atom *katom);
/**
* kbase_js_complete_atom - Complete an atom.
*
* @katom: Pointer to the atom to complete
* @end_timestamp: The time that the atom completed (may be NULL)
*
* Most of the work required to complete an atom will be performed by
* jd_done_worker().
*
* The HW access lock must be held when calling this function.
*
* Return: a atom that has now been unblocked and can now be run, or NULL
* if none
*/
struct kbase_jd_atom *kbase_js_complete_atom(struct kbase_jd_atom *katom,
ktime_t *end_timestamp);
/**
* kbase_js_atom_blocked_on_x_dep - Decide whether to ignore a cross-slot
* dependency
* @katom: Pointer to an atom in the slot ringbuffer
*
* A cross-slot dependency is ignored if necessary to unblock incremental
* rendering. If the atom at the start of a renderpass used too much memory
* and was soft-stopped then the atom at the end of a renderpass is submitted
* to hardware regardless of its dependency on the start-of-renderpass atom.
* This can happen multiple times for the same pair of atoms.
*
* Return: true to block the atom or false to allow it to be submitted to
* hardware.
*/
bool kbase_js_atom_blocked_on_x_dep(struct kbase_jd_atom *katom);
/**
* kbase_js_sched - Submit atoms from all available contexts.
*
* @kbdev: Device pointer
* @js_mask: Mask of job slots to submit to
*
* This will attempt to submit as many jobs as possible to the provided job
* slots. It will exit when either all job slots are full, or all contexts have
* been used.
*
*/
void kbase_js_sched(struct kbase_device *kbdev, int js_mask);
/**
* kbase_jd_zap_context - Attempt to deschedule a context that is being
* destroyed
* @kctx: Context pointer
*
* This will attempt to remove a context from any internal job scheduler queues
* and perform any other actions to ensure a context will not be submitted
* from.
*
* If the context is currently scheduled, then the caller must wait for all
* pending jobs to complete before taking any further action.
*/
void kbase_js_zap_context(struct kbase_context *kctx);
/**
* kbase_js_is_atom_valid - Validate an atom
*
* @kbdev: Device pointer
* @katom: Atom to validate
*
* This will determine whether the atom can be scheduled onto the GPU. Atoms
* with invalid combinations of core requirements will be rejected.
*
* Return: true if atom is valid false otherwise.
*/
bool kbase_js_is_atom_valid(struct kbase_device *kbdev,
struct kbase_jd_atom *katom);
/**
* kbase_js_set_timeouts - update all JS timeouts with user specified data
*
* @kbdev: Device pointer
*
* Timeouts are specified through the 'js_timeouts' sysfs file. If a timeout is
* set to a positive number then that becomes the new value used, if a timeout
* is negative then the default is set.
*/
void kbase_js_set_timeouts(struct kbase_device *kbdev);
/**
* kbase_js_set_ctx_priority - set the context priority
*
* @kctx: Context pointer
* @new_priority: New priority value for the Context
*
* The context priority is set to a new value and it is moved to the
* pullable/unpullable list as per the new priority.
*/
void kbase_js_set_ctx_priority(struct kbase_context *kctx, int new_priority);
/**
* kbase_js_update_ctx_priority - update the context priority
*
* @kctx: Context pointer
*
* The context priority gets updated as per the priority of atoms currently in
* use for that context, but only if system priority mode for context scheduling
* is being used.
*/
void kbase_js_update_ctx_priority(struct kbase_context *kctx);
/*
* Helpers follow
*/
/**
* kbasep_js_is_submit_allowed - Check that a context is allowed to submit
* jobs on this policy
*
* The purpose of this abstraction is to hide the underlying data size,
* and wrap up the long repeated line of code.
*
* As with any bool, never test the return value with true.
*
* The caller must hold hwaccess_lock.
*/
static inline bool kbasep_js_is_submit_allowed(
struct kbasep_js_device_data *js_devdata,
struct kbase_context *kctx)
{
u16 test_bit;
bool is_allowed;
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
test_bit = (u16) (1u << kctx->as_nr);
is_allowed = (bool) (js_devdata->runpool_irq.submit_allowed & test_bit);
dev_dbg(kctx->kbdev->dev, "JS: submit %s allowed on %p (as=%d)",
is_allowed ? "is" : "isn't", (void *)kctx, kctx->as_nr);
return is_allowed;
}
/**
* kbasep_js_set_submit_allowed - Allow a context to submit jobs on this policy
*
* The purpose of this abstraction is to hide the underlying data size,
* and wrap up the long repeated line of code.
*
* The caller must hold hwaccess_lock.
*/
static inline void kbasep_js_set_submit_allowed(
struct kbasep_js_device_data *js_devdata,
struct kbase_context *kctx)
{
u16 set_bit;
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
set_bit = (u16) (1u << kctx->as_nr);
dev_dbg(kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)",
kctx, kctx->as_nr);
js_devdata->runpool_irq.submit_allowed |= set_bit;
}
/**
* kbasep_js_clear_submit_allowed - Prevent a context from submitting more
* jobs on this policy
*
* The purpose of this abstraction is to hide the underlying data size,
* and wrap up the long repeated line of code.
*
* The caller must hold hwaccess_lock.
*/
static inline void kbasep_js_clear_submit_allowed(
struct kbasep_js_device_data *js_devdata,
struct kbase_context *kctx)
{
u16 clear_bit;
u16 clear_mask;
/* Ensure context really is scheduled in */
KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
clear_bit = (u16) (1u << kctx->as_nr);
clear_mask = ~clear_bit;
dev_dbg(kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)",
kctx, kctx->as_nr);
js_devdata->runpool_irq.submit_allowed &= clear_mask;
}
/**
* Create an initial 'invalid' atom retained state, that requires no
* atom-related work to be done on releasing with
* kbasep_js_runpool_release_ctx_and_katom_retained_state()
*/
static inline void kbasep_js_atom_retained_state_init_invalid(
struct kbasep_js_atom_retained_state *retained_state)
{
retained_state->event_code = BASE_JD_EVENT_NOT_STARTED;
retained_state->core_req =
KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID;
}
/**
* Copy atom state that can be made available after jd_done_nolock() is called
* on that atom.
*/
static inline void kbasep_js_atom_retained_state_copy(
struct kbasep_js_atom_retained_state *retained_state,
const struct kbase_jd_atom *katom)
{
retained_state->event_code = katom->event_code;
retained_state->core_req = katom->core_req;
retained_state->sched_priority = katom->sched_priority;
retained_state->device_nr = katom->device_nr;
}
/**
* kbasep_js_has_atom_finished - Determine whether an atom has finished
* (given its retained state),
* and so should be given back to
* userspace/removed from the system.
*
* @katom_retained_state: the retained state of the atom to check
*
* Reasons for an atom not finishing include:
* * Being soft-stopped (and so, the atom should be resubmitted sometime later)
* * It is an end of renderpass atom that was run to consume the output of a
* start-of-renderpass atom that was soft-stopped because it used too much
* memory. In this case, it will have to be run again later.
*
* Return: false if the atom has not finished, true otherwise.
*/
static inline bool kbasep_js_has_atom_finished(
const struct kbasep_js_atom_retained_state *katom_retained_state)
{
return (bool) (katom_retained_state->event_code !=
BASE_JD_EVENT_STOPPED &&
katom_retained_state->event_code !=
BASE_JD_EVENT_REMOVED_FROM_NEXT &&
katom_retained_state->event_code !=
BASE_JD_EVENT_END_RP_DONE);
}
/**
* kbasep_js_atom_retained_state_is_valid - Determine whether a struct
* kbasep_js_atom_retained_state
* is valid
* @katom_retained_state the atom's retained state to check
*
* An invalid struct kbasep_js_atom_retained_state is allowed, and indicates
* that the code should just ignore it.
*
* Return: false if the retained state is invalid, true otherwise.
*/
static inline bool kbasep_js_atom_retained_state_is_valid(
const struct kbasep_js_atom_retained_state *katom_retained_state)
{
return (bool) (katom_retained_state->core_req !=
KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID);
}
/**
* kbase_js_runpool_inc_context_count - Increment number of running contexts.
*
* The following locking conditions are made on the caller:
* * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
* * The caller must hold the kbasep_js_device_data::runpool_mutex
*/
static inline void kbase_js_runpool_inc_context_count(
struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_devdata = &kbdev->js_data;
js_kctx_info = &kctx->jctx.sched_info;
lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
lockdep_assert_held(&js_devdata->runpool_mutex);
/* Track total contexts */
KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running < S8_MAX);
++(js_devdata->nr_all_contexts_running);
if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
/* Track contexts that can submit jobs */
KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running <
S8_MAX);
++(js_devdata->nr_user_contexts_running);
}
}
/**
* kbase_js_runpool_dec_context_count - decrement number of running contexts.
*
* The following locking conditions are made on the caller:
* * The caller must hold the kbasep_js_kctx_info::ctx::jsctx_mutex.
* * The caller must hold the kbasep_js_device_data::runpool_mutex
*/
static inline void kbase_js_runpool_dec_context_count(
struct kbase_device *kbdev,
struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_kctx_info *js_kctx_info;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
js_devdata = &kbdev->js_data;
js_kctx_info = &kctx->jctx.sched_info;
lockdep_assert_held(&js_kctx_info->ctx.jsctx_mutex);
lockdep_assert_held(&js_devdata->runpool_mutex);
/* Track total contexts */
--(js_devdata->nr_all_contexts_running);
KBASE_DEBUG_ASSERT(js_devdata->nr_all_contexts_running >= 0);
if (!kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
/* Track contexts that can submit jobs */
--(js_devdata->nr_user_contexts_running);
KBASE_DEBUG_ASSERT(js_devdata->nr_user_contexts_running >= 0);
}
}
/**
* kbase_js_sched_all - Submit atoms from all available contexts to all
* job slots.
*
* @kbdev: Device pointer
*
* This will attempt to submit as many jobs as possible. It will exit when
* either all job slots are full, or all contexts have been used.
*/
static inline void kbase_js_sched_all(struct kbase_device *kbdev)
{
kbase_js_sched(kbdev, (1 << kbdev->gpu_props.num_job_slots) - 1);
}
extern const int
kbasep_js_atom_priority_to_relative[BASE_JD_NR_PRIO_LEVELS];
extern const base_jd_prio
kbasep_js_relative_priority_to_atom[KBASE_JS_ATOM_SCHED_PRIO_COUNT];
/**
* kbasep_js_atom_prio_to_sched_prio(): - Convert atom priority (base_jd_prio)
* to relative ordering
* @atom_prio: Priority ID to translate.
*
* Atom priority values for @ref base_jd_prio cannot be compared directly to
* find out which are higher or lower.
*
* This function will convert base_jd_prio values for successively lower
* priorities into a monotonically increasing sequence. That is, the lower the
* base_jd_prio priority, the higher the value produced by this function. This
* is in accordance with how the rest of the kernel treats priority.
*
* The mapping is 1:1 and the size of the valid input range is the same as the
* size of the valid output range, i.e.
* KBASE_JS_ATOM_SCHED_PRIO_COUNT == BASE_JD_NR_PRIO_LEVELS
*
* Note This must be kept in sync with BASE_JD_PRIO_<...> definitions
*
* Return: On success: a value in the inclusive range
* 0..KBASE_JS_ATOM_SCHED_PRIO_COUNT-1. On failure:
* KBASE_JS_ATOM_SCHED_PRIO_INVALID
*/
static inline int kbasep_js_atom_prio_to_sched_prio(base_jd_prio atom_prio)
{
if (atom_prio >= BASE_JD_NR_PRIO_LEVELS)
return KBASE_JS_ATOM_SCHED_PRIO_INVALID;
return kbasep_js_atom_priority_to_relative[atom_prio];
}
static inline base_jd_prio kbasep_js_sched_prio_to_atom_prio(int sched_prio)
{
unsigned int prio_idx;
KBASE_DEBUG_ASSERT(sched_prio >= 0 &&
sched_prio < KBASE_JS_ATOM_SCHED_PRIO_COUNT);
prio_idx = (unsigned int)sched_prio;
return kbasep_js_relative_priority_to_atom[prio_idx];
}
#endif /* _KBASE_JM_JS_H_ */

51
drivers/gpu/arm/bifrost/mali_kbase_js_defs.h → drivers/gpu/arm/bifrost/jm/mali_kbase_js_defs.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -21,7 +21,6 @@
*/
/**
* @file mali_kbase_js.h
* Job Scheduler Type Definitions
@ -51,12 +50,6 @@ struct kbase_jd_atom;
typedef u32 kbase_context_flags;
struct kbasep_atom_req {
base_jd_core_req core_req;
kbase_context_flags ctx_req;
u32 device_nr;
};
/** Callback function run on all of a context's jobs registered with the Job
* Scheduler */
typedef void (*kbasep_js_ctx_job_cb)(struct kbase_device *kbdev, struct kbase_jd_atom *katom);
@ -245,24 +238,6 @@ struct kbasep_js_device_data {
s8 slot_affinity_refcount[BASE_JM_MAX_NR_SLOTS][64];
} runpool_irq;
/**
* Run Pool mutex, for managing contexts within the runpool.
* Unless otherwise specified, you must hold this lock whilst accessing any
* members that follow
*
* In addition, this is used to access:
* - the kbasep_js_kctx_info::runpool substructure
*/
struct mutex runpool_mutex;
/**
* Queue Lock, used to access the Policy's queue of contexts independently
* of the Run Pool.
*
* Of course, you don't need the Run Pool lock to access this.
*/
struct mutex queue_mutex;
/**
* Scheduling semaphore. This must be held when calling
* kbase_jm_kick()
@ -299,9 +274,6 @@ struct kbasep_js_device_data {
u32 gpu_reset_ticks_dumping; /*< Value for JS_RESET_TICKS_DUMPING */
u32 ctx_timeslice_ns; /**< Value for JS_CTX_TIMESLICE_NS */
/**< Value for JS_SOFT_JOB_TIMEOUT */
atomic_t soft_job_timeout_ms;
/** List of suspended soft jobs */
struct list_head suspended_soft_jobs_list;
@ -321,6 +293,27 @@ struct kbasep_js_device_data {
/* Number of contexts that can either be pulled from or are currently
* running */
atomic_t nr_contexts_runnable;
/** Value for JS_SOFT_JOB_TIMEOUT */
atomic_t soft_job_timeout_ms;
/**
* Queue Lock, used to access the Policy's queue of contexts
* independently of the Run Pool.
*
* Of course, you don't need the Run Pool lock to access this.
*/
struct mutex queue_mutex;
/**
* Run Pool mutex, for managing contexts within the runpool.
* Unless otherwise specified, you must hold this lock whilst accessing
* any members that follow
*
* In addition, this is used to access:
* * the kbasep_js_kctx_info::runpool substructure
*/
struct mutex runpool_mutex;
};
/**

102
drivers/gpu/arm/bifrost/mali_base_hwconfig_features.h
View File

@ -48,7 +48,6 @@ enum base_hw_feature {
BASE_HW_FEATURE_BRNDOUT_KILL,
BASE_HW_FEATURE_WARPING,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_AARCH64_MMU,
@ -85,7 +84,6 @@ static const enum base_hw_feature base_hw_features_tMIx[] = {
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_THREAD_GROUP_SPLIT,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
BASE_HW_FEATURE_END
@ -112,7 +110,6 @@ static const enum base_hw_feature base_hw_features_tHEx[] = {
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_THREAD_GROUP_SPLIT,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -140,7 +137,6 @@ static const enum base_hw_feature base_hw_features_tSIx[] = {
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_THREAD_GROUP_SPLIT,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -168,7 +164,6 @@ static const enum base_hw_feature base_hw_features_tDVx[] = {
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_THREAD_GROUP_SPLIT,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -196,7 +191,6 @@ static const enum base_hw_feature base_hw_features_tNOx[] = {
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_THREAD_GROUP_SPLIT,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -226,7 +220,6 @@ static const enum base_hw_feature base_hw_features_tGOx[] = {
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_THREAD_GROUP_SPLIT,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -255,7 +248,6 @@ static const enum base_hw_feature base_hw_features_tTRx[] = {
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -284,7 +276,6 @@ static const enum base_hw_feature base_hw_features_tNAx[] = {
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -313,7 +304,6 @@ static const enum base_hw_feature base_hw_features_tBEx[] = {
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -323,35 +313,6 @@ static const enum base_hw_feature base_hw_features_tBEx[] = {
BASE_HW_FEATURE_END
};
static const enum base_hw_feature base_hw_features_tULx[] = {
BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
BASE_HW_FEATURE_XAFFINITY,
BASE_HW_FEATURE_WARPING,
BASE_HW_FEATURE_INTERPIPE_REG_ALIASING,
BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
BASE_HW_FEATURE_BRNDOUT_CC,
BASE_HW_FEATURE_BRNDOUT_KILL,
BASE_HW_FEATURE_LD_ST_LEA_TEX,
BASE_HW_FEATURE_LD_ST_TILEBUFFER,
BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
BASE_HW_FEATURE_MRT,
BASE_HW_FEATURE_MSAA_16X,
BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
BASE_HW_FEATURE_OUT_OF_ORDER_EXEC,
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
BASE_HW_FEATURE_L2_CONFIG,
BASE_HW_FEATURE_CLEAN_ONLY_SAFE,
BASE_HW_FEATURE_END
};
static const enum base_hw_feature base_hw_features_tDUx[] = {
BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
@ -372,7 +333,6 @@ static const enum base_hw_feature base_hw_features_tDUx[] = {
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -402,7 +362,6 @@ static const enum base_hw_feature base_hw_features_tODx[] = {
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -411,7 +370,7 @@ static const enum base_hw_feature base_hw_features_tODx[] = {
BASE_HW_FEATURE_END
};
static const enum base_hw_feature base_hw_features_tIDx[] = {
static const enum base_hw_feature base_hw_features_tGRx[] = {
BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
BASE_HW_FEATURE_XAFFINITY,
@ -431,7 +390,6 @@ static const enum base_hw_feature base_hw_features_tIDx[] = {
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -460,7 +418,6 @@ static const enum base_hw_feature base_hw_features_tVAx[] = {
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_MODE,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
@ -469,4 +426,61 @@ static const enum base_hw_feature base_hw_features_tVAx[] = {
BASE_HW_FEATURE_END
};
static const enum base_hw_feature base_hw_features_tTUx[] = {
BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
BASE_HW_FEATURE_XAFFINITY,
BASE_HW_FEATURE_WARPING,
BASE_HW_FEATURE_INTERPIPE_REG_ALIASING,
BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
BASE_HW_FEATURE_BRNDOUT_CC,
BASE_HW_FEATURE_BRNDOUT_KILL,
BASE_HW_FEATURE_LD_ST_LEA_TEX,
BASE_HW_FEATURE_LD_ST_TILEBUFFER,
BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
BASE_HW_FEATURE_MRT,
BASE_HW_FEATURE_MSAA_16X,
BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
BASE_HW_FEATURE_OUT_OF_ORDER_EXEC,
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
BASE_HW_FEATURE_L2_CONFIG,
BASE_HW_FEATURE_CLEAN_ONLY_SAFE,
BASE_HW_FEATURE_END
};
static const enum base_hw_feature base_hw_features_tE2x[] = {
BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION,
BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS,
BASE_HW_FEATURE_XAFFINITY,
BASE_HW_FEATURE_WARPING,
BASE_HW_FEATURE_INTERPIPE_REG_ALIASING,
BASE_HW_FEATURE_32_BIT_UNIFORM_ADDRESS,
BASE_HW_FEATURE_ATTR_AUTO_TYPE_INFERRAL,
BASE_HW_FEATURE_BRNDOUT_CC,
BASE_HW_FEATURE_BRNDOUT_KILL,
BASE_HW_FEATURE_LD_ST_LEA_TEX,
BASE_HW_FEATURE_LD_ST_TILEBUFFER,
BASE_HW_FEATURE_LINEAR_FILTER_FLOAT,
BASE_HW_FEATURE_MRT,
BASE_HW_FEATURE_MSAA_16X,
BASE_HW_FEATURE_NEXT_INSTRUCTION_TYPE,
BASE_HW_FEATURE_OUT_OF_ORDER_EXEC,
BASE_HW_FEATURE_T7XX_PAIRING_RULES,
BASE_HW_FEATURE_TEST4_DATUM_MODE,
BASE_HW_FEATURE_FLUSH_REDUCTION,
BASE_HW_FEATURE_PROTECTED_DEBUG_MODE,
BASE_HW_FEATURE_COHERENCY_REG,
BASE_HW_FEATURE_AARCH64_MMU,
BASE_HW_FEATURE_IDVS_GROUP_SIZE,
BASE_HW_FEATURE_L2_CONFIG,
BASE_HW_FEATURE_CLEAN_ONLY_SAFE,
BASE_HW_FEATURE_END
};
#endif /* _BASE_HWCONFIG_FEATURES_H_ */

160
drivers/gpu/arm/bifrost/mali_base_hwconfig_issues.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -53,7 +53,12 @@ enum base_hw_issue {
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_3076,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@ -206,6 +211,7 @@ static const enum base_hw_issue base_hw_issues_tSIx_r0p0[] = {
BASE_HW_ISSUE_TSIX_1792,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -218,6 +224,7 @@ static const enum base_hw_issue base_hw_issues_tSIx_r0p1[] = {
BASE_HW_ISSUE_TSIX_1792,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -229,6 +236,7 @@ static const enum base_hw_issue base_hw_issues_tSIx_r1p0[] = {
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -239,6 +247,7 @@ static const enum base_hw_issue base_hw_issues_tSIx_r1p1[] = {
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -248,6 +257,7 @@ static const enum base_hw_issue base_hw_issues_model_tSIx[] = {
BASE_HW_ISSUE_TMIX_8133,
BASE_HW_ISSUE_TSIX_1116,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -258,6 +268,7 @@ static const enum base_hw_issue base_hw_issues_tDVx_r0p0[] = {
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -267,6 +278,7 @@ static const enum base_hw_issue base_hw_issues_model_tDVx[] = {
BASE_HW_ISSUE_TMIX_8133,
BASE_HW_ISSUE_TSIX_1116,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -278,6 +290,7 @@ static const enum base_hw_issue base_hw_issues_tNOx_r0p0[] = {
BASE_HW_ISSUE_TNOX_1194,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -287,6 +300,7 @@ static const enum base_hw_issue base_hw_issues_model_tNOx[] = {
BASE_HW_ISSUE_TMIX_8133,
BASE_HW_ISSUE_TSIX_1116,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -298,6 +312,7 @@ static const enum base_hw_issue base_hw_issues_tGOx_r0p0[] = {
BASE_HW_ISSUE_TNOX_1194,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -309,6 +324,7 @@ static const enum base_hw_issue base_hw_issues_tGOx_r1p0[] = {
BASE_HW_ISSUE_TGOX_R1_1234,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -318,6 +334,7 @@ static const enum base_hw_issue base_hw_issues_model_tGOx[] = {
BASE_HW_ISSUE_TMIX_8133,
BASE_HW_ISSUE_TSIX_1116,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -328,7 +345,12 @@ static const enum base_hw_issue base_hw_issues_tTRx_r0p0[] = {
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_3076,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@ -339,7 +361,27 @@ static const enum base_hw_issue base_hw_issues_tTRx_r0p1[] = {
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_3076,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_tTRx_r0p2[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_3076,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -348,6 +390,10 @@ static const enum base_hw_issue base_hw_issues_model_tTRx[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -358,7 +404,12 @@ static const enum base_hw_issue base_hw_issues_tNAx_r0p0[] = {
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_3076,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
@ -369,7 +420,11 @@ static const enum base_hw_issue base_hw_issues_tNAx_r0p1[] = {
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_3076,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_GPU2017_1336,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -378,6 +433,10 @@ static const enum base_hw_issue base_hw_issues_model_tNAx[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -387,6 +446,24 @@ static const enum base_hw_issue base_hw_issues_tBEx_r0p0[] = {
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_tBEx_r0p1[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -396,6 +473,23 @@ static const enum base_hw_issue base_hw_issues_tBEx_r1p0[] = {
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_tBEx_r1p1[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -404,22 +498,37 @@ static const enum base_hw_issue base_hw_issues_model_tBEx[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_tULx_r0p0[] = {
static const enum base_hw_issue base_hw_issues_lBEx_r1p0[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_TTRX_3485,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_model_tULx[] = {
BASE_HW_ISSUE_5736,
static const enum base_hw_issue base_hw_issues_lBEx_r1p1[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_2968_TTRX_3162,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_TTRX_3470,
BASE_HW_ISSUE_TTRX_3464,
BASE_HW_ISSUE_END
};
@ -428,6 +537,8 @@ static const enum base_hw_issue base_hw_issues_tDUx_r0p0[] = {
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_END
};
@ -436,6 +547,8 @@ static const enum base_hw_issue base_hw_issues_model_tDUx[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_END
};
@ -454,14 +567,14 @@ static const enum base_hw_issue base_hw_issues_model_tODx[] = {
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_tIDx_r0p0[] = {
static const enum base_hw_issue base_hw_issues_tGRx_r0p0[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_model_tIDx[] = {
static const enum base_hw_issue base_hw_issues_model_tGRx[] = {
BASE_HW_ISSUE_5736,
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
@ -484,4 +597,39 @@ static const enum base_hw_issue base_hw_issues_model_tVAx[] = {
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_tTUx_r0p0[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_model_tTUx[] = {
BASE_HW_ISSUE_5736,
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_tE2x_r0p0[] = {
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_921,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_END
};
static const enum base_hw_issue base_hw_issues_model_tE2x[] = {
BASE_HW_ISSUE_5736,
BASE_HW_ISSUE_9435,
BASE_HW_ISSUE_TSIX_2033,
BASE_HW_ISSUE_TTRX_1337,
BASE_HW_ISSUE_TTRX_3414,
BASE_HW_ISSUE_TTRX_3083,
BASE_HW_ISSUE_END
};
#endif /* _BASE_HWCONFIG_ISSUES_H_ */

1361
drivers/gpu/arm/bifrost/mali_base_kernel.h
View File

File diff suppressed because it is too large Load Diff

4
drivers/gpu/arm/bifrost/mali_base_mem_priv.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2014 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2015, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -47,7 +47,7 @@
* - offset is ignored.
*/
struct basep_syncset {
base_mem_handle mem_handle;
struct base_mem_handle mem_handle;
u64 user_addr;
u64 size;
u8 type;

335
drivers/gpu/arm/bifrost/mali_kbase.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -56,19 +56,21 @@
*/
#include "mali_kbase_defs.h"
#include "mali_kbase_context.h"
#include "debug/mali_kbase_debug_ktrace.h"
#include "context/mali_kbase_context.h"
#include "mali_kbase_strings.h"
#include "mali_kbase_mem_lowlevel.h"
#include "mali_kbase_js.h"
#include "mali_kbase_utility.h"
#include "mali_kbase_mem.h"
#include "mmu/mali_kbase_mmu.h"
#include "mali_kbase_gpu_memory_debugfs.h"
#include "mali_kbase_mem_profile_debugfs.h"
#include "mali_kbase_gpuprops.h"
#include "mali_kbase_ioctl.h"
#include "mali_kbase_debug_job_fault.h"
#include "mali_kbase_jd_debugfs.h"
#include "mali_kbase_gpuprops.h"
#include "mali_kbase_jm.h"
#include "mali_kbase_ioctl.h"
#include "mali_kbase_js.h"
#include "ipa/mali_kbase_ipa.h"
@ -76,6 +78,8 @@
#include <trace/events/gpu.h>
#endif
#include "mali_linux_trace.h"
#ifndef u64_to_user_ptr
/* Introduced in Linux v4.6 */
@ -97,16 +101,8 @@ struct kbase_device *kbase_device_alloc(void);
* been setup before calling kbase_device_init
*/
/*
* API to acquire device list semaphore and return pointer
* to the device list head
*/
const struct list_head *kbase_dev_list_get(void);
/* API to release the device list semaphore */
void kbase_dev_list_put(const struct list_head *dev_list);
int kbase_device_init(struct kbase_device * const kbdev);
void kbase_device_term(struct kbase_device *kbdev);
int kbase_device_misc_init(struct kbase_device *kbdev);
void kbase_device_misc_term(struct kbase_device *kbdev);
void kbase_device_free(struct kbase_device *kbdev);
int kbase_device_has_feature(struct kbase_device *kbdev, u32 feature);
@ -156,6 +152,60 @@ unsigned long kbase_context_get_unmapped_area(struct kbase_context *kctx,
const unsigned long addr, const unsigned long len,
const unsigned long pgoff, const unsigned long flags);
int assign_irqs(struct kbase_device *kbdev);
int kbase_sysfs_init(struct kbase_device *kbdev);
void kbase_sysfs_term(struct kbase_device *kbdev);
int kbase_protected_mode_init(struct kbase_device *kbdev);
void kbase_protected_mode_term(struct kbase_device *kbdev);
/**
* kbase_device_pm_init() - Performs power management initialization and
* Verifies device tree configurations.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Return: 0 if successful, otherwise a standard Linux error code
*/
int kbase_device_pm_init(struct kbase_device *kbdev);
/**
* kbase_device_pm_term() - Performs power management deinitialization and
* Free resources.
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Clean up all the resources
*/
void kbase_device_pm_term(struct kbase_device *kbdev);
int power_control_init(struct kbase_device *kbdev);
void power_control_term(struct kbase_device *kbdev);
#ifdef CONFIG_DEBUG_FS
void kbase_device_debugfs_term(struct kbase_device *kbdev);
int kbase_device_debugfs_init(struct kbase_device *kbdev);
#else /* CONFIG_DEBUG_FS */
static inline int kbase_device_debugfs_init(struct kbase_device *kbdev)
{
return 0;
}
static inline void kbase_device_debugfs_term(struct kbase_device *kbdev) { }
#endif /* CONFIG_DEBUG_FS */
int registers_map(struct kbase_device *kbdev);
void registers_unmap(struct kbase_device *kbdev);
int kbase_device_coherency_init(struct kbase_device *kbdev);
#ifdef CONFIG_MALI_BUSLOG
int buslog_init(struct kbase_device *kbdev);
void buslog_term(struct kbase_device *kbdev);
#endif
int kbase_jd_init(struct kbase_context *kctx);
void kbase_jd_exit(struct kbase_context *kctx);
@ -200,9 +250,6 @@ void kbase_jd_zap_context(struct kbase_context *kctx);
bool jd_done_nolock(struct kbase_jd_atom *katom,
struct list_head *completed_jobs_ctx);
void kbase_jd_free_external_resources(struct kbase_jd_atom *katom);
bool jd_submit_atom(struct kbase_context *kctx,
const struct base_jd_atom_v2 *user_atom,
struct kbase_jd_atom *katom);
void kbase_jd_dep_clear_locked(struct kbase_jd_atom *katom);
/**
@ -232,6 +279,22 @@ void kbase_job_done(struct kbase_device *kbdev, u32 done);
void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
struct kbase_jd_atom *katom);
/**
* kbase_job_slot_softstop_start_rp() - Soft-stop the atom at the start
* of a renderpass.
* @kctx: Pointer to a kernel base context.
* @reg: Reference of a growable GPU memory region in the same context.
* Takes ownership of the reference if successful.
*
* Used to switch to incremental rendering if we have nearly run out of
* virtual address space in a growable memory region and the atom currently
* executing on a job slot is the tiler job chain at the start of a renderpass.
*
* Return 0 if successful, otherwise a negative error code.
*/
int kbase_job_slot_softstop_start_rp(struct kbase_context *kctx,
struct kbase_va_region *reg);
void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
struct kbase_jd_atom *target_katom);
void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
@ -243,6 +306,7 @@ void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
struct kbase_jd_atom *target_katom);
void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *event);
int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent);
int kbase_event_pending(struct kbase_context *ctx);
@ -262,6 +326,16 @@ void kbase_event_wakeup(struct kbase_context *kctx);
*/
int kbasep_jit_alloc_validate(struct kbase_context *kctx,
struct base_jit_alloc_info *info);
/**
* kbase_jit_retry_pending_alloc() - Retry blocked just-in-time memory
* allocations.
*
* @kctx: Pointer to the kbase context within which the just-in-time
* memory allocations are to be retried.
*/
void kbase_jit_retry_pending_alloc(struct kbase_context *kctx);
/**
* kbase_free_user_buffer() - Free memory allocated for struct
* @kbase_debug_copy_buffer.
@ -288,23 +362,6 @@ static inline void kbase_free_user_buffer(
}
}
/**
* kbase_mem_copy_from_extres_page() - Copy pages from external resources.
*
* @kctx: kbase context within which the copying is to take place.
* @extres_pages: Pointer to the pages which correspond to the external
* resources from which the copying will take place.
* @pages: Pointer to the pages to which the content is to be
* copied from the provided external resources.
* @nr_pages: Number of pages to copy.
* @target_page_nr: Number of target pages which will be used for copying.
* @offset: Offset into the target pages from which the copying
* is to be performed.
* @to_copy: Size of the chunk to be copied, in bytes.
*/
void kbase_mem_copy_from_extres_page(struct kbase_context *kctx,
void *extres_page, struct page **pages, unsigned int nr_pages,
unsigned int *target_page_nr, size_t offset, size_t *to_copy);
/**
* kbase_mem_copy_from_extres() - Copy from external resources.
*
@ -333,18 +390,6 @@ void kbasep_complete_triggered_soft_events(struct kbase_context *kctx, u64 evt);
void kbasep_as_do_poke(struct work_struct *work);
/** Returns the name associated with a Mali exception code
*
* This function is called from the interrupt handler when a GPU fault occurs.
* It reports the details of the fault using KBASE_DEBUG_PRINT_WARN.
*
* @param[in] kbdev The kbase device that the GPU fault occurred from.
* @param[in] exception_code exception code
* @return name associated with the exception code
*/
const char *kbase_exception_name(struct kbase_device *kbdev,
u32 exception_code);
/**
* Check whether a system suspend is in progress, or has already been suspended
*
@ -360,6 +405,24 @@ static inline bool kbase_pm_is_suspending(struct kbase_device *kbdev)
return kbdev->pm.suspending;
}
#ifdef CONFIG_MALI_ARBITER_SUPPORT
/*
* Check whether a gpu lost is in progress
*
* @kbdev: The kbase device structure for the device (must be a valid pointer)
*
* Indicates whether a gpu lost has been received and jobs are no longer
* being scheduled
*
* Return: false if gpu is lost
* Return: != false otherwise
*/
static inline bool kbase_pm_is_gpu_lost(struct kbase_device *kbdev)
{
return kbdev->pm.gpu_lost;
}
#endif
/**
* kbase_pm_is_active - Determine whether the GPU is active
*
@ -446,6 +509,8 @@ static inline struct kbase_jd_atom *kbase_jd_atom_from_id(
* and during context creation.
*
* @param kbdev The kbase device
*
* Return: 0 on success and non-zero value on failure.
*/
void kbase_disjoint_init(struct kbase_device *kbdev);
@ -507,181 +572,6 @@ void kbase_disjoint_state_down(struct kbase_device *kbdev);
#define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL)
#endif
#if KBASE_TRACE_ENABLE
void kbasep_trace_debugfs_init(struct kbase_device *kbdev);
#ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE
/** Add trace values about a job-slot
*
* @note Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
* functions called to get the parameters supplied to this macro must:
* - be static or static inline
* - must just return 0 and have no other statements present in the body.
*/
#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot) \
kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, 0)
/** Add trace values about a job-slot, with info
*
* @note Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
* functions called to get the parameters supplied to this macro must:
* - be static or static inline
* - must just return 0 and have no other statements present in the body.
*/
#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val) \
kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
KBASE_TRACE_FLAG_JOBSLOT, 0, jobslot, info_val)
/** Add trace values about a ctx refcount
*
* @note Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
* functions called to get the parameters supplied to this macro must:
* - be static or static inline
* - must just return 0 and have no other statements present in the body.
*/
#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount) \
kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, 0)
/** Add trace values about a ctx refcount, and info
*
* @note Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
* functions called to get the parameters supplied to this macro must:
* - be static or static inline
* - must just return 0 and have no other statements present in the body.
*/
#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val) \
kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
KBASE_TRACE_FLAG_REFCOUNT, refcount, 0, info_val)
/** Add trace values (no slot or refcount)
*
* @note Any functions called through this macro will still be evaluated in
* Release builds (CONFIG_MALI_BIFROST_DEBUG not defined). Therefore, when KBASE_TRACE_ENABLE == 0 any
* functions called to get the parameters supplied to this macro must:
* - be static or static inline
* - must just return 0 and have no other statements present in the body.
*/
#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val) \
kbasep_trace_add(kbdev, KBASE_TRACE_CODE(code), ctx, katom, gpu_addr, \
0, 0, 0, info_val)
/** Clear the trace */
#define KBASE_TRACE_CLEAR(kbdev) \
kbasep_trace_clear(kbdev)
/** Dump the slot trace */
#define KBASE_TRACE_DUMP(kbdev) \
kbasep_trace_dump(kbdev)
/** PRIVATE - do not use directly. Use KBASE_TRACE_ADD() instead */
void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val);
/** PRIVATE - do not use directly. Use KBASE_TRACE_CLEAR() instead */
void kbasep_trace_clear(struct kbase_device *kbdev);
#else /* #ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE */
/* Dispatch kbase trace events as system trace events */
#include <mali_linux_kbase_trace.h>
#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
trace_mali_##code(jobslot, 0)
#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
trace_mali_##code(jobslot, info_val)
#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
trace_mali_##code(refcount, 0)
#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
trace_mali_##code(refcount, info_val)
#define KBASE_TRACE_ADD(kbdev, code, ctx, katom, gpu_addr, info_val)\
trace_mali_##code(gpu_addr, info_val)
#define KBASE_TRACE_CLEAR(kbdev)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(0);\
} while (0)
#define KBASE_TRACE_DUMP(kbdev)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(0);\
} while (0)
#endif /* #ifndef CONFIG_MALI_BIFROST_SYSTEM_TRACE */
#else
#define KBASE_TRACE_ADD_SLOT(kbdev, code, ctx, katom, gpu_addr, jobslot)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(ctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(jobslot);\
} while (0)
#define KBASE_TRACE_ADD_SLOT_INFO(kbdev, code, ctx, katom, gpu_addr, jobslot, info_val)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(ctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(jobslot);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#define KBASE_TRACE_ADD_REFCOUNT(kbdev, code, ctx, katom, gpu_addr, refcount)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(ctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(refcount);\
CSTD_NOP(0);\
} while (0)
#define KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, code, ctx, katom, gpu_addr, refcount, info_val)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(ctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(gpu_addr);\
CSTD_UNUSED(info_val);\
CSTD_NOP(0);\
} while (0)
#define KBASE_TRACE_ADD(kbdev, code, subcode, ctx, katom, val)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(code);\
CSTD_UNUSED(subcode);\
CSTD_UNUSED(ctx);\
CSTD_UNUSED(katom);\
CSTD_UNUSED(val);\
CSTD_NOP(0);\
} while (0)
#define KBASE_TRACE_CLEAR(kbdev)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(0);\
} while (0)
#define KBASE_TRACE_DUMP(kbdev)\
do {\
CSTD_UNUSED(kbdev);\
CSTD_NOP(0);\
} while (0)
#endif /* KBASE_TRACE_ENABLE */
/** PRIVATE - do not use directly. Use KBASE_TRACE_DUMP() instead */
void kbasep_trace_dump(struct kbase_device *kbdev);
#if defined(CONFIG_DEBUG_FS) && !defined(CONFIG_MALI_BIFROST_NO_MALI)
/* kbase_io_history_init - initialize data struct for register access history
@ -731,5 +621,4 @@ int kbase_io_history_resize(struct kbase_io_history *h, u16 new_size);
#endif /* CONFIG_DEBUG_FS */
#endif

209
drivers/gpu/arm/bifrost/mali_kbase_10969_workaround.c
View File

@ -1,209 +0,0 @@
/*
*
* (C) COPYRIGHT 2013-2015,2017-2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#include <linux/dma-mapping.h>
#include <mali_kbase.h>
#include <mali_kbase_10969_workaround.h>
/* Mask of X and Y coordinates for the coordinates words in the descriptors*/
#define X_COORDINATE_MASK 0x00000FFF
#define Y_COORDINATE_MASK 0x0FFF0000
/* Max number of words needed from the fragment shader job descriptor */
#define JOB_HEADER_SIZE_IN_WORDS 10
#define JOB_HEADER_SIZE (JOB_HEADER_SIZE_IN_WORDS*sizeof(u32))
/* Word 0: Status Word */
#define JOB_DESC_STATUS_WORD 0
/* Word 1: Restart Index */
#define JOB_DESC_RESTART_INDEX_WORD 1
/* Word 2: Fault address low word */
#define JOB_DESC_FAULT_ADDR_LOW_WORD 2
/* Word 8: Minimum Tile Coordinates */
#define FRAG_JOB_DESC_MIN_TILE_COORD_WORD 8
/* Word 9: Maximum Tile Coordinates */
#define FRAG_JOB_DESC_MAX_TILE_COORD_WORD 9
int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom)
{
struct device *dev = katom->kctx->kbdev->dev;
u32 clamped = 0;
struct kbase_va_region *region;
struct tagged_addr *page_array;
u64 page_index;
u32 offset = katom->jc & (~PAGE_MASK);
u32 *page_1 = NULL;
u32 *page_2 = NULL;
u32 job_header[JOB_HEADER_SIZE_IN_WORDS];
void *dst = job_header;
u32 minX, minY, maxX, maxY;
u32 restartX, restartY;
struct page *p;
u32 copy_size;
dev_warn(dev, "Called TILE_RANGE_FAULT workaround clamping function.\n");
if (!(katom->core_req & BASE_JD_REQ_FS))
return 0;
kbase_gpu_vm_lock(katom->kctx);
region = kbase_region_tracker_find_region_enclosing_address(katom->kctx,
katom->jc);
if (kbase_is_region_invalid_or_free(region))
goto out_unlock;
page_array = kbase_get_cpu_phy_pages(region);
if (!page_array)
goto out_unlock;
page_index = (katom->jc >> PAGE_SHIFT) - region->start_pfn;
p = as_page(page_array[page_index]);
/* we need the first 10 words of the fragment shader job descriptor.
* We need to check that the offset + 10 words is less that the page
* size otherwise we need to load the next page.
* page_size_overflow will be equal to 0 in case the whole descriptor
* is within the page > 0 otherwise.
*/
copy_size = MIN(PAGE_SIZE - offset, JOB_HEADER_SIZE);
page_1 = kmap_atomic(p);
/* page_1 is a u32 pointer, offset is expressed in bytes */
page_1 += offset>>2;
kbase_sync_single_for_cpu(katom->kctx->kbdev,
kbase_dma_addr(p) + offset,
copy_size, DMA_BIDIRECTIONAL);
memcpy(dst, page_1, copy_size);
/* The data needed overflows page the dimension,
* need to map the subsequent page */
if (copy_size < JOB_HEADER_SIZE) {
p = as_page(page_array[page_index + 1]);
page_2 = kmap_atomic(p);
kbase_sync_single_for_cpu(katom->kctx->kbdev,
kbase_dma_addr(p),
JOB_HEADER_SIZE - copy_size, DMA_BIDIRECTIONAL);
memcpy(dst + copy_size, page_2, JOB_HEADER_SIZE - copy_size);
}
/* We managed to correctly map one or two pages (in case of overflow) */
/* Get Bounding Box data and restart index from fault address low word */
minX = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & X_COORDINATE_MASK;
minY = job_header[FRAG_JOB_DESC_MIN_TILE_COORD_WORD] & Y_COORDINATE_MASK;
maxX = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & X_COORDINATE_MASK;
maxY = job_header[FRAG_JOB_DESC_MAX_TILE_COORD_WORD] & Y_COORDINATE_MASK;
restartX = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & X_COORDINATE_MASK;
restartY = job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] & Y_COORDINATE_MASK;
dev_warn(dev, "Before Clamping:\n"
"Jobstatus: %08x\n"
"restartIdx: %08x\n"
"Fault_addr_low: %08x\n"
"minCoordsX: %08x minCoordsY: %08x\n"
"maxCoordsX: %08x maxCoordsY: %08x\n",
job_header[JOB_DESC_STATUS_WORD],
job_header[JOB_DESC_RESTART_INDEX_WORD],
job_header[JOB_DESC_FAULT_ADDR_LOW_WORD],
minX, minY,
maxX, maxY);
/* Set the restart index to the one which generated the fault*/
job_header[JOB_DESC_RESTART_INDEX_WORD] =
job_header[JOB_DESC_FAULT_ADDR_LOW_WORD];
if (restartX < minX) {
job_header[JOB_DESC_RESTART_INDEX_WORD] = (minX) | restartY;
dev_warn(dev,
"Clamping restart X index to minimum. %08x clamped to %08x\n",
restartX, minX);
clamped = 1;
}
if (restartY < minY) {
job_header[JOB_DESC_RESTART_INDEX_WORD] = (minY) | restartX;
dev_warn(dev,
"Clamping restart Y index to minimum. %08x clamped to %08x\n",
restartY, minY);
clamped = 1;
}
if (restartX > maxX) {
job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxX) | restartY;
dev_warn(dev,
"Clamping restart X index to maximum. %08x clamped to %08x\n",
restartX, maxX);
clamped = 1;
}
if (restartY > maxY) {
job_header[JOB_DESC_RESTART_INDEX_WORD] = (maxY) | restartX;
dev_warn(dev,
"Clamping restart Y index to maximum. %08x clamped to %08x\n",
restartY, maxY);
clamped = 1;
}
if (clamped) {
/* Reset the fault address low word
* and set the job status to STOPPED */
job_header[JOB_DESC_FAULT_ADDR_LOW_WORD] = 0x0;
job_header[JOB_DESC_STATUS_WORD] = BASE_JD_EVENT_STOPPED;
dev_warn(dev, "After Clamping:\n"
"Jobstatus: %08x\n"
"restartIdx: %08x\n"
"Fault_addr_low: %08x\n"
"minCoordsX: %08x minCoordsY: %08x\n"
"maxCoordsX: %08x maxCoordsY: %08x\n",
job_header[JOB_DESC_STATUS_WORD],
job_header[JOB_DESC_RESTART_INDEX_WORD],
job_header[JOB_DESC_FAULT_ADDR_LOW_WORD],
minX, minY,
maxX, maxY);
/* Flush CPU cache to update memory for future GPU reads*/
memcpy(page_1, dst, copy_size);
p = as_page(page_array[page_index]);
kbase_sync_single_for_device(katom->kctx->kbdev,
kbase_dma_addr(p) + offset,
copy_size, DMA_TO_DEVICE);
if (copy_size < JOB_HEADER_SIZE) {
memcpy(page_2, dst + copy_size,
JOB_HEADER_SIZE - copy_size);
p = as_page(page_array[page_index + 1]);
kbase_sync_single_for_device(katom->kctx->kbdev,
kbase_dma_addr(p),
JOB_HEADER_SIZE - copy_size,
DMA_TO_DEVICE);
}
}
if (copy_size < JOB_HEADER_SIZE)
kunmap_atomic(page_2);
kunmap_atomic(page_1);
out_unlock:
kbase_gpu_vm_unlock(katom->kctx);
return clamped;
}

37
drivers/gpu/arm/bifrost/mali_kbase_10969_workaround.h
View File

@ -1,37 +0,0 @@
/*
*
* (C) COPYRIGHT 2013-2014, 2018 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_10969_WORKAROUND_
#define _KBASE_10969_WORKAROUND_
/**
* kbasep_10969_workaround_clamp_coordinates - Apply the WA to clamp the restart indices
* @katom: atom representing the fragment job for which the WA has to be applied
*
* This workaround is used to solve an HW issue with single iterator GPUs.
* If a fragment job is soft-stopped on the edge of its bounding box, it can happen
* that the restart index is out of bounds and the rerun causes a tile range
* fault. If this happens we try to clamp the restart index to a correct value.
*/
int kbasep_10969_workaround_clamp_coordinates(struct kbase_jd_atom *katom);
#endif /* _KBASE_10969_WORKAROUND_ */

5
drivers/gpu/arm/bifrost/mali_kbase_as_fault_debugfs.c
View File

@ -24,6 +24,7 @@
#include <mali_kbase.h>
#include <mali_kbase_as_fault_debugfs.h>
#include <device/mali_kbase_device.h>
#ifdef CONFIG_DEBUG_FS
#ifdef CONFIG_MALI_BIFROST_DEBUG
@ -36,7 +37,7 @@ static int kbase_as_fault_read(struct seq_file *sfile, void *data)
const struct list_head *kbdev_list;
struct kbase_device *kbdev = NULL;
kbdev_list = kbase_dev_list_get();
kbdev_list = kbase_device_get_list();
list_for_each(entry, kbdev_list) {
kbdev = list_entry(entry, struct kbase_device, entry);
@ -53,7 +54,7 @@ static int kbase_as_fault_read(struct seq_file *sfile, void *data)
}
kbase_dev_list_put(kbdev_list);
kbase_device_put_list(kbdev_list);
return 0;
}

42
drivers/gpu/arm/bifrost/mali_kbase_config_defaults.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2013-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2013-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -33,27 +33,6 @@
/* Include mandatory definitions per platform */
#include <mali_kbase_config_platform.h>
/**
* Boolean indicating whether the driver is configured to be secure at
* a potential loss of performance.
*
* This currently affects only r0p0-15dev0 HW and earlier.
*
* On r0p0-15dev0 HW and earlier, there are tradeoffs between security and
* performance:
*
* - When this is set to true, the driver remains fully secure,
* but potentially loses performance compared with setting this to
* false.
* - When set to false, the driver is open to certain security
* attacks.
*
* From r0p0-00rel0 and onwards, there is no security loss by setting
* this to false, and no performance loss by setting it to
* true.
*/
#define DEFAULT_SECURE_BUT_LOSS_OF_PERFORMANCE false
enum {
/**
* Use unrestricted Address ID width on the AXI bus.
@ -211,5 +190,24 @@ enum {
*/
#define DEFAULT_GPU_FREQ_KHZ_MAX (5000)
/**
* Default timeout for task execution on an endpoint
*
* Number of GPU clock cycles before the driver terminates a task that is
* making no forward progress on an endpoint (e.g. shader core).
* Value chosen is equivalent to the time after which a job is hard stopped
* which is 5 seconds (assuming the GPU is usually clocked at ~500 MHZ).
*/
#define DEFAULT_PROGRESS_TIMEOUT ((u64)5 * 500 * 1024 * 1024)
/**
* Default threshold at which to switch to incremental rendering
*
* Fraction of the maximum size of an allocation that grows on GPU page fault
* that can be used up before the driver switches to incremental rendering,
* in 256ths. 0 means disable incremental rendering.
*/
#define DEFAULT_IR_THRESHOLD (192)
#endif /* _KBASE_CONFIG_DEFAULTS_H_ */

334
drivers/gpu/arm/bifrost/mali_kbase_context.c
View File

@ -1,334 +0,0 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel context APIs
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <mali_kbase_mem_linux.h>
#include <mali_kbase_dma_fence.h>
#include <mali_kbase_ctx_sched.h>
#include <mali_kbase_mem_pool_group.h>
#include <mali_kbase_tracepoints.h>
struct kbase_context *
kbase_create_context(struct kbase_device *kbdev, bool is_compat,
base_context_create_flags const flags,
unsigned long const api_version,
struct file *const filp)
{
struct kbase_context *kctx;
int err;
struct page *p;
struct kbasep_js_kctx_info *js_kctx_info = NULL;
unsigned long irq_flags = 0;
const unsigned long cookies_mask = KBASE_COOKIE_MASK;
if (WARN_ON(!kbdev))
goto out;
/* Validate flags */
if (WARN_ON(flags != (flags & BASEP_CONTEXT_CREATE_KERNEL_FLAGS)))
goto out;
/* zero-inited as lot of code assume it's zero'ed out on create */
kctx = vzalloc(sizeof(*kctx));
if (!kctx)
goto out;
/* creating a context is considered a disjoint event */
kbase_disjoint_event(kbdev);
kctx->kbdev = kbdev;
kctx->as_nr = KBASEP_AS_NR_INVALID;
atomic_set(&kctx->refcount, 0);
if (is_compat)
kbase_ctx_flag_set(kctx, KCTX_COMPAT);
#if defined(CONFIG_64BIT)
else
kbase_ctx_flag_set(kctx, KCTX_FORCE_SAME_VA);
#endif /* !defined(CONFIG_64BIT) */
spin_lock_init(&kctx->mm_update_lock);
kctx->process_mm = NULL;
atomic_set(&kctx->nonmapped_pages, 0);
atomic_set(&kctx->permanent_mapped_pages, 0);
kctx->slots_pullable = 0;
kctx->tgid = current->tgid;
kctx->pid = current->pid;
err = kbase_mem_pool_group_init(&kctx->mem_pools, kbdev,
&kbdev->mem_pool_defaults, &kbdev->mem_pools);
if (err)
goto free_kctx;
err = kbase_mem_evictable_init(kctx);
if (err)
goto free_both_pools;
atomic_set(&kctx->used_pages, 0);
err = kbase_jd_init(kctx);
if (err)
goto deinit_evictable;
err = kbasep_js_kctx_init(kctx);
if (err)
goto free_jd; /* safe to call kbasep_js_kctx_term in this case */
err = kbase_event_init(kctx);
if (err)
goto free_jd;
mutex_init(&kctx->reg_lock);
spin_lock_init(&kctx->mem_partials_lock);
INIT_LIST_HEAD(&kctx->mem_partials);
INIT_LIST_HEAD(&kctx->waiting_soft_jobs);
spin_lock_init(&kctx->waiting_soft_jobs_lock);
err = kbase_dma_fence_init(kctx);
if (err)
goto free_event;
err = kbase_mmu_init(kbdev, &kctx->mmu, kctx,
base_context_mmu_group_id_get(flags));
if (err)
goto term_dma_fence;
p = kbase_mem_alloc_page(
&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK]);
if (!p)
goto no_sink_page;
kctx->aliasing_sink_page = as_tagged(page_to_phys(p));
init_waitqueue_head(&kctx->event_queue);
bitmap_copy(kctx->cookies, &cookies_mask, BITS_PER_LONG);
/* Make sure page 0 is not used... */
err = kbase_region_tracker_init(kctx);
if (err)
goto no_region_tracker;
err = kbase_sticky_resource_init(kctx);
if (err)
goto no_sticky;
err = kbase_jit_init(kctx);
if (err)
goto no_jit;
#ifdef CONFIG_GPU_TRACEPOINTS
atomic_set(&kctx->jctx.work_id, 0);
#endif
kctx->id = atomic_add_return(1, &(kbdev->ctx_num)) - 1;
mutex_init(&kctx->legacy_hwcnt_lock);
kbase_timer_setup(&kctx->soft_job_timeout,
kbasep_soft_job_timeout_worker);
mutex_lock(&kbdev->kctx_list_lock);
list_add(&kctx->kctx_list_link, &kbdev->kctx_list);
KBASE_TLSTREAM_TL_NEW_CTX(kbdev, kctx, kctx->id, (u32)(kctx->tgid));
mutex_unlock(&kbdev->kctx_list_lock);
kctx->api_version = api_version;
kctx->filp = filp;
js_kctx_info = &kctx->jctx.sched_info;
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, irq_flags);
/* Translate the flags */
if ((flags & BASE_CONTEXT_SYSTEM_MONITOR_SUBMIT_DISABLED) == 0)
kbase_ctx_flag_clear(kctx, KCTX_SUBMIT_DISABLED);
spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, irq_flags);
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
return kctx;
no_jit:
kbase_gpu_vm_lock(kctx);
kbase_sticky_resource_term(kctx);
kbase_gpu_vm_unlock(kctx);
no_sticky:
kbase_region_tracker_term(kctx);
no_region_tracker:
kbase_mem_pool_free(
&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK], p, false);
no_sink_page:
kbase_mmu_term(kbdev, &kctx->mmu);
term_dma_fence:
kbase_dma_fence_term(kctx);
free_event:
kbase_event_cleanup(kctx);
free_jd:
/* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */
kbasep_js_kctx_term(kctx);
kbase_jd_exit(kctx);
deinit_evictable:
kbase_mem_evictable_deinit(kctx);
free_both_pools:
kbase_mem_pool_group_term(&kctx->mem_pools);
free_kctx:
vfree(kctx);
out:
return NULL;
}
KBASE_EXPORT_SYMBOL(kbase_create_context);
static void kbase_reg_pending_dtor(struct kbase_device *kbdev,
struct kbase_va_region *reg)
{
dev_dbg(kbdev->dev, "Freeing pending unmapped region\n");
kbase_mem_phy_alloc_put(reg->cpu_alloc);
kbase_mem_phy_alloc_put(reg->gpu_alloc);
kfree(reg);
}
void kbase_destroy_context(struct kbase_context *kctx)
{
struct kbase_device *kbdev;
int pages;
unsigned long pending_regions_to_clean;
unsigned long flags;
struct page *p;
if (WARN_ON(!kctx))
return;
kbdev = kctx->kbdev;
if (WARN_ON(!kbdev))
return;
mutex_lock(&kbdev->kctx_list_lock);
KBASE_TLSTREAM_TL_DEL_CTX(kbdev, kctx);
list_del(&kctx->kctx_list_link);
mutex_unlock(&kbdev->kctx_list_lock);
KBASE_TRACE_ADD(kbdev, CORE_CTX_DESTROY, kctx, NULL, 0u, 0u);
/* Ensure the core is powered up for the destroy process */
/* A suspend won't happen here, because we're in a syscall from a userspace
* thread. */
kbase_pm_context_active(kbdev);
kbase_mem_pool_group_mark_dying(&kctx->mem_pools);
kbase_jd_zap_context(kctx);
/* We have already waited for the jobs to complete (and hereafter there
* can be no more submissions for the context). However the wait could
* have timedout and there could still be work items in flight that
* would do the completion processing of jobs.
* kbase_jd_exit() will destroy the 'job_done_wq'. And destroying the wq
* will cause it do drain and implicitly wait for those work items to
* complete.
*/
kbase_jd_exit(kctx);
#ifdef CONFIG_DEBUG_FS
/* Removing the rest of the debugfs entries here as we want to keep the
* atom debugfs interface alive until all atoms have completed. This
* is useful for debugging hung contexts. */
debugfs_remove_recursive(kctx->kctx_dentry);
kbase_debug_job_fault_context_term(kctx);
#endif
kbase_event_cleanup(kctx);
/*
* JIT must be terminated before the code below as it must be called
* without the region lock being held.
* The code above ensures no new JIT allocations can be made by
* by the time we get to this point of context tear down.
*/
kbase_jit_term(kctx);
kbase_gpu_vm_lock(kctx);
kbase_sticky_resource_term(kctx);
/* drop the aliasing sink page now that it can't be mapped anymore */
p = as_page(kctx->aliasing_sink_page);
kbase_mem_pool_free(&kctx->mem_pools.small[KBASE_MEM_GROUP_SINK],
p, false);
/* free pending region setups */
pending_regions_to_clean = KBASE_COOKIE_MASK;
bitmap_andnot(&pending_regions_to_clean, &pending_regions_to_clean,
kctx->cookies, BITS_PER_LONG);
while (pending_regions_to_clean) {
unsigned int cookie = find_first_bit(&pending_regions_to_clean,
BITS_PER_LONG);
BUG_ON(!kctx->pending_regions[cookie]);
kbase_reg_pending_dtor(kbdev, kctx->pending_regions[cookie]);
kctx->pending_regions[cookie] = NULL;
bitmap_clear(&pending_regions_to_clean, cookie, 1);
}
kbase_region_tracker_term(kctx);
kbase_gpu_vm_unlock(kctx);
/* Safe to call this one even when didn't initialize (assuming kctx was sufficiently zeroed) */
kbasep_js_kctx_term(kctx);
kbase_dma_fence_term(kctx);
mutex_lock(&kbdev->mmu_hw_mutex);
spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
kbase_ctx_sched_remove_ctx(kctx);
spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
mutex_unlock(&kbdev->mmu_hw_mutex);
kbase_mmu_term(kbdev, &kctx->mmu);
pages = atomic_read(&kctx->used_pages);
if (pages != 0)
dev_warn(kbdev->dev, "%s: %d pages in use!\n", __func__, pages);
kbase_mem_evictable_deinit(kctx);
kbase_mem_pool_group_term(&kctx->mem_pools);
WARN_ON(atomic_read(&kctx->nonmapped_pages) != 0);
vfree(kctx);
kbase_pm_context_idle(kbdev);
}
KBASE_EXPORT_SYMBOL(kbase_destroy_context);

1030
drivers/gpu/arm/bifrost/mali_kbase_core_linux.c
View File

File diff suppressed because it is too large Load Diff

54
drivers/gpu/arm/bifrost/mali_kbase_cs_experimental.h Normal file
View File

@ -0,0 +1,54 @@
/*
*
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
* (C) COPYRIGHT 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
*/
#ifndef _KBASE_CS_EXPERIMENTAL_H_
#define _KBASE_CS_EXPERIMENTAL_H_
#include <linux/kernel.h>
/**
* mali_kbase_print_cs_experimental() - Print a string for every Core Services
* experimental feature that is enabled
*/
static inline void mali_kbase_print_cs_experimental(void)
{
#if MALI_JIT_PRESSURE_LIMIT
pr_info("mali_kbase: JIT_PRESSURE_LIMIT (experimental) enabled");
#endif /* MALI_JIT_PRESSURE_LIMIT */
#if MALI_INCREMENTAL_RENDERING
pr_info("mali_kbase: INCREMENTAL_RENDERING (experimental) enabled");
#endif /* MALI_INCREMENTAL_RENDERING */
}
#endif /* _KBASE_CS_EXPERIMENTAL_H_ */

152
drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2017-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2017-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -23,7 +23,23 @@
#include <mali_kbase.h>
#include <mali_kbase_config_defaults.h>
#include <mali_kbase_defs.h>
#include "mali_kbase_ctx_sched.h"
#include "tl/mali_kbase_tracepoints.h"
/* Helper for ktrace */
#if KBASE_KTRACE_ENABLE
static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx)
{
return atomic_read(&kctx->refcount);
}
#else /* KBASE_KTRACE_ENABLE */
static int kbase_ktrace_get_ctx_refcnt(struct kbase_context *kctx)
{
CSTD_UNUSED(kctx);
return 0;
}
#endif /* KBASE_KTRACE_ENABLE */
int kbase_ctx_sched_init(struct kbase_device *kbdev)
{
@ -106,11 +122,15 @@ int kbase_ctx_sched_retain_ctx(struct kbase_context *kctx)
if (prev_kctx) {
WARN_ON(atomic_read(&prev_kctx->refcount) != 0);
kbase_mmu_disable(prev_kctx);
KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(
kbdev, prev_kctx->id);
prev_kctx->as_nr = KBASEP_AS_NR_INVALID;
}
kctx->as_nr = free_as;
kbdev->as_to_kctx[free_as] = kctx;
KBASE_TLSTREAM_TL_KBASE_CTX_ASSIGN_AS(
kbdev, kctx->id, free_as);
kbase_mmu_update(kbdev, &kctx->mmu,
kctx->as_nr);
}
@ -142,11 +162,23 @@ void kbase_ctx_sched_retain_ctx_refcount(struct kbase_context *kctx)
void kbase_ctx_sched_release_ctx(struct kbase_context *kctx)
{
struct kbase_device *const kbdev = kctx->kbdev;
int new_ref_count;
lockdep_assert_held(&kbdev->hwaccess_lock);
if (atomic_dec_return(&kctx->refcount) == 0)
new_ref_count = atomic_dec_return(&kctx->refcount);
if (new_ref_count == 0) {
kbdev->as_free |= (1u << kctx->as_nr);
if (kbase_ctx_flag(kctx, KCTX_AS_DISABLED_ON_FAULT)) {
KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(
kbdev, kctx->id);
kbdev->as_to_kctx[kctx->as_nr] = NULL;
kctx->as_nr = KBASEP_AS_NR_INVALID;
kbase_ctx_flag_clear(kctx, KCTX_AS_DISABLED_ON_FAULT);
}
}
KBASE_KTRACE_ADD(kbdev, SCHED_RELEASE_CTX, kctx, new_ref_count);
}
void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx)
@ -162,6 +194,7 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx)
if (kbdev->pm.backend.gpu_powered)
kbase_mmu_disable(kctx);
KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(kbdev, kctx->id);
kbdev->as_to_kctx[kctx->as_nr] = NULL;
kctx->as_nr = KBASEP_AS_NR_INVALID;
}
@ -186,15 +219,126 @@ void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev)
kbase_mmu_update(kbdev, &kctx->mmu,
kctx->as_nr);
kbase_ctx_flag_clear(kctx,
KCTX_AS_DISABLED_ON_FAULT);
} else {
/* This context might have been assigned an
* AS before, clear it.
*/
kbdev->as_to_kctx[kctx->as_nr] = NULL;
kctx->as_nr = KBASEP_AS_NR_INVALID;
if (kctx->as_nr != KBASEP_AS_NR_INVALID) {
KBASE_TLSTREAM_TL_KBASE_CTX_UNASSIGN_AS(
kbdev, kctx->id);
kbdev->as_to_kctx[kctx->as_nr] = NULL;
kctx->as_nr = KBASEP_AS_NR_INVALID;
}
}
} else {
kbase_mmu_disable_as(kbdev, i);
}
}
}
struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount(
struct kbase_device *kbdev, size_t as_nr)
{
unsigned long flags;
struct kbase_context *found_kctx = NULL;
if (WARN_ON(kbdev == NULL))
return NULL;
if (WARN_ON(as_nr >= BASE_MAX_NR_AS))
return NULL;
spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
found_kctx = kbdev->as_to_kctx[as_nr];
if (found_kctx != NULL)
kbase_ctx_sched_retain_ctx_refcount(found_kctx);
spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return found_kctx;
}
struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev,
size_t as_nr)
{
struct kbase_context *found_kctx;
if (WARN_ON(kbdev == NULL))
return NULL;
if (WARN_ON(as_nr >= BASE_MAX_NR_AS))
return NULL;
found_kctx = kbdev->as_to_kctx[as_nr];
if (WARN_ON(!found_kctx))
return NULL;
if (WARN_ON(atomic_read(&found_kctx->refcount) <= 0))
return NULL;
return found_kctx;
}
bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx)
{
bool result = false;
int as_nr;
if (WARN_ON(kctx == NULL))
return result;
lockdep_assert_held(&kctx->kbdev->hwaccess_lock);
as_nr = kctx->as_nr;
if (atomic_read(&kctx->refcount) > 0) {
KBASE_DEBUG_ASSERT(as_nr >= 0);
kbase_ctx_sched_retain_ctx_refcount(kctx);
KBASE_KTRACE_ADD(kctx->kbdev, SCHED_RETAIN_CTX_NOLOCK, kctx,
kbase_ktrace_get_ctx_refcnt(kctx));
result = true;
}
return result;
}
bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx)
{
unsigned long flags;
bool result = false;
if (WARN_ON(kctx == NULL))
return result;
if (WARN_ON(kctx->kbdev == NULL))
return result;
mutex_lock(&kctx->kbdev->mmu_hw_mutex);
spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
result = kbase_ctx_sched_inc_refcount_nolock(kctx);
spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
mutex_unlock(&kctx->kbdev->mmu_hw_mutex);
return result;
}
void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx)
{
unsigned long flags;
if (WARN_ON(!kctx))
return;
spin_lock_irqsave(&kctx->kbdev->hwaccess_lock, flags);
if (!WARN_ON(kctx->as_nr == KBASEP_AS_NR_INVALID) &&
!WARN_ON(atomic_read(&kctx->refcount) <= 0))
kbase_ctx_sched_release_ctx(kctx);
spin_unlock_irqrestore(&kctx->kbdev->hwaccess_lock, flags);
}

76
drivers/gpu/arm/bifrost/mali_kbase_ctx_sched.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2017-2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2017-2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -132,4 +132,78 @@ void kbase_ctx_sched_remove_ctx(struct kbase_context *kctx);
*/
void kbase_ctx_sched_restore_all_as(struct kbase_device *kbdev);
/**
* kbase_ctx_sched_as_to_ctx_refcount - Lookup a context based on its current
* address space and ensure that is stays scheduled in
* @kbdev: The device for which the returned context must belong
* @as_nr: address space assigned to the context of interest
*
* The context is refcounted as being busy to prevent it from scheduling
* out. It must be released with kbase_ctx_sched_release_ctx() when it is no
* longer required to stay scheduled in.
*
* This function can safely be called from IRQ context.
*
* The following locking conditions are made on the caller:
* * it must not hold the kbase_device::hwaccess_lock, because it will be used
* internally.
*
* Return: a valid struct kbase_context on success, which has been refcounted
* as being busy or return NULL on failure, indicating that no context was found
* in as_nr.
*/
struct kbase_context *kbase_ctx_sched_as_to_ctx_refcount(
struct kbase_device *kbdev, size_t as_nr);
/**
* kbase_ctx_sched_as_to_ctx - Lookup a context based on its current address
* space
* @kbdev: The device for which the returned context must belong
* @as_nr: address space assigned to the context of interest
*
* Return: a valid struct kbase_context on success or NULL on failure,
* indicating that no context was found in as_nr.
*/
struct kbase_context *kbase_ctx_sched_as_to_ctx(struct kbase_device *kbdev,
size_t as_nr);
/**
* kbase_ctx_sched_inc_refcount_nolock - Refcount a context as being busy,
* preventing it from being scheduled out.
* @kctx: Context to be refcounted
*
* The following locks must be held by the caller:
* * kbase_device::mmu_hw_mutex
* * kbase_device::hwaccess_lock
*
* Return: true if refcount succeeded, and the context will not be scheduled
* out, false if the refcount failed (because the context is being/has been
* scheduled out).
*/
bool kbase_ctx_sched_inc_refcount_nolock(struct kbase_context *kctx);
/**
* kbase_ctx_sched_inc_refcount - Refcount a context as being busy, preventing
* it from being scheduled out.
* @kctx: Context to be refcounted
*
* The following locking conditions are made on the caller:
* * it must not hold kbase_device::mmu_hw_mutex and
* kbase_device::hwaccess_lock, because they will be used internally.
*
* Return: true if refcount succeeded, and the context will not be scheduled
* out, false if the refcount failed (because the context is being/has been
* scheduled out).
*/
bool kbase_ctx_sched_inc_refcount(struct kbase_context *kctx);
/**
* kbase_ctx_sched_release_ctx_lock - Release a reference count of a context
* @kctx: Context for which refcount should be decreased
*
* Effectivelly, this is a wrapper for kbase_ctx_sched_release_ctx, but
* kbase_device::hwaccess_lock is required NOT to be locked.
*/
void kbase_ctx_sched_release_ctx_lock(struct kbase_context *kctx);
#endif /* _KBASE_CTX_SCHED_H_ */

6
drivers/gpu/arm/bifrost/mali_kbase_debug_mem_view.c
View File

@ -233,6 +233,12 @@ static int debug_mem_open(struct inode *i, struct file *file)
goto out;
}
ret = debug_mem_zone_open(&kctx->reg_rbtree_exec, mem_data);
if (0 != ret) {
kbase_gpu_vm_unlock(kctx);
goto out;
}
kbase_gpu_vm_unlock(kctx);
((struct seq_file *)file->private_data)->private = mem_data;

949
drivers/gpu/arm/bifrost/mali_kbase_defs.h
View File

File diff suppressed because it is too large Load Diff

515
drivers/gpu/arm/bifrost/mali_kbase_device.c
View File

@ -1,515 +0,0 @@
/*
*
* (C) COPYRIGHT 2010-2019 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Base kernel device APIs
*/
#include <linux/debugfs.h>
#include <linux/dma-mapping.h>
#include <linux/seq_file.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/of_platform.h>
#include <mali_kbase.h>
#include <mali_kbase_defs.h>
#include <mali_kbase_hwaccess_instr.h>
#include <mali_kbase_hw.h>
#include <mali_kbase_config_defaults.h>
/* NOTE: Magic - 0x45435254 (TRCE in ASCII).
* Supports tracing feature provided in the base module.
* Please keep it in sync with the value of base module.
*/
#define TRACE_BUFFER_HEADER_SPECIAL 0x45435254
#if KBASE_TRACE_ENABLE
static const char *kbasep_trace_code_string[] = {
/* IMPORTANT: USE OF SPECIAL #INCLUDE OF NON-STANDARD HEADER FILE
* THIS MUST BE USED AT THE START OF THE ARRAY */
#define KBASE_TRACE_CODE_MAKE_CODE(X) # X
#include "mali_kbase_trace_defs.h"
#undef KBASE_TRACE_CODE_MAKE_CODE
};
#endif
#define DEBUG_MESSAGE_SIZE 256
static int kbasep_trace_init(struct kbase_device *kbdev);
static void kbasep_trace_term(struct kbase_device *kbdev);
static void kbasep_trace_hook_wrapper(void *param);
struct kbase_device *kbase_device_alloc(void)
{
return kzalloc(sizeof(struct kbase_device), GFP_KERNEL);
}
static int kbase_device_as_init(struct kbase_device *kbdev, int i)
{
kbdev->as[i].number = i;
kbdev->as[i].bf_data.addr = 0ULL;
kbdev->as[i].pf_data.addr = 0ULL;
kbdev->as[i].pf_wq = alloc_workqueue("mali_mmu%d", 0, 1, i);
if (!kbdev->as[i].pf_wq)
return -EINVAL;
INIT_WORK(&kbdev->as[i].work_pagefault, page_fault_worker);
INIT_WORK(&kbdev->as[i].work_busfault, bus_fault_worker);
return 0;
}
static void kbase_device_as_term(struct kbase_device *kbdev, int i)
{
destroy_workqueue(kbdev->as[i].pf_wq);
}
static int kbase_device_all_as_init(struct kbase_device *kbdev)
{
int i, err;
for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
err = kbase_device_as_init(kbdev, i);
if (err)
goto free_workqs;
}
return 0;
free_workqs:
for (; i > 0; i--)
kbase_device_as_term(kbdev, i);
return err;
}
static void kbase_device_all_as_term(struct kbase_device *kbdev)
{
int i;
for (i = 0; i < kbdev->nr_hw_address_spaces; i++)
kbase_device_as_term(kbdev, i);
}
int kbase_device_init(struct kbase_device * const kbdev)
{
int err;
#ifdef CONFIG_ARM64
struct device_node *np = NULL;
#endif /* CONFIG_ARM64 */
spin_lock_init(&kbdev->mmu_mask_change);
mutex_init(&kbdev->mmu_hw_mutex);
#ifdef CONFIG_ARM64
kbdev->cci_snoop_enabled = false;
np = kbdev->dev->of_node;
if (np != NULL) {
if (of_property_read_u32(np, "snoop_enable_smc",
&kbdev->snoop_enable_smc))
kbdev->snoop_enable_smc = 0;
if (of_property_read_u32(np, "snoop_disable_smc",
&kbdev->snoop_disable_smc))
kbdev->snoop_disable_smc = 0;
/* Either both or none of the calls should be provided. */
if (!((kbdev->snoop_disable_smc == 0
&& kbdev->snoop_enable_smc == 0)
|| (kbdev->snoop_disable_smc != 0
&& kbdev->snoop_enable_smc != 0))) {
WARN_ON(1);
err = -EINVAL;
goto fail;
}
}
#endif /* CONFIG_ARM64 */
/* Get the list of workarounds for issues on the current HW
* (identified by the GPU_ID register)
*/
err = kbase_hw_set_issues_mask(kbdev);
if (err)
goto fail;
/* Set the list of features available on the current HW
* (identified by the GPU_ID register)
*/
kbase_hw_set_features_mask(kbdev);
kbase_gpuprops_set_features(kbdev);
/* On Linux 4.0+, dma coherency is determined from device tree */
#if defined(CONFIG_ARM64) && LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
set_dma_ops(kbdev->dev, &noncoherent_swiotlb_dma_ops);
#endif
/* Workaround a pre-3.13 Linux issue, where dma_mask is NULL when our
* device structure was created by device-tree
*/
if (!kbdev->dev->dma_mask)
kbdev->dev->dma_mask = &kbdev->dev->coherent_dma_mask;
err = dma_set_mask(kbdev->dev,
DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits));
if (err)
goto dma_set_mask_failed;
err = dma_set_coherent_mask(kbdev->dev,
DMA_BIT_MASK(kbdev->gpu_props.mmu.pa_bits));
if (err)
goto dma_set_mask_failed;
kbdev->nr_hw_address_spaces = kbdev->gpu_props.num_address_spaces;
err = kbase_device_all_as_init(kbdev);
if (err)
goto as_init_failed;
spin_lock_init(&kbdev->hwcnt.lock);
err = kbasep_trace_init(kbdev);
if (err)
goto term_as;
init_waitqueue_head(&kbdev->cache_clean_wait);
kbase_debug_assert_register_hook(&kbasep_trace_hook_wrapper, kbdev);
atomic_set(&kbdev->ctx_num, 0);
err = kbase_instr_backend_init(kbdev);
if (err)
goto term_trace;
kbdev->pm.dvfs_period = DEFAULT_PM_DVFS_PERIOD;
kbdev->reset_timeout_ms = DEFAULT_RESET_TIMEOUT_MS;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_AARCH64_MMU))
kbdev->mmu_mode = kbase_mmu_mode_get_aarch64();
else
kbdev->mmu_mode = kbase_mmu_mode_get_lpae();
mutex_init(&kbdev->kctx_list_lock);
INIT_LIST_HEAD(&kbdev->kctx_list);
return 0;
term_trace:
kbasep_trace_term(kbdev);
term_as:
kbase_device_all_as_term(kbdev);
as_init_failed:
dma_set_mask_failed:
fail:
return err;
}
void kbase_device_term(struct kbase_device *kbdev)
{
KBASE_DEBUG_ASSERT(kbdev);
WARN_ON(!list_empty(&kbdev->kctx_list));
#if KBASE_TRACE_ENABLE
kbase_debug_assert_register_hook(NULL, NULL);
#endif
kbase_instr_backend_term(kbdev);
kbasep_trace_term(kbdev);
kbase_device_all_as_term(kbdev);
}
void kbase_device_free(struct kbase_device *kbdev)
{
kfree(kbdev);
}
/*
* Device trace functions
*/
#if KBASE_TRACE_ENABLE
static int kbasep_trace_init(struct kbase_device *kbdev)
{
struct kbase_trace *rbuf;
rbuf = kmalloc_array(KBASE_TRACE_SIZE, sizeof(*rbuf), GFP_KERNEL);
if (!rbuf)
return -EINVAL;
kbdev->trace_rbuf = rbuf;
spin_lock_init(&kbdev->trace_lock);
return 0;
}
static void kbasep_trace_term(struct kbase_device *kbdev)
{
kfree(kbdev->trace_rbuf);
}
static void kbasep_trace_format_msg(struct kbase_trace *trace_msg, char *buffer, int len)
{
s32 written = 0;
/* Initial part of message */
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d.%.6d,%d,%d,%s,%p,", (int)trace_msg->timestamp.tv_sec, (int)(trace_msg->timestamp.tv_nsec / 1000), trace_msg->thread_id, trace_msg->cpu, kbasep_trace_code_string[trace_msg->code], trace_msg->ctx), 0);
if (trace_msg->katom)
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "atom %d (ud: 0x%llx 0x%llx)", trace_msg->atom_number, trace_msg->atom_udata[0], trace_msg->atom_udata[1]), 0);
written += MAX(snprintf(buffer + written, MAX(len - written, 0), ",%.8llx,", trace_msg->gpu_addr), 0);
/* NOTE: Could add function callbacks to handle different message types */
/* Jobslot present */
if (trace_msg->flags & KBASE_TRACE_FLAG_JOBSLOT)
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->jobslot), 0);
written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0);
/* Refcount present */
if (trace_msg->flags & KBASE_TRACE_FLAG_REFCOUNT)
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "%d", trace_msg->refcount), 0);
written += MAX(snprintf(buffer + written, MAX(len - written, 0), ","), 0);
/* Rest of message */
written += MAX(snprintf(buffer + written, MAX(len - written, 0), "0x%.8lx", trace_msg->info_val), 0);
}
static void kbasep_trace_dump_msg(struct kbase_device *kbdev, struct kbase_trace *trace_msg)
{
char buffer[DEBUG_MESSAGE_SIZE];
kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE);
dev_dbg(kbdev->dev, "%s", buffer);
}
void kbasep_trace_add(struct kbase_device *kbdev, enum kbase_trace_code code, void *ctx, struct kbase_jd_atom *katom, u64 gpu_addr, u8 flags, int refcount, int jobslot, unsigned long info_val)
{
unsigned long irqflags;
struct kbase_trace *trace_msg;
spin_lock_irqsave(&kbdev->trace_lock, irqflags);
trace_msg = &kbdev->trace_rbuf[kbdev->trace_next_in];
/* Fill the message */
trace_msg->thread_id = task_pid_nr(current);
trace_msg->cpu = task_cpu(current);
getnstimeofday(&trace_msg->timestamp);
trace_msg->code = code;
trace_msg->ctx = ctx;
if (NULL == katom) {
trace_msg->katom = false;
} else {
trace_msg->katom = true;
trace_msg->atom_number = kbase_jd_atom_id(katom->kctx, katom);
trace_msg->atom_udata[0] = katom->udata.blob[0];
trace_msg->atom_udata[1] = katom->udata.blob[1];
}
trace_msg->gpu_addr = gpu_addr;
trace_msg->jobslot = jobslot;
trace_msg->refcount = MIN((unsigned int)refcount, 0xFF);
trace_msg->info_val = info_val;
trace_msg->flags = flags;
/* Update the ringbuffer indices */
kbdev->trace_next_in = (kbdev->trace_next_in + 1) & KBASE_TRACE_MASK;
if (kbdev->trace_next_in == kbdev->trace_first_out)
kbdev->trace_first_out = (kbdev->trace_first_out + 1) & KBASE_TRACE_MASK;
/* Done */
spin_unlock_irqrestore(&kbdev->trace_lock, irqflags);
}
void kbasep_trace_clear(struct kbase_device *kbdev)
{
unsigned long flags;
spin_lock_irqsave(&kbdev->trace_lock, flags);
kbdev->trace_first_out = kbdev->trace_next_in;
spin_unlock_irqrestore(&kbdev->trace_lock, flags);
}
void kbasep_trace_dump(struct kbase_device *kbdev)
{
unsigned long flags;
u32 start;
u32 end;
dev_dbg(kbdev->dev, "Dumping trace:\nsecs,nthread,cpu,code,ctx,katom,gpu_addr,jobslot,refcount,info_val");
spin_lock_irqsave(&kbdev->trace_lock, flags);
start = kbdev->trace_first_out;
end = kbdev->trace_next_in;
while (start != end) {
struct kbase_trace *trace_msg = &kbdev->trace_rbuf[start];
kbasep_trace_dump_msg(kbdev, trace_msg);
start = (start + 1) & KBASE_TRACE_MASK;
}
dev_dbg(kbdev->dev, "TRACE_END");
spin_unlock_irqrestore(&kbdev->trace_lock, flags);
KBASE_TRACE_CLEAR(kbdev);
}
static void kbasep_trace_hook_wrapper(void *param)
{
struct kbase_device *kbdev = (struct kbase_device *)param;
kbasep_trace_dump(kbdev);
}
#ifdef CONFIG_DEBUG_FS
struct trace_seq_state {
struct kbase_trace trace_buf[KBASE_TRACE_SIZE];
u32 start;
u32 end;
};
static void *kbasep_trace_seq_start(struct seq_file *s, loff_t *pos)
{
struct trace_seq_state *state = s->private;
int i;
if (*pos > KBASE_TRACE_SIZE)
return NULL;
i = state->start + *pos;
if ((state->end >= state->start && i >= state->end) ||
i >= state->end + KBASE_TRACE_SIZE)
return NULL;
i &= KBASE_TRACE_MASK;
return &state->trace_buf[i];
}
static void kbasep_trace_seq_stop(struct seq_file *s, void *data)
{
}
static void *kbasep_trace_seq_next(struct seq_file *s, void *data, loff_t *pos)
{
struct trace_seq_state *state = s->private;
int i;
(*pos)++;
i = (state->start + *pos) & KBASE_TRACE_MASK;
if (i == state->end)
return NULL;
return &state->trace_buf[i];
}
static int kbasep_trace_seq_show(struct seq_file *s, void *data)
{
struct kbase_trace *trace_msg = data;
char buffer[DEBUG_MESSAGE_SIZE];
kbasep_trace_format_msg(trace_msg, buffer, DEBUG_MESSAGE_SIZE);
seq_printf(s, "%s\n", buffer);
return 0;
}
static const struct seq_operations kbasep_trace_seq_ops = {
.start = kbasep_trace_seq_start,
.next = kbasep_trace_seq_next,
.stop = kbasep_trace_seq_stop,
.show = kbasep_trace_seq_show,
};
static int kbasep_trace_debugfs_open(struct inode *inode, struct file *file)
{
struct kbase_device *kbdev = inode->i_private;
unsigned long flags;
struct trace_seq_state *state;
state = __seq_open_private(file, &kbasep_trace_seq_ops, sizeof(*state));
if (!state)
return -ENOMEM;
spin_lock_irqsave(&kbdev->trace_lock, flags);
state->start = kbdev->trace_first_out;
state->end = kbdev->trace_next_in;
memcpy(state->trace_buf, kbdev->trace_rbuf, sizeof(state->trace_buf));
spin_unlock_irqrestore(&kbdev->trace_lock, flags);
return 0;
}
static const struct file_operations kbasep_trace_debugfs_fops = {
.owner = THIS_MODULE,
.open = kbasep_trace_debugfs_open,
.read = seq_read,
.llseek = seq_lseek,
.release = seq_release_private,
};
void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
{
debugfs_create_file("mali_trace", S_IRUGO,
kbdev->mali_debugfs_directory, kbdev,
&kbasep_trace_debugfs_fops);
}
#else
void kbasep_trace_debugfs_init(struct kbase_device *kbdev)
{
}
#endif /* CONFIG_DEBUG_FS */
#else /* KBASE_TRACE_ENABLE */
static int kbasep_trace_init(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
return 0;
}
static void kbasep_trace_term(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
}
static void kbasep_trace_hook_wrapper(void *param)
{
CSTD_UNUSED(param);
}
void kbasep_trace_dump(struct kbase_device *kbdev)
{
CSTD_UNUSED(kbdev);
}
#endif /* KBASE_TRACE_ENABLE */

2
drivers/gpu/arm/bifrost/mali_kbase_disjoint_events.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2014 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2014, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software

4
drivers/gpu/arm/bifrost/mali_kbase_dma_fence.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2017 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2017,2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -114,6 +114,8 @@ kbase_dma_fence_unlock_reservations(struct kbase_dma_fence_resv_info *info,
ww_acquire_fini(ctx);
}
/**
* kbase_dma_fence_queue_work() - Queue work to handle @katom
* @katom: Pointer to atom for which to queue work

442
drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.c Normal file
View File

@ -0,0 +1,442 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
/*
* Implementation of the dummy job execution workaround for the GPU hang issue.
*/
#include <mali_kbase.h>
#include <backend/gpu/mali_kbase_device_internal.h>
#include <mali_kbase_dummy_job_wa.h>
#include <linux/firmware.h>
#include <linux/delay.h>
#define DUMMY_JOB_WA_BINARY_NAME "valhall-1691526.wa"
struct wa_header {
u16 signature;
u16 version;
u32 info_offset;
} __packed;
struct wa_v2_info {
u64 jc;
u32 js;
u32 blob_offset;
u64 flags;
} __packed;
struct wa_blob {
u64 base;
u32 size;
u32 map_flags;
u32 payload_offset;
u32 blob_offset;
} __packed;
static bool in_range(const u8 *base, const u8 *end, off_t off, size_t sz)
{
return !(end - base - off < sz);
}
static u32 wait_any(struct kbase_device *kbdev, off_t offset, u32 bits)
{
int loop;
const int timeout = 100;
u32 val;
for (loop = 0; loop < timeout; loop++) {
val = kbase_reg_read(kbdev, offset);
if (val & bits)
break;
udelay(10);
}
if (loop == timeout) {
dev_err(kbdev->dev,
"Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n",
(unsigned long)offset, (unsigned long)bits,
(unsigned long)val);
}
return (val & bits);
}
static int wait(struct kbase_device *kbdev, off_t offset, u32 bits, bool set)
{
int loop;
const int timeout = 100;
u32 val;
u32 target = 0;
if (set)
target = bits;
for (loop = 0; loop < timeout; loop++) {
val = kbase_reg_read(kbdev, (offset));
if ((val & bits) == target)
break;
udelay(10);
}
if (loop == timeout) {
dev_err(kbdev->dev,
"Timeout reading register 0x%lx, bits 0x%lx, last read was 0x%lx\n",
(unsigned long)offset, (unsigned long)bits,
(unsigned long)val);
return -ETIMEDOUT;
}
return 0;
}
static inline int run_job(struct kbase_device *kbdev, int as, int slot,
u64 cores, u64 jc)
{
u32 done;
/* setup job */
kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_HEAD_NEXT_LO),
jc & U32_MAX);
kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_HEAD_NEXT_HI),
jc >> 32);
kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_AFFINITY_NEXT_LO),
cores & U32_MAX);
kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_AFFINITY_NEXT_HI),
cores >> 32);
kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_CONFIG_NEXT),
JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK | as);
/* go */
kbase_reg_write(kbdev, JOB_SLOT_REG(slot, JS_COMMAND_NEXT),
JS_COMMAND_START);
/* wait for the slot to finish (done, error) */
done = wait_any(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT),
(1ul << (16+slot)) | (1ul << slot));
kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), done);
if (done != (1ul << slot)) {
dev_err(kbdev->dev,
"Failed to run WA job on slot %d cores 0x%llx: done 0x%lx\n",
slot, (unsigned long long)cores,
(unsigned long)done);
dev_err(kbdev->dev, "JS_STATUS on failure: 0x%x\n",
kbase_reg_read(kbdev, JOB_SLOT_REG(slot, JS_STATUS)));
return -EFAULT;
} else {
return 0;
}
}
/* To be called after power up & MMU init, but before everything else */
int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores)
{
int as;
int slot;
u64 jc;
int failed = 0;
int runs = 0;
u32 old_gpu_mask;
u32 old_job_mask;
if (!kbdev)
return -EFAULT;
if (!kbdev->dummy_job_wa.ctx)
return -EFAULT;
as = kbdev->dummy_job_wa.ctx->as_nr;
slot = kbdev->dummy_job_wa.slot;
jc = kbdev->dummy_job_wa.jc;
/* mask off all but MMU IRQs */
old_gpu_mask = kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK));
old_job_mask = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK));
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0);
kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0);
/* power up requested cores */
kbase_reg_write(kbdev, SHADER_PWRON_LO, (cores & U32_MAX));
kbase_reg_write(kbdev, SHADER_PWRON_HI, (cores >> 32));
if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP) {
/* wait for power-ups */
wait(kbdev, SHADER_READY_LO, (cores & U32_MAX), true);
if (cores >> 32)
wait(kbdev, SHADER_READY_HI, (cores >> 32), true);
}
if (kbdev->dummy_job_wa.flags & KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE) {
int i;
/* do for each requested core */
for (i = 0; i < sizeof(cores) * 8; i++) {
u64 affinity;
affinity = 1ull << i;
if (!(cores & affinity))
continue;
if (run_job(kbdev, as, slot, affinity, jc))
failed++;
runs++;
}
} else {
if (run_job(kbdev, as, slot, cores, jc))
failed++;
runs++;
}
if (kbdev->dummy_job_wa.flags &
KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER) {
/* power off shader cores (to reduce any dynamic leakage) */
kbase_reg_write(kbdev, SHADER_PWROFF_LO, (cores & U32_MAX));
kbase_reg_write(kbdev, SHADER_PWROFF_HI, (cores >> 32));
/* wait for power off complete */
wait(kbdev, SHADER_READY_LO, (cores & U32_MAX), false);
wait(kbdev, SHADER_PWRTRANS_LO, (cores & U32_MAX), false);
if (cores >> 32) {
wait(kbdev, SHADER_READY_HI, (cores >> 32), false);
wait(kbdev, SHADER_PWRTRANS_HI, (cores >> 32), false);
}
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), U32_MAX);
}
/* restore IRQ masks */
kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), old_gpu_mask);
kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), old_job_mask);
if (failed)
dev_err(kbdev->dev,
"WA complete with %d failures out of %d runs\n", failed,
runs);
return failed ? -EFAULT : 0;
}
static ssize_t show_dummy_job_wa_info(struct device * const dev,
struct device_attribute * const attr, char * const buf)
{
struct kbase_device *const kbdev = dev_get_drvdata(dev);
int err;
if (!kbdev || !kbdev->dummy_job_wa.ctx)
return -ENODEV;
err = scnprintf(buf, PAGE_SIZE, "slot %u flags %llx\n",
kbdev->dummy_job_wa.slot, kbdev->dummy_job_wa.flags);
return err;
}
static DEVICE_ATTR(dummy_job_wa_info, 0444, show_dummy_job_wa_info, NULL);
static bool wa_blob_load_needed(struct kbase_device *kbdev)
{
if (of_machine_is_compatible("arm,juno"))
return false;
if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_TTRX_3485))
return true;
return false;
}
int kbase_dummy_job_wa_load(struct kbase_device *kbdev)
{
const struct firmware *firmware;
static const char wa_name[] = DUMMY_JOB_WA_BINARY_NAME;
const u32 signature = 0x4157;
const u32 version = 2;
const u8 *fw_end;
const u8 *fw;
const struct wa_header *header;
const struct wa_v2_info *v2_info;
u32 blob_offset;
int err;
struct kbase_context *kctx;
if (!wa_blob_load_needed(kbdev))
return 0;
/* load the wa */
err = request_firmware(&firmware, wa_name, kbdev->dev);
if (err) {
dev_err(kbdev->dev, "WA blob missing. Please refer to the Arm Mali DDK Valhall Release Notes, "
"Part number DC-06002 or contact support-mali@arm.com - driver probe will be failed");
return -ENODEV;
}
kctx = kbase_create_context(kbdev, true,
BASE_CONTEXT_CREATE_FLAG_NONE, 0,
NULL);
if (!kctx) {
dev_err(kbdev->dev, "Failed to create WA context\n");
goto no_ctx;
}
fw = firmware->data;
fw_end = fw + firmware->size;
dev_dbg(kbdev->dev, "Loaded firmware of size %zu bytes\n",
firmware->size);
if (!in_range(fw, fw_end, 0, sizeof(*header))) {
dev_err(kbdev->dev, "WA too small\n");
goto bad_fw;
}
header = (const struct wa_header *)(fw + 0);
if (header->signature != signature) {
dev_err(kbdev->dev, "WA signature failure: 0x%lx\n",
(unsigned long)header->signature);
goto bad_fw;
}
if (header->version != version) {
dev_err(kbdev->dev, "WA version 0x%lx not supported\n",
(unsigned long)header->version);
goto bad_fw;
}
if (!in_range(fw, fw_end, header->info_offset, sizeof(*v2_info))) {
dev_err(kbdev->dev, "WA info offset out of bounds\n");
goto bad_fw;
}
v2_info = (const struct wa_v2_info *)(fw + header->info_offset);
if (v2_info->flags & ~KBASE_DUMMY_JOB_WA_FLAGS) {
dev_err(kbdev->dev, "Unsupported WA flag(s): 0x%llx\n",
(unsigned long long)v2_info->flags);
goto bad_fw;
}
kbdev->dummy_job_wa.slot = v2_info->js;
kbdev->dummy_job_wa.jc = v2_info->jc;
kbdev->dummy_job_wa.flags = v2_info->flags;
blob_offset = v2_info->blob_offset;
while (blob_offset) {
const struct wa_blob *blob;
size_t nr_pages;
u64 flags;
u64 gpu_va;
struct kbase_va_region *va_region;
if (!in_range(fw, fw_end, blob_offset, sizeof(*blob))) {
dev_err(kbdev->dev, "Blob offset out-of-range: 0x%lx\n",
(unsigned long)blob_offset);
goto bad_fw;
}
blob = (const struct wa_blob *)(fw + blob_offset);
if (!in_range(fw, fw_end, blob->payload_offset, blob->size)) {
dev_err(kbdev->dev, "Payload out-of-bounds\n");
goto bad_fw;
}
gpu_va = blob->base;
if (PAGE_ALIGN(gpu_va) != gpu_va) {
dev_err(kbdev->dev, "blob not page aligned\n");
goto bad_fw;
}
nr_pages = PFN_UP(blob->size);
flags = blob->map_flags | BASE_MEM_FLAG_MAP_FIXED;
va_region = kbase_mem_alloc(kctx, nr_pages, nr_pages,
0, &flags, &gpu_va);
if (!va_region) {
dev_err(kbdev->dev, "Failed to allocate for blob\n");
} else {
struct kbase_vmap_struct vmap = { 0 };
const u8 *payload;
void *dst;
/* copy the payload, */
payload = fw + blob->payload_offset;
dst = kbase_vmap(kctx,
va_region->start_pfn << PAGE_SHIFT,
nr_pages << PAGE_SHIFT, &vmap);
if (dst) {
memcpy(dst, payload, blob->size);
kbase_vunmap(kctx, &vmap);
} else {
dev_err(kbdev->dev,
"Failed to copy payload\n");
}
}
blob_offset = blob->blob_offset; /* follow chain */
}
release_firmware(firmware);
kbasep_js_schedule_privileged_ctx(kbdev, kctx);
kbdev->dummy_job_wa.ctx = kctx;
err = sysfs_create_file(&kbdev->dev->kobj,
&dev_attr_dummy_job_wa_info.attr);
if (err)
dev_err(kbdev->dev, "SysFS file creation for dummy job wa failed\n");
return 0;
bad_fw:
kbase_destroy_context(kctx);
no_ctx:
release_firmware(firmware);
return -EFAULT;
}
void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev)
{
struct kbase_context *wa_ctx;
/* Can be safely called even if the file wasn't created on probe */
sysfs_remove_file(&kbdev->dev->kobj, &dev_attr_dummy_job_wa_info.attr);
wa_ctx = READ_ONCE(kbdev->dummy_job_wa.ctx);
WRITE_ONCE(kbdev->dummy_job_wa.ctx, NULL);
/* make this write visible before we tear down the ctx */
smp_mb();
if (wa_ctx) {
kbasep_js_release_privileged_ctx(kbdev, wa_ctx);
kbase_destroy_context(wa_ctx);
}
}

45
drivers/gpu/arm/bifrost/mali_kbase_dummy_job_wa.h Normal file
View File

@ -0,0 +1,45 @@
/*
*
* (C) COPYRIGHT 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, you can access it online at
* http://www.gnu.org/licenses/gpl-2.0.html.
*
* SPDX-License-Identifier: GPL-2.0
*
*/
#ifndef _KBASE_DUMMY_JOB_WORKAROUND_
#define _KBASE_DUMMY_JOB_WORKAROUND_
#define KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE (1ull << 0)
#define KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP (1ull << 1)
#define KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER (1ull << 2)
#define KBASE_DUMMY_JOB_WA_FLAGS (KBASE_DUMMY_JOB_WA_FLAG_SERIALIZE | \
KBASE_DUMMY_JOB_WA_FLAG_WAIT_POWERUP | \
KBASE_DUMMY_JOB_WA_FLAG_LOGICAL_SHADER_POWER)
int kbase_dummy_job_wa_load(struct kbase_device *kbdev);
void kbase_dummy_job_wa_cleanup(struct kbase_device *kbdev);
int kbase_dummy_job_wa_execute(struct kbase_device *kbdev, u64 cores);
static inline bool kbase_dummy_job_wa_enabled(struct kbase_device *kbdev)
{
return (kbdev->dummy_job_wa.ctx != NULL);
}
#endif /* _KBASE_DUMMY_JOB_WORKAROUND_ */

32
drivers/gpu/arm/bifrost/mali_kbase_event.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2016,2018-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2016,2018-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -24,7 +24,8 @@
#include <mali_kbase.h>
#include <mali_kbase_debug.h>
#include <mali_kbase_tracepoints.h>
#include <tl/mali_kbase_tracepoints.h>
#include <mali_linux_trace.h>
static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, struct kbase_jd_atom *katom)
{
@ -44,22 +45,12 @@ static struct base_jd_udata kbase_event_process(struct kbase_context *kctx, stru
KBASE_TLSTREAM_TL_DEL_ATOM(kbdev, katom);
katom->status = KBASE_JD_ATOM_STATE_UNUSED;
dev_dbg(kbdev->dev, "Atom %p status to unused\n", (void *)katom);
wake_up(&katom->completed);
return data;
}
int kbase_event_pending(struct kbase_context *ctx)
{
KBASE_DEBUG_ASSERT(ctx);
return (atomic_read(&ctx->event_count) != 0) ||
(atomic_read(&ctx->event_closed) != 0);
}
KBASE_EXPORT_TEST_API(kbase_event_pending);
int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *uevent)
{
struct kbase_jd_atom *atom;
@ -93,6 +84,7 @@ int kbase_event_dequeue(struct kbase_context *ctx, struct base_jd_event_v2 *ueve
dev_dbg(ctx->kbdev->dev, "event dequeuing %p\n", (void *)atom);
uevent->event_code = atom->event_code;
uevent->atom_number = (atom - ctx->jctx.atoms);
if (atom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES)
@ -174,22 +166,25 @@ void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom)
{
struct kbase_device *kbdev = ctx->kbdev;
dev_dbg(kbdev->dev, "Posting event for atom %p\n", (void *)atom);
if (atom->core_req & BASE_JD_REQ_EVENT_ONLY_ON_FAILURE) {
if (atom->event_code == BASE_JD_EVENT_DONE) {
/* Don't report the event */
dev_dbg(kbdev->dev, "Suppressing event (atom done)\n");
kbase_event_process_noreport(ctx, atom);
return;
}
}
if (atom->core_req & BASEP_JD_REQ_EVENT_NEVER) {
/* Don't report the event */
dev_dbg(kbdev->dev, "Suppressing event (never)\n");
kbase_event_process_noreport(ctx, atom);
return;
}
KBASE_TLSTREAM_TL_ATTRIB_ATOM_STATE(kbdev, atom, TL_ATOM_STATE_POSTED);
if (atom->core_req & BASE_JD_REQ_EVENT_COALESCE) {
/* Don't report the event until other event(s) have completed */
dev_dbg(kbdev->dev, "Deferring event (coalesced)\n");
mutex_lock(&ctx->event_mutex);
list_add_tail(&atom->dep_item[0], &ctx->event_coalesce_list);
++ctx->event_coalesce_count;
@ -203,8 +198,13 @@ void kbase_event_post(struct kbase_context *ctx, struct kbase_jd_atom *atom)
list_add_tail(&atom->dep_item[0], &ctx->event_list);
atomic_add(event_count, &ctx->event_count);
mutex_unlock(&ctx->event_mutex);
dev_dbg(kbdev->dev, "Reporting %d events\n", event_count);
kbase_event_wakeup(ctx);
/* Post-completion latency */
trace_sysgraph(SGR_POST, ctx->id,
kbase_jd_atom_id(ctx, atom));
}
}
KBASE_EXPORT_TEST_API(kbase_event_post);
@ -224,9 +224,7 @@ int kbase_event_init(struct kbase_context *kctx)
INIT_LIST_HEAD(&kctx->event_list);
INIT_LIST_HEAD(&kctx->event_coalesce_list);
mutex_init(&kctx->event_mutex);
atomic_set(&kctx->event_count, 0);
kctx->event_coalesce_count = 0;
atomic_set(&kctx->event_closed, false);
kctx->event_workq = alloc_workqueue("kbase_event", WQ_MEM_RECLAIM, 1);
if (NULL == kctx->event_workq)

4
drivers/gpu/arm/bifrost/mali_kbase_fence.c
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -87,6 +87,7 @@ const struct dma_fence_ops kbase_fence_ops = {
.fence_value_str = kbase_fence_fence_value_str
};
#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 10, 0))
struct fence *
kbase_fence_out_new(struct kbase_jd_atom *katom)
@ -210,3 +211,4 @@ kbase_fence_add_callback(struct kbase_jd_atom *katom,
return err;
}

4
drivers/gpu/arm/bifrost/mali_kbase_fence.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2010-2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2010-2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -87,6 +87,7 @@ struct dma_fence *kbase_fence_out_new(struct kbase_jd_atom *katom);
} while (0)
#endif
/**
* kbase_fence_out_remove() - Removes the output fence from atom
* @katom: Atom to remove output fence for
@ -268,6 +269,7 @@ bool kbase_fence_free_callbacks(struct kbase_jd_atom *katom);
*/
#define kbase_fence_out_get(katom) dma_fence_get((katom)->dma_fence.fence)
/**
* kbase_fence_put() - Releases a reference to a fence
* @fence: Fence to release reference for.

5
drivers/gpu/arm/bifrost/mali_kbase_gpu_memory_debugfs.c
View File

@ -21,6 +21,7 @@
*/
#include <mali_kbase.h>
#include <device/mali_kbase_device.h>
#ifdef CONFIG_DEBUG_FS
/** Show callback for the @c gpu_memory debugfs file.
@ -40,7 +41,7 @@ static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data)
struct list_head *entry;
const struct list_head *kbdev_list;
kbdev_list = kbase_dev_list_get();
kbdev_list = kbase_device_get_list();
list_for_each(entry, kbdev_list) {
struct kbase_device *kbdev = NULL;
struct kbase_context *kctx;
@ -61,7 +62,7 @@ static int kbasep_gpu_memory_seq_show(struct seq_file *sfile, void *data)
}
mutex_unlock(&kbdev->kctx_list_lock);
}
kbase_dev_list_put(kbdev_list);
kbase_device_put_list(kbdev_list);
return 0;
}

102
drivers/gpu/arm/bifrost/mali_kbase_gpuprops.c
View File

@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
*
* (C) COPYRIGHT 2011-2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -27,30 +28,19 @@
*/
#include <mali_kbase.h>
#include <mali_midg_regmap.h>
#include <gpu/mali_kbase_gpu_regmap.h>
#include <mali_kbase_gpuprops.h>
#include <mali_kbase_hwaccess_gpuprops.h>
#include <mali_kbase_config_defaults.h>
#include "mali_kbase_ioctl.h"
#include <linux/clk.h>
#include <mali_kbase_pm_internal.h>
#include <linux/of_platform.h>
#include <linux/moduleparam.h>
/**
* KBASE_UBFX32 - Extracts bits from a 32-bit bitfield.
* @value: The value from which to extract bits.
* @offset: The first bit to extract (0 being the LSB).
* @size: The number of bits to extract.
*
* Context: @offset + @size <= 32.
*
* Return: Bits [@offset, @offset + @size) from @value.
*/
/* from mali_cdsb.h */
#define KBASE_UBFX32(value, offset, size) \
(((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1))
static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const props)
static void kbase_gpuprops_construct_coherent_groups(
struct base_gpu_props * const props)
{
struct mali_base_gpu_coherent_group *current_group;
u64 group_present;
@ -119,22 +109,28 @@ static void kbase_gpuprops_construct_coherent_groups(base_gpu_props * const prop
/**
* kbase_gpuprops_get_props - Get the GPU configuration
* @gpu_props: The &base_gpu_props structure
* @gpu_props: The &struct base_gpu_props structure
* @kbdev: The &struct kbase_device structure for the device
*
* Fill the &base_gpu_props structure with values from the GPU configuration
* registers. Only the raw properties are filled in this function
* Fill the &struct base_gpu_props structure with values from the GPU
* configuration registers. Only the raw properties are filled in this function.
*
* Return: Zero on success, Linux error code on failure
*/
static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev)
static int kbase_gpuprops_get_props(struct base_gpu_props * const gpu_props,
struct kbase_device *kbdev)
{
struct kbase_gpuprops_regdump regdump;
int i;
int err;
KBASE_DEBUG_ASSERT(NULL != kbdev);
KBASE_DEBUG_ASSERT(NULL != gpu_props);
/* Dump relevant registers */
kbase_backend_gpuprops_get(kbdev, &regdump);
err = kbase_backend_gpuprops_get(kbdev, &regdump);
if (err)
return err;
gpu_props->raw_props.gpu_id = regdump.gpu_id;
gpu_props->raw_props.tiler_features = regdump.tiler_features;
@ -169,9 +165,12 @@ static void kbase_gpuprops_get_props(base_gpu_props * const gpu_props, struct kb
gpu_props->raw_props.thread_max_workgroup_size = regdump.thread_max_workgroup_size;
gpu_props->raw_props.thread_features = regdump.thread_features;
gpu_props->raw_props.thread_tls_alloc = regdump.thread_tls_alloc;
return 0;
}
void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props)
void kbase_gpuprops_update_core_props_gpu_id(
struct base_gpu_props * const gpu_props)
{
gpu_props->core_props.version_status =
KBASE_UBFX32(gpu_props->raw_props.gpu_id, 0U, 4);
@ -185,13 +184,14 @@ void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props)
/**
* kbase_gpuprops_calculate_props - Calculate the derived properties
* @gpu_props: The &base_gpu_props structure
* @gpu_props: The &struct base_gpu_props structure
* @kbdev: The &struct kbase_device structure for the device
*
* Fill the &base_gpu_props structure with values derived from the GPU
* Fill the &struct base_gpu_props structure with values derived from the GPU
* configuration registers
*/
static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, struct kbase_device *kbdev)
static void kbase_gpuprops_calculate_props(
struct base_gpu_props * const gpu_props, struct kbase_device *kbdev)
{
int i;
u32 gpu_id;
@ -247,8 +247,8 @@ static void kbase_gpuprops_calculate_props(base_gpu_props * const gpu_props, str
gpu_props->thread_props.tls_alloc =
gpu_props->raw_props.thread_tls_alloc;
/* Workaround for GPU2019HW-509. MIDHARC-2364 was wrongfully applied
* to tDUx GPUs.
/* MIDHARC-2364 was intended for tULx.
* Workaround for the incorrectly applied THREAD_FEATURES to tDUx.
*/
gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
product_id = gpu_id & GPU_ID_VERSION_PRODUCT_ID;
@ -320,15 +320,18 @@ void kbase_gpuprops_set(struct kbase_device *kbdev)
gpu_props->num_job_slots = hweight32(raw->js_present);
}
void kbase_gpuprops_set_features(struct kbase_device *kbdev)
int kbase_gpuprops_set_features(struct kbase_device *kbdev)
{
base_gpu_props *gpu_props;
struct base_gpu_props *gpu_props;
struct kbase_gpuprops_regdump regdump;
int err;
gpu_props = &kbdev->gpu_props.props;
/* Dump relevant registers */
kbase_backend_gpuprops_get_features(kbdev, &regdump);
err = kbase_backend_gpuprops_get_features(kbdev, &regdump);
if (err)
return err;
/*
* Copy the raw value from the register, later this will get turned
@ -340,6 +343,8 @@ void kbase_gpuprops_set_features(struct kbase_device *kbdev)
if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_THREAD_GROUP_SPLIT))
gpu_props->thread_props.max_thread_group_split = 0;
return err;
}
/*
@ -391,15 +396,17 @@ static bool kbase_read_l2_config_from_dt(struct kbase_device * const kbdev)
return false;
}
void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
{
int err = 0;
if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_L2_CONFIG)) {
struct kbase_gpuprops_regdump regdump;
base_gpu_props *gpu_props = &kbdev->gpu_props.props;
struct base_gpu_props *gpu_props = &kbdev->gpu_props.props;
/* Check for L2 cache size & hash overrides */
if (!kbase_read_l2_config_from_dt(kbdev))
return;
return 0;
/* Need L2 to get powered to reflect to L2_FEATURES */
kbase_pm_context_active(kbdev);
@ -408,7 +415,9 @@ void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
kbase_pm_wait_for_l2_powered(kbdev);
/* Dump L2_FEATURES register */
kbase_backend_gpuprops_get_l2_features(kbdev, &regdump);
err = kbase_backend_gpuprops_get_l2_features(kbdev, &regdump);
if (err)
goto idle_gpu;
dev_info(kbdev->dev, "Reflected L2_FEATURES is 0x%x\n",
regdump.l2_features);
@ -418,9 +427,12 @@ void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev)
gpu_props->l2_props.log2_cache_size =
KBASE_UBFX32(gpu_props->raw_props.l2_features, 16U, 8);
idle_gpu:
/* Let GPU idle */
kbase_pm_context_idle(kbdev);
}
return err;
}
static struct {
@ -600,3 +612,25 @@ int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev)
return 0;
}
void kbase_gpuprops_free_user_buffer(struct kbase_device *kbdev)
{
kfree(kbdev->gpu_props.prop_buffer);
}
int kbase_device_populate_max_freq(struct kbase_device *kbdev)
{
struct mali_base_gpu_core_props *core_props;
/* obtain max configured gpu frequency, if devfreq is enabled then
* this will be overridden by the highest operating point found
*/
core_props = &(kbdev->gpu_props.props.core_props);
#ifdef GPU_FREQ_KHZ_MAX
core_props->gpu_freq_khz_max = GPU_FREQ_KHZ_MAX;
#else
core_props->gpu_freq_khz_max = DEFAULT_GPU_FREQ_KHZ_MAX;
#endif
return 0;
}

64
drivers/gpu/arm/bifrost/mali_kbase_gpuprops.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2015,2017,2019 ARM Limited. All rights reserved.
* (C) COPYRIGHT ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -18,6 +18,20 @@
*
* SPDX-License-Identifier: GPL-2.0
*
*//* SPDX-License-Identifier: GPL-2.0 */
/*
*
* (C) COPYRIGHT 2011-2015, 2017, 2019-2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
* Foundation, and any use by you of this program is subject to the terms
* of such GNU licence.
*
* A copy of the licence is included with the program, and can also be obtained
* from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
*
*/
@ -35,6 +49,20 @@
/* Forward definition - see mali_kbase.h */
struct kbase_device;
/**
* KBASE_UBFX32 - Extracts bits from a 32-bit bitfield.
* @value: The value from which to extract bits.
* @offset: The first bit to extract (0 being the LSB).
* @size: The number of bits to extract.
*
* Context: @offset + @size <= 32.
*
* Return: Bits [@offset, @offset + @size) from @value.
*/
/* from mali_cdsb.h */
#define KBASE_UBFX32(value, offset, size) \
(((u32)(value) >> (u32)(offset)) & (u32)((1ULL << (u32)(size)) - 1))
/**
* @brief Set up Kbase GPU properties.
*
@ -51,26 +79,48 @@ void kbase_gpuprops_set(struct kbase_device *kbdev);
* This function sets up GPU properties that are dependent on the hardware
* features bitmask. This function must be preceeded by a call to
* kbase_hw_set_features_mask().
*
* Return: Zero on success, Linux error code on failure
*/
void kbase_gpuprops_set_features(struct kbase_device *kbdev);
int kbase_gpuprops_set_features(struct kbase_device *kbdev);
/**
* kbase_gpuprops_update_l2_features - Update GPU property of L2_FEATURES
* @kbdev: Device pointer
*
* This function updates l2_features and the log2 cache size.
*
* Return: Zero on success, Linux error code for failure
*/
void kbase_gpuprops_update_l2_features(struct kbase_device *kbdev);
int kbase_gpuprops_update_l2_features(struct kbase_device *kbdev);
/**
* kbase_gpuprops_populate_user_buffer - Populate the GPU properties buffer
* @kbdev: The kbase device
*
* Fills kbdev->gpu_props->prop_buffer with the GPU properties for user
* space to read.
* Fills prop_buffer with the GPU properties for user space to read.
*/
int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev);
/**
* kbase_gpuprops_free_user_buffer - Free the GPU properties buffer.
* @kbdev: kbase device pointer
*
* Free the GPU properties buffer allocated from
* kbase_gpuprops_populate_user_buffer.
*/
void kbase_gpuprops_free_user_buffer(struct kbase_device *kbdev);
/**
* kbase_device_populate_max_freq - Populate max gpu frequency.
* @kbdev: kbase device pointer
*
* Populate the maximum gpu frequency to be used when devfreq is disabled.
*
* Return: 0 on success and non-zero value on failure.
*/
int kbase_device_populate_max_freq(struct kbase_device *kbdev);
/**
* kbase_gpuprops_update_core_props_gpu_id - break down gpu id value
* @gpu_props: the &base_gpu_props structure
@ -79,7 +129,7 @@ int kbase_gpuprops_populate_user_buffer(struct kbase_device *kbdev);
* separate fields (version_status, minor_revision, major_revision, product_id)
* stored in base_gpu_props::core_props.
*/
void kbase_gpuprops_update_core_props_gpu_id(base_gpu_props * const gpu_props);
void kbase_gpuprops_update_core_props_gpu_id(
struct base_gpu_props * const gpu_props);
#endif /* _KBASE_GPUPROPS_H_ */

4
drivers/gpu/arm/bifrost/mali_kbase_gpuprops_types.h
View File

@ -1,6 +1,6 @@
/*
*
* (C) COPYRIGHT 2011-2018 ARM Limited. All rights reserved.
* (C) COPYRIGHT 2011-2018, 2020 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
@ -89,7 +89,7 @@ struct kbase_gpu_props {
struct kbase_gpu_mmu_props mmu;
/* Properties shared with userspace */
base_gpu_props props;
struct base_gpu_props props;
u32 prop_buffer_size;
void *prop_buffer;

Some files were not shown because too many files have changed in this diff Show More