github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-05-31 18:43:33 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

Merge branch 'for-7.0/cxl-aer-prep' into cxl-for-next

Browse Source 
Preparation for CXL port error protocol handling. First part contains
all the changes centered around setting up the PCI side of error
handling.

cxl: Update RAS handler interfaces to also support CXL Ports
cxl/mem: Clarify @host for devm_cxl_add_nvdimm()
PCI/AER: Update struct aer_err_info with kernel-doc formatting
PCI/AER: Report CXL or PCIe bus type in AER trace logging
PCI/AER: Use guard() in cxl_rch_handle_error_iter()
PCI/AER: Move CXL RCH error handling to aer_cxl_rch.c
PCI/AER: Update is_internal_error() to be non-static is_aer_internal_error()
PCI/AER: Export pci_aer_unmask_internal_errors()
cxl/pci: Move CXL driver's RCH error handling into core/ras_rch.c
PCI/AER: Replace PCIEAER_CXL symbol with CXL_RAS
cxl/pci: Remove CXL VH handling in CONFIG_PCIEAER_CXL conditional blocks from core/pci.c
PCI: Replace cxl_error_is_native() with pcie_aer_is_native()
cxl/pci: Remove unnecessary CXL RCH handling helper functions
cxl/pci: Remove unnecessary CXL Endpoint handling helper functions
PCI: Introduce pcie_is_cxl()
PCI: Update CXL DVSEC definitions
PCI: Move CXL DVSEC definitions into uapi/linux/pci_regs.h
This commit is contained in:
Dave Jiang 2026-01-22 16:59:43 -07:00
commit 914c743509
24 changed files with 666 additions and 565 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
drivers/cxl/Kconfig
View File

@ -233,4 +233,8 @@ config CXL_MCE
def_bool y
depends on X86_MCE && MEMORY_FAILURE
config CXL_RAS
def_bool y
depends on ACPI_APEI_GHES && PCIEAER && CXL_BUS
endif

3
drivers/cxl/core/Makefile
View File

@ -14,9 +14,10 @@ cxl_core-y += pci.o
cxl_core-y += hdm.o
cxl_core-y += pmu.o
cxl_core-y += cdat.o
cxl_core-y += ras.o
cxl_core-$(CONFIG_TRACING) += trace.o
cxl_core-$(CONFIG_CXL_REGION) += region.o
cxl_core-$(CONFIG_CXL_MCE) += mce.o
cxl_core-$(CONFIG_CXL_FEATURES) += features.o
cxl_core-$(CONFIG_CXL_EDAC_MEM_FEATURES) += edac.o
cxl_core-$(CONFIG_CXL_RAS) += ras.o
cxl_core-$(CONFIG_CXL_RAS) += ras_rch.o

22
drivers/cxl/core/core.h
View File

@ -144,8 +144,30 @@ int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c);
int cxl_port_get_switch_dport_bandwidth(struct cxl_port *port,
struct access_coordinate *c);
#ifdef CONFIG_CXL_RAS
int cxl_ras_init(void);
void cxl_ras_exit(void);
bool cxl_handle_ras(struct device *dev, void __iomem *ras_base);
void cxl_handle_cor_ras(struct device *dev, void __iomem *ras_base);
void cxl_dport_map_rch_aer(struct cxl_dport *dport);
void cxl_disable_rch_root_ints(struct cxl_dport *dport);
void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds);
#else
static inline int cxl_ras_init(void)
{
return 0;
}
static inline void cxl_ras_exit(void) { }
static inline bool cxl_handle_ras(struct device *dev, void __iomem *ras_base)
{
return false;
}
static inline void cxl_handle_cor_ras(struct device *dev, void __iomem *ras_base) { }
static inline void cxl_dport_map_rch_aer(struct cxl_dport *dport) { }
static inline void cxl_disable_rch_root_ints(struct cxl_dport *dport) { }
static inline void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { }
#endif /* CONFIG_CXL_RAS */
int cxl_gpf_port_setup(struct cxl_dport *dport);
struct cxl_hdm;

376
drivers/cxl/core/pci.c
View File

@ -86,12 +86,12 @@ static int cxl_dvsec_mem_range_valid(struct cxl_dev_state *cxlds, int id)
i = 1;
do {
rc = pci_read_config_dword(pdev,
d + CXL_DVSEC_RANGE_SIZE_LOW(id),
d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id),
&temp);
if (rc)
return rc;
valid = FIELD_GET(CXL_DVSEC_MEM_INFO_VALID, temp);
valid = FIELD_GET(PCI_DVSEC_CXL_MEM_INFO_VALID, temp);
if (valid)
break;
msleep(1000);
@ -121,11 +121,11 @@ static int cxl_dvsec_mem_range_active(struct cxl_dev_state *cxlds, int id)
/* Check MEM ACTIVE bit, up to 60s timeout by default */
for (i = media_ready_timeout; i; i--) {
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(id), &temp);
pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(id), &temp);
if (rc)
return rc;
active = FIELD_GET(CXL_DVSEC_MEM_ACTIVE, temp);
active = FIELD_GET(PCI_DVSEC_CXL_MEM_ACTIVE, temp);
if (active)
break;
msleep(1000);
@ -154,11 +154,11 @@ int cxl_await_media_ready(struct cxl_dev_state *cxlds)
u16 cap;
rc = pci_read_config_word(pdev,
d + CXL_DVSEC_CAP_OFFSET, &cap);
d + PCI_DVSEC_CXL_CAP, &cap);
if (rc)
return rc;
hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
hdm_count = FIELD_GET(PCI_DVSEC_CXL_HDM_COUNT, cap);
for (i = 0; i < hdm_count; i++) {
rc = cxl_dvsec_mem_range_valid(cxlds, i);
if (rc)
@ -186,16 +186,16 @@ static int cxl_set_mem_enable(struct cxl_dev_state *cxlds, u16 val)
u16 ctrl;
int rc;
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl);
if (rc < 0)
return rc;
if ((ctrl & CXL_DVSEC_MEM_ENABLE) == val)
if ((ctrl & PCI_DVSEC_CXL_MEM_ENABLE) == val)
return 1;
ctrl &= ~CXL_DVSEC_MEM_ENABLE;
ctrl &= ~PCI_DVSEC_CXL_MEM_ENABLE;
ctrl |= val;
rc = pci_write_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, ctrl);
rc = pci_write_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, ctrl);
if (rc < 0)
return rc;
@ -211,7 +211,7 @@ static int devm_cxl_enable_mem(struct device *host, struct cxl_dev_state *cxlds)
{
int rc;
rc = cxl_set_mem_enable(cxlds, CXL_DVSEC_MEM_ENABLE);
rc = cxl_set_mem_enable(cxlds, PCI_DVSEC_CXL_MEM_ENABLE);
if (rc < 0)
return rc;
if (rc > 0)
@ -273,11 +273,11 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
return -ENXIO;
}
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CAP_OFFSET, &cap);
rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CAP, &cap);
if (rc)
return rc;
if (!(cap & CXL_DVSEC_MEM_CAPABLE)) {
if (!(cap & PCI_DVSEC_CXL_MEM_CAPABLE)) {
dev_dbg(dev, "Not MEM Capable\n");
return -ENXIO;
}
@ -288,7 +288,7 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
* driver is for a spec defined class code which must be CXL.mem
* capable, there is no point in continuing to enable CXL.mem.
*/
hdm_count = FIELD_GET(CXL_DVSEC_HDM_COUNT_MASK, cap);
hdm_count = FIELD_GET(PCI_DVSEC_CXL_HDM_COUNT, cap);
if (!hdm_count || hdm_count > 2)
return -EINVAL;
@ -297,11 +297,11 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
* disabled, and they will remain moot after the HDM Decoder
* capability is enabled.
*/
rc = pci_read_config_word(pdev, d + CXL_DVSEC_CTRL_OFFSET, &ctrl);
rc = pci_read_config_word(pdev, d + PCI_DVSEC_CXL_CTRL, &ctrl);
if (rc)
return rc;
info->mem_enabled = FIELD_GET(CXL_DVSEC_MEM_ENABLE, ctrl);
info->mem_enabled = FIELD_GET(PCI_DVSEC_CXL_MEM_ENABLE, ctrl);
if (!info->mem_enabled)
return 0;
@ -314,35 +314,35 @@ int cxl_dvsec_rr_decode(struct cxl_dev_state *cxlds,
return rc;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_HIGH(i), &temp);
pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_HIGH(i), &temp);
if (rc)
return rc;
size = (u64)temp << 32;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_SIZE_LOW(i), &temp);
pdev, d + PCI_DVSEC_CXL_RANGE_SIZE_LOW(i), &temp);
if (rc)
return rc;
size |= temp & CXL_DVSEC_MEM_SIZE_LOW_MASK;
size |= temp & PCI_DVSEC_CXL_MEM_SIZE_LOW;
if (!size) {
continue;
}
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_BASE_HIGH(i), &temp);
pdev, d + PCI_DVSEC_CXL_RANGE_BASE_HIGH(i), &temp);
if (rc)
return rc;
base = (u64)temp << 32;
rc = pci_read_config_dword(
pdev, d + CXL_DVSEC_RANGE_BASE_LOW(i), &temp);
pdev, d + PCI_DVSEC_CXL_RANGE_BASE_LOW(i), &temp);
if (rc)
return rc;
base |= temp & CXL_DVSEC_MEM_BASE_LOW_MASK;
base |= temp & PCI_DVSEC_CXL_MEM_BASE_LOW;
info->dvsec_range[ranges++] = (struct range) {
.start = base,
@ -632,324 +632,6 @@ void read_cdat_data(struct cxl_port *port)
}
EXPORT_SYMBOL_NS_GPL(read_cdat_data, "CXL");
static void __cxl_handle_cor_ras(struct cxl_dev_state *cxlds,
void __iomem *ras_base)
{
void __iomem *addr;
u32 status;
if (!ras_base)
return;
addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET;
status = readl(addr);
if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) {
writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr);
trace_cxl_aer_correctable_error(cxlds->cxlmd, status);
}
}
static void cxl_handle_endpoint_cor_ras(struct cxl_dev_state *cxlds)
{
return __cxl_handle_cor_ras(cxlds, cxlds->regs.ras);
}
/* CXL spec rev3.0 8.2.4.16.1 */
static void header_log_copy(void __iomem *ras_base, u32 *log)
{
void __iomem *addr;
u32 *log_addr;
int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32);
addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET;
log_addr = log;
for (i = 0; i < log_u32_size; i++) {
*log_addr = readl(addr);
log_addr++;
addr += sizeof(u32);
}
}
/*
* Log the state of the RAS status registers and prepare them to log the
* next error status. Return 1 if reset needed.
*/
static bool __cxl_handle_ras(struct cxl_dev_state *cxlds,
void __iomem *ras_base)
{
u32 hl[CXL_HEADERLOG_SIZE_U32];
void __iomem *addr;
u32 status;
u32 fe;
if (!ras_base)
return false;
addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET;
status = readl(addr);
if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK))
return false;
/* If multiple errors, log header points to first error from ctrl reg */
if (hweight32(status) > 1) {
void __iomem *rcc_addr =
ras_base + CXL_RAS_CAP_CONTROL_OFFSET;
fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK,
readl(rcc_addr)));
} else {
fe = status;
}
header_log_copy(ras_base, hl);
trace_cxl_aer_uncorrectable_error(cxlds->cxlmd, status, fe, hl);
writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr);
return true;
}
static bool cxl_handle_endpoint_ras(struct cxl_dev_state *cxlds)
{
return __cxl_handle_ras(cxlds, cxlds->regs.ras);
}
#ifdef CONFIG_PCIEAER_CXL
static void cxl_dport_map_rch_aer(struct cxl_dport *dport)
{
resource_size_t aer_phys;
struct device *host;
u16 aer_cap;
aer_cap = cxl_rcrb_to_aer(dport->dport_dev, dport->rcrb.base);
if (aer_cap) {
host = dport->reg_map.host;
aer_phys = aer_cap + dport->rcrb.base;
dport->regs.dport_aer = devm_cxl_iomap_block(host, aer_phys,
sizeof(struct aer_capability_regs));
}
}
static void cxl_dport_map_ras(struct cxl_dport *dport)
{
struct cxl_register_map *map = &dport->reg_map;
struct device *dev = dport->dport_dev;
if (!map->component_map.ras.valid)
dev_dbg(dev, "RAS registers not found\n");
else if (cxl_map_component_regs(map, &dport->regs.component,
BIT(CXL_CM_CAP_CAP_ID_RAS)))
dev_dbg(dev, "Failed to map RAS capability.\n");
}
static void cxl_disable_rch_root_ints(struct cxl_dport *dport)
{
void __iomem *aer_base = dport->regs.dport_aer;
u32 aer_cmd_mask, aer_cmd;
if (!aer_base)
return;
/*
* Disable RCH root port command interrupts.
* CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors
*
* This sequence may not be necessary. CXL spec states disabling
* the root cmd register's interrupts is required. But, PCI spec
* shows these are disabled by default on reset.
*/
aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN |
PCI_ERR_ROOT_CMD_NONFATAL_EN |
PCI_ERR_ROOT_CMD_FATAL_EN);
aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND);
aer_cmd &= ~aer_cmd_mask;
writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND);
}
/**
* cxl_dport_init_ras_reporting - Setup CXL RAS report on this dport
* @dport: the cxl_dport that needs to be initialized
* @host: host device for devm operations
*/
void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host)
{
dport->reg_map.host = host;
cxl_dport_map_ras(dport);
if (dport->rch) {
struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport->dport_dev);
if (!host_bridge->native_aer)
return;
cxl_dport_map_rch_aer(dport);
cxl_disable_rch_root_ints(dport);
}
}
EXPORT_SYMBOL_NS_GPL(cxl_dport_init_ras_reporting, "CXL");
static void cxl_handle_rdport_cor_ras(struct cxl_dev_state *cxlds,
struct cxl_dport *dport)
{
return __cxl_handle_cor_ras(cxlds, dport->regs.ras);
}
static bool cxl_handle_rdport_ras(struct cxl_dev_state *cxlds,
struct cxl_dport *dport)
{
return __cxl_handle_ras(cxlds, dport->regs.ras);
}
/*
* Copy the AER capability registers using 32 bit read accesses.
* This is necessary because RCRB AER capability is MMIO mapped. Clear the
* status after copying.
*
* @aer_base: base address of AER capability block in RCRB
* @aer_regs: destination for copying AER capability
*/
static bool cxl_rch_get_aer_info(void __iomem *aer_base,
struct aer_capability_regs *aer_regs)
{
int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32);
u32 *aer_regs_buf = (u32 *)aer_regs;
int n;
if (!aer_base)
return false;
/* Use readl() to guarantee 32-bit accesses */
for (n = 0; n < read_cnt; n++)
aer_regs_buf[n] = readl(aer_base + n * sizeof(u32));
writel(aer_regs->uncor_status, aer_base + PCI_ERR_UNCOR_STATUS);
writel(aer_regs->cor_status, aer_base + PCI_ERR_COR_STATUS);
return true;
}
/* Get AER severity. Return false if there is no error. */
static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs,
int *severity)
{
if (aer_regs->uncor_status & ~aer_regs->uncor_mask) {
if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV)
*severity = AER_FATAL;
else
*severity = AER_NONFATAL;
return true;
}
if (aer_regs->cor_status & ~aer_regs->cor_mask) {
*severity = AER_CORRECTABLE;
return true;
}
return false;
}
static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
struct aer_capability_regs aer_regs;
struct cxl_dport *dport;
int severity;
struct cxl_port *port __free(put_cxl_port) =
cxl_pci_find_port(pdev, &dport);
if (!port)
return;
if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs))
return;
if (!cxl_rch_get_aer_severity(&aer_regs, &severity))
return;
pci_print_aer(pdev, severity, &aer_regs);
if (severity == AER_CORRECTABLE)
cxl_handle_rdport_cor_ras(cxlds, dport);
else
cxl_handle_rdport_ras(cxlds, dport);
}
#else
static void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) { }
#endif
void cxl_cor_error_detected(struct pci_dev *pdev)
{
struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
struct device *dev = &cxlds->cxlmd->dev;
scoped_guard(device, dev) {
if (!dev->driver) {
dev_warn(&pdev->dev,
"%s: memdev disabled, abort error handling\n",
dev_name(dev));
return;
}
if (cxlds->rcd)
cxl_handle_rdport_errors(cxlds);
cxl_handle_endpoint_cor_ras(cxlds);
}
}
EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL");
pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state)
{
struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
struct cxl_memdev *cxlmd = cxlds->cxlmd;
struct device *dev = &cxlmd->dev;
bool ue;
scoped_guard(device, dev) {
if (!dev->driver) {
dev_warn(&pdev->dev,
"%s: memdev disabled, abort error handling\n",
dev_name(dev));
return PCI_ERS_RESULT_DISCONNECT;
}
if (cxlds->rcd)
cxl_handle_rdport_errors(cxlds);
/*
* A frozen channel indicates an impending reset which is fatal to
* CXL.mem operation, and will likely crash the system. On the off
* chance the situation is recoverable dump the status of the RAS
* capability registers and bounce the active state of the memdev.
*/
ue = cxl_handle_endpoint_ras(cxlds);
}
switch (state) {
case pci_channel_io_normal:
if (ue) {
device_release_driver(dev);
return PCI_ERS_RESULT_NEED_RESET;
}
return PCI_ERS_RESULT_CAN_RECOVER;
case pci_channel_io_frozen:
dev_warn(&pdev->dev,
"%s: frozen state error detected, disable CXL.mem\n",
dev_name(dev));
device_release_driver(dev);
return PCI_ERS_RESULT_NEED_RESET;
case pci_channel_io_perm_failure:
dev_warn(&pdev->dev,
"failure state error detected, request disconnect\n");
return PCI_ERS_RESULT_DISCONNECT;
}
return PCI_ERS_RESULT_NEED_RESET;
}
EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL");
static int cxl_flit_size(struct pci_dev *pdev)
{
if (cxl_pci_flit_256(pdev))
@ -1068,7 +750,7 @@ u16 cxl_gpf_get_dvsec(struct device *dev)
is_port = false;
dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
is_port ? CXL_DVSEC_PORT_GPF : CXL_DVSEC_DEVICE_GPF);
is_port ? PCI_DVSEC_CXL_PORT_GPF : PCI_DVSEC_CXL_DEVICE_GPF);
if (!dvsec)
dev_warn(dev, "%s GPF DVSEC not present\n",
is_port ? "Port" : "Device");
@ -1084,14 +766,14 @@ static int update_gpf_port_dvsec(struct pci_dev *pdev, int dvsec, int phase)
switch (phase) {
case 1:
offset = CXL_DVSEC_PORT_GPF_PHASE_1_CONTROL_OFFSET;
base = CXL_DVSEC_PORT_GPF_PHASE_1_TMO_BASE_MASK;
scale = CXL_DVSEC_PORT_GPF_PHASE_1_TMO_SCALE_MASK;
offset = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_CONTROL;
base = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_BASE;
scale = PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_SCALE;
break;
case 2:
offset = CXL_DVSEC_PORT_GPF_PHASE_2_CONTROL_OFFSET;
base = CXL_DVSEC_PORT_GPF_PHASE_2_TMO_BASE_MASK;
scale = CXL_DVSEC_PORT_GPF_PHASE_2_TMO_SCALE_MASK;
offset = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_CONTROL;
base = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_BASE;
scale = PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_SCALE;
break;
default:
return -EINVAL;

13
drivers/cxl/core/pmem.c
View File

@ -237,12 +237,13 @@ static void cxlmd_release_nvdimm(void *_cxlmd)
/**
* devm_cxl_add_nvdimm() - add a bridge between a cxl_memdev and an nvdimm
* @parent_port: parent port for the (to be added) @cxlmd endpoint port
* @cxlmd: cxl_memdev instance that will perform LIBNVDIMM operations
* @host: host device for devm operations
* @port: any port in the CXL topology to find the nvdimm-bridge device
* @cxlmd: parent of the to be created cxl_nvdimm device
*
* Return: 0 on success negative error code on failure.
*/
int devm_cxl_add_nvdimm(struct cxl_port *parent_port,
int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
struct cxl_memdev *cxlmd)
{
struct cxl_nvdimm_bridge *cxl_nvb;
@ -250,7 +251,7 @@ int devm_cxl_add_nvdimm(struct cxl_port *parent_port,
struct device *dev;
int rc;
cxl_nvb = cxl_find_nvdimm_bridge(parent_port);
cxl_nvb = cxl_find_nvdimm_bridge(port);
if (!cxl_nvb)
return -ENODEV;
@ -270,10 +271,10 @@ int devm_cxl_add_nvdimm(struct cxl_port *parent_port,
if (rc)
goto err;
dev_dbg(&cxlmd->dev, "register %s\n", dev_name(dev));
dev_dbg(host, "register %s\n", dev_name(dev));
/* @cxlmd carries a reference on @cxl_nvb until cxlmd_release_nvdimm */
return devm_add_action_or_reset(&cxlmd->dev, cxlmd_release_nvdimm, cxlmd);
return devm_add_action_or_reset(host, cxlmd_release_nvdimm, cxlmd);
err:
put_device(dev);

176
drivers/cxl/core/ras.c
View File

@ -5,6 +5,7 @@
#include <linux/aer.h>
#include <cxl/event.h>
#include <cxlmem.h>
#include <cxlpci.h>
#include "trace.h"
static void cxl_cper_trace_corr_port_prot_err(struct pci_dev *pdev,
@ -124,3 +125,178 @@ void cxl_ras_exit(void)
cxl_cper_unregister_prot_err_work(&cxl_cper_prot_err_work);
cancel_work_sync(&cxl_cper_prot_err_work);
}
static void cxl_dport_map_ras(struct cxl_dport *dport)
{
struct cxl_register_map *map = &dport->reg_map;
struct device *dev = dport->dport_dev;
if (!map->component_map.ras.valid)
dev_dbg(dev, "RAS registers not found\n");
else if (cxl_map_component_regs(map, &dport->regs.component,
BIT(CXL_CM_CAP_CAP_ID_RAS)))
dev_dbg(dev, "Failed to map RAS capability.\n");
}
/**
* cxl_dport_init_ras_reporting - Setup CXL RAS report on this dport
* @dport: the cxl_dport that needs to be initialized
* @host: host device for devm operations
*/
void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host)
{
dport->reg_map.host = host;
cxl_dport_map_ras(dport);
if (dport->rch) {
struct pci_host_bridge *host_bridge = to_pci_host_bridge(dport->dport_dev);
if (!host_bridge->native_aer)
return;
cxl_dport_map_rch_aer(dport);
cxl_disable_rch_root_ints(dport);
}
}
EXPORT_SYMBOL_NS_GPL(cxl_dport_init_ras_reporting, "CXL");
void cxl_handle_cor_ras(struct device *dev, void __iomem *ras_base)
{
void __iomem *addr;
u32 status;
if (!ras_base)
return;
addr = ras_base + CXL_RAS_CORRECTABLE_STATUS_OFFSET;
status = readl(addr);
if (status & CXL_RAS_CORRECTABLE_STATUS_MASK) {
writel(status & CXL_RAS_CORRECTABLE_STATUS_MASK, addr);
trace_cxl_aer_correctable_error(to_cxl_memdev(dev), status);
}
}
/* CXL spec rev3.0 8.2.4.16.1 */
static void header_log_copy(void __iomem *ras_base, u32 *log)
{
void __iomem *addr;
u32 *log_addr;
int i, log_u32_size = CXL_HEADERLOG_SIZE / sizeof(u32);
addr = ras_base + CXL_RAS_HEADER_LOG_OFFSET;
log_addr = log;
for (i = 0; i < log_u32_size; i++) {
*log_addr = readl(addr);
log_addr++;
addr += sizeof(u32);
}
}
/*
* Log the state of the RAS status registers and prepare them to log the
* next error status. Return 1 if reset needed.
*/
bool cxl_handle_ras(struct device *dev, void __iomem *ras_base)
{
u32 hl[CXL_HEADERLOG_SIZE_U32];
void __iomem *addr;
u32 status;
u32 fe;
if (!ras_base)
return false;
addr = ras_base + CXL_RAS_UNCORRECTABLE_STATUS_OFFSET;
status = readl(addr);
if (!(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK))
return false;
/* If multiple errors, log header points to first error from ctrl reg */
if (hweight32(status) > 1) {
void __iomem *rcc_addr =
ras_base + CXL_RAS_CAP_CONTROL_OFFSET;
fe = BIT(FIELD_GET(CXL_RAS_CAP_CONTROL_FE_MASK,
readl(rcc_addr)));
} else {
fe = status;
}
header_log_copy(ras_base, hl);
trace_cxl_aer_uncorrectable_error(to_cxl_memdev(dev), status, fe, hl);
writel(status & CXL_RAS_UNCORRECTABLE_STATUS_MASK, addr);
return true;
}
void cxl_cor_error_detected(struct pci_dev *pdev)
{
struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
struct device *dev = &cxlds->cxlmd->dev;
scoped_guard(device, dev) {
if (!dev->driver) {
dev_warn(&pdev->dev,
"%s: memdev disabled, abort error handling\n",
dev_name(dev));
return;
}
if (cxlds->rcd)
cxl_handle_rdport_errors(cxlds);
cxl_handle_cor_ras(&cxlds->cxlmd->dev, cxlds->regs.ras);
}
}
EXPORT_SYMBOL_NS_GPL(cxl_cor_error_detected, "CXL");
pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state)
{
struct cxl_dev_state *cxlds = pci_get_drvdata(pdev);
struct cxl_memdev *cxlmd = cxlds->cxlmd;
struct device *dev = &cxlmd->dev;
bool ue;
scoped_guard(device, dev) {
if (!dev->driver) {
dev_warn(&pdev->dev,
"%s: memdev disabled, abort error handling\n",
dev_name(dev));
return PCI_ERS_RESULT_DISCONNECT;
}
if (cxlds->rcd)
cxl_handle_rdport_errors(cxlds);
/*
* A frozen channel indicates an impending reset which is fatal to
* CXL.mem operation, and will likely crash the system. On the off
* chance the situation is recoverable dump the status of the RAS
* capability registers and bounce the active state of the memdev.
*/
ue = cxl_handle_ras(&cxlds->cxlmd->dev, cxlds->regs.ras);
}
switch (state) {
case pci_channel_io_normal:
if (ue) {
device_release_driver(dev);
return PCI_ERS_RESULT_NEED_RESET;
}
return PCI_ERS_RESULT_CAN_RECOVER;
case pci_channel_io_frozen:
dev_warn(&pdev->dev,
"%s: frozen state error detected, disable CXL.mem\n",
dev_name(dev));
device_release_driver(dev);
return PCI_ERS_RESULT_NEED_RESET;
case pci_channel_io_perm_failure:
dev_warn(&pdev->dev,
"failure state error detected, request disconnect\n");
return PCI_ERS_RESULT_DISCONNECT;
}
return PCI_ERS_RESULT_NEED_RESET;
}
EXPORT_SYMBOL_NS_GPL(cxl_error_detected, "CXL");

121
drivers/cxl/core/ras_rch.c Normal file
View File

@ -0,0 +1,121 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2025 AMD Corporation. All rights reserved. */
#include <linux/types.h>
#include <linux/aer.h>
#include "cxl.h"
#include "core.h"
#include "cxlmem.h"
void cxl_dport_map_rch_aer(struct cxl_dport *dport)
{
resource_size_t aer_phys;
struct device *host;
u16 aer_cap;
aer_cap = cxl_rcrb_to_aer(dport->dport_dev, dport->rcrb.base);
if (aer_cap) {
host = dport->reg_map.host;
aer_phys = aer_cap + dport->rcrb.base;
dport->regs.dport_aer =
devm_cxl_iomap_block(host, aer_phys,
sizeof(struct aer_capability_regs));
}
}
void cxl_disable_rch_root_ints(struct cxl_dport *dport)
{
void __iomem *aer_base = dport->regs.dport_aer;
u32 aer_cmd_mask, aer_cmd;
if (!aer_base)
return;
/*
* Disable RCH root port command interrupts.
* CXL 3.0 12.2.1.1 - RCH Downstream Port-detected Errors
*
* This sequence may not be necessary. CXL spec states disabling
* the root cmd register's interrupts is required. But, PCI spec
* shows these are disabled by default on reset.
*/
aer_cmd_mask = (PCI_ERR_ROOT_CMD_COR_EN |
PCI_ERR_ROOT_CMD_NONFATAL_EN |
PCI_ERR_ROOT_CMD_FATAL_EN);
aer_cmd = readl(aer_base + PCI_ERR_ROOT_COMMAND);
aer_cmd &= ~aer_cmd_mask;
writel(aer_cmd, aer_base + PCI_ERR_ROOT_COMMAND);
}
/*
* Copy the AER capability registers using 32 bit read accesses.
* This is necessary because RCRB AER capability is MMIO mapped. Clear the
* status after copying.
*
* @aer_base: base address of AER capability block in RCRB
* @aer_regs: destination for copying AER capability
*/
static bool cxl_rch_get_aer_info(void __iomem *aer_base,
struct aer_capability_regs *aer_regs)
{
int read_cnt = sizeof(struct aer_capability_regs) / sizeof(u32);
u32 *aer_regs_buf = (u32 *)aer_regs;
int n;
if (!aer_base)
return false;
/* Use readl() to guarantee 32-bit accesses */
for (n = 0; n < read_cnt; n++)
aer_regs_buf[n] = readl(aer_base + n * sizeof(u32));
writel(aer_regs->uncor_status, aer_base + PCI_ERR_UNCOR_STATUS);
writel(aer_regs->cor_status, aer_base + PCI_ERR_COR_STATUS);
return true;
}
/* Get AER severity. Return false if there is no error. */
static bool cxl_rch_get_aer_severity(struct aer_capability_regs *aer_regs,
int *severity)
{
if (aer_regs->uncor_status & ~aer_regs->uncor_mask) {
if (aer_regs->uncor_status & PCI_ERR_ROOT_FATAL_RCV)
*severity = AER_FATAL;
else
*severity = AER_NONFATAL;
return true;
}
if (aer_regs->cor_status & ~aer_regs->cor_mask) {
*severity = AER_CORRECTABLE;
return true;
}
return false;
}
void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds)
{
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
struct aer_capability_regs aer_regs;
struct cxl_dport *dport;
int severity;
struct cxl_port *port __free(put_cxl_port) =
cxl_pci_find_port(pdev, &dport);
if (!port)
return;
if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs))
return;
if (!cxl_rch_get_aer_severity(&aer_regs, &severity))
return;
pci_print_aer(pdev, severity, &aer_regs);
if (severity == AER_CORRECTABLE)
cxl_handle_cor_ras(&cxlds->cxlmd->dev, dport->regs.ras);
else
cxl_handle_ras(&cxlds->cxlmd->dev, dport->regs.ras);
}

14
drivers/cxl/core/regs.c
View File

@ -271,10 +271,10 @@ EXPORT_SYMBOL_NS_GPL(cxl_map_device_regs, "CXL");
static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi,
struct cxl_register_map *map)
{
u8 reg_type = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BLOCK_ID_MASK, reg_lo);
int bar = FIELD_GET(CXL_DVSEC_REG_LOCATOR_BIR_MASK, reg_lo);
u8 reg_type = FIELD_GET(PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_ID, reg_lo);
int bar = FIELD_GET(PCI_DVSEC_CXL_REG_LOCATOR_BIR, reg_lo);
u64 offset = ((u64)reg_hi << 32) |
(reg_lo & CXL_DVSEC_REG_LOCATOR_BLOCK_OFF_LOW_MASK);
(reg_lo & PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW);
if (offset > pci_resource_len(pdev, bar)) {
dev_warn(&pdev->dev,
@ -311,15 +311,15 @@ static int __cxl_find_regblock_instance(struct pci_dev *pdev, enum cxl_regloc_ty
};
regloc = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
CXL_DVSEC_REG_LOCATOR);
PCI_DVSEC_CXL_REG_LOCATOR);
if (!regloc)
return -ENXIO;
pci_read_config_dword(pdev, regloc + PCI_DVSEC_HEADER1, &regloc_size);
regloc_size = FIELD_GET(PCI_DVSEC_HEADER1_LENGTH_MASK, regloc_size);
regloc_size = PCI_DVSEC_HEADER1_LEN(regloc_size);
regloc += CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET;
regblocks = (regloc_size - CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET) / 8;
regloc += PCI_DVSEC_CXL_REG_LOCATOR_BLOCK1;
regblocks = (regloc_size - PCI_DVSEC_CXL_REG_LOCATOR_BLOCK1) / 8;
for (i = 0; i < regblocks; i++, regloc += 8) {
u32 reg_lo, reg_hi;

11
drivers/cxl/cxl.h
View File

@ -803,14 +803,6 @@ struct cxl_dport *devm_cxl_add_rch_dport(struct cxl_port *port,
struct device *dport_dev, int port_id,
resource_size_t rcrb);
#ifdef CONFIG_PCIEAER_CXL
void cxl_setup_parent_dport(struct device *host, struct cxl_dport *dport);
void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host);
#else
static inline void cxl_dport_init_ras_reporting(struct cxl_dport *dport,
struct device *host) { }
#endif
struct cxl_decoder *to_cxl_decoder(struct device *dev);
struct cxl_root_decoder *to_cxl_root_decoder(struct device *dev);
struct cxl_switch_decoder *to_cxl_switch_decoder(struct device *dev);
@ -895,7 +887,8 @@ struct cxl_nvdimm_bridge *devm_cxl_add_nvdimm_bridge(struct device *host,
struct cxl_port *port);
struct cxl_nvdimm *to_cxl_nvdimm(struct device *dev);
bool is_cxl_nvdimm(struct device *dev);
int devm_cxl_add_nvdimm(struct cxl_port *parent_port, struct cxl_memdev *cxlmd);
int devm_cxl_add_nvdimm(struct device *host, struct cxl_port *port,
struct cxl_memdev *cxlmd);
struct cxl_nvdimm_bridge *cxl_find_nvdimm_bridge(struct cxl_port *port);
#ifdef CONFIG_CXL_REGION

69
drivers/cxl/cxlpci.h
View File

@ -7,59 +7,6 @@
#define CXL_MEMORY_PROGIF 0x10
/*
* See section 8.1 Configuration Space Registers in the CXL 2.0
* Specification. Names are taken straight from the specification with "CXL" and
* "DVSEC" redundancies removed. When obvious, abbreviations may be used.
*/
#define PCI_DVSEC_HEADER1_LENGTH_MASK GENMASK(31, 20)
/* CXL 2.0 8.1.3: PCIe DVSEC for CXL Device */
#define CXL_DVSEC_PCIE_DEVICE 0
#define CXL_DVSEC_CAP_OFFSET 0xA
#define CXL_DVSEC_MEM_CAPABLE BIT(2)
#define CXL_DVSEC_HDM_COUNT_MASK GENMASK(5, 4)
#define CXL_DVSEC_CTRL_OFFSET 0xC
#define CXL_DVSEC_MEM_ENABLE BIT(2)
#define CXL_DVSEC_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
#define CXL_DVSEC_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
#define CXL_DVSEC_MEM_INFO_VALID BIT(0)
#define CXL_DVSEC_MEM_ACTIVE BIT(1)
#define CXL_DVSEC_MEM_SIZE_LOW_MASK GENMASK(31, 28)
#define CXL_DVSEC_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
#define CXL_DVSEC_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
#define CXL_DVSEC_MEM_BASE_LOW_MASK GENMASK(31, 28)
#define CXL_DVSEC_RANGE_MAX 2
/* CXL 2.0 8.1.4: Non-CXL Function Map DVSEC */
#define CXL_DVSEC_FUNCTION_MAP 2
/* CXL 2.0 8.1.5: CXL 2.0 Extensions DVSEC for Ports */
#define CXL_DVSEC_PORT_EXTENSIONS 3
/* CXL 2.0 8.1.6: GPF DVSEC for CXL Port */
#define CXL_DVSEC_PORT_GPF 4
#define CXL_DVSEC_PORT_GPF_PHASE_1_CONTROL_OFFSET 0x0C
#define CXL_DVSEC_PORT_GPF_PHASE_1_TMO_BASE_MASK GENMASK(3, 0)
#define CXL_DVSEC_PORT_GPF_PHASE_1_TMO_SCALE_MASK GENMASK(11, 8)
#define CXL_DVSEC_PORT_GPF_PHASE_2_CONTROL_OFFSET 0xE
#define CXL_DVSEC_PORT_GPF_PHASE_2_TMO_BASE_MASK GENMASK(3, 0)
#define CXL_DVSEC_PORT_GPF_PHASE_2_TMO_SCALE_MASK GENMASK(11, 8)
/* CXL 2.0 8.1.7: GPF DVSEC for CXL Device */
#define CXL_DVSEC_DEVICE_GPF 5
/* CXL 2.0 8.1.8: PCIe DVSEC for Flex Bus Port */
#define CXL_DVSEC_PCIE_FLEXBUS_PORT 7
/* CXL 2.0 8.1.9: Register Locator DVSEC */
#define CXL_DVSEC_REG_LOCATOR 8
#define CXL_DVSEC_REG_LOCATOR_BLOCK1_OFFSET 0xC
#define CXL_DVSEC_REG_LOCATOR_BIR_MASK GENMASK(2, 0)
#define CXL_DVSEC_REG_LOCATOR_BLOCK_ID_MASK GENMASK(15, 8)
#define CXL_DVSEC_REG_LOCATOR_BLOCK_OFF_LOW_MASK GENMASK(31, 16)
/*
* NOTE: Currently all the functions which are enabled for CXL require their
* vectors to be in the first 16. Use this as the default max.
@ -129,7 +76,23 @@ static inline bool cxl_pci_flit_256(struct pci_dev *pdev)
struct cxl_dev_state;
void read_cdat_data(struct cxl_port *port);
#ifdef CONFIG_CXL_RAS
void cxl_cor_error_detected(struct pci_dev *pdev);
pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state);
void cxl_dport_init_ras_reporting(struct cxl_dport *dport, struct device *host);
#else
static inline void cxl_cor_error_detected(struct pci_dev *pdev) { }
static inline pci_ers_result_t cxl_error_detected(struct pci_dev *pdev,
pci_channel_state_t state)
{
return PCI_ERS_RESULT_NONE;
}
static inline void cxl_dport_init_ras_reporting(struct cxl_dport *dport,
struct device *host) { }
#endif
#endif /* __CXL_PCI_H__ */

2
drivers/cxl/mem.c
View File

@ -153,7 +153,7 @@ static int cxl_mem_probe(struct device *dev)
}
if (cxl_pmem_size(cxlds) && IS_ENABLED(CONFIG_CXL_PMEM)) {
rc = devm_cxl_add_nvdimm(parent_port, cxlmd);
rc = devm_cxl_add_nvdimm(dev, parent_port, cxlmd);
if (rc) {
if (rc == -ENODEV)
dev_info(dev, "PMEM disabled by platform\n");

2
drivers/cxl/pci.c
View File

@ -926,7 +926,7 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
cxlds->rcd = is_cxl_restricted(pdev);
cxlds->serial = pci_get_dsn(pdev);
cxlds->cxl_dvsec = pci_find_dvsec_capability(
pdev, PCI_VENDOR_ID_CXL, CXL_DVSEC_PCIE_DEVICE);
pdev, PCI_VENDOR_ID_CXL, PCI_DVSEC_CXL_DEVICE);
if (!cxlds->cxl_dvsec)
dev_warn(&pdev->dev,
"Device DVSEC not present, skip CXL.mem init\n");

39
drivers/pci/pci.h
View File

@ -724,31 +724,56 @@ static inline bool pci_dev_binding_disallowed(struct pci_dev *dev)
#define AER_MAX_MULTI_ERR_DEVICES 5 /* Not likely to have more */
/**
* struct aer_err_info - AER Error Information
* @dev: Devices reporting error
* @ratelimit_print: Flag to log or not log the devices' error. 0=NotLog/1=Log
* @__pad1: Padding for alignment
* @error_dev_num: Number of devices reporting an error
* @level: printk level to use in logging
* @id: Value from register PCI_ERR_ROOT_ERR_SRC
* @severity: AER severity, 0-UNCOR Non-fatal, 1-UNCOR fatal, 2-COR
* @root_ratelimit_print: Flag to log or not log the root's error. 0=NotLog/1=Log
* @multi_error_valid: If multiple errors are reported
* @first_error: First reported error
* @__pad2: Padding for alignment
* @is_cxl: Bus type error: 0-PCI Bus error, 1-CXL Bus error
* @tlp_header_valid: Indicates if TLP field contains error information
* @status: COR/UNCOR error status
* @mask: COR/UNCOR mask
* @tlp: Transaction packet information
*/
struct aer_err_info {
struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES];
int ratelimit_print[AER_MAX_MULTI_ERR_DEVICES];
int error_dev_num;
const char *level; /* printk level */
const char *level;
unsigned int id:16;
unsigned int severity:2; /* 0:NONFATAL | 1:FATAL | 2:COR */
unsigned int root_ratelimit_print:1; /* 0=skip, 1=print */
unsigned int severity:2;
unsigned int root_ratelimit_print:1;
unsigned int __pad1:4;
unsigned int multi_error_valid:1;
unsigned int first_error:5;
unsigned int __pad2:2;
unsigned int __pad2:1;
unsigned int is_cxl:1;
unsigned int tlp_header_valid:1;
unsigned int status; /* COR/UNCOR Error Status */
unsigned int mask; /* COR/UNCOR Error Mask */
struct pcie_tlp_log tlp; /* TLP Header */
unsigned int status;
unsigned int mask;
struct pcie_tlp_log tlp;
};
int aer_get_device_error_info(struct aer_err_info *info, int i);
void aer_print_error(struct aer_err_info *info, int i);
static inline const char *aer_err_bus(struct aer_err_info *info)
{
return info->is_cxl ? "CXL" : "PCIe";
}
int pcie_read_tlp_log(struct pci_dev *dev, int where, int where2,
unsigned int tlp_len, bool flit,
struct pcie_tlp_log *log);

9
drivers/pci/pcie/Kconfig
View File

@ -49,15 +49,6 @@ config PCIEAER_INJECT
gotten from:
https://github.com/intel/aer-inject.git
config PCIEAER_CXL
bool "PCI Express CXL RAS support"
default y
depends on PCIEAER && CXL_PCI
help
Enables CXL error handling.
If unsure, say Y.
#
# PCI Express ECRC
#

1
drivers/pci/pcie/Makefile
View File

@ -8,6 +8,7 @@ obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o bwctrl.o
obj-y += aspm.o
obj-$(CONFIG_PCIEAER) += aer.o err.o tlp.o
obj-$(CONFIG_CXL_RAS) += aer_cxl_rch.o
obj-$(CONFIG_PCIEAER_INJECT) += aer_inject.o
obj-$(CONFIG_PCIE_PME) += pme.o
obj-$(CONFIG_PCIE_DPC) += dpc.o

135
drivers/pci/pcie/aer.c
View File

@ -870,6 +870,7 @@ void aer_print_error(struct aer_err_info *info, int i)
struct pci_dev *dev;
int layer, agent, id;
const char *level = info->level;
const char *bus_type = aer_err_bus(info);
if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES))
return;
@ -879,22 +880,22 @@ void aer_print_error(struct aer_err_info *info, int i)
pci_dev_aer_stats_incr(dev, info);
trace_aer_event(pci_name(dev), (info->status & ~info->mask),
info->severity, info->tlp_header_valid, &info->tlp);
info->severity, info->tlp_header_valid, &info->tlp, bus_type);
if (!info->ratelimit_print[i])
return;
if (!info->status) {
pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",
aer_error_severity_string[info->severity]);
pci_err(dev, "%s Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",
bus_type, aer_error_severity_string[info->severity]);
goto out;
}
layer = AER_GET_LAYER_ERROR(info->severity, info->status);
agent = AER_GET_AGENT(info->severity, info->status);
aer_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",
aer_error_severity_string[info->severity],
aer_printk(level, dev, "%s Bus Error: severity=%s, type=%s, (%s)\n",
bus_type, aer_error_severity_string[info->severity],
aer_error_layer[layer], aer_agent_string[agent]);
aer_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n",
@ -928,6 +929,7 @@ EXPORT_SYMBOL_GPL(cper_severity_to_aer);
void pci_print_aer(struct pci_dev *dev, int aer_severity,
struct aer_capability_regs *aer)
{
const char *bus_type;
int layer, agent, tlp_header_valid = 0;
u32 status, mask;
struct aer_err_info info = {
@ -948,10 +950,13 @@ void pci_print_aer(struct pci_dev *dev, int aer_severity,
info.status = status;
info.mask = mask;
info.is_cxl = pcie_is_cxl(dev);
bus_type = aer_err_bus(&info);
pci_dev_aer_stats_incr(dev, &info);
trace_aer_event(pci_name(dev), (status & ~mask),
aer_severity, tlp_header_valid, &aer->header_log);
trace_aer_event(pci_name(dev), (status & ~mask), aer_severity,
tlp_header_valid, &aer->header_log, bus_type);
if (!aer_ratelimit(dev, info.severity))
return;
@ -1120,8 +1125,6 @@ static bool find_source_device(struct pci_dev *parent,
return true;
}
#ifdef CONFIG_PCIEAER_CXL
/**
* pci_aer_unmask_internal_errors - unmask internal errors
* @dev: pointer to the pci_dev data structure
@ -1132,7 +1135,7 @@ static bool find_source_device(struct pci_dev *parent,
* Note: AER must be enabled and supported by the device which must be
* checked in advance, e.g. with pcie_aer_is_native().
*/
static void pci_aer_unmask_internal_errors(struct pci_dev *dev)
void pci_aer_unmask_internal_errors(struct pci_dev *dev)
{
int aer = dev->aer_cap;
u32 mask;
@ -1146,117 +1149,20 @@ static void pci_aer_unmask_internal_errors(struct pci_dev *dev)
pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, mask);
}
static bool is_cxl_mem_dev(struct pci_dev *dev)
{
/*
* The capability, status, and control fields in Device 0,
* Function 0 DVSEC control the CXL functionality of the
* entire device (CXL 3.0, 8.1.3).
*/
if (dev->devfn != PCI_DEVFN(0, 0))
return false;
/*
* Internal errors are too device-specific to enable generally, however for CXL
* their behavior is standardized for conveying CXL protocol errors.
*/
EXPORT_SYMBOL_FOR_MODULES(pci_aer_unmask_internal_errors, "cxl_core");
/*
* CXL Memory Devices must have the 502h class code set (CXL
* 3.0, 8.1.12.1).
*/
if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)
return false;
return true;
}
static bool cxl_error_is_native(struct pci_dev *dev)
{
struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
return (pcie_ports_native || host->native_aer);
}
static bool is_internal_error(struct aer_err_info *info)
#ifdef CONFIG_CXL_RAS
bool is_aer_internal_error(struct aer_err_info *info)
{
if (info->severity == AER_CORRECTABLE)
return info->status & PCI_ERR_COR_INTERNAL;
return info->status & PCI_ERR_UNC_INTN;
}
static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data)
{
struct aer_err_info *info = (struct aer_err_info *)data;
const struct pci_error_handlers *err_handler;
if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev))
return 0;
/* Protect dev->driver */
device_lock(&dev->dev);
err_handler = dev->driver ? dev->driver->err_handler : NULL;
if (!err_handler)
goto out;
if (info->severity == AER_CORRECTABLE) {
if (err_handler->cor_error_detected)
err_handler->cor_error_detected(dev);
} else if (err_handler->error_detected) {
if (info->severity == AER_NONFATAL)
err_handler->error_detected(dev, pci_channel_io_normal);
else if (info->severity == AER_FATAL)
err_handler->error_detected(dev, pci_channel_io_frozen);
}
out:
device_unlock(&dev->dev);
return 0;
}
static void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info)
{
/*
* Internal errors of an RCEC indicate an AER error in an
* RCH's downstream port. Check and handle them in the CXL.mem
* device driver.
*/
if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC &&
is_internal_error(info))
pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info);
}
static int handles_cxl_error_iter(struct pci_dev *dev, void *data)
{
bool *handles_cxl = data;
if (!*handles_cxl)
*handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev);
/* Non-zero terminates iteration */
return *handles_cxl;
}
static bool handles_cxl_errors(struct pci_dev *rcec)
{
bool handles_cxl = false;
if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC &&
pcie_aer_is_native(rcec))
pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl);
return handles_cxl;
}
static void cxl_rch_enable_rcec(struct pci_dev *rcec)
{
if (!handles_cxl_errors(rcec))
return;
pci_aer_unmask_internal_errors(rcec);
pci_info(rcec, "CXL: Internal errors unmasked");
}
#else
static inline void cxl_rch_enable_rcec(struct pci_dev *dev) { }
static inline void cxl_rch_handle_error(struct pci_dev *dev,
struct aer_err_info *info) { }
#endif
/**
@ -1405,6 +1311,7 @@ int aer_get_device_error_info(struct aer_err_info *info, int i)
/* Must reset in this function */
info->status = 0;
info->tlp_header_valid = 0;
info->is_cxl = pcie_is_cxl(dev);
/* The device might not support AER */
if (!aer)

104
drivers/pci/pcie/aer_cxl_rch.c Normal file
View File

@ -0,0 +1,104 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2023 AMD Corporation. All rights reserved. */
#include <linux/pci.h>
#include <linux/aer.h>
#include <linux/bitfield.h>
#include "../pci.h"
#include "portdrv.h"
static bool is_cxl_mem_dev(struct pci_dev *dev)
{
/*
* The capability, status, and control fields in Device 0,
* Function 0 DVSEC control the CXL functionality of the
* entire device (CXL 3.0, 8.1.3).
*/
if (dev->devfn != PCI_DEVFN(0, 0))
return false;
/*
* CXL Memory Devices must have the 502h class code set (CXL
* 3.0, 8.1.12.1).
*/
if ((dev->class >> 8) != PCI_CLASS_MEMORY_CXL)
return false;
return true;
}
static bool cxl_error_is_native(struct pci_dev *dev)
{
struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
return (pcie_ports_native || host->native_aer);
}
static int cxl_rch_handle_error_iter(struct pci_dev *dev, void *data)
{
struct aer_err_info *info = (struct aer_err_info *)data;
const struct pci_error_handlers *err_handler;
if (!is_cxl_mem_dev(dev) || !cxl_error_is_native(dev))
return 0;
guard(device)(&dev->dev);
err_handler = dev->driver ? dev->driver->err_handler : NULL;
if (!err_handler)
return 0;
if (info->severity == AER_CORRECTABLE) {
if (err_handler->cor_error_detected)
err_handler->cor_error_detected(dev);
} else if (err_handler->error_detected) {
if (info->severity == AER_NONFATAL)
err_handler->error_detected(dev, pci_channel_io_normal);
else if (info->severity == AER_FATAL)
err_handler->error_detected(dev, pci_channel_io_frozen);
}
return 0;
}
void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info)
{
/*
* Internal errors of an RCEC indicate an AER error in an
* RCH's downstream port. Check and handle them in the CXL.mem
* device driver.
*/
if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_EC &&
is_aer_internal_error(info))
pcie_walk_rcec(dev, cxl_rch_handle_error_iter, info);
}
static int handles_cxl_error_iter(struct pci_dev *dev, void *data)
{
bool *handles_cxl = data;
if (!*handles_cxl)
*handles_cxl = is_cxl_mem_dev(dev) && cxl_error_is_native(dev);
/* Non-zero terminates iteration */
return *handles_cxl;
}
static bool handles_cxl_errors(struct pci_dev *rcec)
{
bool handles_cxl = false;
if (pci_pcie_type(rcec) == PCI_EXP_TYPE_RC_EC &&
pcie_aer_is_native(rcec))
pcie_walk_rcec(rcec, handles_cxl_error_iter, &handles_cxl);
return handles_cxl;
}
void cxl_rch_enable_rcec(struct pci_dev *rcec)
{
if (!handles_cxl_errors(rcec))
return;
pci_aer_unmask_internal_errors(rcec);
pci_info(rcec, "CXL: Internal errors unmasked");
}

12
drivers/pci/pcie/portdrv.h
View File

@ -123,4 +123,16 @@ static inline void pcie_pme_interrupt_enable(struct pci_dev *dev, bool en) {}
#endif /* !CONFIG_PCIE_PME */
struct device *pcie_port_find_device(struct pci_dev *dev, u32 service);
struct aer_err_info;
#ifdef CONFIG_CXL_RAS
bool is_aer_internal_error(struct aer_err_info *info);
void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info);
void cxl_rch_enable_rcec(struct pci_dev *rcec);
#else
static inline bool is_aer_internal_error(struct aer_err_info *info) { return false; }
static inline void cxl_rch_handle_error(struct pci_dev *dev, struct aer_err_info *info) { }
static inline void cxl_rch_enable_rcec(struct pci_dev *rcec) { }
#endif /* CONFIG_CXL_RAS */
#endif /* _PORTDRV_H_ */

31
drivers/pci/probe.c
View File

@ -1735,6 +1735,35 @@ static void set_pcie_thunderbolt(struct pci_dev *dev)
dev->is_thunderbolt = 1;
}
static void set_pcie_cxl(struct pci_dev *dev)
{
struct pci_dev *bridge;
u16 dvsec, cap;
if (!pci_is_pcie(dev))
return;
/*
* Update parent's CXL state because alternate protocol training
* may have changed
*/
bridge = pci_upstream_bridge(dev);
if (bridge)
set_pcie_cxl(bridge);
dvsec = pci_find_dvsec_capability(dev, PCI_VENDOR_ID_CXL,
PCI_DVSEC_CXL_FLEXBUS_PORT);
if (!dvsec)
return;
pci_read_config_word(dev, dvsec + PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS,
&cap);
dev->is_cxl = FIELD_GET(PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_CACHE, cap) ||
FIELD_GET(PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_MEM, cap);
}
static void set_pcie_untrusted(struct pci_dev *dev)
{
struct pci_dev *parent = pci_upstream_bridge(dev);
@ -2065,6 +2094,8 @@ int pci_setup_device(struct pci_dev *dev)
/* Need to have dev->cfg_size ready */
set_pcie_thunderbolt(dev);
set_pcie_cxl(dev);
set_pcie_untrusted(dev);
if (pci_is_pcie(dev))

2
include/linux/aer.h
View File

@ -56,12 +56,14 @@ struct aer_capability_regs {
#if defined(CONFIG_PCIEAER)
int pci_aer_clear_nonfatal_status(struct pci_dev *dev);
int pcie_aer_is_native(struct pci_dev *dev);
void pci_aer_unmask_internal_errors(struct pci_dev *dev);
#else
static inline int pci_aer_clear_nonfatal_status(struct pci_dev *dev)
{
return -EINVAL;
}
static inline int pcie_aer_is_native(struct pci_dev *dev) { return 0; }
static inline void pci_aer_unmask_internal_errors(struct pci_dev *dev) { }
#endif
void pci_print_aer(struct pci_dev *dev, int aer_severity,

6
include/linux/pci.h
View File

@ -463,6 +463,7 @@ struct pci_dev {
unsigned int is_pciehp:1;
unsigned int shpc_managed:1; /* SHPC owned by shpchp */
unsigned int is_thunderbolt:1; /* Thunderbolt controller */
unsigned int is_cxl:1; /* Compute Express Link (CXL) */
/*
* Devices marked being untrusted are the ones that can potentially
* execute DMA attacks and similar. They are typically connected
@ -791,6 +792,11 @@ static inline bool pci_is_display(struct pci_dev *pdev)
return (pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY;
}
static inline bool pcie_is_cxl(struct pci_dev *pci_dev)
{
return pci_dev->is_cxl;
}
#define for_each_pci_bridge(dev, bus) \
list_for_each_entry(dev, &bus->devices, bus_list) \
if (!pci_is_bridge(dev)) {} else

12
include/ras/ras_event.h
View File

@ -339,9 +339,11 @@ TRACE_EVENT(aer_event,
const u32 status,
const u8 severity,
const u8 tlp_header_valid,
struct pcie_tlp_log *tlp),
struct pcie_tlp_log *tlp,
const char *bus_type),
TP_ARGS(dev_name, status, severity, tlp_header_valid, tlp),
TP_ARGS(dev_name, status, severity, tlp_header_valid, tlp, bus_type),
TP_STRUCT__entry(
__string( dev_name, dev_name )
@ -349,10 +351,12 @@ TRACE_EVENT(aer_event,
__field( u8, severity )
__field( u8, tlp_header_valid)
__array( u32, tlp_header, PCIE_STD_MAX_TLP_HEADERLOG)
__string( bus_type, bus_type )
),
TP_fast_assign(
__assign_str(dev_name);
__assign_str(bus_type);
__entry->status = status;
__entry->severity = severity;
__entry->tlp_header_valid = tlp_header_valid;
@ -364,8 +368,8 @@ TRACE_EVENT(aer_event,
}
),
TP_printk("%s PCIe Bus Error: severity=%s, %s, TLP Header=%s\n",
__get_str(dev_name),
TP_printk("%s %s Bus Error: severity=%s, %s, TLP Header=%s\n",
__get_str(dev_name), __get_str(bus_type),
__entry->severity == AER_CORRECTABLE ? "Corrected" :
__entry->severity == AER_FATAL ?
"Fatal" : "Uncorrected, non-fatal",

64
include/uapi/linux/pci_regs.h
View File

@ -1253,11 +1253,6 @@
#define PCI_DEV3_STA 0x0c /* Device 3 Status Register */
#define PCI_DEV3_STA_SEGMENT 0x8 /* Segment Captured (end-to-end flit-mode detected) */
/* Compute Express Link (CXL r3.1, sec 8.1.5) */
#define PCI_DVSEC_CXL_PORT 3
#define PCI_DVSEC_CXL_PORT_CTL 0x0c
#define PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR 0x00000001
/* Integrity and Data Encryption Extended Capability */
#define PCI_IDE_CAP 0x04
#define PCI_IDE_CAP_LINK 0x1 /* Link IDE Stream Supported */
@ -1338,4 +1333,63 @@
#define PCI_IDE_SEL_ADDR_3(x) (28 + (x) * PCI_IDE_SEL_ADDR_BLOCK_SIZE)
#define PCI_IDE_SEL_BLOCK_SIZE(nr_assoc) (20 + PCI_IDE_SEL_ADDR_BLOCK_SIZE * (nr_assoc))
/*
* Compute Express Link (CXL r4.0, sec 8.1)
*
* Note that CXL DVSEC id 3 and 7 to be ignored when the CXL link state
* is "disconnected" (CXL r4.0, sec 9.12.3). Re-enumerate these
* registers on downstream link-up events.
*/
/* CXL r4.0, 8.1.3: PCIe DVSEC for CXL Device */
#define PCI_DVSEC_CXL_DEVICE 0
#define PCI_DVSEC_CXL_CAP 0xA
#define PCI_DVSEC_CXL_MEM_CAPABLE _BITUL(2)
#define PCI_DVSEC_CXL_HDM_COUNT __GENMASK(5, 4)
#define PCI_DVSEC_CXL_CTRL 0xC
#define PCI_DVSEC_CXL_MEM_ENABLE _BITUL(2)
#define PCI_DVSEC_CXL_RANGE_SIZE_HIGH(i) (0x18 + (i * 0x10))
#define PCI_DVSEC_CXL_RANGE_SIZE_LOW(i) (0x1C + (i * 0x10))
#define PCI_DVSEC_CXL_MEM_INFO_VALID _BITUL(0)
#define PCI_DVSEC_CXL_MEM_ACTIVE _BITUL(1)
#define PCI_DVSEC_CXL_MEM_SIZE_LOW __GENMASK(31, 28)
#define PCI_DVSEC_CXL_RANGE_BASE_HIGH(i) (0x20 + (i * 0x10))
#define PCI_DVSEC_CXL_RANGE_BASE_LOW(i) (0x24 + (i * 0x10))
#define PCI_DVSEC_CXL_MEM_BASE_LOW __GENMASK(31, 28)
#define CXL_DVSEC_RANGE_MAX 2
/* CXL r4.0, 8.1.4: Non-CXL Function Map DVSEC */
#define PCI_DVSEC_CXL_FUNCTION_MAP 2
/* CXL r4.0, 8.1.5: Extensions DVSEC for Ports */
#define PCI_DVSEC_CXL_PORT 3
#define PCI_DVSEC_CXL_PORT_CTL 0x0c
#define PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR 0x00000001
/* CXL r4.0, 8.1.6: GPF DVSEC for CXL Port */
#define PCI_DVSEC_CXL_PORT_GPF 4
#define PCI_DVSEC_CXL_PORT_GPF_PHASE_1_CONTROL 0x0C
#define PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_BASE __GENMASK(3, 0)
#define PCI_DVSEC_CXL_PORT_GPF_PHASE_1_TMO_SCALE __GENMASK(11, 8)
#define PCI_DVSEC_CXL_PORT_GPF_PHASE_2_CONTROL 0xE
#define PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_BASE __GENMASK(3, 0)
#define PCI_DVSEC_CXL_PORT_GPF_PHASE_2_TMO_SCALE __GENMASK(11, 8)
/* CXL r4.0, 8.1.7: GPF DVSEC for CXL Device */
#define PCI_DVSEC_CXL_DEVICE_GPF 5
/* CXL r4.0, 8.1.8: Flex Bus DVSEC */
#define PCI_DVSEC_CXL_FLEXBUS_PORT 7
#define PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS 0xE
#define PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_CACHE _BITUL(0)
#define PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_MEM _BITUL(2)
/* CXL r4.0, 8.1.9: Register Locator DVSEC */
#define PCI_DVSEC_CXL_REG_LOCATOR 8
#define PCI_DVSEC_CXL_REG_LOCATOR_BLOCK1 0xC
#define PCI_DVSEC_CXL_REG_LOCATOR_BIR __GENMASK(2, 0)
#define PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_ID __GENMASK(15, 8)
#define PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW __GENMASK(31, 16)
#endif /* LINUX_PCI_REGS_H */

3
tools/testing/cxl/Kbuild
View File

@ -57,12 +57,13 @@ cxl_core-y += $(CXL_CORE_SRC)/pci.o
cxl_core-y += $(CXL_CORE_SRC)/hdm.o
cxl_core-y += $(CXL_CORE_SRC)/pmu.o
cxl_core-y += $(CXL_CORE_SRC)/cdat.o
cxl_core-y += $(CXL_CORE_SRC)/ras.o
cxl_core-$(CONFIG_TRACING) += $(CXL_CORE_SRC)/trace.o
cxl_core-$(CONFIG_CXL_REGION) += $(CXL_CORE_SRC)/region.o
cxl_core-$(CONFIG_CXL_MCE) += $(CXL_CORE_SRC)/mce.o
cxl_core-$(CONFIG_CXL_FEATURES) += $(CXL_CORE_SRC)/features.o
cxl_core-$(CONFIG_CXL_EDAC_MEM_FEATURES) += $(CXL_CORE_SRC)/edac.o
cxl_core-$(CONFIG_CXL_RAS) += $(CXL_CORE_SRC)/ras.o
cxl_core-$(CONFIG_CXL_RAS) += $(CXL_CORE_SRC)/ras_rch.o
cxl_core-y += config_check.o
cxl_core-y += cxl_core_test.o
cxl_core-y += cxl_core_exports.o