github-mirror/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2026-11-06 21:58:01 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity Graph

CXL for v7.4

Browse Source 
CXL Dynamic Capacity Device (DCD) support (part 1):
 cxl/mbox: Flag support for Dynamic Capacity Devices (DCD)
 cxl/mem: Read dynamic capacity configuration from the device
 cxl/cdat: Gather DSMAS data for DCD partitions
 cxl/events: Split event msgnum configuration from irq setup
 cxl/pci: Factor out interrupt policy check
 cxl/mem: Configure dynamic capacity interrupts
 cxl/core: Enforce partition order/simplify partition calls
 Documentation/cxl: Document DPA partition layout and ordering rules
 
 CXL zero sized decoder support:
 cxl/hdm: Allow zero sized HDM decoders
 tools/testing/cxl: Enable zero sized decoders under hb0
 cxl/region: Simplify poison_by_decoder() error handling
 
 CXL decoder hardening:
 cxl/hdm: Reject switch decoder interleave ways that overflow targets
 cxl/hdm: Make switch decoder target parsing endian-safe
 cxl/hdm: Restore commit_end when decoder enumeration fails
 cxl/port: Bound switch decoder target array access by nr_targets
 cxl/pci: Skip reset detection for DVSEC emulated decoders
 
 CXL latency and GPF handling:
 cxl/port: Convert PCIe link latency to nanoseconds
 cxl: Account for link width in latency calculation
 cxl/acpi: Convert QTG _DSM latencies to picoseconds
 cxl/pci: Use the Device GPF DVSEC for restricted endpoints
 cxl/pci: Program the Port GPF timeouts before caching the DVSEC
 cxl/pci: Update only the Port GPF timeout fields
 
 CXL misc patches:
 MAINTAINERS: Add Richard Cheng to CXL subsystem as a Reviewer
 cxl: docs/platform/acpi: Fix brackets
 cxl/ras: Pass the PCI device's struct device to match_memdev_by_parent()
 cxl/test: Map mock device nodes to an online node
 tools/testing/cxl: Don't wrap cxl_core's own exported symbols
 cxl/region: Fix the return value in devm_cxl_add_region() kernel-doc
 cxl/region: Warn on user region position mismatch
 cxl/region: Create node access attributes for CFMWS-only NUMA nodes
 
 CXL fixes:
 cxl/events: Test CME counter expiry with equality, not bitwise AND
 cxl/events: Return IRQ_NONE when no event is pending
 cxl/port: Attach endpoints to a dport under the lock that pins it
 cxl/core: Hold the dport host lock across dport lookup and use
 cxl/port: Clear cached dport pointers when a dport is removed
 cxl/acpi: Check ACPI companion before use
 driver core: Add conditional guard support for device_trylock()
 cxl/memdev: Fix deadlock between poison debugfs and cxl_mem unbind
 cxl/core: Skip einj_inject creation when devm_add_action_or_reset() fails
 cxl/region: Unregister the pmem region when the bridge is unbound
 cxl/core: Fix dport use-after-free via the einj_inject debugfs file
 cxl/port: Fix uninitialized coordinates reported for RCDs
 cxl/cdat: Fix uninitialized stack use in bandwidth gathering
 cxl/features: Ensure that count is set before access to ent[] __counted_by()
 cxl/features: Reject feature offset that overflows 16-bit field
 cxl/fwctl: Propagate feature RPC delivery errors
 cxl/regs: Reject register blocks in an unassigned BAR
 cxl/region: Guard against a missing peer mapping
 cxl/mce: Avoid alias page retirement for corrected errors
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEE5DAy15EJMCV1R6v9YGjFFmlTOEoFAmraKeQACgkQYGjFFmlT
 OErV6w/+LuvHDcm3JhD8NJqqCRB7/gOCkM26n+dLuL6ZIo81rMhi4ZOq55/dkpbo
 fhdMtCUQUJoTFOHrLrTiU1IyHlv0EI+Uj5EvVfdrcowSRAzKGZsbB/mxSunWDl+W
 84kQN8lcTxT9dUh4BsYZu6KqaiNyHuNZ+qFRzvUfM15dDq6qrfgKtg/ZOQXJ2Uyj
 MovYZSzLn4h2sBSWSE8639RgxtDGw4I6fLlEJOGXX9zHm20Th1fGECFCvVW5GIq8
 XglnqD9b2vueGQDAmDMNm1QztCBz/6J13SQU6fEVeB7wmO6TfaIEv9Cl9M2xlS41
 FvNlZ+vtI/3/vhBZwto/uzOhCtmt9o0KeVFQcT9eDv2nB5655Mv1wPKd0M3YSbEh
 5aY5OHMSPBZvuuZp7d19hVxNFBcOpAALXpgp5Lv1zFLZyVJxHd7duz3AGG1fmz2z
 Ohqc7dBvj3SF14IYp6pfCy1cynXtYK4o+UbNt9PsbZgThs97s4dDK6UCWXw6buKH
 ZRkiXt0OPH+8+2QrtVi6N4rV/03xPDc90CEpW5zzCrbVRL3kn1w0GK4a9m5n1I2N
 RbHd6hyWIRHAQC9IwzaVIFp2k6CqgBeunygPYKYmGnhoHI0HV+4VIFNey6bVmzw4
 h0sD3GoQ5pNFJZyakJRyWBYRGaxx5AiH8eKH1ZfTGCgsDbZC3j8=
 =ym/U
 -----END PGP SIGNATURE-----

Merge tag 'cxl-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl

Pull CXL updates from Dave Jiang:
 "The headline item is the first part of support for Dynamic Capacity
  Devices (DCD). It covers flagging DCD support in the mailbox, reading
  the dynamic capacity configuration from the device, gathering DSMAS
  data for the DCD partitions, and configuring the dynamic capacity
  interrupts. It also adds a documentation update and a cleanup that
  enforces partition ordering. DCD is not enabled yet with these code
  changes. Patches are applied to reduce the whole set in order to
  relief the burden of very large patch set review.

  Zero sized HDM decoders are now allowed, along with cxl_test support
  for them. There is also a set of decoder hardening patches for switch
  decoder enumeration. Latency calculations now account for link width
  and use consistent units. The GPF timeout programming has been
  reworked. The pull also has a number of misc patches.

  A large number of bug fixes against pre-existing issues make up the
  rest of the pull. Many of these were flagged by review of patches
  posted to the mailing list by sashiko.

  Also adding Richard Cheng as a reviewer (R) to the CXL sub-system"

* tag 'cxl-for-7.4' of git://git.kernel.org/pub/scm/linux/kernel/git/cxl/cxl: (49 commits)
  cxl/events: Test CME counter expiry with equality, not bitwise AND
  Documentation/cxl: Document DPA partition layout and ordering rules
  cxl/core: Enforce partition order/simplify partition calls
  cxl/mem: Configure dynamic capacity interrupts
  cxl/pci: Factor out interrupt policy check
  cxl/events: Split event msgnum configuration from irq setup
  cxl/cdat: Gather DSMAS data for DCD partitions
  cxl/mem: Read dynamic capacity configuration from the device
  cxl/mbox: Flag support for Dynamic Capacity Devices (DCD)
  cxl/acpi: Convert QTG _DSM latencies to picoseconds
  cxl: Account for link width in latency calculation
  cxl/port: Convert PCIe link latency to nanoseconds
  cxl/pci: Update only the Port GPF timeout fields
  cxl/pci: Program the Port GPF timeouts before caching the DVSEC
  cxl/pci: Use the Device GPF DVSEC for restricted endpoints
  cxl/port: Attach endpoints to a dport under the lock that pins it
  cxl/core: Hold the dport host lock across dport lookup and use
  cxl/port: Clear cached dport pointers when a dport is removed
  cxl/region: Warn on user region position mismatch
  cxl/acpi: Check ACPI companion before use
  ...
This commit is contained in:
Linus Torvalds 2026-10-26 06:50:38 -10:00
commit d6bbdb67a4
32 changed files with 1497 additions and 385 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
Documentation/driver-api/cxl/linux/cxl-driver.rst
View File

@ -181,6 +181,41 @@ A Memory Device is a discrete base object that is not a port. While the
physical device it belongs to may also host an `endpoint`, the relationship
between an `endpoint` and a `memdev` is not captured in sysfs.
DPA Partitions
~~~~~~~~~~~~~~
A memory device presents its capacity as one flat `Device Physical Address`
(DPA) space divided into `partitions`, which Linux lays out in a fixed
order::
DPA 0 end
+---------------+---------------+---------------------------+
| ram | pmem | dynamic_ram_1 |
+---------------+---------------+---------------------------+
part[0] part[1] part[2]
Part of that order is required by the CXL specification and part of it is a
Linux choice.
The `ram` and `pmem` order is mandated. CXL r4.0 section 8.2.10.9.2.1 "Get
Partition Info" (4100h), Table 8-310, mandates that volatile capacity starts
at DPA 0 and pmem starts at the DPA immediately following it.
Dynamic Capacity partitions only need to be 256MB aligned according to
CXL r4.0 section 8.2.10.9.9.1 "Get Dynamic Capacity Configuration"
(opcode 4800h), Table 8-347. So a device could leave a gap between ram/pmem
(static) capacity and its first DC partition, or between one DC partition
and the next.
Linux chooses not to support gaps between static and dynamic capacity: the
first DC partition must begin at the DPA immediately following static
capacity -- after pmem, after ram on a device with no pmem, or at DPA 0 on
a device with no static capacity at all.
Currently, only one dynamic partition is supported. A device may report up
to eight (CXL r4.0 Table 8-346); Linux configures the first as its
`dynamic_ram_1` partition. Gaps between DC partitions are not checked, since
only the first is used.
Port Relationships
~~~~~~~~~~~~~~~~~~
In our example described above, there are four host bridges attached to the

4
Documentation/driver-api/cxl/maturity-map.rst
View File

@ -150,6 +150,10 @@ Memory-pooling
* [1] Hotplug of LDs (via PCI hotplug)
* [0] Dynamic Capacity Device (DCD) Support
* [1] DC partition enumeration / configuration (Get DC Config, CDAT DSMAS, event interrupts)
* [0] Extent add / release event handling
* [0] DC-backed DAX regions
Multi-host sharing
------------------

3
Documentation/driver-api/cxl/platform/acpi/dsdt.rst
View File

@ -25,4 +25,5 @@ Example Compute Express Link Host Bridge ::
...
Name (_UID, 0x05) // _UID: Unique ID
...
}
}
}

1
MAINTAINERS
View File

@ -6666,6 +6666,7 @@ M: Vishal Verma <vishal.l.verma@intel.com>
M: Dan Williams <djbw@kernel.org>
R: Ira Weiny <iweiny@kernel.org>
R: Li Ming <ming.li@zohomail.com>
R: Richard Cheng <icheng@nvidia.com>
L: linux-cxl@vger.kernel.org
S: Maintained
F: Documentation/driver-api/cxl

20
drivers/cxl/acpi.c
View File

@ -231,9 +231,10 @@ cxl_acpi_evaluate_qtg_dsm(acpi_handle handle, struct access_coordinate *coord,
int entries, int *qos_class)
{
union acpi_object *out_obj, *out_buf, *obj;
/* QTG _DSM expects latencies in picoseconds. */
union acpi_object in_array[4] = {
[0].integer = { ACPI_TYPE_INTEGER, coord->read_latency },
[1].integer = { ACPI_TYPE_INTEGER, coord->write_latency },
[0].integer = { ACPI_TYPE_INTEGER, coord->read_latency * 1000ULL },
[1].integer = { ACPI_TYPE_INTEGER, coord->write_latency * 1000ULL },
[2].integer = { ACPI_TYPE_INTEGER, coord->read_bandwidth },
[3].integer = { ACPI_TYPE_INTEGER, coord->write_bandwidth },
};
@ -251,6 +252,11 @@ cxl_acpi_evaluate_qtg_dsm(acpi_handle handle, struct access_coordinate *coord,
if (!entries)
return -EINVAL;
/* Latency inputs must fit in the DWORD fields defined by the spec. */
if (coord->read_latency > U32_MAX / 1000 ||
coord->write_latency > U32_MAX / 1000)
return -ERANGE;
out_obj = acpi_evaluate_dsm(handle, &acpi_cxl_qtg_id_guid, 1, 1, &in_obj);
if (!out_obj)
return -ENXIO;
@ -885,13 +891,17 @@ static int pair_cxl_resource(struct device *dev, void *data)
static int cxl_acpi_probe(struct platform_device *pdev)
{
int rc;
struct cxl_cfmws_context ctx;
struct acpi_device *adev;
struct resource *cxl_res;
struct cxl_root *cxl_root;
struct cxl_port *root_port;
struct device *host = &pdev->dev;
struct acpi_device *adev = ACPI_COMPANION(host);
struct cxl_cfmws_context ctx;
int rc;
adev = ACPI_COMPANION(host);
if (!adev)
return -ENODEV;
device_lock_set_class(&pdev->dev, &cxl_root_key);
rc = devm_add_action_or_reset(&pdev->dev, cxl_acpi_lock_reset_class,

107
drivers/cxl/core/cdat.c
View File

@ -17,6 +17,8 @@ struct dsmas_entry {
struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX];
int entries;
int qos_class;
bool shareable;
bool read_only;
};
static u32 cdat_normalize(u16 entry, u64 base, u8 type)
@ -73,7 +75,13 @@ static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg,
if (!dent)
return -ENOMEM;
/* Stays invalid unless the QTG lookup assigns a class */
dent->qos_class = CXL_QOS_CLASS_INVALID;
dent->handle = dsmas->dsmad_handle;
/* Shareable is CDAT 1.03 and later, DSMAS Flags bit 3 */
dent->shareable = dsmas->flags & ACPI_CDAT_DSMAS_SHAREABLE;
/* Read Only is CDAT 1.04 and later, DSMAS Flags bit 6 */
dent->read_only = dsmas->flags & ACPI_CDAT_DSMAS_READ_ONLY;
dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address);
dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) +
le64_to_cpu((__force __le64)dsmas->dpa_length) - 1;
@ -188,9 +196,10 @@ static int cxl_cdat_endpoint_process(struct cxl_port *port,
if (rc)
return rc;
/* DSLBIS is optional; zero entries is not an error */
rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler,
dsmas_xa, port->cdat.table, port->cdat.length);
return cdat_table_parse_output(rc);
return rc < 0 ? rc : 0;
}
static int cxl_port_perf_data_calculate(struct cxl_port *port,
@ -219,6 +228,13 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port,
xa_for_each(dsmas_xa, index, dent) {
int qos_class;
/* No DSLBIS for this DSMAS; nothing to hand the QTG _DSM */
if (!dent->cdat_coord[ACCESS_COORDINATE_CPU].read_bandwidth &&
!dent->cdat_coord[ACCESS_COORDINATE_CPU].write_bandwidth &&
!dent->cdat_coord[ACCESS_COORDINATE_CPU].read_latency &&
!dent->cdat_coord[ACCESS_COORDINATE_CPU].write_latency)
continue;
cxl_coordinates_combine(dent->coord, dent->cdat_coord, ep_c);
dent->entries = 1;
rc = cxl_root->ops.qos_class(cxl_root,
@ -255,35 +271,90 @@ static void update_perf_entry(struct device *dev, struct dsmas_entry *dent,
dent->coord[ACCESS_COORDINATE_CPU].write_latency);
}
static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds,
struct xarray *dsmas_xa)
/*
* A DCD shall describe its DC partition in a DSMAS entry (CXL r4.0 9.13.3).
* Without one the partition's attributes are unknown, so disable DCD.
*/
static void cxl_dcd_dsmas_missing(struct cxl_dev_state *cxlds, const char *why)
{
struct cxl_memdev_state *mds;
if (cxlds->type != CXL_DEVTYPE_CLASSMEM)
return;
mds = to_cxl_memdev_state(cxlds);
if (!cxl_dcd_supported(mds) ||
!cxl_part_size(cxlds, CXL_PARTMODE_DYNAMIC_RAM))
return;
dev_warn(cxlds->dev, "DC partition has no valid DSMAS entry (%s), disabling DCD\n",
why);
cxl_disable_dcd(mds);
}
enum cxl_dsmas_apply {
CXL_DSMAS_APPLY_FLAGS, /* shareable etc.; no QTG data */
CXL_DSMAS_APPLY_FLAGS_PERF, /* perf coordinates and qos_class */
};
/*
* DSMAS flags such as shareable are properties of the CDAT entry and are
* applied regardless; perf data is applied only if the QTG lookup produced
* recommendations.
*/
static void cxl_memdev_apply_dsmas(struct cxl_dev_state *cxlds,
struct xarray *dsmas_xa,
enum cxl_dsmas_apply attrs)
{
struct device *dev = cxlds->dev;
struct dsmas_entry *dent;
bool dc_matched = false;
unsigned long index;
xa_for_each(dsmas_xa, index, dent) {
bool found = false;
for (int i = 0; i < cxlds->nr_partitions; i++) {
struct resource *res = &cxlds->part[i].res;
struct cxl_dpa_partition *part = &cxlds->part[i];
struct range range = {
.start = res->start,
.end = res->end,
.start = part->res.start,
.end = part->res.end,
};
if (range_contains(&range, &dent->dpa_range)) {
update_perf_entry(dev, dent,
&cxlds->part[i].perf);
found = true;
if (!range_contains(&range, &dent->dpa_range))
continue;
found = true;
/*
* part->handle is from Get DC Config, dent->handle
* from the CDAT DSMAS entry.
*/
if (part->mode == CXL_PARTMODE_DYNAMIC_RAM &&
dent->handle != part->handle) {
dev_warn(dev,
"DSMAD handle mismatch: %pra has %u, DSMAS %pra has %u\n",
&range, part->handle,
&dent->dpa_range, dent->handle);
break;
}
if (attrs == CXL_DSMAS_APPLY_FLAGS_PERF)
update_perf_entry(dev, dent, &part->perf);
if (part->mode == CXL_PARTMODE_DYNAMIC_RAM) {
part->shareable = dent->shareable;
part->read_only = dent->read_only;
dc_matched = true;
}
break;
}
if (!found)
dev_dbg(dev, "no partition for dsmas dpa: %pra\n",
&dent->dpa_range);
}
if (!dc_matched)
cxl_dcd_dsmas_missing(cxlds, "no matching entry");
}
static int match_cxlrd_qos_class(struct device *dev, void *data)
@ -409,22 +480,26 @@ void cxl_endpoint_parse_cdat(struct cxl_port *port)
int rc;
xa_init(&__dsmas_xa);
if (!port->cdat.table)
if (!port->cdat.table) {
cxl_dcd_dsmas_missing(cxlds, "no CDAT");
return;
}
rc = cxl_cdat_endpoint_process(port, dsmas_xa);
if (rc < 0) {
dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc);
cxl_dcd_dsmas_missing(cxlds, "CDAT parse failed");
return;
}
rc = cxl_port_perf_data_calculate(port, dsmas_xa);
if (rc) {
dev_dbg(&port->dev, "Failed to do perf coord calculations.\n");
cxl_memdev_apply_dsmas(cxlds, dsmas_xa, CXL_DSMAS_APPLY_FLAGS);
return;
}
cxl_memdev_set_qos_class(cxlds, dsmas_xa);
cxl_memdev_apply_dsmas(cxlds, dsmas_xa, CXL_DSMAS_APPLY_FLAGS_PERF);
cxl_qos_class_verify(cxlmd);
cxl_memdev_update_perf(cxlmd);
}
@ -633,9 +708,9 @@ static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr,
struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent);
struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent);
struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent);
struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX];
struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX];
struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX];
struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX] = { };
struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX] = { };
struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX] = { };
struct cxl_memdev *cxlmd = cxled_to_memdev(cxled);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
struct pci_dev *pdev = to_pci_dev(cxlds->dev);
@ -757,7 +832,7 @@ static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr,
{
struct xarray *res_xa __free(free_perf_xa) =
kzalloc_obj(*res_xa);
struct access_coordinate coords[ACCESS_COORDINATE_MAX];
struct access_coordinate coords[ACCESS_COORDINATE_MAX] = { };
struct cxl_perf_ctx *ctx, *us_ctx;
unsigned long index, us_index;
int dev_count = 0;

14
drivers/cxl/core/core.h
View File

@ -178,11 +178,7 @@ static inline struct device *port_to_host(struct cxl_port *port)
static inline struct device *dport_to_host(struct cxl_dport *dport)
{
struct cxl_port *port = dport->port;
if (is_cxl_root(port))
return port->uport_dev;
return &port->dev;
return cxl_port_dport_host(dport->port);
}
#ifdef CONFIG_CXL_RAS
void cxl_ras_init(void);
@ -217,10 +213,10 @@ int cxl_port_get_possible_dports(struct cxl_port *port);
#ifdef CONFIG_CXL_FEATURES
struct cxl_feat_entry *
cxl_feature_info(struct cxl_features_state *cxlfs, const uuid_t *uuid);
size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
enum cxl_get_feat_selection selection,
void *feat_out, size_t feat_out_size, u16 offset,
u16 *return_code);
ssize_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
enum cxl_get_feat_selection selection,
void *feat_out, size_t feat_out_size, u16 offset,
u16 *return_code);
int cxl_set_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
u8 feat_version, const void *feat_data,
size_t feat_data_size, u32 feat_flag, u16 offset,

30
drivers/cxl/core/edac.c
View File

@ -78,7 +78,7 @@ static int cxl_mem_scrub_get_attrbs(struct cxl_mailbox *cxl_mbox, u8 *cap,
u16 *cycle, u8 *flags, u8 *min_cycle)
{
size_t rd_data_size = sizeof(struct cxl_scrub_rd_attrbs);
size_t data_size;
ssize_t data_size;
struct cxl_scrub_rd_attrbs *rd_attrbs __free(kfree) =
kzalloc(rd_data_size, GFP_KERNEL);
if (!rd_attrbs)
@ -87,7 +87,9 @@ static int cxl_mem_scrub_get_attrbs(struct cxl_mailbox *cxl_mbox, u8 *cap,
data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_PATROL_SCRUB_UUID,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, NULL);
if (!data_size)
if (data_size < 0)
return data_size;
if ((size_t)data_size != rd_data_size)
return -EIO;
*cap = rd_attrbs->scrub_cycle_cap;
@ -551,7 +553,7 @@ static int cxl_mem_ecs_get_attrbs(struct device *dev,
struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
struct cxl_ecs_fru_rd_attrbs *fru_rd_attrbs;
size_t rd_data_size;
size_t data_size;
ssize_t data_size;
rd_data_size = cxl_ecs_ctx->get_feat_size;
@ -563,7 +565,9 @@ static int cxl_mem_ecs_get_attrbs(struct device *dev,
data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, NULL);
if (!data_size)
if (data_size < 0)
return data_size;
if ((size_t)data_size != rd_data_size)
return -EIO;
fru_rd_attrbs = rd_attrbs->fru_attrbs;
@ -583,7 +587,7 @@ static int cxl_mem_ecs_set_attrbs(struct device *dev,
struct cxl_ecs_fru_wr_attrbs *fru_wr_attrbs;
size_t rd_data_size, wr_data_size;
u16 num_media_frus, count;
size_t data_size;
ssize_t data_size;
num_media_frus = cxl_ecs_ctx->num_media_frus;
rd_data_size = cxl_ecs_ctx->get_feat_size;
@ -596,7 +600,9 @@ static int cxl_mem_ecs_set_attrbs(struct device *dev,
data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, NULL);
if (!data_size)
if (data_size < 0)
return data_size;
if ((size_t)data_size != rd_data_size)
return -EIO;
struct cxl_ecs_wr_attrbs *wr_attrbs __free(kvfree) =
@ -1264,7 +1270,7 @@ cxl_mem_sparing_get_attrbs(struct cxl_mem_sparing_context *cxl_sparing_ctx)
struct cxl_memdev *cxlmd = cxl_sparing_ctx->cxlmd;
struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
u16 restriction_flags;
size_t data_size;
ssize_t data_size;
u16 return_code;
struct cxl_memdev_sparing_rd_attrbs *rd_attrbs __free(kfree) =
kzalloc(rd_data_size, GFP_KERNEL);
@ -1274,7 +1280,9 @@ cxl_mem_sparing_get_attrbs(struct cxl_mem_sparing_context *cxl_sparing_ctx)
data_size = cxl_get_feature(cxl_mbox, &cxl_sparing_ctx->repair_uuid,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, &return_code);
if (!data_size)
if (data_size < 0)
return data_size;
if ((size_t)data_size != rd_data_size)
return -EIO;
cxl_sparing_ctx->op_class = rd_attrbs->hdr.op_class;
@ -1750,7 +1758,7 @@ static int cxl_mem_ppr_get_attrbs(struct cxl_ppr_context *cxl_ppr_ctx)
struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd;
struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox;
u16 restriction_flags;
size_t data_size;
ssize_t data_size;
u16 return_code;
struct cxl_memdev_ppr_rd_attrbs *rd_attrbs __free(kfree) =
@ -1761,7 +1769,9 @@ static int cxl_mem_ppr_get_attrbs(struct cxl_ppr_context *cxl_ppr_ctx)
data_size = cxl_get_feature(cxl_mbox, &cxl_ppr_ctx->repair_uuid,
CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs,
rd_data_size, 0, &return_code);
if (!data_size)
if (data_size < 0)
return data_size;
if ((size_t)data_size != rd_data_size)
return -EIO;
cxl_ppr_ctx->op_class = rd_attrbs->hdr.op_class;

58
drivers/cxl/core/features.c
View File

@ -97,6 +97,7 @@ get_supported_features(struct cxl_features_state *cxlfs)
kvmalloc_flex(*entries, ent, count);
if (!entries)
return NULL;
entries->num_features = count;
struct cxl_mbox_get_sup_feats_out *mbox_out __free(kvfree) =
kvmalloc(cxl_mbox->payload_size, GFP_KERNEL);
@ -174,7 +175,6 @@ get_supported_features(struct cxl_features_state *cxlfs)
start += num_entries;
} while (remain_feats);
entries->num_features = count;
entries->num_user_features = user_feats;
return no_free_ptr(entries);
@ -220,10 +220,10 @@ int devm_cxl_setup_features(struct cxl_dev_state *cxlds)
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_features, "CXL");
size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
enum cxl_get_feat_selection selection,
void *feat_out, size_t feat_out_size, u16 offset,
u16 *return_code)
ssize_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
enum cxl_get_feat_selection selection,
void *feat_out, size_t feat_out_size, u16 offset,
u16 *return_code)
{
size_t data_to_rd_size;
struct cxl_mbox_get_feat_in pi;
@ -232,10 +232,13 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
int rc;
if (return_code)
*return_code = CXL_MBOX_CMD_RC_INPUT;
*return_code = CXL_MBOX_CMD_RC_SUCCESS;
if (!feat_out || !feat_out_size)
return 0;
return -EINVAL;
if (feat_out_size > U16_MAX - offset)
return -EINVAL;
uuid_copy(&pi.uuid, feat_uuid);
pi.selection = selection;
@ -256,17 +259,25 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid,
.min_out = data_to_rd_size,
};
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
/*
* Per CXL r4.0 8.2.10.6.2, when Offset + Count runs past the
* end of the Feature the device returns only the bytes up to
* the Feature size. cxl_internal_send_cmd() reports that as
* -EIO with a short payload, so stop and return what arrived.
*/
if (rc == -EIO && mbox_cmd.size_out &&
mbox_cmd.size_out < data_to_rd_size) {
data_rcvd_size += mbox_cmd.size_out;
break;
}
if (rc < 0 || !mbox_cmd.size_out) {
if (return_code)
*return_code = mbox_cmd.return_code;
return 0;
return rc < 0 ? rc : -EIO;
}
data_rcvd_size += mbox_cmd.size_out;
} while (data_rcvd_size < feat_out_size);
if (return_code)
*return_code = CXL_MBOX_CMD_RC_SUCCESS;
return data_rcvd_size;
}
@ -286,7 +297,10 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox,
size_t hdr_size;
if (return_code)
*return_code = CXL_MBOX_CMD_RC_INPUT;
*return_code = CXL_MBOX_CMD_RC_SUCCESS;
if (feat_data_size > U16_MAX - offset)
return -EINVAL;
struct cxl_mbox_set_feat_in *pi __free(kfree) =
kzalloc(cxl_mbox->payload_size, GFP_KERNEL);
@ -334,11 +348,8 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox,
}
data_sent_size += data_in_size;
if (data_sent_size >= feat_data_size) {
if (return_code)
*return_code = CXL_MBOX_CMD_RC_SUCCESS;
if (data_sent_size >= feat_data_size)
return 0;
}
if ((feat_data_size - data_sent_size) <= (cxl_mbox->payload_size - hdr_size)) {
data_in_size = feat_data_size - data_sent_size;
@ -462,6 +473,7 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs,
const struct cxl_mbox_get_feat_in *feat_in;
u16 offset, count, return_code;
size_t out_size = *out_len;
ssize_t data_size;
if (rpc_in->op_size != sizeof(*feat_in))
return ERR_PTR(-EINVAL);
@ -482,16 +494,20 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs,
if (!rpc_out)
return ERR_PTR(-ENOMEM);
out_size = cxl_get_feature(cxl_mbox, &feat_in->uuid,
feat_in->selection, rpc_out->payload,
count, offset, &return_code);
data_size = cxl_get_feature(cxl_mbox, &feat_in->uuid,
feat_in->selection, rpc_out->payload,
count, offset, &return_code);
if (data_size <= 0 &&
return_code == CXL_MBOX_CMD_RC_SUCCESS)
return ERR_PTR(data_size ?: -EIO);
*out_len = sizeof(struct fwctl_rpc_cxl_out);
if (!out_size) {
if (data_size <= 0) {
rpc_out->size = 0;
rpc_out->retval = return_code;
return no_free_ptr(rpc_out);
}
out_size = data_size;
rpc_out->size = out_size;
rpc_out->retval = CXL_MBOX_CMD_RC_SUCCESS;
*out_len += out_size;
@ -536,6 +552,8 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs,
rc = cxl_set_feature(cxl_mbox, &feat_in->uuid,
feat_in->version, feat_in->feat_data,
data_size, flags, offset, &return_code);
if (rc && return_code == CXL_MBOX_CMD_RC_SUCCESS)
return ERR_PTR(rc);
*out_len = sizeof(*rpc_out);
if (rc) {
rpc_out->retval = return_code;

144
drivers/cxl/core/hdm.c
View File

@ -1,6 +1,7 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
#include <linux/seq_file.h>
#include <linux/unaligned.h>
#include <linux/device.h>
#include <linux/delay.h>
@ -76,6 +77,8 @@ static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm)
hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET);
cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap);
/* target_count is a direct count (1h..8h), not 0-based like decoder_count */
cxlhdm->target_count =
FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap);
if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap))
@ -240,6 +243,18 @@ static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds,
}
#define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL)
static void cxl_dpa_release_region(struct resource *parent,
struct resource *res)
{
/* zero sized decoders are not tracked in the resource tree */
if (resource_size(res) == 0) {
kfree(res);
return;
}
__release_region(parent, res->start, resource_size(res));
}
/*
* Must be called in a context that synchronizes against this decoder's
* port ->remove() callback (like an endpoint decoder sysfs attribute)
@ -256,7 +271,7 @@ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled)
/* save @skip_start, before @res is released */
skip_start = res->start - cxled->skip;
__release_region(&cxlds->dpa_res, res->start, resource_size(res));
cxl_dpa_release_region(&cxlds->dpa_res, res);
if (cxled->skip)
release_skip(cxlds, skip_start, cxled->skip);
cxled->skip = 0;
@ -336,6 +351,27 @@ static int request_skip(struct cxl_dev_state *cxlds,
return -EBUSY;
}
static struct resource *cxl_dpa_request_region(struct resource *parent,
resource_size_t start,
resource_size_t n,
const char *name)
{
struct resource *res;
if (!n) {
res = kmalloc_obj(*res);
if (!res)
return ERR_PTR(-ENOMEM);
*res = DEFINE_RES_NAMED(start, 0, name, IORESOURCE_MEM);
return res;
}
res = __request_region(parent, start, n, name, 0);
return res ?: ERR_PTR(-EBUSY);
}
static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
resource_size_t base, resource_size_t len,
resource_size_t skipped)
@ -349,12 +385,6 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
lockdep_assert_held_write(&cxl_rwsem.dpa);
if (!len) {
dev_warn(dev, "decoder%d.%d: empty reservation attempted\n",
port->id, cxled->cxld.id);
return -EINVAL;
}
if (cxled->dpa_res) {
dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n",
port->id, cxled->cxld.id, cxled->dpa_res);
@ -378,14 +408,14 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
if (rc)
return rc;
}
res = __request_region(&cxlds->dpa_res, base, len,
dev_name(&cxled->cxld.dev), 0);
if (!res) {
res = cxl_dpa_request_region(&cxlds->dpa_res, base, len,
dev_name(&cxled->cxld.dev));
if (IS_ERR(res)) {
dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n",
port->id, cxled->cxld.id);
if (skipped)
release_skip(cxlds, base - skipped, skipped);
return -EBUSY;
return PTR_ERR(res);
}
cxled->dpa_res = res;
cxled->skip = skipped;
@ -402,7 +432,8 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled,
break;
}
if (cxled->part < 0)
/* Empty decoders may not be contained by a partition boundary */
if (cxled->part < 0 && resource_size(res))
dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n",
port->id, cxled->cxld.id, res);
@ -446,6 +477,8 @@ static const char *cxl_mode_name(enum cxl_partition_mode mode)
return "ram";
case CXL_PARTMODE_PMEM:
return "pmem";
case CXL_PARTMODE_DYNAMIC_RAM:
return "dynamic_ram";
default:
return "";
};
@ -455,6 +488,7 @@ static const char *cxl_mode_name(enum cxl_partition_mode mode)
int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info)
{
struct device *dev = cxlds->dev;
int dc_nth = 0;
guard(rwsem_write)(&cxl_rwsem.dpa);
@ -467,14 +501,33 @@ int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info)
return 0;
}
/* Verify partitions are in expected order. */
for (int i = 1; i < info->nr_partitions; i++) {
if (info->part[i].mode < info->part[i - 1].mode) {
dev_err(dev, "Partition order mismatch: %d (%s) follows %d (%s)\n",
i, cxl_mode_name(info->part[i].mode),
i - 1, cxl_mode_name(info->part[i - 1].mode));
return -EINVAL;
}
}
cxlds->dpa_res = DEFINE_RES_MEM(0, info->size);
for (int i = 0; i < info->nr_partitions; i++) {
const struct cxl_dpa_part_info *part = &info->part[i];
const char *name = cxl_mode_name(part->mode);
int rc;
if (part->mode == CXL_PARTMODE_DYNAMIC_RAM) {
name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", name,
++dc_nth);
if (!name)
return -ENOMEM;
}
cxlds->part[i].perf.qos_class = CXL_QOS_CLASS_INVALID;
cxlds->part[i].mode = part->mode;
cxlds->part[i].handle = part->handle;
/* Require ordered + contiguous partitions */
if (i) {
@ -485,7 +538,7 @@ int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info)
}
rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->part[i].res,
part->range.start, range_len(&part->range),
cxl_mode_name(part->mode));
name);
if (rc)
return rc;
cxlds->nr_partitions++;
@ -953,7 +1006,8 @@ static int cxl_setup_hdm_decoder_from_dvsec(
* change the range registers at run time.
*/
cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
port->commit_end = cxld->id;
scoped_guard(rwsem_write, &cxl_rwsem.region)
port->commit_end = cxld->id;
rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0);
if (rc) {
@ -973,15 +1027,12 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
u64 *dpa_base, struct cxl_endpoint_dvsec_info *info)
{
struct cxl_endpoint_decoder *cxled = NULL;
u8 target_id[CXL_HDM_DECODER0_TL_TARGETS];
u64 size, base, skip, dpa_size, lo, hi;
bool committed;
u32 remainder;
int i, rc;
u32 ctrl;
union {
u64 value;
unsigned char target_id[8];
} target_list;
if (should_emulate_decoders(info))
return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base,
@ -1031,12 +1082,6 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
return -ENXIO;
}
if (size == 0) {
dev_warn(&port->dev,
"decoder%d.%d: Committed with zero size\n",
port->id, cxld->id);
return -ENXIO;
}
port->commit_end = cxld->id;
} else {
if (cxled) {
@ -1084,11 +1129,22 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld,
cxld->interleave_ways, cxld->interleave_granularity);
if (!cxled) {
lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which));
hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which));
target_list.value = (hi << 32) + lo;
struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(&cxld->dev);
if (cxld->interleave_ways > cxlsd->nr_targets) {
dev_err(&port->dev,
"decoder%d.%d: interleave ways: %d exceeds targets: %d\n",
port->id, cxld->id, cxld->interleave_ways,
cxlsd->nr_targets);
return -ENXIO;
}
put_unaligned_le32(readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)),
&target_id[0]);
put_unaligned_le32(readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)),
&target_id[4]);
for (i = 0; i < cxld->interleave_ways; i++)
cxld->target_map[i] = target_list.target_id[i];
cxld->target_map[i] = target_id[i];
return 0;
}
@ -1148,6 +1204,14 @@ static void cxl_settle_decoders(struct cxl_hdm *cxlhdm)
msleep(20);
}
static void cxl_reset_commit_end(void *data)
{
struct cxl_port *port = data;
guard(rwsem_write)(&cxl_rwsem.region);
port->commit_end = -1;
}
/**
* devm_cxl_enumerate_decoders - add decoder objects per HDM register set
* @cxlhdm: Structure to populate with HDM capabilities
@ -1158,13 +1222,33 @@ static int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm,
{
void __iomem *hdm = cxlhdm->regs.hdm_decoder;
struct cxl_port *port = cxlhdm->port;
int i;
u64 dpa_base = 0;
int i, rc;
/*
* Per CXL 4.0 8.2.4.20.1 Target Count is the number of target ports
* per decoder, max 8. Endpoint decoders have no targets.
*/
if (!is_cxl_endpoint(port) &&
(cxlhdm->target_count < 1 ||
cxlhdm->target_count > CXL_HDM_DECODER0_TL_TARGETS)) {
dev_err(&port->dev, "Invalid decoder target count: %u\n",
cxlhdm->target_count);
return -ENXIO;
}
cxl_settle_decoders(cxlhdm);
/*
* Reset commit_end after all decoders have been torn down so a
* subsequent probe rebuilds it from scratch.
*/
rc = devm_add_action(&port->dev, cxl_reset_commit_end, port);
if (rc)
return rc;
for (i = 0; i < cxlhdm->decoder_count; i++) {
int rc, target_count = cxlhdm->target_count;
int target_count = cxlhdm->target_count;
struct cxl_decoder *cxld;
if (is_cxl_endpoint(port)) {

294
drivers/cxl/core/mbox.c
View File

@ -171,6 +171,33 @@ static void cxl_set_security_cmd_enabled(struct cxl_security_state *security,
}
}
static bool cxl_is_dcd_command(u16 opcode)
{
#define CXL_MBOX_OP_DCD_CMDS 0x48
return (opcode >> 8) == CXL_MBOX_OP_DCD_CMDS;
}
static void cxl_set_dcd_cmd_enabled(u16 opcode, unsigned long *cmd_mask)
{
switch (opcode) {
case CXL_MBOX_OP_GET_DC_CONFIG:
set_bit(CXL_DCD_ENABLED_GET_CONFIG, cmd_mask);
break;
case CXL_MBOX_OP_GET_DC_EXTENT_LIST:
set_bit(CXL_DCD_ENABLED_GET_EXTENT_LIST, cmd_mask);
break;
case CXL_MBOX_OP_ADD_DC_RESPONSE:
set_bit(CXL_DCD_ENABLED_ADD_RESPONSE, cmd_mask);
break;
case CXL_MBOX_OP_RELEASE_DC:
set_bit(CXL_DCD_ENABLED_RELEASE, cmd_mask);
break;
default:
break;
}
}
static bool cxl_is_poison_command(u16 opcode)
{
#define CXL_MBOX_OP_POISON_CMDS 0x43
@ -759,6 +786,7 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_cel_entry *cel_entry;
const int cel_entries = size / sizeof(*cel_entry);
DECLARE_BITMAP(dcd_cmds, CXL_DCD_ENABLED_MAX) = {};
struct device *dev = mds->cxlds.dev;
int i, ro_cmds = 0, wr_cmds = 0;
@ -787,11 +815,22 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel)
enabled++;
}
if (cxl_is_dcd_command(opcode)) {
cxl_set_dcd_cmd_enabled(opcode, dcd_cmds);
enabled++;
}
dev_dbg(dev, "Opcode 0x%04x %s\n", opcode,
enabled ? "enabled" : "unsupported by driver");
}
set_features_cap(cxl_mbox, ro_cmds, wr_cmds);
/*
* Disabled until event handling implemented.
*/
if (bitmap_full(dcd_cmds, CXL_DCD_ENABLED_MAX))
dev_dbg(dev, "Device supports DCD; capability disabled\n");
mds->dcd_supported = false;
}
static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds)
@ -942,11 +981,11 @@ void cxl_event_trace_record(struct cxl_memdev *cxlmd,
if (evt->gen_media.media_hdr.descriptor &
CXL_GMER_EVT_DESC_THRESHOLD_EVENT)
WARN_ON_ONCE((evt->gen_media.media_hdr.type &
WARN_ON_ONCE((evt->gen_media.media_hdr.type ==
CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) &&
!get_unaligned_le24(evt->gen_media.cme_count));
else
WARN_ON_ONCE(evt->gen_media.media_hdr.type &
WARN_ON_ONCE(evt->gen_media.media_hdr.type ==
CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE);
trace_cxl_general_media(cxlmd, type, cxlr, hpa,
@ -957,11 +996,11 @@ void cxl_event_trace_record(struct cxl_memdev *cxlmd,
if (evt->dram.media_hdr.descriptor &
CXL_GMER_EVT_DESC_THRESHOLD_EVENT)
WARN_ON_ONCE((evt->dram.media_hdr.type &
WARN_ON_ONCE((evt->dram.media_hdr.type ==
CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) &&
!get_unaligned_le24(evt->dram.cvme_count));
else
WARN_ON_ONCE(evt->dram.media_hdr.type &
WARN_ON_ONCE(evt->dram.media_hdr.type ==
CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE);
trace_cxl_dram(cxlmd, type, cxlr, hpa, hpa_alias,
@ -1310,6 +1349,250 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd)
return -EBUSY;
}
/*
* A partition unavailable for Dynamic Capacity reports zeros for decode_length,
* length, and block_size.
*/
static bool cxl_dc_partition_unavailable(struct cxl_dc_partition *dev_part)
{
return !le64_to_cpu(dev_part->decode_length) &&
!le64_to_cpu(dev_part->length) &&
!le64_to_cpu(dev_part->block_size);
}
static int cxl_dc_check(struct device *dev, struct cxl_dc_partition_info *part,
struct cxl_dc_partition *dev_part)
{
u64 decode_len = le64_to_cpu(dev_part->decode_length);
u64 blk_size = le64_to_cpu(dev_part->block_size);
u64 len = le64_to_cpu(dev_part->length);
u32 handle = le32_to_cpu(dev_part->dsmad_handle);
u64 size;
/*
* Not an error; leave the entry empty. A partially zeroed partition
* is rejected by the checks below. CXL r4.0 Table 8-347.
*/
if (cxl_dc_partition_unavailable(dev_part)) {
*part = (struct cxl_dc_partition_info) { };
dev_dbg(dev, "Partition 0 unavailable for DC\n");
return 0;
}
/* The CDAT DSMAD handle this refers to is 8 bits */
if (handle > U8_MAX) {
dev_warn(dev, "DSMAD handle 0x%x exceeds the 8 bit CDAT DSMAD handle\n",
handle);
return -EINVAL;
}
/*
* Decode length is a count of 256M units, so a device reporting 2^36
* or more wraps.
*/
if (check_mul_overflow(decode_len, (u64)CXL_CAPACITY_MULTIPLIER, &size)) {
dev_err(dev, "DC partition 0 decode length %#llx overflows\n",
decode_len);
return -EINVAL;
}
*part = (struct cxl_dc_partition_info) {
.start = le64_to_cpu(dev_part->base),
.size = size,
.handle = handle,
};
/*
* Block size is a power of 2 and a multiple of 40h. is_power_of_2()
* takes an unsigned long, which truncates blk_size on 32 bit.
*/
if (blk_size == 0 || (blk_size & (blk_size - 1)) ||
blk_size % CXL_DCD_BLOCK_LINE_SIZE) {
dev_err(dev, "DC partition 0 invalid block size %#llx\n", blk_size);
return -EINVAL;
}
if (part->size == 0) {
dev_err(dev, "DC partition 0 zero decode length\n");
return -EINVAL;
}
if (len == 0) {
dev_err(dev, "DC partition 0 zero length\n");
return -EINVAL;
}
if (len > part->size) {
dev_err(dev, "DC partition 0 length %#llx exceeds decode length %#llx\n",
len, part->size);
return -EINVAL;
}
if (!IS_ALIGNED(len, blk_size)) {
dev_err(dev, "DC partition 0 length %#llx not a multiple of block size %#llx\n",
len, blk_size);
return -EINVAL;
}
if (!IS_ALIGNED(part->start, SZ_256M)) {
dev_err(dev, "DC partition 0 base %#llx not aligned to 256M\n",
part->start);
return -EINVAL;
}
dev_dbg(dev, "DC partition 0 start %#llx size %#llx blk_size: %#llx\n",
part->start, part->size, blk_size);
return 0;
}
/* Returns the number of partitions in dc_resp or -ERRNO */
static int cxl_get_dc_config(struct cxl_mailbox *mbox, u8 start_partition,
u8 partition_count,
struct cxl_mbox_get_dc_config_out *dc_resp,
size_t dc_resp_size)
{
struct cxl_mbox_get_dc_config_in get_dc = (struct cxl_mbox_get_dc_config_in) {
.partition_count = partition_count,
.start_partition_index = start_partition,
};
struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd) {
.opcode = CXL_MBOX_OP_GET_DC_CONFIG,
.payload_in = &get_dc,
.size_in = sizeof(get_dc),
.size_out = dc_resp_size,
.payload_out = dc_resp,
/* The device must return at least the fixed header */
.min_out = sizeof(*dc_resp),
};
size_t expected_sz;
int rc;
rc = cxl_internal_send_cmd(mbox, &mbox_cmd);
if (rc < 0)
return rc;
if (dc_resp->partitions_returned > partition_count) {
dev_err(mbox->host, "Device returned %u partitions, requested %u\n",
dc_resp->partitions_returned, partition_count);
return -EIO;
}
/* The trailing extent/tag counts (CXL r4.0 Table 8-346) are not read */
expected_sz = struct_size(dc_resp, partition,
dc_resp->partitions_returned);
if (mbox_cmd.size_out < expected_sz) {
dev_err(mbox->host,
"Payload size %zu less than expected %zu for %u partitions\n",
mbox_cmd.size_out, expected_sz,
dc_resp->partitions_returned);
return -EIO;
}
dev_dbg(mbox->host, "Read %d/%d DC partitions\n",
dc_resp->partitions_returned, dc_resp->avail_partition_count);
return dc_resp->partitions_returned;
}
/**
* cxl_dev_dc_identify() - Reads the dynamic capacity information from the
* device.
* @mbox: Mailbox to query
* @dc_info: The dynamic partition information to return
*
* Read every partition the device reports, but validate only the first:
* Linux maps partition 0 and nothing else, so a defect in capacity the
* driver never touches is not a reason to refuse the device dynamic
* capacity. The remaining partitions are read but not recorded.
*
* Return: 0 if identify was executed successfully, -ERRNO on error.
* On error, @dc_info is left unchanged.
*/
int cxl_dev_dc_identify(struct cxl_mailbox *mbox,
struct cxl_dc_partition_info *dc_info)
{
struct cxl_dc_partition_info partitions[CXL_MAX_DC_PARTITIONS] = { };
struct device *dev = mbox->host;
u8 avail_partitions = 0;
u8 num_partitions;
u8 partition_count;
size_t dc_resp_size;
/*
* Bound requested number of partitions by mailbox payload size. The
* 256 byte spec minimum, verified in cxl_pci_setup_mailbox(), keeps
* the subtraction below from underflowing.
*/
partition_count = min(CXL_MAX_DC_PARTITIONS,
(mbox->payload_size -
sizeof(struct cxl_mbox_get_dc_config_out) -
sizeof(struct cxl_mbox_get_dc_config_tail)) /
sizeof(struct cxl_dc_partition));
dc_resp_size = struct_size_t(struct cxl_mbox_get_dc_config_out,
partition, partition_count) +
sizeof(struct cxl_mbox_get_dc_config_tail);
struct cxl_mbox_get_dc_config_out *dc_resp __free(kfree) =
kmalloc(dc_resp_size, GFP_KERNEL);
if (!dc_resp)
return -ENOMEM;
num_partitions = 0;
do {
int returned, rc;
returned = cxl_get_dc_config(mbox, num_partitions,
partition_count, dc_resp,
dc_resp_size);
if (returned < 0) {
dev_err(dev, "Failed to get DC config: %d\n", returned);
return returned;
}
if (returned == 0) {
dev_err(dev,
"Device reported %u partitions available but returned none at index %u\n",
dc_resp->avail_partition_count, num_partitions);
return -EIO;
}
/* A DCD reports at least one partition (CXL r4.0 Table 8-346) */
if (dc_resp->avail_partition_count == 0) {
dev_err(dev, "Device reported no DC partitions\n");
return -EIO;
}
avail_partitions = min(dc_resp->avail_partition_count,
CXL_MAX_DC_PARTITIONS);
/*
* Only partition 0 is mapped, so it is the only one checked.
* The rest are read to complete the enumeration and dropped.
*/
if (num_partitions == 0) {
rc = cxl_dc_check(dev, &partitions[0],
&dc_resp->partition[0]);
if (rc)
return rc;
}
num_partitions += returned;
} while (num_partitions < avail_partitions);
/* Linux only supports the 1st partition; nothing to do if it is unavailable */
if (partitions[0].size == 0)
return -ENODEV;
*dc_info = partitions[0];
dev_dbg(dev, "Returning partition 0 %#llx size %#llx\n",
dc_info->start, dc_info->size);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_dev_dc_identify, "CXL");
int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count)
{
struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
@ -1386,6 +1669,9 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,
int nr_records = 0;
int rc;
if (!len)
return 0;
ACQUIRE(mutex_intr, lock)(&mds->poison.mutex);
if ((rc = ACQUIRE_ERR(mutex_intr, &lock)))
return rc;

8
drivers/cxl/core/mce.c
View File

@ -18,7 +18,13 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val,
u64 spa, spa_alias;
unsigned long pfn;
if (!mce || !mce_usable_address(mce))
if (!mce)
return NOTIFY_DONE;
if (mce_is_correctable(mce))
return NOTIFY_DONE;
if (!mce_usable_address(mce))
return NOTIFY_DONE;
spa = mce->addr & MCI_ADDR_PHYSADDR;

100
drivers/cxl/core/memdev.c
View File

@ -77,20 +77,12 @@ static ssize_t label_storage_size_show(struct device *dev,
}
static DEVICE_ATTR_RO(label_storage_size);
static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds)
{
/* Static RAM is only expected at partition 0. */
if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
return 0;
return resource_size(&cxlds->part[0].res);
}
static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr,
char *buf)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
unsigned long long len = cxl_ram_size(cxlds);
unsigned long long len = cxl_part_size(cxlds, CXL_PARTMODE_RAM);
return sysfs_emit(buf, "%#llx\n", len);
}
@ -103,7 +95,7 @@ static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr,
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
unsigned long long len = cxl_pmem_size(cxlds);
unsigned long long len = cxl_part_size(cxlds, CXL_PARTMODE_PMEM);
return sysfs_emit(buf, "%#llx\n", len);
}
@ -426,10 +418,11 @@ static struct attribute *cxl_memdev_attributes[] = {
NULL,
};
static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds)
static struct cxl_dpa_perf *part_perf(struct cxl_dev_state *cxlds,
enum cxl_partition_mode mode)
{
for (int i = 0; i < cxlds->nr_partitions; i++)
if (cxlds->part[i].mode == CXL_PARTMODE_PMEM)
if (cxlds->part[i].mode == mode)
return &cxlds->part[i].perf;
return NULL;
}
@ -440,7 +433,7 @@ static ssize_t pmem_qos_class_show(struct device *dev,
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class);
return sysfs_emit(buf, "%d\n", part_perf(cxlds, CXL_PARTMODE_PMEM)->qos_class);
}
static struct device_attribute dev_attr_pmem_qos_class =
@ -452,20 +445,13 @@ static struct attribute *cxl_memdev_pmem_attributes[] = {
NULL,
};
static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds)
{
if (cxlds->part[0].mode != CXL_PARTMODE_RAM)
return NULL;
return &cxlds->part[0].perf;
}
static ssize_t ram_qos_class_show(struct device *dev,
struct device_attribute *attr, char *buf)
{
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dev_state *cxlds = cxlmd->cxlds;
return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class);
return sysfs_emit(buf, "%d\n", part_perf(cxlds, CXL_PARTMODE_RAM)->qos_class);
}
static struct device_attribute dev_attr_ram_qos_class =
@ -501,7 +487,7 @@ static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n)
{
struct device *dev = kobj_to_dev(kobj);
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds);
struct cxl_dpa_perf *perf = part_perf(cxlmd->cxlds, CXL_PARTMODE_RAM);
if (a == &dev_attr_ram_qos_class.attr &&
(!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
@ -520,7 +506,7 @@ static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n
{
struct device *dev = kobj_to_dev(kobj);
struct cxl_memdev *cxlmd = to_cxl_memdev(dev);
struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds);
struct cxl_dpa_perf *perf = part_perf(cxlmd->cxlds, CXL_PARTMODE_PMEM);
if (a == &dev_attr_pmem_qos_class.attr &&
(!perf || perf->qos_class == CXL_QOS_CLASS_INVALID))
@ -594,7 +580,8 @@ bool is_cxl_memdev(const struct device *dev)
}
EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL");
static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode)
static void add_part(struct cxl_dpa_info *info, u64 start, u64 size,
enum cxl_partition_mode mode, u8 handle)
{
int i = info->nr_partitions;
@ -606,6 +593,7 @@ static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_pa
.end = start + size - 1,
};
info->part[i].mode = mode;
info->part[i].handle = handle;
info->nr_partitions++;
}
@ -623,9 +611,9 @@ int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
info->size = mds->total_bytes;
if (mds->partition_align_bytes == 0) {
add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM);
add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM, 0);
add_part(info, mds->volatile_only_bytes,
mds->persistent_only_bytes, CXL_PARTMODE_PMEM);
mds->persistent_only_bytes, CXL_PARTMODE_PMEM, 0);
return 0;
}
@ -635,14 +623,68 @@ int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
return rc;
}
add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM);
add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM, 0);
add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes,
CXL_PARTMODE_PMEM);
CXL_PARTMODE_PMEM, 0);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL");
int cxl_configure_dcd(struct cxl_memdev_state *mds, struct cxl_dpa_info *info)
{
struct cxl_dc_partition_info dc_info = { };
struct device *dev = mds->cxlds.dev;
u64 static_end = 0;
u64 dc_end;
int rc;
if (!mds->cxlds.media_ready)
return 0;
rc = cxl_dev_dc_identify(&mds->cxlds.cxl_mbox, &dc_info);
if (rc) {
dev_warn(dev,
"Failed to read Dynamic Capacity config: %d\n", rc);
return rc;
}
if (info->nr_partitions)
static_end = info->part[info->nr_partitions - 1].range.end + 1;
if (dc_info.start < static_end) {
dev_err(dev,
"DC partition 0 base %#llx overlaps static capacity ending at %#llx\n",
dc_info.start, static_end);
return -EINVAL;
}
/* A gap between static capacity and the DC partition is not supported */
if (dc_info.start > static_end) {
dev_warn(dev,
"DC partition 0 base %#llx leaves a gap from static capacity ending at %#llx\n",
dc_info.start, static_end);
return -EOPNOTSUPP;
}
if (check_add_overflow(dc_info.start, dc_info.size, &dc_end)) {
dev_err(dev,
"DC partition 0 size %#llx overflows DPA space at base %#llx\n",
dc_info.size, dc_info.start);
return -EINVAL;
}
/* The DPA space must reach the end of the DC partition */
if (dc_end > info->size)
info->size = dc_end;
dev_dbg(dev, "Adding dynamic ram partition; %#llx size %#llx\n",
dc_info.start, dc_info.size);
add_part(info, dc_info.start, dc_info.size, CXL_PARTMODE_DYNAMIC_RAM,
dc_info.handle);
return 0;
}
EXPORT_SYMBOL_NS_GPL(cxl_configure_dcd, "CXL");
/**
* cxl_set_capacity: initialize dpa by a driver without a mailbox.
@ -656,7 +698,7 @@ int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity)
.size = capacity,
};
add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM);
add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM, 0);
return cxl_dpa_setup(cxlds, &range_info);
}
EXPORT_SYMBOL_NS_GPL(cxl_set_capacity, "CXL");

74
drivers/cxl/core/pci.c
View File

@ -660,8 +660,8 @@ long cxl_pci_get_latency(struct pci_dev *pdev)
{
long bw;
bw = pcie_link_speed_mbps(pdev);
if (bw < 0)
bw = pcie_link_bandwidth_mbps(pdev);
if (bw <= 0)
return 0;
bw /= BITS_PER_BYTE;
@ -683,6 +683,15 @@ static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data)
if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0)
return 0;
/*
* Decoders emulated from the DVSEC range registers have no commit
* callback and no HDM decoder registers to consult.
*/
if (!cxld->commit) {
dev_warn(dev, "DVSEC emulated decode may have been cleared by reset\n");
return 0;
}
cxlhdm = dev_get_drvdata(&port->dev);
hdm = cxlhdm->regs.hdm_decoder;
ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id));
@ -734,10 +743,16 @@ int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type,
if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev)) {
struct cxl_dport *dport;
struct cxl_port *port __free(put_cxl_port) =
cxl_pci_find_port(pdev, &dport);
cxl_pci_find_port(pdev);
if (!port)
return -EPROBE_DEFER;
guard(device)(cxl_port_dport_host(port));
dport = cxl_pci_find_dport(pdev, port);
if (!dport)
return -EPROBE_DEFER;
rc = cxl_rcrb_get_comp_regs(pdev, map, dport);
if (rc)
return rc;
@ -756,18 +771,12 @@ EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, "CXL");
int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c)
{
int speed, bw;
u16 lnksta;
u32 width;
int bw;
speed = pcie_link_speed_mbps(pdev);
if (speed < 0)
return speed;
speed /= BITS_PER_BYTE;
pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
bw = speed * width;
bw = pcie_link_bandwidth_mbps(pdev);
if (bw < 0)
return bw;
bw /= BITS_PER_BYTE;
for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
c[i].read_bandwidth = bw;
@ -804,7 +813,8 @@ u16 cxl_gpf_get_dvsec(struct device *dev)
return 0;
pdev = to_pci_dev(dev);
if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ENDPOINT)
if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ENDPOINT ||
is_cxl_restricted(pdev))
is_port = false;
dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
@ -839,21 +849,27 @@ static int update_gpf_port_dvsec(struct pci_dev *pdev, int dvsec, int phase)
rc = pci_read_config_word(pdev, dvsec + offset, &ctrl);
if (rc)
return rc;
return pcibios_err_to_errno(rc);
if (FIELD_GET(base, ctrl) == GPF_TIMEOUT_BASE_MAX &&
FIELD_GET(scale, ctrl) == GPF_TIMEOUT_SCALE_MAX)
return 0;
ctrl = FIELD_PREP(base, GPF_TIMEOUT_BASE_MAX);
ctrl |= FIELD_PREP(scale, GPF_TIMEOUT_SCALE_MAX);
FIELD_MODIFY(base, &ctrl, GPF_TIMEOUT_BASE_MAX);
FIELD_MODIFY(scale, &ctrl, GPF_TIMEOUT_SCALE_MAX);
rc = pci_write_config_word(pdev, dvsec + offset, ctrl);
if (!rc)
pci_dbg(pdev, "Port GPF phase %d timeout: %d0 secs\n",
phase, GPF_TIMEOUT_BASE_MAX);
if (rc) {
rc = pcibios_err_to_errno(rc);
pci_warn(pdev, "Port GPF phase %d timeout write failed: %d\n",
phase, rc);
return rc;
}
return rc;
pci_dbg(pdev, "Port GPF phase %d timeout: %d0 secs\n",
phase, GPF_TIMEOUT_BASE_MAX);
return 0;
}
int cxl_gpf_port_setup(struct cxl_dport *dport)
@ -863,16 +879,22 @@ int cxl_gpf_port_setup(struct cxl_dport *dport)
if (!dport->gpf_dvsec) {
struct pci_dev *pdev;
int dvsec;
int dvsec, rc;
dvsec = cxl_gpf_get_dvsec(dport->dport_dev);
if (!dvsec)
return -EINVAL;
dport->gpf_dvsec = dvsec;
pdev = to_pci_dev(dport->dport_dev);
update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 1);
update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 2);
rc = update_gpf_port_dvsec(pdev, dvsec, 1);
if (rc)
return rc;
rc = update_gpf_port_dvsec(pdev, dvsec, 2);
if (rc)
return rc;
dport->gpf_dvsec = dvsec;
}
return 0;

250
drivers/cxl/core/port.c
View File

@ -148,20 +148,29 @@ static ssize_t target_type_show(struct device *dev,
}
static DEVICE_ATTR_RO(target_type);
/*
* Interleave ways selects how many targets a decoder uses, but the target
* array is only nr_targets long. Bound array access by both.
*/
static int cxlsd_nr_used_targets(struct cxl_switch_decoder *cxlsd)
{
return min(cxlsd->cxld.interleave_ways, cxlsd->nr_targets);
}
static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf)
{
struct cxl_decoder *cxld = &cxlsd->cxld;
int nr_used = cxlsd_nr_used_targets(cxlsd);
ssize_t offset = 0;
int i, rc = 0;
for (i = 0; i < cxld->interleave_ways; i++) {
for (i = 0; i < nr_used; i++) {
struct cxl_dport *dport = cxlsd->target[i];
struct cxl_dport *next = NULL;
if (!dport)
break;
if (i + 1 < cxld->interleave_ways)
if (i + 1 < nr_used)
next = cxlsd->target[i + 1];
rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id,
next ? "," : "");
@ -814,6 +823,11 @@ static int cxl_einj_inject(void *data, u64 type)
DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject,
"0x%llx\n");
static void remove_debugfs(void *dentry)
{
debugfs_remove_recursive(dentry);
}
static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
{
struct cxl_port *parent = parent_port_of(dport->port);
@ -832,6 +846,9 @@ static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport)
dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev));
if (devm_add_action_or_reset(dport_to_host(dport), remove_debugfs, dir))
return;
debugfs_create_file("einj_inject", 0200, dir, dport,
&cxl_einj_inject_fops);
}
@ -1089,11 +1106,40 @@ static void cond_cxl_root_unlock(struct cxl_port *port)
device_unlock(&port->dev);
}
static int clear_decoder_target(struct device *dev, void *data)
{
struct cxl_dport *dport = data;
struct cxl_switch_decoder *cxlsd;
if (!is_switch_decoder(dev))
return 0;
cxlsd = to_cxl_switch_decoder(dev);
guard(rwsem_write)(&cxl_rwsem.region);
/*
* A dport can occupy more than one position of an interleave, so
* scan the whole target list rather than stopping at the first hit.
*/
for (int i = 0; i < cxlsd->nr_targets; i++) {
if (cxlsd->target[i] != dport)
continue;
cxlsd->target[i] = NULL;
dev_dbg(dev, "dport%d removed from target list, index %d\n",
dport->port_id, i);
}
return 0;
}
static void cxl_dport_remove(void *data)
{
struct cxl_dport *dport = data;
struct cxl_port *port = dport->port;
/* counterpart of cxl_port_update_decoder_targets() */
device_for_each_child(&port->dev, dport, clear_decoder_target);
port->nr_dports--;
xa_erase(&port->dports, (unsigned long) dport->dport_dev);
put_device(dport->dport_dev);
@ -1320,7 +1366,7 @@ static int add_ep(struct cxl_ep *new)
{
struct cxl_port *port = new->dport->port;
guard(device)(&port->dev);
device_lock_assert(&port->dev);
if (port->dead)
return -ENXIO;
@ -1336,6 +1382,11 @@ static int add_ep(struct cxl_ep *new)
* Intermediate CXL ports are scanned based on the arrival of endpoints.
* When those endpoints depart the port can be destroyed once all
* endpoints that care about that port have been removed.
*
* Context: Caller must hold the device lock of @dport->port, which pins
* @dport, across the lookup of @dport and this call. @dport->port
* is never the CXL root port, whose dports are pinned by its
* uport_dev instead.
*/
static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
{
@ -1358,13 +1409,11 @@ static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev)
struct cxl_find_port_ctx {
const struct device *dport_dev;
const struct cxl_port *parent_port;
struct cxl_dport **dport;
};
static int match_port_by_dport(struct device *dev, const void *data)
{
const struct cxl_find_port_ctx *ctx = data;
struct cxl_dport *dport;
struct cxl_port *port;
if (!is_cxl_port(dev))
@ -1373,10 +1422,7 @@ static int match_port_by_dport(struct device *dev, const void *data)
return 0;
port = to_cxl_port(dev);
dport = cxl_find_dport_by_dev(port, ctx->dport_dev);
if (ctx->dport)
*ctx->dport = dport;
return dport != NULL;
return !!cxl_find_dport_by_dev(port, ctx->dport_dev);
}
static struct cxl_port *__find_cxl_port_by_dport(struct cxl_find_port_ctx *ctx)
@ -1395,22 +1441,21 @@ static struct cxl_port *__find_cxl_port_by_dport(struct cxl_find_port_ctx *ctx)
/**
* find_cxl_port_by_dport - find a cxl_port by one of its targets
* @dport_dev: device representing the dport target
* @dport: optional output of the 'struct cxl_dport' companion of the @dport_dev
*
* Return a 'struct cxl_port' with an elevated reference if found. Use
* __free(put_cxl_port) to release.
*
* The port reference does not pin the dport, which is a devm allocation of
* cxl_port_dport_host(). Use cxl_pci_find_dport() or cxl_mem_find_dport()
* under that device's lock to get it.
*/
static struct cxl_port *find_cxl_port_by_dport(struct device *dport_dev,
struct cxl_dport **dport)
static struct cxl_port *find_cxl_port_by_dport(struct device *dport_dev)
{
struct cxl_find_port_ctx ctx = {
.dport_dev = dport_dev,
.dport = dport,
};
struct cxl_port *port;
port = __find_cxl_port_by_dport(&ctx);
return port;
return __find_cxl_port_by_dport(&ctx);
}
/*
@ -1611,7 +1656,7 @@ static int update_decoder_targets(struct device *dev, void *data)
struct cxl_dport *dport = data;
struct cxl_switch_decoder *cxlsd;
struct cxl_decoder *cxld;
int i;
int i, nr_used;
if (!is_switch_decoder(dev))
return 0;
@ -1619,8 +1664,9 @@ static int update_decoder_targets(struct device *dev, void *data)
cxlsd = to_cxl_switch_decoder(dev);
cxld = &cxlsd->cxld;
guard(rwsem_write)(&cxl_rwsem.region);
nr_used = cxlsd_nr_used_targets(cxlsd);
for (i = 0; i < cxld->interleave_ways; i++) {
for (i = 0; i < nr_used; i++) {
if (cxld->target_map[i] == dport->port_id) {
cxlsd->target[i] = dport;
dev_dbg(dev, "dport%d found in target list, index %d\n",
@ -1672,13 +1718,15 @@ static struct cxl_dport *probe_dport(struct cxl_port *port,
return drv->add_dport(port, dport_dev);
}
static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
struct cxl_port *parent_port,
struct cxl_dport *parent_dport,
struct device *uport_dev,
struct device *dport_dev)
static int devm_cxl_create_port(struct device *ep_dev,
struct cxl_port *parent_port,
struct cxl_dport *parent_dport,
struct device *uport_dev,
struct device *dport_dev)
{
resource_size_t component_reg_phys;
struct cxl_dport *dport;
int rc;
device_lock_assert(&parent_port->dev);
if (!parent_port->dev.driver) {
@ -1695,7 +1743,7 @@ static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
port = devm_cxl_add_port(&parent_port->dev, uport_dev,
component_reg_phys, parent_dport);
if (IS_ERR(port))
return ERR_CAST(port);
return PTR_ERR(port);
/*
* retry to make sure a port is found. a port device
@ -1703,7 +1751,7 @@ static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
*/
port = find_cxl_port_by_uport(uport_dev);
if (!port)
return ERR_PTR(-ENODEV);
return -ENODEV;
dev_dbg(ep_dev, "created port %s:%s\n",
dev_name(&port->dev), dev_name(port->uport_dev));
@ -1712,11 +1760,25 @@ static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev,
* Port was created before right before this function is
* called. Signal the caller to deal with it.
*/
return ERR_PTR(-EAGAIN);
return -EAGAIN;
}
/* @dport is only valid while @port's lock is held */
guard(device)(&port->dev);
return probe_dport(port, dport_dev);
dport = probe_dport(port, dport_dev);
if (IS_ERR(dport))
return PTR_ERR(dport);
rc = cxl_add_ep(dport, ep_dev);
if (rc == -EBUSY) {
/*
* "can't" happen, but this error code means
* something to the caller, so translate it.
*/
rc = -ENXIO;
}
return rc;
}
static int add_port_attach_ep(struct cxl_memdev *cxlmd,
@ -1724,7 +1786,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
struct device *dport_dev)
{
struct device *dparent = grandparent(dport_dev);
struct cxl_dport *dport, *parent_dport;
struct cxl_dport *parent_dport;
int rc;
if (is_cxl_host_bridge(dparent)) {
@ -1753,33 +1815,23 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd,
return PTR_ERR(parent_dport);
}
dport = devm_cxl_create_port(&cxlmd->dev, parent_port,
parent_dport, uport_dev,
dport_dev);
if (IS_ERR(dport)) {
/* Port or dport already exists, restart iteration */
if (PTR_ERR(dport) == -EAGAIN || PTR_ERR(dport) == -EBUSY)
return 0;
return PTR_ERR(dport);
}
rc = devm_cxl_create_port(&cxlmd->dev, parent_port,
parent_dport, uport_dev, dport_dev);
}
rc = cxl_add_ep(dport, &cxlmd->dev);
if (rc == -EBUSY) {
/*
* "can't" happen, but this error code means
* something to the caller, so translate it.
*/
rc = -ENXIO;
}
/* Port or dport already exists, restart iteration */
if (rc == -EAGAIN || rc == -EBUSY)
return 0;
return rc;
}
static struct cxl_dport *find_or_add_dport(struct cxl_port *port,
struct device *dport_dev)
static int find_or_add_dport_attach_ep(struct cxl_memdev *cxlmd,
struct cxl_port *port,
struct device *dport_dev)
{
struct cxl_dport *dport;
int rc;
/*
* The port is already visible in CXL hierarchy, but it may still
@ -1795,13 +1847,28 @@ static struct cxl_dport *find_or_add_dport(struct cxl_port *port,
if (!dport) {
dport = probe_dport(port, dport_dev);
if (IS_ERR(dport))
return dport;
return PTR_ERR(dport);
/* New dport added, restart iteration */
return ERR_PTR(-EAGAIN);
return -EAGAIN;
}
return dport;
/* @dport is only valid while @port's lock is held */
rc = cxl_add_ep(dport, &cxlmd->dev);
/*
* If the endpoint already exists in the port's list,
* that's ok, it was added on a previous pass.
* Otherwise, retry in add_port_attach_ep() after taking
* the parent_port lock as the current port may be being
* reaped.
*/
if (rc && rc != -EBUSY)
return rc;
cxl_gpf_port_setup(dport);
return 0;
}
int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
@ -1830,7 +1897,6 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
for (iter = dev; iter; iter = grandparent(iter)) {
struct device *dport_dev = grandparent(iter);
struct device *uport_dev;
struct cxl_dport *dport;
if (is_cxl_host_bridge(dport_dev))
return 0;
@ -1857,27 +1923,13 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
* RP port enumerated by cxl_acpi without dport will
* have the dport added here.
*/
dport = find_or_add_dport(port, dport_dev);
if (IS_ERR(dport)) {
if (PTR_ERR(dport) == -EAGAIN)
goto retry;
return PTR_ERR(dport);
}
rc = cxl_add_ep(dport, &cxlmd->dev);
/*
* If the endpoint already exists in the port's list,
* that's ok, it was added on a previous pass.
* Otherwise, retry in add_port_attach_ep() after taking
* the parent_port lock as the current port may be being
* reaped.
*/
if (rc && rc != -EBUSY)
rc = find_or_add_dport_attach_ep(cxlmd, port,
dport_dev);
if (rc == -EAGAIN)
goto retry;
if (rc)
return rc;
cxl_gpf_port_setup(dport);
/* Any more ports to add between this one and the root? */
if (!dev_is_cxl_root_child(&port->dev))
continue;
@ -1900,25 +1952,55 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd)
}
EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL");
struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
struct cxl_dport **dport)
struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev)
{
return find_cxl_port_by_dport(pdev->dev.parent, dport);
return find_cxl_port_by_dport(pdev->dev.parent);
}
EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL");
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
struct cxl_dport **dport)
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd)
{
return find_cxl_port_by_dport(grandparent(&cxlmd->dev), dport);
return find_cxl_port_by_dport(grandparent(&cxlmd->dev));
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL");
/**
* cxl_pci_find_dport - find the dport of @port that @pdev is below
* @pdev: PCI device below the dport
* @port: port to search, typically from cxl_pci_find_port()
*
* Context: Caller must hold the cxl_port_dport_host() lock of @port, and
* must not use the result after dropping it.
*/
struct cxl_dport *cxl_pci_find_dport(struct pci_dev *pdev,
struct cxl_port *port)
{
device_lock_assert(cxl_port_dport_host(port));
return cxl_find_dport_by_dev(port, pdev->dev.parent);
}
EXPORT_SYMBOL_NS_GPL(cxl_pci_find_dport, "CXL");
/**
* cxl_mem_find_dport - find the dport of @port that @cxlmd is below
* @cxlmd: memdev below the dport
* @port: port to search, typically from cxl_mem_find_port()
*
* Context: Caller must hold the cxl_port_dport_host() lock of @port, and
* must not use the result after dropping it.
*/
struct cxl_dport *cxl_mem_find_dport(struct cxl_memdev *cxlmd,
struct cxl_port *port)
{
device_lock_assert(cxl_port_dport_host(port));
return cxl_find_dport_by_dev(port, grandparent(&cxlmd->dev));
}
EXPORT_SYMBOL_NS_GPL(cxl_mem_find_dport, "CXL");
static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
struct cxl_port *port)
{
struct cxl_decoder *cxld = &cxlsd->cxld;
int i;
int i, nr_used;
device_lock_assert(&port->dev);
@ -1926,7 +2008,8 @@ static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd,
return 0;
guard(rwsem_write)(&cxl_rwsem.region);
for (i = 0; i < cxlsd->cxld.interleave_ways; i++) {
nr_used = cxlsd_nr_used_targets(cxlsd);
for (i = 0; i < nr_used; i++) {
struct cxl_dport *dport = find_dport(port, cxld->target_map[i]);
if (!dport) {
@ -1987,7 +2070,7 @@ static int cxl_switch_decoder_init(struct cxl_port *port,
struct cxl_switch_decoder *cxlsd,
int nr_targets)
{
if (nr_targets > CXL_DECODER_MAX_INTERLEAVE)
if (nr_targets < 1 || nr_targets > CXL_DECODER_MAX_INTERLEAVE)
return -EINVAL;
cxlsd->nr_targets = nr_targets;
@ -2320,6 +2403,9 @@ EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL");
static void add_latency(struct access_coordinate *c, long latency)
{
/* Convert link latency from picoseconds to nanoseconds. */
latency = DIV_ROUND_UP(latency, 1000);
for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
c[i].write_latency += latency;
c[i].read_latency += latency;
@ -2394,8 +2480,10 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port,
* Skip calculation for RCD. Expectation is HMAT already covers RCD case
* since RCH does not support hotplug.
*/
if (cxlmd->cxlds->rcd)
if (cxlmd->cxlds->rcd) {
memset(coord, 0, sizeof(*coord) * ACCESS_COORDINATE_MAX);
return 0;
}
/*
* Exit the loop when the parent port of the current iter port is cxl

6
drivers/cxl/core/ras.c
View File

@ -70,9 +70,9 @@ cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd,
trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl);
}
static int match_memdev_by_parent(struct device *dev, const void *uport)
static int match_memdev_by_parent(struct device *dev, const void *parent)
{
if (is_cxl_memdev(dev) && dev->parent == uport)
if (is_cxl_memdev(dev) && dev->parent == parent)
return 1;
return 0;
}
@ -108,7 +108,7 @@ static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data)
return;
struct device *mem_dev __free(put_device) = bus_find_device(
&cxl_bus_type, NULL, pdev, match_memdev_by_parent);
&cxl_bus_type, NULL, &pdev->dev, match_memdev_by_parent);
if (!mem_dev)
return;

12
drivers/cxl/core/ras_rch.c
View File

@ -107,10 +107,20 @@ void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds)
int severity;
struct cxl_port *port __free(put_cxl_port) =
cxl_pci_find_port(pdev, &dport);
cxl_pci_find_port(pdev);
if (!port)
return;
/*
* The dport may have been freed since the lookup above, so take the
* lock that pins it and find it again.
*/
guard(device)(cxl_port_dport_host(port));
dport = cxl_pci_find_dport(pdev, port);
if (!dport)
return;
if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs))
return;

100
drivers/cxl/core/region.c
View File

@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright(c) 2022 Intel Corporation. All rights reserved. */
#include <linux/acpi.h>
#include <linux/memregion.h>
#include <linux/genalloc.h>
#include <linux/debugfs.h>
@ -1367,7 +1368,8 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled,
/*
* If this position wants to share a dport with the last endpoint mapped
* then that endpoint, at index 'position - distance', must also be
* mapped by this dport.
* mapped by this dport. An endpoint that this port does not map at all
* fails that requirement.
*/
if (pos < distance) {
dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n",
@ -1378,7 +1380,7 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled,
cxled_peer = p->targets[pos - distance];
cxlmd_peer = cxled_to_memdev(cxled_peer);
ep_peer = cxl_ep_load(port, cxlmd_peer);
if (ep->dport != ep_peer->dport) {
if (!ep_peer || ep->dport != ep_peer->dport) {
dev_dbg(&cxlr->dev,
"%s:%s: %s:%s pos %d mismatched peer %s:%s\n",
dev_name(port->uport_dev), dev_name(&port->dev),
@ -2112,7 +2114,7 @@ static int cxl_region_attach(struct cxl_region *cxlr,
return -ENXIO;
}
if (!cxled->dpa_res) {
if (cxled_empty(cxled)) {
dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n",
dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev));
return -ENXIO;
@ -2215,10 +2217,11 @@ static int cxl_region_attach(struct cxl_region *cxlr,
int test_pos;
test_pos = cxl_calc_interleave_pos(target, &cxlr->hpa_range);
dev_dbg(&target->cxld.dev,
"Test cxl_calc_interleave_pos(): %s test_pos:%d target->pos:%d\n",
(test_pos == target->pos) ? "success" : "fail",
test_pos, target->pos);
if (test_pos != target->pos)
dev_warn(&cxlr->dev,
"%s: position mismatch: calculated:%d assigned:%d\n",
dev_name(&target->cxld.dev), test_pos,
target->pos);
}
return 0;
@ -2630,7 +2633,19 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid)
for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) {
if (cxlr->coord[i].read_bandwidth) {
node_update_perf_attrs(nid, &cxlr->coord[i], i);
/*
* A CFMWS-only node (not backed by a real SRAT pxm)
* has no HMAT memory_target, so hmat_callback()
* never created the access attributes for it:
* create them here instead of only updating them.
* This is called at most once per node (see
* nodemask_region_seen), so the attributes are not
* created twice.
*/
if (!acpi_node_backed_by_real_pxm(nid))
node_set_perf_attrs(nid, &cxlr->coord[i], i);
else
node_update_perf_attrs(nid, &cxlr->coord[i], i);
cset++;
}
}
@ -2727,7 +2742,7 @@ void kill_regions(struct cxl_root_decoder *cxlrd)
* This is the second step of region initialization. Regions exist within an
* address space which is mapped by a @cxlrd.
*
* Return: 0 if the region was added to the @cxlrd, else returns negative error
* Return: the new region if it was added to the @cxlrd, else an ERR_PTR() with the error
* code. The region will be named "regionZ" where Z is the unique region number.
*/
static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd,
@ -2902,6 +2917,16 @@ struct cxl_poison_context {
u64 offset;
};
/*
* A device may answer a Get Poison List request with "physical address
* specified is invalid" (-EFAULT). That answer is tolerated for a RAM
* partition and the poison walk continues.
*/
static inline bool poison_efault_forgiven(int rc, enum cxl_partition_mode mode)
{
return rc == -EFAULT && mode == CXL_PARTMODE_RAM;
}
static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
struct cxl_poison_context *ctx)
{
@ -2930,7 +2955,7 @@ static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd,
if (!length)
break;
rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM)
if (poison_efault_forgiven(rc, cxlds->part[i].mode))
continue;
if (rc)
break;
@ -2947,37 +2972,40 @@ static int poison_by_decoder(struct device *dev, void *arg)
struct cxl_dev_state *cxlds;
struct cxl_memdev *cxlmd;
u64 offset, length;
int rc = 0;
int rc;
if (!is_endpoint_decoder(dev))
return rc;
return 0;
cxled = to_cxl_endpoint_decoder(dev);
if (!cxled->dpa_res)
return rc;
return 0;
cxlmd = cxled_to_memdev(cxled);
cxlds = cxlmd->cxlds;
mode = cxlds->part[cxled->part].mode;
/*
* Handle the degenerate case of a device with only empty decoders. An
* empty decoder can still map a non-zero skip range, so advance the
* walk to commit_end either way.
*/
if (cxled->part >= 0) {
cxlmd = cxled_to_memdev(cxled);
cxlds = cxlmd->cxlds;
mode = cxlds->part[cxled->part].mode;
if (cxled->skip) {
offset = cxled->dpa_res->start - cxled->skip;
length = cxled->skip;
rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
rc = 0;
if (rc)
if (cxled->skip) {
offset = cxled->dpa_res->start - cxled->skip;
length = cxled->skip;
rc = cxl_mem_get_poison(cxlmd, offset, length, NULL);
if (rc && !poison_efault_forgiven(rc, mode))
return rc;
}
offset = cxled->dpa_res->start;
length = cxled->dpa_res->end - offset + 1;
rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
if (rc && !poison_efault_forgiven(rc, mode))
return rc;
}
offset = cxled->dpa_res->start;
length = cxled->dpa_res->end - offset + 1;
rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region);
if (rc == -EFAULT && mode == CXL_PARTMODE_RAM)
rc = 0;
if (rc)
return rc;
/* Iterate until commit_end is reached */
if (cxled->cxld.id == ctx->port->commit_end) {
ctx->offset = cxled->dpa_res->end + 1;
@ -2999,9 +3027,17 @@ int cxl_get_poison_by_endpoint(struct cxl_port *port)
};
rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder);
if (rc == 1)
if (rc == 1) {
/*
* No decoder with a sized DPA reservation was walked
* (every committed decoder is zero-size): scan all
* partitions in full.
*/
if (ctx.part < 0)
ctx.part = 0;
rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev),
&ctx);
}
return rc;
}

6
drivers/cxl/core/region_pmem.c
View File

@ -168,12 +168,14 @@ int devm_cxl_add_pmem_region(struct cxl_region *cxlr)
dev_name(dev));
scoped_guard(device, &cxl_nvb->dev) {
if (cxl_nvb->dev.driver)
if (cxl_nvb->dev.driver) {
rc = devm_add_action_or_reset(&cxl_nvb->dev,
cxlr_pmem_unregister,
cxlr_pmem);
else
} else {
rc = -ENXIO;
cxlr_pmem_unregister(cxlr_pmem);
}
}
if (rc)

14
drivers/cxl/core/regs.c
View File

@ -277,6 +277,20 @@ static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi,
u64 offset = ((u64)reg_hi << 32) |
(reg_lo & PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW);
if (reg_type == CXL_REGLOC_RBI_EMPTY)
return false;
/*
* A BAR the PCI core could not place is reset to zero; decoding it
* would map the block at physical address 0.
*/
if (!pci_resource_len(pdev, bar) ||
(pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)) {
dev_warn(&pdev->dev, "BAR%d: not assigned (type: %d)\n", bar,
reg_type);
return false;
}
if (offset > pci_resource_len(pdev, bar)) {
dev_warn(&pdev->dev,
"BAR%d: %pr: too small (offset: %pa, type: %d)\n", bar,

48
drivers/cxl/cxl.h
View File

@ -67,6 +67,8 @@ extern const struct nvdimm_security_ops *cxl_security_ops;
#define CXL_HDM_DECODER0_CTRL_HOSTONLY BIT(12)
#define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24)
#define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28)
/* Target list capacity, i.e. max interleave ways for a non-endpoint decoder */
#define CXL_HDM_DECODER0_TL_TARGETS 8
#define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i)
#define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i)
@ -192,11 +194,15 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw)
#define CXLDEV_EVENT_STATUS_WARN BIT(1)
#define CXLDEV_EVENT_STATUS_FAIL BIT(2)
#define CXLDEV_EVENT_STATUS_FATAL BIT(3)
#define CXLDEV_EVENT_STATUS_ALL (CXLDEV_EVENT_STATUS_INFO | \
CXLDEV_EVENT_STATUS_WARN | \
CXLDEV_EVENT_STATUS_FAIL | \
CXLDEV_EVENT_STATUS_FATAL)
#define CXLDEV_EVENT_STATUS_DCD BIT(4)
/*
* Logs whose ownership follows native_cxl_error; the DCD log is always
* OS owned.
*/
#define CXLDEV_EVENT_STATUS_STANDARD_LOGS (CXLDEV_EVENT_STATUS_INFO | \
CXLDEV_EVENT_STATUS_WARN | \
CXLDEV_EVENT_STATUS_FAIL | \
CXLDEV_EVENT_STATUS_FATAL)
/* CXL rev 3.0 section 8.2.9.2.4; Table 8-52 */
#define CXLDEV_EVENT_INT_MODE_MASK GENMASK(1, 0)
@ -336,6 +342,16 @@ struct cxl_endpoint_decoder {
int pos;
};
/*
* The common case is decoders with no reservation, but also handle
* decoders with a zero-sized reservation that firmware may install for
* security lockdown purposes.
*/
static inline bool cxled_empty(struct cxl_endpoint_decoder *cxled)
{
return !cxled->dpa_res || !resource_size(cxled->dpa_res);
}
/**
* struct cxl_switch_decoder - Switch specific CXL HDM Decoder
* @cxld: base cxl_decoder object
@ -701,6 +717,18 @@ static inline bool is_cxl_root(struct cxl_port *port)
return port->uport_dev == port->dev.parent;
}
/*
* Return the owner device of the devm allocations of @port's dports. They
* are freed when that device unbinds, so hold device_lock() on the returned
* device across any lookup and use of a 'struct cxl_dport'.
*/
static inline struct device *cxl_port_dport_host(struct cxl_port *port)
{
if (is_cxl_root(port))
return port->uport_dev;
return &port->dev;
}
/* Address translation functions exported to cxl_translate test module only */
int cxl_validate_translation_params(u8 eiw, u16 eig, int pos);
u64 cxl_calculate_hpa_offset(u64 dpa_offset, int pos, u8 eiw, u16 eig);
@ -747,10 +775,12 @@ DEFINE_FREE(put_cxl_dax_region, struct cxl_dax_region *, if (!IS_ERR_OR_NULL(_T)
int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd);
void cxl_bus_rescan(void);
void cxl_bus_drain(void);
struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev,
struct cxl_dport **dport);
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd,
struct cxl_dport **dport);
struct cxl_dport *cxl_pci_find_dport(struct pci_dev *pdev,
struct cxl_port *port);
struct cxl_dport *cxl_mem_find_dport(struct cxl_memdev *cxlmd,
struct cxl_port *port);
struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev);
struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd);
bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd);
struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port,

88
drivers/cxl/cxlmem.h
View File

@ -140,6 +140,7 @@ struct cxl_dpa_info {
struct cxl_dpa_part_info {
struct range range;
enum cxl_partition_mode mode;
u8 handle;
} part[CXL_NR_PARTITIONS_MAX];
int nr_partitions;
};
@ -239,7 +240,9 @@ struct cxl_event_interrupt_policy {
u8 warn_settings;
u8 failure_settings;
u8 fatal_settings;
u8 dcd_settings;
} __packed;
#define CXL_EVENT_INT_POLICY_BASE_SIZE 4 /* info, warn, failure, fatal */
/**
* struct cxl_event_state - Event log driver state
@ -252,6 +255,20 @@ struct cxl_event_state {
struct mutex log_lock;
};
/*
* CXL r4.0 Section 8.2.10.9 - Memory Device Command Sets. See Table 8-308.
*
* The 48h Command Set (Opcodes 4800h - 4803h) defines the device-enabled DCD
* commands.
*/
enum dcd_cmd_enabled_bits {
CXL_DCD_ENABLED_GET_CONFIG,
CXL_DCD_ENABLED_GET_EXTENT_LIST,
CXL_DCD_ENABLED_ADD_RESPONSE,
CXL_DCD_ENABLED_RELEASE,
CXL_DCD_ENABLED_MAX
};
/* Device enabled poison commands */
enum poison_cmd_enabled_bits {
CXL_POISON_ENABLED_LIST,
@ -393,14 +410,13 @@ struct cxl_security_state {
struct kernfs_node *sanitize_node;
};
static inline resource_size_t cxl_pmem_size(struct cxl_dev_state *cxlds)
#define CXL_MAX_DC_PARTITIONS 8
static inline resource_size_t cxl_part_size(struct cxl_dev_state *cxlds,
enum cxl_partition_mode mode)
{
/*
* Static PMEM may be at partition index 0 when there is no static RAM
* capacity.
*/
for (int i = 0; i < cxlds->nr_partitions; i++)
if (cxlds->part[i].mode == CXL_PARTMODE_PMEM)
if (cxlds->part[i].mode == mode)
return resource_size(&cxlds->part[i].res);
return 0;
}
@ -427,6 +443,7 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox)
* @partition_align_bytes: alignment size for partition-able capacity
* @active_volatile_bytes: sum of hard + soft volatile
* @active_persistent_bytes: sum of hard + soft persistent
* @dcd_supported: all DCD commands are supported
* @event: event log driver state
* @poison: poison driver state info
* @security: security driver state info
@ -445,6 +462,7 @@ struct cxl_memdev_state {
u64 partition_align_bytes;
u64 active_volatile_bytes;
u64 active_persistent_bytes;
bool dcd_supported;
struct cxl_event_state event;
struct cxl_poison_state poison;
@ -505,6 +523,10 @@ enum cxl_opcode {
CXL_MBOX_OP_UNLOCK = 0x4503,
CXL_MBOX_OP_FREEZE_SECURITY = 0x4504,
CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE = 0x4505,
CXL_MBOX_OP_GET_DC_CONFIG = 0x4800,
CXL_MBOX_OP_GET_DC_EXTENT_LIST = 0x4801,
CXL_MBOX_OP_ADD_DC_RESPONSE = 0x4802,
CXL_MBOX_OP_RELEASE_DC = 0x4803,
CXL_MBOX_OP_MAX = 0x10000
};
@ -669,6 +691,39 @@ struct cxl_mbox_set_shutdown_state_in {
u8 state;
} __packed;
/* See CXL r4.0 Table 8-345 get dynamic capacity config Input Payload */
struct cxl_mbox_get_dc_config_in {
u8 partition_count;
u8 start_partition_index;
} __packed;
/* See CXL r4.0 Table 8-346 get dynamic capacity config Output Payload */
struct cxl_mbox_get_dc_config_out {
u8 avail_partition_count;
u8 partitions_returned;
u8 rsvd[6];
/* See CXL r4.0 Table 8-347 */
struct cxl_dc_partition {
__le64 base;
__le64 decode_length;
__le64 length;
__le64 block_size;
__le32 dsmad_handle;
u8 flags;
u8 rsvd[3];
} __packed partition[] __counted_by(partitions_returned);
/* Trailing extent/tag count fields unused */
} __packed;
/* Trailing counts; cannot be a member after the flex array above */
struct cxl_mbox_get_dc_config_tail {
__le32 num_extents_supported;
__le32 num_extents_available;
__le32 num_tags_supported;
__le32 num_tags_available;
} __packed;
#define CXL_DCD_BLOCK_LINE_SIZE 0x40
/* Set Timestamp CXL 3.0 Spec 8.2.9.4.2 */
struct cxl_mbox_set_timestamp_in {
__le64 timestamp;
@ -792,9 +847,19 @@ enum {
int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox,
struct cxl_mbox_cmd *cmd);
int cxl_dev_state_identify(struct cxl_memdev_state *mds);
struct cxl_dc_partition_info {
u64 start;
u64 size;
u8 handle;
};
int cxl_dev_dc_identify(struct cxl_mailbox *mbox,
struct cxl_dc_partition_info *dc_info);
int cxl_await_media_ready(struct cxl_dev_state *cxlds);
int cxl_enumerate_cmds(struct cxl_memdev_state *mds);
int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info);
int cxl_configure_dcd(struct cxl_memdev_state *mds, struct cxl_dpa_info *info);
struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial,
u16 dvsec);
void set_exclusive_cxl_commands(struct cxl_memdev_state *mds,
@ -808,6 +873,17 @@ void cxl_event_trace_record(struct cxl_memdev *cxlmd,
const uuid_t *uuid, union cxl_event *evt);
int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count);
int cxl_arm_dirty_shutdown(struct cxl_memdev_state *mds);
static inline bool cxl_dcd_supported(struct cxl_memdev_state *mds)
{
return mds->dcd_supported;
}
static inline void cxl_disable_dcd(struct cxl_memdev_state *mds)
{
mds->dcd_supported = false;
}
int cxl_set_timestamp(struct cxl_memdev_state *mds);
int cxl_poison_state_init(struct cxl_memdev_state *mds);
int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len,

30
drivers/cxl/mem.c
View File

@ -50,8 +50,13 @@ static int cxl_debugfs_poison_inject(void *data, u64 dpa)
struct cxl_memdev *cxlmd = data;
int rc;
ACQUIRE(device_intr, devlock)(&cxlmd->dev);
if ((rc = ACQUIRE_ERR(device_intr, &devlock)))
/*
* Never wait for this lock: the debugfs proxy holds a file reference
* across the callback and unbind removes the file under the same
* device lock, so waiting here deadlocks against unbind.
*/
ACQUIRE(device_try, devlock)(&cxlmd->dev);
if ((rc = ACQUIRE_ERR(device_try, &devlock)))
return rc;
return cxl_inject_poison(cxlmd, dpa);
@ -65,8 +70,9 @@ static int cxl_debugfs_poison_clear(void *data, u64 dpa)
struct cxl_memdev *cxlmd = data;
int rc;
ACQUIRE(device_intr, devlock)(&cxlmd->dev);
if ((rc = ACQUIRE_ERR(device_intr, &devlock)))
/* Never wait, per the inject path above. */
ACQUIRE(device_try, devlock)(&cxlmd->dev);
if ((rc = ACQUIRE_ERR(device_try, &devlock)))
return rc;
return cxl_clear_poison(cxlmd, dpa);
@ -133,13 +139,13 @@ static int cxl_mem_probe(struct device *dev)
return rc;
struct cxl_port *parent_port __free(put_cxl_port) =
cxl_mem_find_port(cxlmd, &dport);
cxl_mem_find_port(cxlmd);
if (!parent_port) {
dev_err(dev, "CXL port topology not found\n");
return -ENXIO;
}
if (cxl_pmem_size(cxlds) && IS_ENABLED(CONFIG_CXL_PMEM)) {
if (cxl_part_size(cxlds, CXL_PARTMODE_PMEM) && IS_ENABLED(CONFIG_CXL_PMEM)) {
rc = devm_cxl_add_nvdimm(dev, parent_port, cxlmd);
if (rc) {
if (rc == -ENODEV)
@ -148,10 +154,7 @@ static int cxl_mem_probe(struct device *dev)
}
}
if (dport->rch)
endpoint_parent = parent_port->uport_dev;
else
endpoint_parent = &parent_port->dev;
endpoint_parent = cxl_port_dport_host(parent_port);
scoped_guard(device, endpoint_parent) {
if (!endpoint_parent->driver) {
@ -160,6 +163,13 @@ static int cxl_mem_probe(struct device *dev)
return -ENXIO;
}
/* @dport is only valid while @endpoint_parent stays bound */
dport = cxl_mem_find_dport(cxlmd, parent_port);
if (!dport) {
dev_err(dev, "CXL topology dport not found\n");
return -ENXIO;
}
rc = devm_cxl_add_endpoint(endpoint_parent, cxlmd, dport);
if (rc)
return rc;

243
drivers/cxl/pci.c
View File

@ -510,11 +510,23 @@ static bool cxl_alloc_irq_vectors(struct pci_dev *pdev)
return true;
}
/* Event logs the driver drains: standard logs when native_cxl */
static u32 cxl_event_drain_mask(struct pci_host_bridge *host_bridge)
{
if (host_bridge->native_cxl_error)
return CXLDEV_EVENT_STATUS_STANDARD_LOGS;
return 0;
}
static irqreturn_t cxl_event_thread(int irq, void *id)
{
struct cxl_dev_id *dev_id = id;
struct cxl_dev_state *cxlds = dev_id->cxlds;
struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds);
struct pci_host_bridge *host_bridge =
pci_find_host_bridge(to_pci_dev(cxlds->dev)->bus);
u32 mask = cxl_event_drain_mask(host_bridge);
bool handled = false;
u32 status;
do {
@ -523,15 +535,17 @@ static irqreturn_t cxl_event_thread(int irq, void *id)
* ignore the reserved upper 32 bits
*/
status = readl(cxlds->regs.status + CXLDEV_DEV_EVENT_STATUS_OFFSET);
/* Ignore logs unknown to the driver */
status &= CXLDEV_EVENT_STATUS_ALL;
/* Ignore logs unknown to the driver or owned by BIOS */
status &= mask;
if (!status)
break;
handled = true;
cxl_mem_get_event_records(mds, status);
cond_resched();
} while (status);
return IRQ_HANDLED;
return handled ? IRQ_HANDLED : IRQ_NONE;
}
static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting)
@ -551,84 +565,108 @@ static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting)
}
static int cxl_event_get_int_policy(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy)
struct cxl_event_interrupt_policy *policy,
size_t *policy_size)
{
struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd = {
.opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY,
.payload_out = policy,
.size_out = sizeof(*policy),
/* CXL 2.0 firmware omits dcd_settings; accept the shorter reply */
.min_out = CXL_EVENT_INT_POLICY_BASE_SIZE,
};
int rc;
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0) {
dev_err(mds->cxlds.dev,
"Failed to get event interrupt policy : %d", rc);
return rc;
}
if (policy_size)
*policy_size = mbox_cmd.size_out;
return rc;
}
static int cxl_event_set_int_policy(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy,
size_t policy_size)
{
struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd = {
.opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY,
.payload_in = policy,
.size_in = policy_size,
};
int rc;
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0)
dev_err(mds->cxlds.dev,
"Failed to get event interrupt policy : %d", rc);
dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d",
rc);
return rc;
}
static int cxl_event_config_msgnums(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy)
struct cxl_event_interrupt_policy *policy,
bool native_cxl, size_t policy_size)
{
struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox;
struct cxl_mbox_cmd mbox_cmd;
int rc;
*policy = (struct cxl_event_interrupt_policy) {
.info_settings = CXL_INT_MSI_MSIX,
.warn_settings = CXL_INT_MSI_MSIX,
.failure_settings = CXL_INT_MSI_MSIX,
.fatal_settings = CXL_INT_MSI_MSIX,
};
mbox_cmd = (struct cxl_mbox_cmd) {
.opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY,
.payload_in = policy,
.size_in = sizeof(*policy),
};
rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd);
if (rc < 0) {
dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d",
rc);
return rc;
/* memory event policy is left if FW has control */
if (native_cxl) {
*policy = (struct cxl_event_interrupt_policy) {
.info_settings = CXL_INT_MSI_MSIX,
.warn_settings = CXL_INT_MSI_MSIX,
.failure_settings = CXL_INT_MSI_MSIX,
.fatal_settings = CXL_INT_MSI_MSIX,
.dcd_settings = 0,
};
}
/* Retrieve final interrupt settings */
return cxl_event_get_int_policy(mds, policy);
}
/*
* A CXL 3.0+ device can carry dcd_settings field without DCD command
* support, so size the request by the device's policy_size and only
* enable the DCD interrupt when DCD commands are supported.
*/
if (cxl_dcd_supported(mds))
policy->dcd_settings = CXL_INT_MSI_MSIX;
static int cxl_event_irqsetup(struct cxl_memdev_state *mds)
{
struct cxl_dev_state *cxlds = &mds->cxlds;
struct cxl_event_interrupt_policy policy;
int rc;
rc = cxl_event_config_msgnums(mds, &policy);
rc = cxl_event_set_int_policy(mds, policy, policy_size);
if (rc)
return rc;
rc = cxl_event_req_irq(cxlds, policy.info_settings);
/* Retrieve final interrupt settings */
return cxl_event_get_int_policy(mds, policy, NULL);
}
static int cxl_event_irqsetup(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy)
{
struct cxl_dev_state *cxlds = &mds->cxlds;
int rc;
rc = cxl_event_req_irq(cxlds, policy->info_settings);
if (rc) {
dev_err(cxlds->dev, "Failed to get interrupt for event Info log\n");
return rc;
}
rc = cxl_event_req_irq(cxlds, policy.warn_settings);
rc = cxl_event_req_irq(cxlds, policy->warn_settings);
if (rc) {
dev_err(cxlds->dev, "Failed to get interrupt for event Warn log\n");
return rc;
}
rc = cxl_event_req_irq(cxlds, policy.failure_settings);
rc = cxl_event_req_irq(cxlds, policy->failure_settings);
if (rc) {
dev_err(cxlds->dev, "Failed to get interrupt for event Failure log\n");
return rc;
}
rc = cxl_event_req_irq(cxlds, policy.fatal_settings);
rc = cxl_event_req_irq(cxlds, policy->fatal_settings);
if (rc) {
dev_err(cxlds->dev, "Failed to get interrupt for event Fatal log\n");
return rc;
@ -637,6 +675,37 @@ static int cxl_event_irqsetup(struct cxl_memdev_state *mds)
return 0;
}
static int cxl_irqsetup(struct cxl_memdev_state *mds,
struct cxl_event_interrupt_policy *policy,
bool native_cxl, size_t policy_size)
{
struct cxl_dev_state *cxlds = &mds->cxlds;
int rc;
if (native_cxl) {
rc = cxl_event_irqsetup(mds, policy);
if (rc)
return rc;
}
if (cxl_dcd_supported(mds)) {
rc = cxl_event_req_irq(cxlds, policy->dcd_settings);
if (rc) {
dev_err(cxlds->dev, "Failed to get interrupt for DCD event log\n");
cxl_disable_dcd(mds);
/* No handler will service the DCD vector; tell the device */
policy->dcd_settings = CXL_INT_NONE;
rc = cxl_event_set_int_policy(mds, policy, policy_size);
if (rc)
dev_warn(cxlds->dev,
"DCD disabled but its interrupt policy could not be cleared: %d\n",
rc);
}
}
return 0;
}
static bool cxl_event_int_is_fw(u8 setting)
{
u8 mode = FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting);
@ -644,48 +713,101 @@ static bool cxl_event_int_is_fw(u8 setting)
return mode == CXL_INT_FW;
}
static bool cxl_event_fw_owns_mem_logs(struct cxl_event_interrupt_policy *policy)
{
return cxl_event_int_is_fw(policy->info_settings) ||
cxl_event_int_is_fw(policy->warn_settings) ||
cxl_event_int_is_fw(policy->failure_settings) ||
cxl_event_int_is_fw(policy->fatal_settings);
}
static int cxl_event_config(struct pci_host_bridge *host_bridge,
struct cxl_memdev_state *mds, bool irq_avail)
{
struct cxl_event_interrupt_policy policy;
struct cxl_event_interrupt_policy policy = { 0 };
bool native_cxl = host_bridge->native_cxl_error;
size_t policy_size;
u32 mask;
int rc;
/*
* When BIOS maintains CXL error reporting control, it will process
* event records. Only one agent can do so.
*
* If BIOS has control of events and DCD is not supported skip event
* configuration.
*/
if (!host_bridge->native_cxl_error)
if (!native_cxl && !cxl_dcd_supported(mds))
return 0;
if (!irq_avail) {
dev_info(mds->cxlds.dev, "No interrupt support, disable event processing.\n");
if (cxl_dcd_supported(mds)) {
dev_info(mds->cxlds.dev, "DCD requires interrupts, disable DCD\n");
cxl_disable_dcd(mds);
}
return 0;
}
rc = cxl_event_get_int_policy(mds, &policy);
rc = cxl_event_get_int_policy(mds, &policy, &policy_size);
if (rc)
return rc;
goto err;
if (cxl_event_int_is_fw(policy.info_settings) ||
cxl_event_int_is_fw(policy.warn_settings) ||
cxl_event_int_is_fw(policy.failure_settings) ||
cxl_event_int_is_fw(policy.fatal_settings)) {
/*
* dcd_settings cannot be sent to a device whose policy is too short
* to hold it, so the device would never accept the setting
* cxl_irqsetup() arms the DCD interrupt with.
*/
if (cxl_dcd_supported(mds) && policy_size < sizeof(policy)) {
dev_warn(mds->cxlds.dev,
"DCD supported but interrupt policy is only %zu bytes\n",
policy_size);
cxl_disable_dcd(mds);
/* DCD was the only reason to be here */
if (!native_cxl)
return 0;
}
if (native_cxl && cxl_event_fw_owns_mem_logs(&policy)) {
dev_err(mds->cxlds.dev,
"FW still in control of Event Logs despite _OSC settings\n");
return -EBUSY;
}
rc = cxl_event_config_msgnums(mds, &policy, native_cxl, policy_size);
if (rc)
goto err;
rc = cxl_mem_alloc_event_buf(mds);
if (rc)
return rc;
rc = cxl_event_irqsetup(mds);
rc = cxl_irqsetup(mds, &policy, native_cxl, policy_size);
if (rc)
return rc;
cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
mask = cxl_event_drain_mask(host_bridge);
if (mask)
cxl_mem_get_event_records(mds, mask);
dev_dbg(mds->cxlds.dev, "Event config : %s DCD %s\n",
native_cxl ? "OS" : "BIOS",
cxl_dcd_supported(mds) ? "supported" : "not supported");
return 0;
err:
/*
* Failed to Get/Set interrupt policy for DCD while BIOS owns the
* other logs. Disable DCD and continue.
*/
if (!native_cxl) {
dev_warn(mds->cxlds.dev,
"Event interrupt policy failed (%d), disabling DCD\n", rc);
cxl_disable_dcd(mds);
return 0;
}
return rc;
}
static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds)
@ -706,22 +828,23 @@ static ssize_t rcd_pcie_cap_emit(struct device *dev, u16 offset, char *buf, size
struct device *root_dev;
struct cxl_dport *dport;
struct cxl_port *root __free(put_cxl_port) =
cxl_mem_find_port(cxlmd, &dport);
cxl_mem_find_port(cxlmd);
if (!root)
return -ENXIO;
root_dev = root->uport_dev;
root_dev = cxl_port_dport_host(root);
if (!root_dev)
return -ENXIO;
if (!dport->regs.rcd_pcie_cap)
return -ENXIO;
guard(device)(root_dev);
if (!root_dev->driver)
return -ENXIO;
dport = cxl_mem_find_dport(cxlmd, root);
if (!dport || !dport->regs.rcd_pcie_cap)
return -ENXIO;
switch (width) {
case 2:
return sysfs_emit(buf, "%#x\n",
@ -868,6 +991,12 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
if (rc)
return rc;
if (cxl_dcd_supported(mds)) {
rc = cxl_configure_dcd(mds, &range_info);
if (rc)
cxl_disable_dcd(mds);
}
rc = cxl_dpa_setup(cxlds, &range_info);
if (rc)
return rc;

3
drivers/cxl/port.c
View File

@ -46,6 +46,9 @@ static int discover_region(struct device *dev, void *unused)
if (cxled->state != CXL_DECODER_STATE_AUTO)
return 0;
if (cxled_empty(cxled))
return 0;
/*
* Region enumeration is opportunistic, if this add-event fails,
* continue to the next endpoint decoder.

23
drivers/pci/pci.c
View File

@ -6030,18 +6030,31 @@ static enum pci_bus_speed to_pcie_link_speed(u16 lnksta)
return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)];
}
int pcie_link_speed_mbps(struct pci_dev *pdev)
/**
* pcie_link_bandwidth_mbps - get the current bandwidth of a single PCIe link
* @pdev: PCI device to query
*
* The bandwidth includes all negotiated lanes and does not account for
* encoding overhead. A negotiated width of zero gives zero bandwidth.
*
* Return: Bandwidth in Mb/s, or a negative errno on failure.
*/
int pcie_link_bandwidth_mbps(struct pci_dev *pdev)
{
u16 lnksta;
int err;
int speed, err;
err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta);
if (err)
return err;
return pcibios_err_to_errno(err);
return pcie_dev_speed_mbps(to_pcie_link_speed(lnksta));
speed = pcie_dev_speed_mbps(to_pcie_link_speed(lnksta));
if (speed < 0)
return speed;
return speed * FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
}
EXPORT_SYMBOL(pcie_link_speed_mbps);
EXPORT_SYMBOL(pcie_link_bandwidth_mbps);
/**
* pcie_bandwidth_available - determine minimum link settings of a PCIe

29
include/cxl/cxl.h
View File

@ -130,9 +130,30 @@ struct cxl_dpa_perf {
int qos_class;
};
/*
* Partitions share one flat DPA space in a fixed order:
*
* DPA 0 end
* +---------------+---------------+---------------------------+
* | ram | pmem | dynamic_ram_1 |
* +---------------+---------------+---------------------------+
* part[0] part[1] part[2]
*
* The static ram and pmem partition order is mandated by CXL r4.0
* 8.2.10.9.2.1 "Get Partition Info" (opcode 4100h), Table 8-310: volatile
* capacity starts at DPA 0 and persistent capacity starts at the DPA
* immediately following it. A dynamic partition has no such
* requirement -- 8.2.10.9.9.1 "Get Dynamic Capacity Configuration"
* (opcode 4800h), Table 8-347, only requires the DC Region Base to be
* 256MB aligned -- but Linux follows that precedent and requires it to
* start at the DPA immediately following static capacity: after pmem,
* after ram on a device with no pmem, or at DPA 0 on a device with no
* static capacity at all. cxl_configure_dcd() enforces it.
*/
enum cxl_partition_mode {
CXL_PARTMODE_RAM,
CXL_PARTMODE_PMEM,
CXL_PARTMODE_DYNAMIC_RAM,
};
/**
@ -140,14 +161,20 @@ enum cxl_partition_mode {
* @res: shortcut to the partition in the DPA resource tree (cxlds->dpa_res)
* @perf: performance attributes of the partition from CDAT
* @mode: operation mode for the DPA capacity, e.g. ram, pmem, dynamic...
* @handle: DSMAS handle that represents this partition
* @shareable: Is the partition shareable (from its CDAT DSMAS entry)
* @read_only: Is the partition read-only (from its CDAT DSMAS entry)
*/
struct cxl_dpa_partition {
struct resource res;
struct cxl_dpa_perf perf;
enum cxl_partition_mode mode;
u8 handle;
bool shareable;
bool read_only;
};
#define CXL_NR_PARTITIONS_MAX 2
#define CXL_NR_PARTITIONS_MAX 3
/**
* struct cxl_dev_state - The driver device state

1
include/linux/device.h
View File

@ -1121,6 +1121,7 @@ static inline void device_unlock(struct device *dev)
DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T))
DEFINE_GUARD_COND(device, _intr, device_lock_interruptible(_T), _RET == 0)
DEFINE_GUARD_COND(device, _try, device_trylock(_T))
static inline void device_lock_assert(struct device *dev)
{

2
include/linux/pci.h
View File

@ -1478,7 +1478,7 @@ int pcie_set_mps(struct pci_dev *dev, int mps);
u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
enum pci_bus_speed *speed,
enum pcie_link_width *width);
int pcie_link_speed_mbps(struct pci_dev *pdev);
int pcie_link_bandwidth_mbps(struct pci_dev *pdev);
void pcie_print_link_status(struct pci_dev *dev);
int pcie_reset_flr(struct pci_dev *dev, bool probe);
int pcie_flr(struct pci_dev *dev);

16
tools/testing/cxl/Kbuild
View File

@ -4,16 +4,20 @@ ldflags-y += --wrap=is_acpi_device_node
ldflags-y += --wrap=acpi_evaluate_integer
ldflags-y += --wrap=acpi_pci_find_root
ldflags-y += --wrap=nvdimm_bus_register
ldflags-y += --wrap=cxl_await_media_ready
ldflags-y += --wrap=devm_cxl_add_rch_dport
ldflags-y += --wrap=cxl_endpoint_parse_cdat
ldflags-y += --wrap=devm_cxl_endpoint_decoders_setup
ldflags-y += --wrap=hmat_get_extended_linear_cache_size
ldflags-y += --wrap=devm_cxl_add_dport_by_dev
ldflags-y += --wrap=devm_cxl_switch_port_decoders_setup
ldflags-y += --wrap=walk_hmem_resources
ldflags-y += --wrap=region_intersects
ldflags-y += --wrap=region_intersects_soft_reserve
wrap_cxl_core := --wrap=cxl_await_media_ready
wrap_cxl_core += --wrap=devm_cxl_add_rch_dport
wrap_cxl_core += --wrap=devm_cxl_add_dport_by_dev
wrap_cxl_core += --wrap=devm_cxl_switch_port_decoders_setup
wrap_cxl_core += --wrap=devm_cxl_endpoint_decoders_setup
wrap_cxl_core += --wrap=cxl_endpoint_parse_cdat
LDFLAGS_cxl_acpi.o += $(wrap_cxl_core)
LDFLAGS_cxl_pmem.o += $(wrap_cxl_core)
LDFLAGS_cxl_port.o += $(wrap_cxl_core)
LDFLAGS_cxl_mem.o += $(wrap_cxl_core)
DRIVERS := ../../../drivers
DAX_HMEM_SRC := $(DRIVERS)/dax/hmem

117
tools/testing/cxl/test/cxl.c
View File

@ -18,6 +18,7 @@ static int interleave_arithmetic;
static bool extended_linear_cache;
static bool fail_autoassemble;
static bool type2_test;
static bool mock_zero_size_decoders;
#define FAKE_QTG_ID 42
@ -872,14 +873,13 @@ static int cxld_registry_restore(struct cxl_decoder *cxld,
cxld_copy(cxld, &td->cxled.cxld);
cxled->state = td->cxled.state;
cxled->skip = td->cxled.skip;
if (range_len(&td->dpa_range)) {
rc = devm_cxl_dpa_reserve(cxled, td->dpa_range.start,
range_len(&td->dpa_range),
td->cxled.skip);
if (rc) {
init_disabled_mock_decoder(cxld);
return rc;
}
/* enabled endpoint decoders hold a reservation, sized or not */
rc = devm_cxl_dpa_reserve(cxled, td->dpa_range.start,
range_len(&td->dpa_range),
td->cxled.skip);
if (rc) {
init_disabled_mock_decoder(cxld);
return rc;
}
port->commit_end = cxld->id;
}
@ -1072,16 +1072,49 @@ static void default_mock_decoder(struct cxl_decoder *cxld)
WARN_ON_ONCE(!cxld_registry_new(cxld));
}
static int first_decoder(struct device *dev, const void *data)
static int match_decoder_by_index(struct device *dev, const void *data)
{
int target_id = *(const int *)data;
struct cxl_decoder *cxld;
if (!is_switch_decoder(dev))
return 0;
cxld = to_cxl_decoder(dev);
if (cxld->id == 0)
return 1;
return 0;
return cxld->id == target_id;
}
/*
* Mock a committed, locked, empty decoder
* (CXL r4.0 8.2.4.20.12). Gated by the mock_zero_size_decoders module
* param so the default cxl_test topology, shared by the region test
* suite, is left undisturbed.
*/
static void size_zero_mock_decoder_ep(struct cxl_decoder *cxld, u64 base)
{
struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(&cxld->dev);
cxld->hpa_range = DEFINE_RANGE(base, base - 1);
cxld->interleave_ways = 2;
cxld->interleave_granularity = 4096;
cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->flags = CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
cxled->state = CXL_DECODER_STATE_AUTO;
/* decoder[0] reserved [0, size/2), empty decoders sit at that watermark */
devm_cxl_dpa_reserve(cxled, mock_auto_region_size / 2, 0, 0);
cxld->commit = mock_decoder_commit;
cxld->reset = mock_decoder_reset;
}
static void size_zero_mock_decoder_sw(struct cxl_decoder *cxld, u64 base,
int level)
{
cxld->flags = CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK;
cxld->target_type = CXL_DECODER_HOSTONLYMEM;
cxld->interleave_ways = level == 0 ? 2 : 1;
cxld->interleave_granularity = 4096;
cxld->hpa_range = DEFINE_RANGE(base, base - 1);
cxld->commit = mock_decoder_commit;
cxld->reset = mock_decoder_reset;
}
enum cxld_init_type {
@ -1114,7 +1147,7 @@ static enum cxld_init_type get_decoder_init_type(struct cxl_decoder *cxld,
* See 'cxl list -BMPu -m cxl_mem.0,cxl_mem.4'
*/
if (!is_endpoint_decoder(&cxld->dev) || !hb0 || pdev->id % 4 ||
pdev->id > 4 || cxld->id > 0)
pdev->id > 4 || cxld->id > (mock_zero_size_decoders ? 2 : 0))
return MOCK_DECODER_INIT_DEFAULT;
return type2_test ? MOCK_DECODER_INIT_TYPE2_AUTO :
@ -1147,6 +1180,7 @@ static void mock_init_hdm_type2_cxled(struct cxl_endpoint_decoder *cxled,
struct cxl_port *root_port;
struct device *dev;
u64 base;
int id = 0;
base = window->base_hpa;
cxld->hpa_range = (struct range) {
@ -1172,7 +1206,7 @@ static void mock_init_hdm_type2_cxled(struct cxl_endpoint_decoder *cxled,
*/
dport = port->parent_dport;
root_port = dport->port;
dev = device_find_child(&root_port->dev, NULL, first_decoder);
dev = device_find_child(&root_port->dev, &id, match_decoder_by_index);
/*
* Ancestor ports are guaranteed to be enumerated before
* @port, and all ports have at least one decoder.
@ -1232,6 +1266,20 @@ static void mock_init_hdm_type3_cxled(struct cxl_endpoint_decoder *cxled,
base = window->base_hpa;
if (extended_linear_cache)
base += mock_auto_region_size;
/*
* With mock_zero_size_decoders, decoders 1 and 2 of the special
* endpoints mock committed, locked, empty decoders above the
* decoder[0] auto-region (CXL r4.0 8.2.4.20.12). commit_end then
* points at a zero-size decoder, exercising the zero-size
* reservation and poison-by-endpoint code paths.
*/
if (cxld->id == 1 || cxld->id == 2) {
size_zero_mock_decoder_ep(cxld, base);
port->commit_end = cxld->id;
WARN_ON_ONCE(!cxld_registry_new(cxld));
return;
}
cxld->hpa_range = (struct range) {
.start = base,
.end = base + mock_auto_region_size - 1,
@ -1255,9 +1303,11 @@ static void mock_init_hdm_type3_cxled(struct cxl_endpoint_decoder *cxled,
*/
iter = port;
for (i = 0; i < 2; i++) {
int id = 0;
dport = iter->parent_dport;
iter = dport->port;
dev = device_find_child(&iter->dev, NULL, first_decoder);
dev = device_find_child(&iter->dev, &id, match_decoder_by_index);
/*
* Ancestor ports are guaranteed to be enumerated before
* @port, and all ports have at least one decoder.
@ -1307,6 +1357,26 @@ static void mock_init_hdm_type3_cxled(struct cxl_endpoint_decoder *cxled,
cxld_registry_update(cxld);
put_device(dev);
if (!mock_zero_size_decoders)
continue;
/*
* Mirror the endpoint: commit the next two switch decoders
* as zero-size + locked so the empty-decoder layout extends
* end-to-end through the switch and host bridge.
*/
for (id = 1; id <= 2; id++) {
dev = device_find_child(&iter->dev, &id,
match_decoder_by_index);
if (WARN_ON(!dev))
continue;
cxld = to_cxl_decoder(dev);
size_zero_mock_decoder_sw(cxld, base, i);
iter->commit_end = id;
cxld_registry_update(cxld);
put_device(dev);
}
}
}
@ -1945,7 +2015,7 @@ static int cxl_type2_mem_init(void)
goto err_mem;
}
pdev->dev.parent = &dport->dev;
set_dev_node(&pdev->dev, i % 2);
set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2));
rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]);
if (rc)
@ -1974,7 +2044,7 @@ static int cxl_type3_mem_init(void)
goto err_mem;
}
pdev->dev.parent = &dport->dev;
set_dev_node(&pdev->dev, i % 2);
set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2));
rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]);
if (rc)
@ -1991,7 +2061,7 @@ static int cxl_type3_mem_init(void)
goto err_single;
}
pdev->dev.parent = &dport->dev;
set_dev_node(&pdev->dev, i % 2);
set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2));
rc = cxl_mock_platform_device_add(pdev, &cxl_mem_single[i]);
if (rc)
@ -2009,7 +2079,7 @@ static int cxl_type3_mem_init(void)
goto err_rcd;
}
pdev->dev.parent = &rch->dev;
set_dev_node(&pdev->dev, i % 2);
set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2));
rc = cxl_mock_platform_device_add(pdev, &cxl_rcd[i]);
if (rc)
@ -2076,6 +2146,12 @@ static bool __init have_multiple_modparms(void)
count++;
if (type2_test)
count++;
if (mock_zero_size_decoders)
count++;
/* Zero-size decoders require decoder[0] to preserve commit order. */
if (fail_autoassemble && mock_zero_size_decoders)
return true;
return count > 1;
}
@ -2484,6 +2560,9 @@ module_param(fail_autoassemble, bool, 0444);
MODULE_PARM_DESC(fail_autoassemble, "Simulate missing member of an auto-region");
module_param(type2_test, bool, 0444);
MODULE_PARM_DESC(type2_test, "Enable type 2 support testing");
module_param(mock_zero_size_decoders, bool, 0444);
MODULE_PARM_DESC(mock_zero_size_decoders,
"Mock committed, locked, empty decoders under host-bridge0");
module_init(cxl_test_init);
module_exit(cxl_test_exit);
MODULE_LICENSE("GPL v2");

2
tools/testing/cxl/test/mem.c
View File

@ -1818,7 +1818,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev)
if (rc)
dev_dbg(dev, "No CXL FWCTL setup\n");
cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL);
cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_STANDARD_LOGS);
cxl_mock_test_feat_init(mdata);
return 0;