diff --git a/Documentation/driver-api/cxl/linux/cxl-driver.rst b/Documentation/driver-api/cxl/linux/cxl-driver.rst index dd6dd17dc536..212df4fb8d63 100644 --- a/Documentation/driver-api/cxl/linux/cxl-driver.rst +++ b/Documentation/driver-api/cxl/linux/cxl-driver.rst @@ -181,6 +181,41 @@ A Memory Device is a discrete base object that is not a port. While the physical device it belongs to may also host an `endpoint`, the relationship between an `endpoint` and a `memdev` is not captured in sysfs. +DPA Partitions +~~~~~~~~~~~~~~ +A memory device presents its capacity as one flat `Device Physical Address` +(DPA) space divided into `partitions`, which Linux lays out in a fixed +order:: + + DPA 0 end + +---------------+---------------+---------------------------+ + | ram | pmem | dynamic_ram_1 | + +---------------+---------------+---------------------------+ + part[0] part[1] part[2] + +Part of that order is required by the CXL specification and part of it is a +Linux choice. + +The `ram` and `pmem` order is mandated. CXL r4.0 section 8.2.10.9.2.1 "Get +Partition Info" (4100h), Table 8-310, mandates that volatile capacity starts +at DPA 0 and pmem starts at the DPA immediately following it. + +Dynamic Capacity partitions only need to be 256MB aligned according to +CXL r4.0 section 8.2.10.9.9.1 "Get Dynamic Capacity Configuration" +(opcode 4800h), Table 8-347. So a device could leave a gap between ram/pmem +(static) capacity and its first DC partition, or between one DC partition +and the next. + +Linux chooses not to support gaps between static and dynamic capacity: the +first DC partition must begin at the DPA immediately following static +capacity -- after pmem, after ram on a device with no pmem, or at DPA 0 on +a device with no static capacity at all. + +Currently, only one dynamic partition is supported. A device may report up +to eight (CXL r4.0 Table 8-346); Linux configures the first as its +`dynamic_ram_1` partition. Gaps between DC partitions are not checked, since +only the first is used. + Port Relationships ~~~~~~~~~~~~~~~~~~ In our example described above, there are four host bridges attached to the diff --git a/Documentation/driver-api/cxl/maturity-map.rst b/Documentation/driver-api/cxl/maturity-map.rst index 282c1102dd81..fefd31899229 100644 --- a/Documentation/driver-api/cxl/maturity-map.rst +++ b/Documentation/driver-api/cxl/maturity-map.rst @@ -150,6 +150,10 @@ Memory-pooling * [1] Hotplug of LDs (via PCI hotplug) * [0] Dynamic Capacity Device (DCD) Support + * [1] DC partition enumeration / configuration (Get DC Config, CDAT DSMAS, event interrupts) + * [0] Extent add / release event handling + * [0] DC-backed DAX regions + Multi-host sharing ------------------ diff --git a/Documentation/driver-api/cxl/platform/acpi/dsdt.rst b/Documentation/driver-api/cxl/platform/acpi/dsdt.rst index b4583b01d67d..73b23d82aff0 100644 --- a/Documentation/driver-api/cxl/platform/acpi/dsdt.rst +++ b/Documentation/driver-api/cxl/platform/acpi/dsdt.rst @@ -25,4 +25,5 @@ Example Compute Express Link Host Bridge :: ... Name (_UID, 0x05) // _UID: Unique ID ... - } + } + } diff --git a/MAINTAINERS b/MAINTAINERS index 1e3f9f2751fe..0c75ccdbaee1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6666,6 +6666,7 @@ M: Vishal Verma M: Dan Williams R: Ira Weiny R: Li Ming +R: Richard Cheng L: linux-cxl@vger.kernel.org S: Maintained F: Documentation/driver-api/cxl diff --git a/drivers/cxl/acpi.c b/drivers/cxl/acpi.c index 3b818adbd38b..b7ce6b59c680 100644 --- a/drivers/cxl/acpi.c +++ b/drivers/cxl/acpi.c @@ -231,9 +231,10 @@ cxl_acpi_evaluate_qtg_dsm(acpi_handle handle, struct access_coordinate *coord, int entries, int *qos_class) { union acpi_object *out_obj, *out_buf, *obj; + /* QTG _DSM expects latencies in picoseconds. */ union acpi_object in_array[4] = { - [0].integer = { ACPI_TYPE_INTEGER, coord->read_latency }, - [1].integer = { ACPI_TYPE_INTEGER, coord->write_latency }, + [0].integer = { ACPI_TYPE_INTEGER, coord->read_latency * 1000ULL }, + [1].integer = { ACPI_TYPE_INTEGER, coord->write_latency * 1000ULL }, [2].integer = { ACPI_TYPE_INTEGER, coord->read_bandwidth }, [3].integer = { ACPI_TYPE_INTEGER, coord->write_bandwidth }, }; @@ -251,6 +252,11 @@ cxl_acpi_evaluate_qtg_dsm(acpi_handle handle, struct access_coordinate *coord, if (!entries) return -EINVAL; + /* Latency inputs must fit in the DWORD fields defined by the spec. */ + if (coord->read_latency > U32_MAX / 1000 || + coord->write_latency > U32_MAX / 1000) + return -ERANGE; + out_obj = acpi_evaluate_dsm(handle, &acpi_cxl_qtg_id_guid, 1, 1, &in_obj); if (!out_obj) return -ENXIO; @@ -885,13 +891,17 @@ static int pair_cxl_resource(struct device *dev, void *data) static int cxl_acpi_probe(struct platform_device *pdev) { - int rc; + struct cxl_cfmws_context ctx; + struct acpi_device *adev; struct resource *cxl_res; struct cxl_root *cxl_root; struct cxl_port *root_port; struct device *host = &pdev->dev; - struct acpi_device *adev = ACPI_COMPANION(host); - struct cxl_cfmws_context ctx; + int rc; + + adev = ACPI_COMPANION(host); + if (!adev) + return -ENODEV; device_lock_set_class(&pdev->dev, &cxl_root_key); rc = devm_add_action_or_reset(&pdev->dev, cxl_acpi_lock_reset_class, diff --git a/drivers/cxl/core/cdat.c b/drivers/cxl/core/cdat.c index 5c9f07262513..f5f86b69c86d 100644 --- a/drivers/cxl/core/cdat.c +++ b/drivers/cxl/core/cdat.c @@ -17,6 +17,8 @@ struct dsmas_entry { struct access_coordinate cdat_coord[ACCESS_COORDINATE_MAX]; int entries; int qos_class; + bool shareable; + bool read_only; }; static u32 cdat_normalize(u16 entry, u64 base, u8 type) @@ -73,7 +75,13 @@ static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, if (!dent) return -ENOMEM; + /* Stays invalid unless the QTG lookup assigns a class */ + dent->qos_class = CXL_QOS_CLASS_INVALID; dent->handle = dsmas->dsmad_handle; + /* Shareable is CDAT 1.03 and later, DSMAS Flags bit 3 */ + dent->shareable = dsmas->flags & ACPI_CDAT_DSMAS_SHAREABLE; + /* Read Only is CDAT 1.04 and later, DSMAS Flags bit 6 */ + dent->read_only = dsmas->flags & ACPI_CDAT_DSMAS_READ_ONLY; dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address); dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) + le64_to_cpu((__force __le64)dsmas->dpa_length) - 1; @@ -188,9 +196,10 @@ static int cxl_cdat_endpoint_process(struct cxl_port *port, if (rc) return rc; + /* DSLBIS is optional; zero entries is not an error */ rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler, dsmas_xa, port->cdat.table, port->cdat.length); - return cdat_table_parse_output(rc); + return rc < 0 ? rc : 0; } static int cxl_port_perf_data_calculate(struct cxl_port *port, @@ -219,6 +228,13 @@ static int cxl_port_perf_data_calculate(struct cxl_port *port, xa_for_each(dsmas_xa, index, dent) { int qos_class; + /* No DSLBIS for this DSMAS; nothing to hand the QTG _DSM */ + if (!dent->cdat_coord[ACCESS_COORDINATE_CPU].read_bandwidth && + !dent->cdat_coord[ACCESS_COORDINATE_CPU].write_bandwidth && + !dent->cdat_coord[ACCESS_COORDINATE_CPU].read_latency && + !dent->cdat_coord[ACCESS_COORDINATE_CPU].write_latency) + continue; + cxl_coordinates_combine(dent->coord, dent->cdat_coord, ep_c); dent->entries = 1; rc = cxl_root->ops.qos_class(cxl_root, @@ -255,35 +271,90 @@ static void update_perf_entry(struct device *dev, struct dsmas_entry *dent, dent->coord[ACCESS_COORDINATE_CPU].write_latency); } -static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, - struct xarray *dsmas_xa) +/* + * A DCD shall describe its DC partition in a DSMAS entry (CXL r4.0 9.13.3). + * Without one the partition's attributes are unknown, so disable DCD. + */ +static void cxl_dcd_dsmas_missing(struct cxl_dev_state *cxlds, const char *why) +{ + struct cxl_memdev_state *mds; + + if (cxlds->type != CXL_DEVTYPE_CLASSMEM) + return; + + mds = to_cxl_memdev_state(cxlds); + if (!cxl_dcd_supported(mds) || + !cxl_part_size(cxlds, CXL_PARTMODE_DYNAMIC_RAM)) + return; + + dev_warn(cxlds->dev, "DC partition has no valid DSMAS entry (%s), disabling DCD\n", + why); + cxl_disable_dcd(mds); +} + +enum cxl_dsmas_apply { + CXL_DSMAS_APPLY_FLAGS, /* shareable etc.; no QTG data */ + CXL_DSMAS_APPLY_FLAGS_PERF, /* perf coordinates and qos_class */ +}; + +/* + * DSMAS flags such as shareable are properties of the CDAT entry and are + * applied regardless; perf data is applied only if the QTG lookup produced + * recommendations. + */ +static void cxl_memdev_apply_dsmas(struct cxl_dev_state *cxlds, + struct xarray *dsmas_xa, + enum cxl_dsmas_apply attrs) { struct device *dev = cxlds->dev; struct dsmas_entry *dent; + bool dc_matched = false; unsigned long index; xa_for_each(dsmas_xa, index, dent) { bool found = false; for (int i = 0; i < cxlds->nr_partitions; i++) { - struct resource *res = &cxlds->part[i].res; + struct cxl_dpa_partition *part = &cxlds->part[i]; struct range range = { - .start = res->start, - .end = res->end, + .start = part->res.start, + .end = part->res.end, }; - if (range_contains(&range, &dent->dpa_range)) { - update_perf_entry(dev, dent, - &cxlds->part[i].perf); - found = true; + if (!range_contains(&range, &dent->dpa_range)) + continue; + + found = true; + /* + * part->handle is from Get DC Config, dent->handle + * from the CDAT DSMAS entry. + */ + if (part->mode == CXL_PARTMODE_DYNAMIC_RAM && + dent->handle != part->handle) { + dev_warn(dev, + "DSMAD handle mismatch: %pra has %u, DSMAS %pra has %u\n", + &range, part->handle, + &dent->dpa_range, dent->handle); break; } + + if (attrs == CXL_DSMAS_APPLY_FLAGS_PERF) + update_perf_entry(dev, dent, &part->perf); + if (part->mode == CXL_PARTMODE_DYNAMIC_RAM) { + part->shareable = dent->shareable; + part->read_only = dent->read_only; + dc_matched = true; + } + break; } if (!found) dev_dbg(dev, "no partition for dsmas dpa: %pra\n", &dent->dpa_range); } + + if (!dc_matched) + cxl_dcd_dsmas_missing(cxlds, "no matching entry"); } static int match_cxlrd_qos_class(struct device *dev, void *data) @@ -409,22 +480,26 @@ void cxl_endpoint_parse_cdat(struct cxl_port *port) int rc; xa_init(&__dsmas_xa); - if (!port->cdat.table) + if (!port->cdat.table) { + cxl_dcd_dsmas_missing(cxlds, "no CDAT"); return; + } rc = cxl_cdat_endpoint_process(port, dsmas_xa); if (rc < 0) { dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc); + cxl_dcd_dsmas_missing(cxlds, "CDAT parse failed"); return; } rc = cxl_port_perf_data_calculate(port, dsmas_xa); if (rc) { dev_dbg(&port->dev, "Failed to do perf coord calculations.\n"); + cxl_memdev_apply_dsmas(cxlds, dsmas_xa, CXL_DSMAS_APPLY_FLAGS); return; } - cxl_memdev_set_qos_class(cxlds, dsmas_xa); + cxl_memdev_apply_dsmas(cxlds, dsmas_xa, CXL_DSMAS_APPLY_FLAGS_PERF); cxl_qos_class_verify(cxlmd); cxl_memdev_update_perf(cxlmd); } @@ -633,9 +708,9 @@ static int cxl_endpoint_gather_bandwidth(struct cxl_region *cxlr, struct cxl_port *endpoint = to_cxl_port(cxled->cxld.dev.parent); struct cxl_port *parent_port = to_cxl_port(endpoint->dev.parent); struct cxl_port *gp_port = to_cxl_port(parent_port->dev.parent); - struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX]; - struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX]; - struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX]; + struct access_coordinate pci_coord[ACCESS_COORDINATE_MAX] = { }; + struct access_coordinate sw_coord[ACCESS_COORDINATE_MAX] = { }; + struct access_coordinate ep_coord[ACCESS_COORDINATE_MAX] = { }; struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); struct cxl_dev_state *cxlds = cxlmd->cxlds; struct pci_dev *pdev = to_pci_dev(cxlds->dev); @@ -757,7 +832,7 @@ static struct xarray *cxl_switch_gather_bandwidth(struct cxl_region *cxlr, { struct xarray *res_xa __free(free_perf_xa) = kzalloc_obj(*res_xa); - struct access_coordinate coords[ACCESS_COORDINATE_MAX]; + struct access_coordinate coords[ACCESS_COORDINATE_MAX] = { }; struct cxl_perf_ctx *ctx, *us_ctx; unsigned long index, us_index; int dev_count = 0; diff --git a/drivers/cxl/core/core.h b/drivers/cxl/core/core.h index 35eaf636adc9..79918fde1575 100644 --- a/drivers/cxl/core/core.h +++ b/drivers/cxl/core/core.h @@ -178,11 +178,7 @@ static inline struct device *port_to_host(struct cxl_port *port) static inline struct device *dport_to_host(struct cxl_dport *dport) { - struct cxl_port *port = dport->port; - - if (is_cxl_root(port)) - return port->uport_dev; - return &port->dev; + return cxl_port_dport_host(dport->port); } #ifdef CONFIG_CXL_RAS void cxl_ras_init(void); @@ -217,10 +213,10 @@ int cxl_port_get_possible_dports(struct cxl_port *port); #ifdef CONFIG_CXL_FEATURES struct cxl_feat_entry * cxl_feature_info(struct cxl_features_state *cxlfs, const uuid_t *uuid); -size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, - enum cxl_get_feat_selection selection, - void *feat_out, size_t feat_out_size, u16 offset, - u16 *return_code); +ssize_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, + enum cxl_get_feat_selection selection, + void *feat_out, size_t feat_out_size, u16 offset, + u16 *return_code); int cxl_set_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, u8 feat_version, const void *feat_data, size_t feat_data_size, u32 feat_flag, u16 offset, diff --git a/drivers/cxl/core/edac.c b/drivers/cxl/core/edac.c index b321971fef58..34b81e8dfbbb 100644 --- a/drivers/cxl/core/edac.c +++ b/drivers/cxl/core/edac.c @@ -78,7 +78,7 @@ static int cxl_mem_scrub_get_attrbs(struct cxl_mailbox *cxl_mbox, u8 *cap, u16 *cycle, u8 *flags, u8 *min_cycle) { size_t rd_data_size = sizeof(struct cxl_scrub_rd_attrbs); - size_t data_size; + ssize_t data_size; struct cxl_scrub_rd_attrbs *rd_attrbs __free(kfree) = kzalloc(rd_data_size, GFP_KERNEL); if (!rd_attrbs) @@ -87,7 +87,9 @@ static int cxl_mem_scrub_get_attrbs(struct cxl_mailbox *cxl_mbox, u8 *cap, data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_PATROL_SCRUB_UUID, CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs, rd_data_size, 0, NULL); - if (!data_size) + if (data_size < 0) + return data_size; + if ((size_t)data_size != rd_data_size) return -EIO; *cap = rd_attrbs->scrub_cycle_cap; @@ -551,7 +553,7 @@ static int cxl_mem_ecs_get_attrbs(struct device *dev, struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; struct cxl_ecs_fru_rd_attrbs *fru_rd_attrbs; size_t rd_data_size; - size_t data_size; + ssize_t data_size; rd_data_size = cxl_ecs_ctx->get_feat_size; @@ -563,7 +565,9 @@ static int cxl_mem_ecs_get_attrbs(struct device *dev, data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID, CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs, rd_data_size, 0, NULL); - if (!data_size) + if (data_size < 0) + return data_size; + if ((size_t)data_size != rd_data_size) return -EIO; fru_rd_attrbs = rd_attrbs->fru_attrbs; @@ -583,7 +587,7 @@ static int cxl_mem_ecs_set_attrbs(struct device *dev, struct cxl_ecs_fru_wr_attrbs *fru_wr_attrbs; size_t rd_data_size, wr_data_size; u16 num_media_frus, count; - size_t data_size; + ssize_t data_size; num_media_frus = cxl_ecs_ctx->num_media_frus; rd_data_size = cxl_ecs_ctx->get_feat_size; @@ -596,7 +600,9 @@ static int cxl_mem_ecs_set_attrbs(struct device *dev, data_size = cxl_get_feature(cxl_mbox, &CXL_FEAT_ECS_UUID, CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs, rd_data_size, 0, NULL); - if (!data_size) + if (data_size < 0) + return data_size; + if ((size_t)data_size != rd_data_size) return -EIO; struct cxl_ecs_wr_attrbs *wr_attrbs __free(kvfree) = @@ -1264,7 +1270,7 @@ cxl_mem_sparing_get_attrbs(struct cxl_mem_sparing_context *cxl_sparing_ctx) struct cxl_memdev *cxlmd = cxl_sparing_ctx->cxlmd; struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; u16 restriction_flags; - size_t data_size; + ssize_t data_size; u16 return_code; struct cxl_memdev_sparing_rd_attrbs *rd_attrbs __free(kfree) = kzalloc(rd_data_size, GFP_KERNEL); @@ -1274,7 +1280,9 @@ cxl_mem_sparing_get_attrbs(struct cxl_mem_sparing_context *cxl_sparing_ctx) data_size = cxl_get_feature(cxl_mbox, &cxl_sparing_ctx->repair_uuid, CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs, rd_data_size, 0, &return_code); - if (!data_size) + if (data_size < 0) + return data_size; + if ((size_t)data_size != rd_data_size) return -EIO; cxl_sparing_ctx->op_class = rd_attrbs->hdr.op_class; @@ -1750,7 +1758,7 @@ static int cxl_mem_ppr_get_attrbs(struct cxl_ppr_context *cxl_ppr_ctx) struct cxl_memdev *cxlmd = cxl_ppr_ctx->cxlmd; struct cxl_mailbox *cxl_mbox = &cxlmd->cxlds->cxl_mbox; u16 restriction_flags; - size_t data_size; + ssize_t data_size; u16 return_code; struct cxl_memdev_ppr_rd_attrbs *rd_attrbs __free(kfree) = @@ -1761,7 +1769,9 @@ static int cxl_mem_ppr_get_attrbs(struct cxl_ppr_context *cxl_ppr_ctx) data_size = cxl_get_feature(cxl_mbox, &cxl_ppr_ctx->repair_uuid, CXL_GET_FEAT_SEL_CURRENT_VALUE, rd_attrbs, rd_data_size, 0, &return_code); - if (!data_size) + if (data_size < 0) + return data_size; + if ((size_t)data_size != rd_data_size) return -EIO; cxl_ppr_ctx->op_class = rd_attrbs->hdr.op_class; diff --git a/drivers/cxl/core/features.c b/drivers/cxl/core/features.c index ba6d2a5acb74..8024c2dcae1d 100644 --- a/drivers/cxl/core/features.c +++ b/drivers/cxl/core/features.c @@ -97,6 +97,7 @@ get_supported_features(struct cxl_features_state *cxlfs) kvmalloc_flex(*entries, ent, count); if (!entries) return NULL; + entries->num_features = count; struct cxl_mbox_get_sup_feats_out *mbox_out __free(kvfree) = kvmalloc(cxl_mbox->payload_size, GFP_KERNEL); @@ -174,7 +175,6 @@ get_supported_features(struct cxl_features_state *cxlfs) start += num_entries; } while (remain_feats); - entries->num_features = count; entries->num_user_features = user_feats; return no_free_ptr(entries); @@ -220,10 +220,10 @@ int devm_cxl_setup_features(struct cxl_dev_state *cxlds) } EXPORT_SYMBOL_NS_GPL(devm_cxl_setup_features, "CXL"); -size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, - enum cxl_get_feat_selection selection, - void *feat_out, size_t feat_out_size, u16 offset, - u16 *return_code) +ssize_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, + enum cxl_get_feat_selection selection, + void *feat_out, size_t feat_out_size, u16 offset, + u16 *return_code) { size_t data_to_rd_size; struct cxl_mbox_get_feat_in pi; @@ -232,10 +232,13 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, int rc; if (return_code) - *return_code = CXL_MBOX_CMD_RC_INPUT; + *return_code = CXL_MBOX_CMD_RC_SUCCESS; if (!feat_out || !feat_out_size) - return 0; + return -EINVAL; + + if (feat_out_size > U16_MAX - offset) + return -EINVAL; uuid_copy(&pi.uuid, feat_uuid); pi.selection = selection; @@ -256,17 +259,25 @@ size_t cxl_get_feature(struct cxl_mailbox *cxl_mbox, const uuid_t *feat_uuid, .min_out = data_to_rd_size, }; rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); + /* + * Per CXL r4.0 8.2.10.6.2, when Offset + Count runs past the + * end of the Feature the device returns only the bytes up to + * the Feature size. cxl_internal_send_cmd() reports that as + * -EIO with a short payload, so stop and return what arrived. + */ + if (rc == -EIO && mbox_cmd.size_out && + mbox_cmd.size_out < data_to_rd_size) { + data_rcvd_size += mbox_cmd.size_out; + break; + } if (rc < 0 || !mbox_cmd.size_out) { if (return_code) *return_code = mbox_cmd.return_code; - return 0; + return rc < 0 ? rc : -EIO; } data_rcvd_size += mbox_cmd.size_out; } while (data_rcvd_size < feat_out_size); - if (return_code) - *return_code = CXL_MBOX_CMD_RC_SUCCESS; - return data_rcvd_size; } @@ -286,7 +297,10 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox, size_t hdr_size; if (return_code) - *return_code = CXL_MBOX_CMD_RC_INPUT; + *return_code = CXL_MBOX_CMD_RC_SUCCESS; + + if (feat_data_size > U16_MAX - offset) + return -EINVAL; struct cxl_mbox_set_feat_in *pi __free(kfree) = kzalloc(cxl_mbox->payload_size, GFP_KERNEL); @@ -334,11 +348,8 @@ int cxl_set_feature(struct cxl_mailbox *cxl_mbox, } data_sent_size += data_in_size; - if (data_sent_size >= feat_data_size) { - if (return_code) - *return_code = CXL_MBOX_CMD_RC_SUCCESS; + if (data_sent_size >= feat_data_size) return 0; - } if ((feat_data_size - data_sent_size) <= (cxl_mbox->payload_size - hdr_size)) { data_in_size = feat_data_size - data_sent_size; @@ -462,6 +473,7 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs, const struct cxl_mbox_get_feat_in *feat_in; u16 offset, count, return_code; size_t out_size = *out_len; + ssize_t data_size; if (rpc_in->op_size != sizeof(*feat_in)) return ERR_PTR(-EINVAL); @@ -482,16 +494,20 @@ static void *cxlctl_get_feature(struct cxl_features_state *cxlfs, if (!rpc_out) return ERR_PTR(-ENOMEM); - out_size = cxl_get_feature(cxl_mbox, &feat_in->uuid, - feat_in->selection, rpc_out->payload, - count, offset, &return_code); + data_size = cxl_get_feature(cxl_mbox, &feat_in->uuid, + feat_in->selection, rpc_out->payload, + count, offset, &return_code); + if (data_size <= 0 && + return_code == CXL_MBOX_CMD_RC_SUCCESS) + return ERR_PTR(data_size ?: -EIO); *out_len = sizeof(struct fwctl_rpc_cxl_out); - if (!out_size) { + if (data_size <= 0) { rpc_out->size = 0; rpc_out->retval = return_code; return no_free_ptr(rpc_out); } + out_size = data_size; rpc_out->size = out_size; rpc_out->retval = CXL_MBOX_CMD_RC_SUCCESS; *out_len += out_size; @@ -536,6 +552,8 @@ static void *cxlctl_set_feature(struct cxl_features_state *cxlfs, rc = cxl_set_feature(cxl_mbox, &feat_in->uuid, feat_in->version, feat_in->feat_data, data_size, flags, offset, &return_code); + if (rc && return_code == CXL_MBOX_CMD_RC_SUCCESS) + return ERR_PTR(rc); *out_len = sizeof(*rpc_out); if (rc) { rpc_out->retval = return_code; diff --git a/drivers/cxl/core/hdm.c b/drivers/cxl/core/hdm.c index 0c80b76a5f9b..fdcef20e01ba 100644 --- a/drivers/cxl/core/hdm.c +++ b/drivers/cxl/core/hdm.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ #include +#include #include #include @@ -76,6 +77,8 @@ static void parse_hdm_decoder_caps(struct cxl_hdm *cxlhdm) hdm_cap = readl(cxlhdm->regs.hdm_decoder + CXL_HDM_DECODER_CAP_OFFSET); cxlhdm->decoder_count = cxl_hdm_decoder_count(hdm_cap); + + /* target_count is a direct count (1h..8h), not 0-based like decoder_count */ cxlhdm->target_count = FIELD_GET(CXL_HDM_DECODER_TARGET_COUNT_MASK, hdm_cap); if (FIELD_GET(CXL_HDM_DECODER_INTERLEAVE_11_8, hdm_cap)) @@ -240,6 +243,18 @@ static resource_size_t __adjust_skip(struct cxl_dev_state *cxlds, } #define release_skip(c, b, l) __adjust_skip((c), (b), (l), NULL) +static void cxl_dpa_release_region(struct resource *parent, + struct resource *res) +{ + /* zero sized decoders are not tracked in the resource tree */ + if (resource_size(res) == 0) { + kfree(res); + return; + } + + __release_region(parent, res->start, resource_size(res)); +} + /* * Must be called in a context that synchronizes against this decoder's * port ->remove() callback (like an endpoint decoder sysfs attribute) @@ -256,7 +271,7 @@ static void __cxl_dpa_release(struct cxl_endpoint_decoder *cxled) /* save @skip_start, before @res is released */ skip_start = res->start - cxled->skip; - __release_region(&cxlds->dpa_res, res->start, resource_size(res)); + cxl_dpa_release_region(&cxlds->dpa_res, res); if (cxled->skip) release_skip(cxlds, skip_start, cxled->skip); cxled->skip = 0; @@ -336,6 +351,27 @@ static int request_skip(struct cxl_dev_state *cxlds, return -EBUSY; } +static struct resource *cxl_dpa_request_region(struct resource *parent, + resource_size_t start, + resource_size_t n, + const char *name) +{ + struct resource *res; + + if (!n) { + res = kmalloc_obj(*res); + if (!res) + return ERR_PTR(-ENOMEM); + + *res = DEFINE_RES_NAMED(start, 0, name, IORESOURCE_MEM); + + return res; + } + + res = __request_region(parent, start, n, name, 0); + return res ?: ERR_PTR(-EBUSY); +} + static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, resource_size_t base, resource_size_t len, resource_size_t skipped) @@ -349,12 +385,6 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, lockdep_assert_held_write(&cxl_rwsem.dpa); - if (!len) { - dev_warn(dev, "decoder%d.%d: empty reservation attempted\n", - port->id, cxled->cxld.id); - return -EINVAL; - } - if (cxled->dpa_res) { dev_dbg(dev, "decoder%d.%d: existing allocation %pr assigned\n", port->id, cxled->cxld.id, cxled->dpa_res); @@ -378,14 +408,14 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, if (rc) return rc; } - res = __request_region(&cxlds->dpa_res, base, len, - dev_name(&cxled->cxld.dev), 0); - if (!res) { + res = cxl_dpa_request_region(&cxlds->dpa_res, base, len, + dev_name(&cxled->cxld.dev)); + if (IS_ERR(res)) { dev_dbg(dev, "decoder%d.%d: failed to reserve allocation\n", port->id, cxled->cxld.id); if (skipped) release_skip(cxlds, base - skipped, skipped); - return -EBUSY; + return PTR_ERR(res); } cxled->dpa_res = res; cxled->skip = skipped; @@ -402,7 +432,8 @@ static int __cxl_dpa_reserve(struct cxl_endpoint_decoder *cxled, break; } - if (cxled->part < 0) + /* Empty decoders may not be contained by a partition boundary */ + if (cxled->part < 0 && resource_size(res)) dev_warn(dev, "decoder%d.%d: %pr does not map any partition\n", port->id, cxled->cxld.id, res); @@ -446,6 +477,8 @@ static const char *cxl_mode_name(enum cxl_partition_mode mode) return "ram"; case CXL_PARTMODE_PMEM: return "pmem"; + case CXL_PARTMODE_DYNAMIC_RAM: + return "dynamic_ram"; default: return ""; }; @@ -455,6 +488,7 @@ static const char *cxl_mode_name(enum cxl_partition_mode mode) int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info) { struct device *dev = cxlds->dev; + int dc_nth = 0; guard(rwsem_write)(&cxl_rwsem.dpa); @@ -467,14 +501,33 @@ int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info) return 0; } + /* Verify partitions are in expected order. */ + for (int i = 1; i < info->nr_partitions; i++) { + if (info->part[i].mode < info->part[i - 1].mode) { + dev_err(dev, "Partition order mismatch: %d (%s) follows %d (%s)\n", + i, cxl_mode_name(info->part[i].mode), + i - 1, cxl_mode_name(info->part[i - 1].mode)); + return -EINVAL; + } + } + cxlds->dpa_res = DEFINE_RES_MEM(0, info->size); for (int i = 0; i < info->nr_partitions; i++) { const struct cxl_dpa_part_info *part = &info->part[i]; + const char *name = cxl_mode_name(part->mode); int rc; + if (part->mode == CXL_PARTMODE_DYNAMIC_RAM) { + name = devm_kasprintf(dev, GFP_KERNEL, "%s_%d", name, + ++dc_nth); + if (!name) + return -ENOMEM; + } + cxlds->part[i].perf.qos_class = CXL_QOS_CLASS_INVALID; cxlds->part[i].mode = part->mode; + cxlds->part[i].handle = part->handle; /* Require ordered + contiguous partitions */ if (i) { @@ -485,7 +538,7 @@ int cxl_dpa_setup(struct cxl_dev_state *cxlds, const struct cxl_dpa_info *info) } rc = add_dpa_res(dev, &cxlds->dpa_res, &cxlds->part[i].res, part->range.start, range_len(&part->range), - cxl_mode_name(part->mode)); + name); if (rc) return rc; cxlds->nr_partitions++; @@ -953,7 +1006,8 @@ static int cxl_setup_hdm_decoder_from_dvsec( * change the range registers at run time. */ cxld->flags |= CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; - port->commit_end = cxld->id; + scoped_guard(rwsem_write, &cxl_rwsem.region) + port->commit_end = cxld->id; rc = devm_cxl_dpa_reserve(cxled, *dpa_base, len, 0); if (rc) { @@ -973,15 +1027,12 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, u64 *dpa_base, struct cxl_endpoint_dvsec_info *info) { struct cxl_endpoint_decoder *cxled = NULL; + u8 target_id[CXL_HDM_DECODER0_TL_TARGETS]; u64 size, base, skip, dpa_size, lo, hi; bool committed; u32 remainder; int i, rc; u32 ctrl; - union { - u64 value; - unsigned char target_id[8]; - } target_list; if (should_emulate_decoders(info)) return cxl_setup_hdm_decoder_from_dvsec(port, cxld, dpa_base, @@ -1031,12 +1082,6 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, return -ENXIO; } - if (size == 0) { - dev_warn(&port->dev, - "decoder%d.%d: Committed with zero size\n", - port->id, cxld->id); - return -ENXIO; - } port->commit_end = cxld->id; } else { if (cxled) { @@ -1084,11 +1129,22 @@ static int init_hdm_decoder(struct cxl_port *port, struct cxl_decoder *cxld, cxld->interleave_ways, cxld->interleave_granularity); if (!cxled) { - lo = readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)); - hi = readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)); - target_list.value = (hi << 32) + lo; + struct cxl_switch_decoder *cxlsd = to_cxl_switch_decoder(&cxld->dev); + + if (cxld->interleave_ways > cxlsd->nr_targets) { + dev_err(&port->dev, + "decoder%d.%d: interleave ways: %d exceeds targets: %d\n", + port->id, cxld->id, cxld->interleave_ways, + cxlsd->nr_targets); + return -ENXIO; + } + + put_unaligned_le32(readl(hdm + CXL_HDM_DECODER0_TL_LOW(which)), + &target_id[0]); + put_unaligned_le32(readl(hdm + CXL_HDM_DECODER0_TL_HIGH(which)), + &target_id[4]); for (i = 0; i < cxld->interleave_ways; i++) - cxld->target_map[i] = target_list.target_id[i]; + cxld->target_map[i] = target_id[i]; return 0; } @@ -1148,6 +1204,14 @@ static void cxl_settle_decoders(struct cxl_hdm *cxlhdm) msleep(20); } +static void cxl_reset_commit_end(void *data) +{ + struct cxl_port *port = data; + + guard(rwsem_write)(&cxl_rwsem.region); + port->commit_end = -1; +} + /** * devm_cxl_enumerate_decoders - add decoder objects per HDM register set * @cxlhdm: Structure to populate with HDM capabilities @@ -1158,13 +1222,33 @@ static int devm_cxl_enumerate_decoders(struct cxl_hdm *cxlhdm, { void __iomem *hdm = cxlhdm->regs.hdm_decoder; struct cxl_port *port = cxlhdm->port; - int i; u64 dpa_base = 0; + int i, rc; + + /* + * Per CXL 4.0 8.2.4.20.1 Target Count is the number of target ports + * per decoder, max 8. Endpoint decoders have no targets. + */ + if (!is_cxl_endpoint(port) && + (cxlhdm->target_count < 1 || + cxlhdm->target_count > CXL_HDM_DECODER0_TL_TARGETS)) { + dev_err(&port->dev, "Invalid decoder target count: %u\n", + cxlhdm->target_count); + return -ENXIO; + } cxl_settle_decoders(cxlhdm); + /* + * Reset commit_end after all decoders have been torn down so a + * subsequent probe rebuilds it from scratch. + */ + rc = devm_add_action(&port->dev, cxl_reset_commit_end, port); + if (rc) + return rc; + for (i = 0; i < cxlhdm->decoder_count; i++) { - int rc, target_count = cxlhdm->target_count; + int target_count = cxlhdm->target_count; struct cxl_decoder *cxld; if (is_cxl_endpoint(port)) { diff --git a/drivers/cxl/core/mbox.c b/drivers/cxl/core/mbox.c index 55828a836c01..39485b6bcb97 100644 --- a/drivers/cxl/core/mbox.c +++ b/drivers/cxl/core/mbox.c @@ -171,6 +171,33 @@ static void cxl_set_security_cmd_enabled(struct cxl_security_state *security, } } +static bool cxl_is_dcd_command(u16 opcode) +{ +#define CXL_MBOX_OP_DCD_CMDS 0x48 + + return (opcode >> 8) == CXL_MBOX_OP_DCD_CMDS; +} + +static void cxl_set_dcd_cmd_enabled(u16 opcode, unsigned long *cmd_mask) +{ + switch (opcode) { + case CXL_MBOX_OP_GET_DC_CONFIG: + set_bit(CXL_DCD_ENABLED_GET_CONFIG, cmd_mask); + break; + case CXL_MBOX_OP_GET_DC_EXTENT_LIST: + set_bit(CXL_DCD_ENABLED_GET_EXTENT_LIST, cmd_mask); + break; + case CXL_MBOX_OP_ADD_DC_RESPONSE: + set_bit(CXL_DCD_ENABLED_ADD_RESPONSE, cmd_mask); + break; + case CXL_MBOX_OP_RELEASE_DC: + set_bit(CXL_DCD_ENABLED_RELEASE, cmd_mask); + break; + default: + break; + } +} + static bool cxl_is_poison_command(u16 opcode) { #define CXL_MBOX_OP_POISON_CMDS 0x43 @@ -759,6 +786,7 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel) struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_cel_entry *cel_entry; const int cel_entries = size / sizeof(*cel_entry); + DECLARE_BITMAP(dcd_cmds, CXL_DCD_ENABLED_MAX) = {}; struct device *dev = mds->cxlds.dev; int i, ro_cmds = 0, wr_cmds = 0; @@ -787,11 +815,22 @@ static void cxl_walk_cel(struct cxl_memdev_state *mds, size_t size, u8 *cel) enabled++; } + if (cxl_is_dcd_command(opcode)) { + cxl_set_dcd_cmd_enabled(opcode, dcd_cmds); + enabled++; + } + dev_dbg(dev, "Opcode 0x%04x %s\n", opcode, enabled ? "enabled" : "unsupported by driver"); } set_features_cap(cxl_mbox, ro_cmds, wr_cmds); + /* + * Disabled until event handling implemented. + */ + if (bitmap_full(dcd_cmds, CXL_DCD_ENABLED_MAX)) + dev_dbg(dev, "Device supports DCD; capability disabled\n"); + mds->dcd_supported = false; } static struct cxl_mbox_get_supported_logs *cxl_get_gsl(struct cxl_memdev_state *mds) @@ -942,11 +981,11 @@ void cxl_event_trace_record(struct cxl_memdev *cxlmd, if (evt->gen_media.media_hdr.descriptor & CXL_GMER_EVT_DESC_THRESHOLD_EVENT) - WARN_ON_ONCE((evt->gen_media.media_hdr.type & + WARN_ON_ONCE((evt->gen_media.media_hdr.type == CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) && !get_unaligned_le24(evt->gen_media.cme_count)); else - WARN_ON_ONCE(evt->gen_media.media_hdr.type & + WARN_ON_ONCE(evt->gen_media.media_hdr.type == CXL_GMER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE); trace_cxl_general_media(cxlmd, type, cxlr, hpa, @@ -957,11 +996,11 @@ void cxl_event_trace_record(struct cxl_memdev *cxlmd, if (evt->dram.media_hdr.descriptor & CXL_GMER_EVT_DESC_THRESHOLD_EVENT) - WARN_ON_ONCE((evt->dram.media_hdr.type & + WARN_ON_ONCE((evt->dram.media_hdr.type == CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE) && !get_unaligned_le24(evt->dram.cvme_count)); else - WARN_ON_ONCE(evt->dram.media_hdr.type & + WARN_ON_ONCE(evt->dram.media_hdr.type == CXL_DER_MEM_EVT_TYPE_AP_CME_COUNTER_EXPIRE); trace_cxl_dram(cxlmd, type, cxlr, hpa, hpa_alias, @@ -1310,6 +1349,250 @@ int cxl_mem_sanitize(struct cxl_memdev *cxlmd, u16 cmd) return -EBUSY; } +/* + * A partition unavailable for Dynamic Capacity reports zeros for decode_length, + * length, and block_size. + */ +static bool cxl_dc_partition_unavailable(struct cxl_dc_partition *dev_part) +{ + return !le64_to_cpu(dev_part->decode_length) && + !le64_to_cpu(dev_part->length) && + !le64_to_cpu(dev_part->block_size); +} + +static int cxl_dc_check(struct device *dev, struct cxl_dc_partition_info *part, + struct cxl_dc_partition *dev_part) +{ + u64 decode_len = le64_to_cpu(dev_part->decode_length); + u64 blk_size = le64_to_cpu(dev_part->block_size); + u64 len = le64_to_cpu(dev_part->length); + u32 handle = le32_to_cpu(dev_part->dsmad_handle); + u64 size; + + /* + * Not an error; leave the entry empty. A partially zeroed partition + * is rejected by the checks below. CXL r4.0 Table 8-347. + */ + if (cxl_dc_partition_unavailable(dev_part)) { + *part = (struct cxl_dc_partition_info) { }; + dev_dbg(dev, "Partition 0 unavailable for DC\n"); + return 0; + } + + /* The CDAT DSMAD handle this refers to is 8 bits */ + if (handle > U8_MAX) { + dev_warn(dev, "DSMAD handle 0x%x exceeds the 8 bit CDAT DSMAD handle\n", + handle); + return -EINVAL; + } + + /* + * Decode length is a count of 256M units, so a device reporting 2^36 + * or more wraps. + */ + if (check_mul_overflow(decode_len, (u64)CXL_CAPACITY_MULTIPLIER, &size)) { + dev_err(dev, "DC partition 0 decode length %#llx overflows\n", + decode_len); + return -EINVAL; + } + + *part = (struct cxl_dc_partition_info) { + .start = le64_to_cpu(dev_part->base), + .size = size, + .handle = handle, + }; + + /* + * Block size is a power of 2 and a multiple of 40h. is_power_of_2() + * takes an unsigned long, which truncates blk_size on 32 bit. + */ + if (blk_size == 0 || (blk_size & (blk_size - 1)) || + blk_size % CXL_DCD_BLOCK_LINE_SIZE) { + dev_err(dev, "DC partition 0 invalid block size %#llx\n", blk_size); + return -EINVAL; + } + + if (part->size == 0) { + dev_err(dev, "DC partition 0 zero decode length\n"); + return -EINVAL; + } + + if (len == 0) { + dev_err(dev, "DC partition 0 zero length\n"); + return -EINVAL; + } + + if (len > part->size) { + dev_err(dev, "DC partition 0 length %#llx exceeds decode length %#llx\n", + len, part->size); + return -EINVAL; + } + + if (!IS_ALIGNED(len, blk_size)) { + dev_err(dev, "DC partition 0 length %#llx not a multiple of block size %#llx\n", + len, blk_size); + return -EINVAL; + } + + if (!IS_ALIGNED(part->start, SZ_256M)) { + dev_err(dev, "DC partition 0 base %#llx not aligned to 256M\n", + part->start); + return -EINVAL; + } + + dev_dbg(dev, "DC partition 0 start %#llx size %#llx blk_size: %#llx\n", + part->start, part->size, blk_size); + + return 0; +} + +/* Returns the number of partitions in dc_resp or -ERRNO */ +static int cxl_get_dc_config(struct cxl_mailbox *mbox, u8 start_partition, + u8 partition_count, + struct cxl_mbox_get_dc_config_out *dc_resp, + size_t dc_resp_size) +{ + struct cxl_mbox_get_dc_config_in get_dc = (struct cxl_mbox_get_dc_config_in) { + .partition_count = partition_count, + .start_partition_index = start_partition, + }; + struct cxl_mbox_cmd mbox_cmd = (struct cxl_mbox_cmd) { + .opcode = CXL_MBOX_OP_GET_DC_CONFIG, + .payload_in = &get_dc, + .size_in = sizeof(get_dc), + .size_out = dc_resp_size, + .payload_out = dc_resp, + /* The device must return at least the fixed header */ + .min_out = sizeof(*dc_resp), + }; + size_t expected_sz; + int rc; + + rc = cxl_internal_send_cmd(mbox, &mbox_cmd); + if (rc < 0) + return rc; + + if (dc_resp->partitions_returned > partition_count) { + dev_err(mbox->host, "Device returned %u partitions, requested %u\n", + dc_resp->partitions_returned, partition_count); + return -EIO; + } + + /* The trailing extent/tag counts (CXL r4.0 Table 8-346) are not read */ + expected_sz = struct_size(dc_resp, partition, + dc_resp->partitions_returned); + + if (mbox_cmd.size_out < expected_sz) { + dev_err(mbox->host, + "Payload size %zu less than expected %zu for %u partitions\n", + mbox_cmd.size_out, expected_sz, + dc_resp->partitions_returned); + return -EIO; + } + + dev_dbg(mbox->host, "Read %d/%d DC partitions\n", + dc_resp->partitions_returned, dc_resp->avail_partition_count); + return dc_resp->partitions_returned; +} + +/** + * cxl_dev_dc_identify() - Reads the dynamic capacity information from the + * device. + * @mbox: Mailbox to query + * @dc_info: The dynamic partition information to return + * + * Read every partition the device reports, but validate only the first: + * Linux maps partition 0 and nothing else, so a defect in capacity the + * driver never touches is not a reason to refuse the device dynamic + * capacity. The remaining partitions are read but not recorded. + * + * Return: 0 if identify was executed successfully, -ERRNO on error. + * On error, @dc_info is left unchanged. + */ +int cxl_dev_dc_identify(struct cxl_mailbox *mbox, + struct cxl_dc_partition_info *dc_info) +{ + struct cxl_dc_partition_info partitions[CXL_MAX_DC_PARTITIONS] = { }; + struct device *dev = mbox->host; + u8 avail_partitions = 0; + u8 num_partitions; + u8 partition_count; + size_t dc_resp_size; + + /* + * Bound requested number of partitions by mailbox payload size. The + * 256 byte spec minimum, verified in cxl_pci_setup_mailbox(), keeps + * the subtraction below from underflowing. + */ + partition_count = min(CXL_MAX_DC_PARTITIONS, + (mbox->payload_size - + sizeof(struct cxl_mbox_get_dc_config_out) - + sizeof(struct cxl_mbox_get_dc_config_tail)) / + sizeof(struct cxl_dc_partition)); + dc_resp_size = struct_size_t(struct cxl_mbox_get_dc_config_out, + partition, partition_count) + + sizeof(struct cxl_mbox_get_dc_config_tail); + + struct cxl_mbox_get_dc_config_out *dc_resp __free(kfree) = + kmalloc(dc_resp_size, GFP_KERNEL); + if (!dc_resp) + return -ENOMEM; + + num_partitions = 0; + do { + int returned, rc; + + returned = cxl_get_dc_config(mbox, num_partitions, + partition_count, dc_resp, + dc_resp_size); + if (returned < 0) { + dev_err(dev, "Failed to get DC config: %d\n", returned); + return returned; + } + + if (returned == 0) { + dev_err(dev, + "Device reported %u partitions available but returned none at index %u\n", + dc_resp->avail_partition_count, num_partitions); + return -EIO; + } + + /* A DCD reports at least one partition (CXL r4.0 Table 8-346) */ + if (dc_resp->avail_partition_count == 0) { + dev_err(dev, "Device reported no DC partitions\n"); + return -EIO; + } + + avail_partitions = min(dc_resp->avail_partition_count, + CXL_MAX_DC_PARTITIONS); + + /* + * Only partition 0 is mapped, so it is the only one checked. + * The rest are read to complete the enumeration and dropped. + */ + if (num_partitions == 0) { + rc = cxl_dc_check(dev, &partitions[0], + &dc_resp->partition[0]); + if (rc) + return rc; + } + + num_partitions += returned; + + } while (num_partitions < avail_partitions); + + /* Linux only supports the 1st partition; nothing to do if it is unavailable */ + if (partitions[0].size == 0) + return -ENODEV; + + *dc_info = partitions[0]; + dev_dbg(dev, "Returning partition 0 %#llx size %#llx\n", + dc_info->start, dc_info->size); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_dev_dc_identify, "CXL"); + int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count) { struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; @@ -1386,6 +1669,9 @@ int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, int nr_records = 0; int rc; + if (!len) + return 0; + ACQUIRE(mutex_intr, lock)(&mds->poison.mutex); if ((rc = ACQUIRE_ERR(mutex_intr, &lock))) return rc; diff --git a/drivers/cxl/core/mce.c b/drivers/cxl/core/mce.c index 65fed913b221..3ac6802e750d 100644 --- a/drivers/cxl/core/mce.c +++ b/drivers/cxl/core/mce.c @@ -18,7 +18,13 @@ static int cxl_handle_mce(struct notifier_block *nb, unsigned long val, u64 spa, spa_alias; unsigned long pfn; - if (!mce || !mce_usable_address(mce)) + if (!mce) + return NOTIFY_DONE; + + if (mce_is_correctable(mce)) + return NOTIFY_DONE; + + if (!mce_usable_address(mce)) return NOTIFY_DONE; spa = mce->addr & MCI_ADDR_PHYSADDR; diff --git a/drivers/cxl/core/memdev.c b/drivers/cxl/core/memdev.c index b3419df586b9..787988e92965 100644 --- a/drivers/cxl/core/memdev.c +++ b/drivers/cxl/core/memdev.c @@ -77,20 +77,12 @@ static ssize_t label_storage_size_show(struct device *dev, } static DEVICE_ATTR_RO(label_storage_size); -static resource_size_t cxl_ram_size(struct cxl_dev_state *cxlds) -{ - /* Static RAM is only expected at partition 0. */ - if (cxlds->part[0].mode != CXL_PARTMODE_RAM) - return 0; - return resource_size(&cxlds->part[0].res); -} - static ssize_t ram_size_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; - unsigned long long len = cxl_ram_size(cxlds); + unsigned long long len = cxl_part_size(cxlds, CXL_PARTMODE_RAM); return sysfs_emit(buf, "%#llx\n", len); } @@ -103,7 +95,7 @@ static ssize_t pmem_size_show(struct device *dev, struct device_attribute *attr, { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; - unsigned long long len = cxl_pmem_size(cxlds); + unsigned long long len = cxl_part_size(cxlds, CXL_PARTMODE_PMEM); return sysfs_emit(buf, "%#llx\n", len); } @@ -426,10 +418,11 @@ static struct attribute *cxl_memdev_attributes[] = { NULL, }; -static struct cxl_dpa_perf *to_pmem_perf(struct cxl_dev_state *cxlds) +static struct cxl_dpa_perf *part_perf(struct cxl_dev_state *cxlds, + enum cxl_partition_mode mode) { for (int i = 0; i < cxlds->nr_partitions; i++) - if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) + if (cxlds->part[i].mode == mode) return &cxlds->part[i].perf; return NULL; } @@ -440,7 +433,7 @@ static ssize_t pmem_qos_class_show(struct device *dev, struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; - return sysfs_emit(buf, "%d\n", to_pmem_perf(cxlds)->qos_class); + return sysfs_emit(buf, "%d\n", part_perf(cxlds, CXL_PARTMODE_PMEM)->qos_class); } static struct device_attribute dev_attr_pmem_qos_class = @@ -452,20 +445,13 @@ static struct attribute *cxl_memdev_pmem_attributes[] = { NULL, }; -static struct cxl_dpa_perf *to_ram_perf(struct cxl_dev_state *cxlds) -{ - if (cxlds->part[0].mode != CXL_PARTMODE_RAM) - return NULL; - return &cxlds->part[0].perf; -} - static ssize_t ram_qos_class_show(struct device *dev, struct device_attribute *attr, char *buf) { struct cxl_memdev *cxlmd = to_cxl_memdev(dev); struct cxl_dev_state *cxlds = cxlmd->cxlds; - return sysfs_emit(buf, "%d\n", to_ram_perf(cxlds)->qos_class); + return sysfs_emit(buf, "%d\n", part_perf(cxlds, CXL_PARTMODE_RAM)->qos_class); } static struct device_attribute dev_attr_ram_qos_class = @@ -501,7 +487,7 @@ static umode_t cxl_ram_visible(struct kobject *kobj, struct attribute *a, int n) { struct device *dev = kobj_to_dev(kobj); struct cxl_memdev *cxlmd = to_cxl_memdev(dev); - struct cxl_dpa_perf *perf = to_ram_perf(cxlmd->cxlds); + struct cxl_dpa_perf *perf = part_perf(cxlmd->cxlds, CXL_PARTMODE_RAM); if (a == &dev_attr_ram_qos_class.attr && (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) @@ -520,7 +506,7 @@ static umode_t cxl_pmem_visible(struct kobject *kobj, struct attribute *a, int n { struct device *dev = kobj_to_dev(kobj); struct cxl_memdev *cxlmd = to_cxl_memdev(dev); - struct cxl_dpa_perf *perf = to_pmem_perf(cxlmd->cxlds); + struct cxl_dpa_perf *perf = part_perf(cxlmd->cxlds, CXL_PARTMODE_PMEM); if (a == &dev_attr_pmem_qos_class.attr && (!perf || perf->qos_class == CXL_QOS_CLASS_INVALID)) @@ -594,7 +580,8 @@ bool is_cxl_memdev(const struct device *dev) } EXPORT_SYMBOL_NS_GPL(is_cxl_memdev, "CXL"); -static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_partition_mode mode) +static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, + enum cxl_partition_mode mode, u8 handle) { int i = info->nr_partitions; @@ -606,6 +593,7 @@ static void add_part(struct cxl_dpa_info *info, u64 start, u64 size, enum cxl_pa .end = start + size - 1, }; info->part[i].mode = mode; + info->part[i].handle = handle; info->nr_partitions++; } @@ -623,9 +611,9 @@ int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info) info->size = mds->total_bytes; if (mds->partition_align_bytes == 0) { - add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM); + add_part(info, 0, mds->volatile_only_bytes, CXL_PARTMODE_RAM, 0); add_part(info, mds->volatile_only_bytes, - mds->persistent_only_bytes, CXL_PARTMODE_PMEM); + mds->persistent_only_bytes, CXL_PARTMODE_PMEM, 0); return 0; } @@ -635,14 +623,68 @@ int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info) return rc; } - add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM); + add_part(info, 0, mds->active_volatile_bytes, CXL_PARTMODE_RAM, 0); add_part(info, mds->active_volatile_bytes, mds->active_persistent_bytes, - CXL_PARTMODE_PMEM); + CXL_PARTMODE_PMEM, 0); return 0; } EXPORT_SYMBOL_NS_GPL(cxl_mem_dpa_fetch, "CXL"); +int cxl_configure_dcd(struct cxl_memdev_state *mds, struct cxl_dpa_info *info) +{ + struct cxl_dc_partition_info dc_info = { }; + struct device *dev = mds->cxlds.dev; + u64 static_end = 0; + u64 dc_end; + int rc; + + if (!mds->cxlds.media_ready) + return 0; + + rc = cxl_dev_dc_identify(&mds->cxlds.cxl_mbox, &dc_info); + if (rc) { + dev_warn(dev, + "Failed to read Dynamic Capacity config: %d\n", rc); + return rc; + } + + if (info->nr_partitions) + static_end = info->part[info->nr_partitions - 1].range.end + 1; + + if (dc_info.start < static_end) { + dev_err(dev, + "DC partition 0 base %#llx overlaps static capacity ending at %#llx\n", + dc_info.start, static_end); + return -EINVAL; + } + + /* A gap between static capacity and the DC partition is not supported */ + if (dc_info.start > static_end) { + dev_warn(dev, + "DC partition 0 base %#llx leaves a gap from static capacity ending at %#llx\n", + dc_info.start, static_end); + return -EOPNOTSUPP; + } + + if (check_add_overflow(dc_info.start, dc_info.size, &dc_end)) { + dev_err(dev, + "DC partition 0 size %#llx overflows DPA space at base %#llx\n", + dc_info.size, dc_info.start); + return -EINVAL; + } + + /* The DPA space must reach the end of the DC partition */ + if (dc_end > info->size) + info->size = dc_end; + dev_dbg(dev, "Adding dynamic ram partition; %#llx size %#llx\n", + dc_info.start, dc_info.size); + add_part(info, dc_info.start, dc_info.size, CXL_PARTMODE_DYNAMIC_RAM, + dc_info.handle); + + return 0; +} +EXPORT_SYMBOL_NS_GPL(cxl_configure_dcd, "CXL"); /** * cxl_set_capacity: initialize dpa by a driver without a mailbox. @@ -656,7 +698,7 @@ int cxl_set_capacity(struct cxl_dev_state *cxlds, u64 capacity) .size = capacity, }; - add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM); + add_part(&range_info, 0, capacity, CXL_PARTMODE_RAM, 0); return cxl_dpa_setup(cxlds, &range_info); } EXPORT_SYMBOL_NS_GPL(cxl_set_capacity, "CXL"); diff --git a/drivers/cxl/core/pci.c b/drivers/cxl/core/pci.c index 9d807c1a002c..125abb36df7f 100644 --- a/drivers/cxl/core/pci.c +++ b/drivers/cxl/core/pci.c @@ -660,8 +660,8 @@ long cxl_pci_get_latency(struct pci_dev *pdev) { long bw; - bw = pcie_link_speed_mbps(pdev); - if (bw < 0) + bw = pcie_link_bandwidth_mbps(pdev); + if (bw <= 0) return 0; bw /= BITS_PER_BYTE; @@ -683,6 +683,15 @@ static int __cxl_endpoint_decoder_reset_detected(struct device *dev, void *data) if ((cxld->flags & CXL_DECODER_F_ENABLE) == 0) return 0; + /* + * Decoders emulated from the DVSEC range registers have no commit + * callback and no HDM decoder registers to consult. + */ + if (!cxld->commit) { + dev_warn(dev, "DVSEC emulated decode may have been cleared by reset\n"); + return 0; + } + cxlhdm = dev_get_drvdata(&port->dev); hdm = cxlhdm->regs.hdm_decoder; ctrl = readl(hdm + CXL_HDM_DECODER0_CTRL_OFFSET(cxld->id)); @@ -734,10 +743,16 @@ int cxl_pci_setup_regs(struct pci_dev *pdev, enum cxl_regloc_type type, if (rc && type == CXL_REGLOC_RBI_COMPONENT && is_cxl_restricted(pdev)) { struct cxl_dport *dport; struct cxl_port *port __free(put_cxl_port) = - cxl_pci_find_port(pdev, &dport); + cxl_pci_find_port(pdev); if (!port) return -EPROBE_DEFER; + guard(device)(cxl_port_dport_host(port)); + + dport = cxl_pci_find_dport(pdev, port); + if (!dport) + return -EPROBE_DEFER; + rc = cxl_rcrb_get_comp_regs(pdev, map, dport); if (rc) return rc; @@ -756,18 +771,12 @@ EXPORT_SYMBOL_NS_GPL(cxl_pci_setup_regs, "CXL"); int cxl_pci_get_bandwidth(struct pci_dev *pdev, struct access_coordinate *c) { - int speed, bw; - u16 lnksta; - u32 width; + int bw; - speed = pcie_link_speed_mbps(pdev); - if (speed < 0) - return speed; - speed /= BITS_PER_BYTE; - - pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); - width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); - bw = speed * width; + bw = pcie_link_bandwidth_mbps(pdev); + if (bw < 0) + return bw; + bw /= BITS_PER_BYTE; for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { c[i].read_bandwidth = bw; @@ -804,7 +813,8 @@ u16 cxl_gpf_get_dvsec(struct device *dev) return 0; pdev = to_pci_dev(dev); - if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ENDPOINT) + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ENDPOINT || + is_cxl_restricted(pdev)) is_port = false; dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, @@ -839,21 +849,27 @@ static int update_gpf_port_dvsec(struct pci_dev *pdev, int dvsec, int phase) rc = pci_read_config_word(pdev, dvsec + offset, &ctrl); if (rc) - return rc; + return pcibios_err_to_errno(rc); if (FIELD_GET(base, ctrl) == GPF_TIMEOUT_BASE_MAX && FIELD_GET(scale, ctrl) == GPF_TIMEOUT_SCALE_MAX) return 0; - ctrl = FIELD_PREP(base, GPF_TIMEOUT_BASE_MAX); - ctrl |= FIELD_PREP(scale, GPF_TIMEOUT_SCALE_MAX); + FIELD_MODIFY(base, &ctrl, GPF_TIMEOUT_BASE_MAX); + FIELD_MODIFY(scale, &ctrl, GPF_TIMEOUT_SCALE_MAX); rc = pci_write_config_word(pdev, dvsec + offset, ctrl); - if (!rc) - pci_dbg(pdev, "Port GPF phase %d timeout: %d0 secs\n", - phase, GPF_TIMEOUT_BASE_MAX); + if (rc) { + rc = pcibios_err_to_errno(rc); + pci_warn(pdev, "Port GPF phase %d timeout write failed: %d\n", + phase, rc); + return rc; + } - return rc; + pci_dbg(pdev, "Port GPF phase %d timeout: %d0 secs\n", + phase, GPF_TIMEOUT_BASE_MAX); + + return 0; } int cxl_gpf_port_setup(struct cxl_dport *dport) @@ -863,16 +879,22 @@ int cxl_gpf_port_setup(struct cxl_dport *dport) if (!dport->gpf_dvsec) { struct pci_dev *pdev; - int dvsec; + int dvsec, rc; dvsec = cxl_gpf_get_dvsec(dport->dport_dev); if (!dvsec) return -EINVAL; - dport->gpf_dvsec = dvsec; pdev = to_pci_dev(dport->dport_dev); - update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 1); - update_gpf_port_dvsec(pdev, dport->gpf_dvsec, 2); + rc = update_gpf_port_dvsec(pdev, dvsec, 1); + if (rc) + return rc; + + rc = update_gpf_port_dvsec(pdev, dvsec, 2); + if (rc) + return rc; + + dport->gpf_dvsec = dvsec; } return 0; diff --git a/drivers/cxl/core/port.c b/drivers/cxl/core/port.c index 625e4aa427db..f3705ea44db7 100644 --- a/drivers/cxl/core/port.c +++ b/drivers/cxl/core/port.c @@ -148,20 +148,29 @@ static ssize_t target_type_show(struct device *dev, } static DEVICE_ATTR_RO(target_type); +/* + * Interleave ways selects how many targets a decoder uses, but the target + * array is only nr_targets long. Bound array access by both. + */ +static int cxlsd_nr_used_targets(struct cxl_switch_decoder *cxlsd) +{ + return min(cxlsd->cxld.interleave_ways, cxlsd->nr_targets); +} + static ssize_t emit_target_list(struct cxl_switch_decoder *cxlsd, char *buf) { - struct cxl_decoder *cxld = &cxlsd->cxld; + int nr_used = cxlsd_nr_used_targets(cxlsd); ssize_t offset = 0; int i, rc = 0; - for (i = 0; i < cxld->interleave_ways; i++) { + for (i = 0; i < nr_used; i++) { struct cxl_dport *dport = cxlsd->target[i]; struct cxl_dport *next = NULL; if (!dport) break; - if (i + 1 < cxld->interleave_ways) + if (i + 1 < nr_used) next = cxlsd->target[i + 1]; rc = sysfs_emit_at(buf, offset, "%d%s", dport->port_id, next ? "," : ""); @@ -814,6 +823,11 @@ static int cxl_einj_inject(void *data, u64 type) DEFINE_DEBUGFS_ATTRIBUTE(cxl_einj_inject_fops, NULL, cxl_einj_inject, "0x%llx\n"); +static void remove_debugfs(void *dentry) +{ + debugfs_remove_recursive(dentry); +} + static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) { struct cxl_port *parent = parent_port_of(dport->port); @@ -832,6 +846,9 @@ static void cxl_debugfs_create_dport_dir(struct cxl_dport *dport) dir = cxl_debugfs_create_dir(dev_name(dport->dport_dev)); + if (devm_add_action_or_reset(dport_to_host(dport), remove_debugfs, dir)) + return; + debugfs_create_file("einj_inject", 0200, dir, dport, &cxl_einj_inject_fops); } @@ -1089,11 +1106,40 @@ static void cond_cxl_root_unlock(struct cxl_port *port) device_unlock(&port->dev); } +static int clear_decoder_target(struct device *dev, void *data) +{ + struct cxl_dport *dport = data; + struct cxl_switch_decoder *cxlsd; + + if (!is_switch_decoder(dev)) + return 0; + + cxlsd = to_cxl_switch_decoder(dev); + guard(rwsem_write)(&cxl_rwsem.region); + + /* + * A dport can occupy more than one position of an interleave, so + * scan the whole target list rather than stopping at the first hit. + */ + for (int i = 0; i < cxlsd->nr_targets; i++) { + if (cxlsd->target[i] != dport) + continue; + cxlsd->target[i] = NULL; + dev_dbg(dev, "dport%d removed from target list, index %d\n", + dport->port_id, i); + } + + return 0; +} + static void cxl_dport_remove(void *data) { struct cxl_dport *dport = data; struct cxl_port *port = dport->port; + /* counterpart of cxl_port_update_decoder_targets() */ + device_for_each_child(&port->dev, dport, clear_decoder_target); + port->nr_dports--; xa_erase(&port->dports, (unsigned long) dport->dport_dev); put_device(dport->dport_dev); @@ -1320,7 +1366,7 @@ static int add_ep(struct cxl_ep *new) { struct cxl_port *port = new->dport->port; - guard(device)(&port->dev); + device_lock_assert(&port->dev); if (port->dead) return -ENXIO; @@ -1336,6 +1382,11 @@ static int add_ep(struct cxl_ep *new) * Intermediate CXL ports are scanned based on the arrival of endpoints. * When those endpoints depart the port can be destroyed once all * endpoints that care about that port have been removed. + * + * Context: Caller must hold the device lock of @dport->port, which pins + * @dport, across the lookup of @dport and this call. @dport->port + * is never the CXL root port, whose dports are pinned by its + * uport_dev instead. */ static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev) { @@ -1358,13 +1409,11 @@ static int cxl_add_ep(struct cxl_dport *dport, struct device *ep_dev) struct cxl_find_port_ctx { const struct device *dport_dev; const struct cxl_port *parent_port; - struct cxl_dport **dport; }; static int match_port_by_dport(struct device *dev, const void *data) { const struct cxl_find_port_ctx *ctx = data; - struct cxl_dport *dport; struct cxl_port *port; if (!is_cxl_port(dev)) @@ -1373,10 +1422,7 @@ static int match_port_by_dport(struct device *dev, const void *data) return 0; port = to_cxl_port(dev); - dport = cxl_find_dport_by_dev(port, ctx->dport_dev); - if (ctx->dport) - *ctx->dport = dport; - return dport != NULL; + return !!cxl_find_dport_by_dev(port, ctx->dport_dev); } static struct cxl_port *__find_cxl_port_by_dport(struct cxl_find_port_ctx *ctx) @@ -1395,22 +1441,21 @@ static struct cxl_port *__find_cxl_port_by_dport(struct cxl_find_port_ctx *ctx) /** * find_cxl_port_by_dport - find a cxl_port by one of its targets * @dport_dev: device representing the dport target - * @dport: optional output of the 'struct cxl_dport' companion of the @dport_dev * * Return a 'struct cxl_port' with an elevated reference if found. Use * __free(put_cxl_port) to release. + * + * The port reference does not pin the dport, which is a devm allocation of + * cxl_port_dport_host(). Use cxl_pci_find_dport() or cxl_mem_find_dport() + * under that device's lock to get it. */ -static struct cxl_port *find_cxl_port_by_dport(struct device *dport_dev, - struct cxl_dport **dport) +static struct cxl_port *find_cxl_port_by_dport(struct device *dport_dev) { struct cxl_find_port_ctx ctx = { .dport_dev = dport_dev, - .dport = dport, }; - struct cxl_port *port; - port = __find_cxl_port_by_dport(&ctx); - return port; + return __find_cxl_port_by_dport(&ctx); } /* @@ -1611,7 +1656,7 @@ static int update_decoder_targets(struct device *dev, void *data) struct cxl_dport *dport = data; struct cxl_switch_decoder *cxlsd; struct cxl_decoder *cxld; - int i; + int i, nr_used; if (!is_switch_decoder(dev)) return 0; @@ -1619,8 +1664,9 @@ static int update_decoder_targets(struct device *dev, void *data) cxlsd = to_cxl_switch_decoder(dev); cxld = &cxlsd->cxld; guard(rwsem_write)(&cxl_rwsem.region); + nr_used = cxlsd_nr_used_targets(cxlsd); - for (i = 0; i < cxld->interleave_ways; i++) { + for (i = 0; i < nr_used; i++) { if (cxld->target_map[i] == dport->port_id) { cxlsd->target[i] = dport; dev_dbg(dev, "dport%d found in target list, index %d\n", @@ -1672,13 +1718,15 @@ static struct cxl_dport *probe_dport(struct cxl_port *port, return drv->add_dport(port, dport_dev); } -static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev, - struct cxl_port *parent_port, - struct cxl_dport *parent_dport, - struct device *uport_dev, - struct device *dport_dev) +static int devm_cxl_create_port(struct device *ep_dev, + struct cxl_port *parent_port, + struct cxl_dport *parent_dport, + struct device *uport_dev, + struct device *dport_dev) { resource_size_t component_reg_phys; + struct cxl_dport *dport; + int rc; device_lock_assert(&parent_port->dev); if (!parent_port->dev.driver) { @@ -1695,7 +1743,7 @@ static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev, port = devm_cxl_add_port(&parent_port->dev, uport_dev, component_reg_phys, parent_dport); if (IS_ERR(port)) - return ERR_CAST(port); + return PTR_ERR(port); /* * retry to make sure a port is found. a port device @@ -1703,7 +1751,7 @@ static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev, */ port = find_cxl_port_by_uport(uport_dev); if (!port) - return ERR_PTR(-ENODEV); + return -ENODEV; dev_dbg(ep_dev, "created port %s:%s\n", dev_name(&port->dev), dev_name(port->uport_dev)); @@ -1712,11 +1760,25 @@ static struct cxl_dport *devm_cxl_create_port(struct device *ep_dev, * Port was created before right before this function is * called. Signal the caller to deal with it. */ - return ERR_PTR(-EAGAIN); + return -EAGAIN; } + /* @dport is only valid while @port's lock is held */ guard(device)(&port->dev); - return probe_dport(port, dport_dev); + dport = probe_dport(port, dport_dev); + if (IS_ERR(dport)) + return PTR_ERR(dport); + + rc = cxl_add_ep(dport, ep_dev); + if (rc == -EBUSY) { + /* + * "can't" happen, but this error code means + * something to the caller, so translate it. + */ + rc = -ENXIO; + } + + return rc; } static int add_port_attach_ep(struct cxl_memdev *cxlmd, @@ -1724,7 +1786,7 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, struct device *dport_dev) { struct device *dparent = grandparent(dport_dev); - struct cxl_dport *dport, *parent_dport; + struct cxl_dport *parent_dport; int rc; if (is_cxl_host_bridge(dparent)) { @@ -1753,33 +1815,23 @@ static int add_port_attach_ep(struct cxl_memdev *cxlmd, return PTR_ERR(parent_dport); } - dport = devm_cxl_create_port(&cxlmd->dev, parent_port, - parent_dport, uport_dev, - dport_dev); - if (IS_ERR(dport)) { - /* Port or dport already exists, restart iteration */ - if (PTR_ERR(dport) == -EAGAIN || PTR_ERR(dport) == -EBUSY) - return 0; - return PTR_ERR(dport); - } + rc = devm_cxl_create_port(&cxlmd->dev, parent_port, + parent_dport, uport_dev, dport_dev); } - rc = cxl_add_ep(dport, &cxlmd->dev); - if (rc == -EBUSY) { - /* - * "can't" happen, but this error code means - * something to the caller, so translate it. - */ - rc = -ENXIO; - } + /* Port or dport already exists, restart iteration */ + if (rc == -EAGAIN || rc == -EBUSY) + return 0; return rc; } -static struct cxl_dport *find_or_add_dport(struct cxl_port *port, - struct device *dport_dev) +static int find_or_add_dport_attach_ep(struct cxl_memdev *cxlmd, + struct cxl_port *port, + struct device *dport_dev) { struct cxl_dport *dport; + int rc; /* * The port is already visible in CXL hierarchy, but it may still @@ -1795,13 +1847,28 @@ static struct cxl_dport *find_or_add_dport(struct cxl_port *port, if (!dport) { dport = probe_dport(port, dport_dev); if (IS_ERR(dport)) - return dport; + return PTR_ERR(dport); /* New dport added, restart iteration */ - return ERR_PTR(-EAGAIN); + return -EAGAIN; } - return dport; + /* @dport is only valid while @port's lock is held */ + rc = cxl_add_ep(dport, &cxlmd->dev); + + /* + * If the endpoint already exists in the port's list, + * that's ok, it was added on a previous pass. + * Otherwise, retry in add_port_attach_ep() after taking + * the parent_port lock as the current port may be being + * reaped. + */ + if (rc && rc != -EBUSY) + return rc; + + cxl_gpf_port_setup(dport); + + return 0; } int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) @@ -1830,7 +1897,6 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) for (iter = dev; iter; iter = grandparent(iter)) { struct device *dport_dev = grandparent(iter); struct device *uport_dev; - struct cxl_dport *dport; if (is_cxl_host_bridge(dport_dev)) return 0; @@ -1857,27 +1923,13 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) * RP port enumerated by cxl_acpi without dport will * have the dport added here. */ - dport = find_or_add_dport(port, dport_dev); - if (IS_ERR(dport)) { - if (PTR_ERR(dport) == -EAGAIN) - goto retry; - return PTR_ERR(dport); - } - - rc = cxl_add_ep(dport, &cxlmd->dev); - - /* - * If the endpoint already exists in the port's list, - * that's ok, it was added on a previous pass. - * Otherwise, retry in add_port_attach_ep() after taking - * the parent_port lock as the current port may be being - * reaped. - */ - if (rc && rc != -EBUSY) + rc = find_or_add_dport_attach_ep(cxlmd, port, + dport_dev); + if (rc == -EAGAIN) + goto retry; + if (rc) return rc; - cxl_gpf_port_setup(dport); - /* Any more ports to add between this one and the root? */ if (!dev_is_cxl_root_child(&port->dev)) continue; @@ -1900,25 +1952,55 @@ int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd) } EXPORT_SYMBOL_NS_GPL(devm_cxl_enumerate_ports, "CXL"); -struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev, - struct cxl_dport **dport) +struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev) { - return find_cxl_port_by_dport(pdev->dev.parent, dport); + return find_cxl_port_by_dport(pdev->dev.parent); } EXPORT_SYMBOL_NS_GPL(cxl_pci_find_port, "CXL"); -struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd, - struct cxl_dport **dport) +struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd) { - return find_cxl_port_by_dport(grandparent(&cxlmd->dev), dport); + return find_cxl_port_by_dport(grandparent(&cxlmd->dev)); } EXPORT_SYMBOL_NS_GPL(cxl_mem_find_port, "CXL"); +/** + * cxl_pci_find_dport - find the dport of @port that @pdev is below + * @pdev: PCI device below the dport + * @port: port to search, typically from cxl_pci_find_port() + * + * Context: Caller must hold the cxl_port_dport_host() lock of @port, and + * must not use the result after dropping it. + */ +struct cxl_dport *cxl_pci_find_dport(struct pci_dev *pdev, + struct cxl_port *port) +{ + device_lock_assert(cxl_port_dport_host(port)); + return cxl_find_dport_by_dev(port, pdev->dev.parent); +} +EXPORT_SYMBOL_NS_GPL(cxl_pci_find_dport, "CXL"); + +/** + * cxl_mem_find_dport - find the dport of @port that @cxlmd is below + * @cxlmd: memdev below the dport + * @port: port to search, typically from cxl_mem_find_port() + * + * Context: Caller must hold the cxl_port_dport_host() lock of @port, and + * must not use the result after dropping it. + */ +struct cxl_dport *cxl_mem_find_dport(struct cxl_memdev *cxlmd, + struct cxl_port *port) +{ + device_lock_assert(cxl_port_dport_host(port)); + return cxl_find_dport_by_dev(port, grandparent(&cxlmd->dev)); +} +EXPORT_SYMBOL_NS_GPL(cxl_mem_find_dport, "CXL"); + static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd, struct cxl_port *port) { struct cxl_decoder *cxld = &cxlsd->cxld; - int i; + int i, nr_used; device_lock_assert(&port->dev); @@ -1926,7 +2008,8 @@ static int decoder_populate_targets(struct cxl_switch_decoder *cxlsd, return 0; guard(rwsem_write)(&cxl_rwsem.region); - for (i = 0; i < cxlsd->cxld.interleave_ways; i++) { + nr_used = cxlsd_nr_used_targets(cxlsd); + for (i = 0; i < nr_used; i++) { struct cxl_dport *dport = find_dport(port, cxld->target_map[i]); if (!dport) { @@ -1987,7 +2070,7 @@ static int cxl_switch_decoder_init(struct cxl_port *port, struct cxl_switch_decoder *cxlsd, int nr_targets) { - if (nr_targets > CXL_DECODER_MAX_INTERLEAVE) + if (nr_targets < 1 || nr_targets > CXL_DECODER_MAX_INTERLEAVE) return -EINVAL; cxlsd->nr_targets = nr_targets; @@ -2320,6 +2403,9 @@ EXPORT_SYMBOL_NS_GPL(schedule_cxl_memdev_detach, "CXL"); static void add_latency(struct access_coordinate *c, long latency) { + /* Convert link latency from picoseconds to nanoseconds. */ + latency = DIV_ROUND_UP(latency, 1000); + for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { c[i].write_latency += latency; c[i].read_latency += latency; @@ -2394,8 +2480,10 @@ int cxl_endpoint_get_perf_coordinates(struct cxl_port *port, * Skip calculation for RCD. Expectation is HMAT already covers RCD case * since RCH does not support hotplug. */ - if (cxlmd->cxlds->rcd) + if (cxlmd->cxlds->rcd) { + memset(coord, 0, sizeof(*coord) * ACCESS_COORDINATE_MAX); return 0; + } /* * Exit the loop when the parent port of the current iter port is cxl diff --git a/drivers/cxl/core/ras.c b/drivers/cxl/core/ras.c index c91db125a971..a2c7c9479f2b 100644 --- a/drivers/cxl/core/ras.c +++ b/drivers/cxl/core/ras.c @@ -70,9 +70,9 @@ cxl_cper_trace_uncorr_prot_err(struct cxl_memdev *cxlmd, trace_cxl_aer_uncorrectable_error(cxlmd, status, fe, hl); } -static int match_memdev_by_parent(struct device *dev, const void *uport) +static int match_memdev_by_parent(struct device *dev, const void *parent) { - if (is_cxl_memdev(dev) && dev->parent == uport) + if (is_cxl_memdev(dev) && dev->parent == parent) return 1; return 0; } @@ -108,7 +108,7 @@ static void cxl_cper_handle_prot_err(struct cxl_cper_prot_err_work_data *data) return; struct device *mem_dev __free(put_device) = bus_find_device( - &cxl_bus_type, NULL, pdev, match_memdev_by_parent); + &cxl_bus_type, NULL, &pdev->dev, match_memdev_by_parent); if (!mem_dev) return; diff --git a/drivers/cxl/core/ras_rch.c b/drivers/cxl/core/ras_rch.c index 50abbdc80722..34a1eb2169ba 100644 --- a/drivers/cxl/core/ras_rch.c +++ b/drivers/cxl/core/ras_rch.c @@ -107,10 +107,20 @@ void cxl_handle_rdport_errors(struct cxl_dev_state *cxlds) int severity; struct cxl_port *port __free(put_cxl_port) = - cxl_pci_find_port(pdev, &dport); + cxl_pci_find_port(pdev); if (!port) return; + /* + * The dport may have been freed since the lookup above, so take the + * lock that pins it and find it again. + */ + guard(device)(cxl_port_dport_host(port)); + + dport = cxl_pci_find_dport(pdev, port); + if (!dport) + return; + if (!cxl_rch_get_aer_info(dport->regs.dport_aer, &aer_regs)) return; diff --git a/drivers/cxl/core/region.c b/drivers/cxl/core/region.c index 27e63e6dab7c..a048bd5c06cb 100644 --- a/drivers/cxl/core/region.c +++ b/drivers/cxl/core/region.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2022 Intel Corporation. All rights reserved. */ +#include #include #include #include @@ -1367,7 +1368,8 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled, /* * If this position wants to share a dport with the last endpoint mapped * then that endpoint, at index 'position - distance', must also be - * mapped by this dport. + * mapped by this dport. An endpoint that this port does not map at all + * fails that requirement. */ if (pos < distance) { dev_dbg(&cxlr->dev, "%s:%s: cannot host %s:%s at %d\n", @@ -1378,7 +1380,7 @@ static int check_last_peer(struct cxl_endpoint_decoder *cxled, cxled_peer = p->targets[pos - distance]; cxlmd_peer = cxled_to_memdev(cxled_peer); ep_peer = cxl_ep_load(port, cxlmd_peer); - if (ep->dport != ep_peer->dport) { + if (!ep_peer || ep->dport != ep_peer->dport) { dev_dbg(&cxlr->dev, "%s:%s: %s:%s pos %d mismatched peer %s:%s\n", dev_name(port->uport_dev), dev_name(&port->dev), @@ -2112,7 +2114,7 @@ static int cxl_region_attach(struct cxl_region *cxlr, return -ENXIO; } - if (!cxled->dpa_res) { + if (cxled_empty(cxled)) { dev_dbg(&cxlr->dev, "%s:%s: missing DPA allocation.\n", dev_name(&cxlmd->dev), dev_name(&cxled->cxld.dev)); return -ENXIO; @@ -2215,10 +2217,11 @@ static int cxl_region_attach(struct cxl_region *cxlr, int test_pos; test_pos = cxl_calc_interleave_pos(target, &cxlr->hpa_range); - dev_dbg(&target->cxld.dev, - "Test cxl_calc_interleave_pos(): %s test_pos:%d target->pos:%d\n", - (test_pos == target->pos) ? "success" : "fail", - test_pos, target->pos); + if (test_pos != target->pos) + dev_warn(&cxlr->dev, + "%s: position mismatch: calculated:%d assigned:%d\n", + dev_name(&target->cxld.dev), test_pos, + target->pos); } return 0; @@ -2630,7 +2633,19 @@ static bool cxl_region_update_coordinates(struct cxl_region *cxlr, int nid) for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { if (cxlr->coord[i].read_bandwidth) { - node_update_perf_attrs(nid, &cxlr->coord[i], i); + /* + * A CFMWS-only node (not backed by a real SRAT pxm) + * has no HMAT memory_target, so hmat_callback() + * never created the access attributes for it: + * create them here instead of only updating them. + * This is called at most once per node (see + * nodemask_region_seen), so the attributes are not + * created twice. + */ + if (!acpi_node_backed_by_real_pxm(nid)) + node_set_perf_attrs(nid, &cxlr->coord[i], i); + else + node_update_perf_attrs(nid, &cxlr->coord[i], i); cset++; } } @@ -2727,7 +2742,7 @@ void kill_regions(struct cxl_root_decoder *cxlrd) * This is the second step of region initialization. Regions exist within an * address space which is mapped by a @cxlrd. * - * Return: 0 if the region was added to the @cxlrd, else returns negative error + * Return: the new region if it was added to the @cxlrd, else an ERR_PTR() with the error * code. The region will be named "regionZ" where Z is the unique region number. */ static struct cxl_region *devm_cxl_add_region(struct cxl_root_decoder *cxlrd, @@ -2902,6 +2917,16 @@ struct cxl_poison_context { u64 offset; }; +/* + * A device may answer a Get Poison List request with "physical address + * specified is invalid" (-EFAULT). That answer is tolerated for a RAM + * partition and the poison walk continues. + */ +static inline bool poison_efault_forgiven(int rc, enum cxl_partition_mode mode) +{ + return rc == -EFAULT && mode == CXL_PARTMODE_RAM; +} + static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd, struct cxl_poison_context *ctx) { @@ -2930,7 +2955,7 @@ static int cxl_get_poison_unmapped(struct cxl_memdev *cxlmd, if (!length) break; rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); - if (rc == -EFAULT && cxlds->part[i].mode == CXL_PARTMODE_RAM) + if (poison_efault_forgiven(rc, cxlds->part[i].mode)) continue; if (rc) break; @@ -2947,37 +2972,40 @@ static int poison_by_decoder(struct device *dev, void *arg) struct cxl_dev_state *cxlds; struct cxl_memdev *cxlmd; u64 offset, length; - int rc = 0; + int rc; if (!is_endpoint_decoder(dev)) - return rc; + return 0; cxled = to_cxl_endpoint_decoder(dev); if (!cxled->dpa_res) - return rc; + return 0; - cxlmd = cxled_to_memdev(cxled); - cxlds = cxlmd->cxlds; - mode = cxlds->part[cxled->part].mode; + /* + * Handle the degenerate case of a device with only empty decoders. An + * empty decoder can still map a non-zero skip range, so advance the + * walk to commit_end either way. + */ + if (cxled->part >= 0) { + cxlmd = cxled_to_memdev(cxled); + cxlds = cxlmd->cxlds; + mode = cxlds->part[cxled->part].mode; - if (cxled->skip) { - offset = cxled->dpa_res->start - cxled->skip; - length = cxled->skip; - rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); - if (rc == -EFAULT && mode == CXL_PARTMODE_RAM) - rc = 0; - if (rc) + if (cxled->skip) { + offset = cxled->dpa_res->start - cxled->skip; + length = cxled->skip; + rc = cxl_mem_get_poison(cxlmd, offset, length, NULL); + if (rc && !poison_efault_forgiven(rc, mode)) + return rc; + } + + offset = cxled->dpa_res->start; + length = cxled->dpa_res->end - offset + 1; + rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region); + if (rc && !poison_efault_forgiven(rc, mode)) return rc; } - offset = cxled->dpa_res->start; - length = cxled->dpa_res->end - offset + 1; - rc = cxl_mem_get_poison(cxlmd, offset, length, cxled->cxld.region); - if (rc == -EFAULT && mode == CXL_PARTMODE_RAM) - rc = 0; - if (rc) - return rc; - /* Iterate until commit_end is reached */ if (cxled->cxld.id == ctx->port->commit_end) { ctx->offset = cxled->dpa_res->end + 1; @@ -2999,9 +3027,17 @@ int cxl_get_poison_by_endpoint(struct cxl_port *port) }; rc = device_for_each_child(&port->dev, &ctx, poison_by_decoder); - if (rc == 1) + if (rc == 1) { + /* + * No decoder with a sized DPA reservation was walked + * (every committed decoder is zero-size): scan all + * partitions in full. + */ + if (ctx.part < 0) + ctx.part = 0; rc = cxl_get_poison_unmapped(to_cxl_memdev(port->uport_dev), &ctx); + } return rc; } diff --git a/drivers/cxl/core/region_pmem.c b/drivers/cxl/core/region_pmem.c index 23d97e3d78b6..7ab1373a95e0 100644 --- a/drivers/cxl/core/region_pmem.c +++ b/drivers/cxl/core/region_pmem.c @@ -168,12 +168,14 @@ int devm_cxl_add_pmem_region(struct cxl_region *cxlr) dev_name(dev)); scoped_guard(device, &cxl_nvb->dev) { - if (cxl_nvb->dev.driver) + if (cxl_nvb->dev.driver) { rc = devm_add_action_or_reset(&cxl_nvb->dev, cxlr_pmem_unregister, cxlr_pmem); - else + } else { rc = -ENXIO; + cxlr_pmem_unregister(cxlr_pmem); + } } if (rc) diff --git a/drivers/cxl/core/regs.c b/drivers/cxl/core/regs.c index 20c2d9fbcfe7..41416fa2ce4a 100644 --- a/drivers/cxl/core/regs.c +++ b/drivers/cxl/core/regs.c @@ -277,6 +277,20 @@ static bool cxl_decode_regblock(struct pci_dev *pdev, u32 reg_lo, u32 reg_hi, u64 offset = ((u64)reg_hi << 32) | (reg_lo & PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW); + if (reg_type == CXL_REGLOC_RBI_EMPTY) + return false; + + /* + * A BAR the PCI core could not place is reset to zero; decoding it + * would map the block at physical address 0. + */ + if (!pci_resource_len(pdev, bar) || + (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)) { + dev_warn(&pdev->dev, "BAR%d: not assigned (type: %d)\n", bar, + reg_type); + return false; + } + if (offset > pci_resource_len(pdev, bar)) { dev_warn(&pdev->dev, "BAR%d: %pr: too small (offset: %pa, type: %d)\n", bar, diff --git a/drivers/cxl/cxl.h b/drivers/cxl/cxl.h index cab8ce39f465..baa6df2d362a 100644 --- a/drivers/cxl/cxl.h +++ b/drivers/cxl/cxl.h @@ -67,6 +67,8 @@ extern const struct nvdimm_security_ops *cxl_security_ops; #define CXL_HDM_DECODER0_CTRL_HOSTONLY BIT(12) #define CXL_HDM_DECODER0_TL_LOW(i) (0x20 * (i) + 0x24) #define CXL_HDM_DECODER0_TL_HIGH(i) (0x20 * (i) + 0x28) +/* Target list capacity, i.e. max interleave ways for a non-endpoint decoder */ +#define CXL_HDM_DECODER0_TL_TARGETS 8 #define CXL_HDM_DECODER0_SKIP_LOW(i) CXL_HDM_DECODER0_TL_LOW(i) #define CXL_HDM_DECODER0_SKIP_HIGH(i) CXL_HDM_DECODER0_TL_HIGH(i) @@ -192,11 +194,15 @@ static inline int ways_to_eiw(unsigned int ways, u8 *eiw) #define CXLDEV_EVENT_STATUS_WARN BIT(1) #define CXLDEV_EVENT_STATUS_FAIL BIT(2) #define CXLDEV_EVENT_STATUS_FATAL BIT(3) - -#define CXLDEV_EVENT_STATUS_ALL (CXLDEV_EVENT_STATUS_INFO | \ - CXLDEV_EVENT_STATUS_WARN | \ - CXLDEV_EVENT_STATUS_FAIL | \ - CXLDEV_EVENT_STATUS_FATAL) +#define CXLDEV_EVENT_STATUS_DCD BIT(4) +/* + * Logs whose ownership follows native_cxl_error; the DCD log is always + * OS owned. + */ +#define CXLDEV_EVENT_STATUS_STANDARD_LOGS (CXLDEV_EVENT_STATUS_INFO | \ + CXLDEV_EVENT_STATUS_WARN | \ + CXLDEV_EVENT_STATUS_FAIL | \ + CXLDEV_EVENT_STATUS_FATAL) /* CXL rev 3.0 section 8.2.9.2.4; Table 8-52 */ #define CXLDEV_EVENT_INT_MODE_MASK GENMASK(1, 0) @@ -336,6 +342,16 @@ struct cxl_endpoint_decoder { int pos; }; +/* + * The common case is decoders with no reservation, but also handle + * decoders with a zero-sized reservation that firmware may install for + * security lockdown purposes. + */ +static inline bool cxled_empty(struct cxl_endpoint_decoder *cxled) +{ + return !cxled->dpa_res || !resource_size(cxled->dpa_res); +} + /** * struct cxl_switch_decoder - Switch specific CXL HDM Decoder * @cxld: base cxl_decoder object @@ -701,6 +717,18 @@ static inline bool is_cxl_root(struct cxl_port *port) return port->uport_dev == port->dev.parent; } +/* + * Return the owner device of the devm allocations of @port's dports. They + * are freed when that device unbinds, so hold device_lock() on the returned + * device across any lookup and use of a 'struct cxl_dport'. + */ +static inline struct device *cxl_port_dport_host(struct cxl_port *port) +{ + if (is_cxl_root(port)) + return port->uport_dev; + return &port->dev; +} + /* Address translation functions exported to cxl_translate test module only */ int cxl_validate_translation_params(u8 eiw, u16 eig, int pos); u64 cxl_calculate_hpa_offset(u64 dpa_offset, int pos, u8 eiw, u16 eig); @@ -747,10 +775,12 @@ DEFINE_FREE(put_cxl_dax_region, struct cxl_dax_region *, if (!IS_ERR_OR_NULL(_T) int devm_cxl_enumerate_ports(struct cxl_memdev *cxlmd); void cxl_bus_rescan(void); void cxl_bus_drain(void); -struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev, - struct cxl_dport **dport); -struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd, - struct cxl_dport **dport); +struct cxl_dport *cxl_pci_find_dport(struct pci_dev *pdev, + struct cxl_port *port); +struct cxl_dport *cxl_mem_find_dport(struct cxl_memdev *cxlmd, + struct cxl_port *port); +struct cxl_port *cxl_pci_find_port(struct pci_dev *pdev); +struct cxl_port *cxl_mem_find_port(struct cxl_memdev *cxlmd); bool schedule_cxl_memdev_detach(struct cxl_memdev *cxlmd); struct cxl_dport *devm_cxl_add_dport(struct cxl_port *port, diff --git a/drivers/cxl/cxlmem.h b/drivers/cxl/cxlmem.h index c401e3a1af06..35268d1c387c 100644 --- a/drivers/cxl/cxlmem.h +++ b/drivers/cxl/cxlmem.h @@ -140,6 +140,7 @@ struct cxl_dpa_info { struct cxl_dpa_part_info { struct range range; enum cxl_partition_mode mode; + u8 handle; } part[CXL_NR_PARTITIONS_MAX]; int nr_partitions; }; @@ -239,7 +240,9 @@ struct cxl_event_interrupt_policy { u8 warn_settings; u8 failure_settings; u8 fatal_settings; + u8 dcd_settings; } __packed; +#define CXL_EVENT_INT_POLICY_BASE_SIZE 4 /* info, warn, failure, fatal */ /** * struct cxl_event_state - Event log driver state @@ -252,6 +255,20 @@ struct cxl_event_state { struct mutex log_lock; }; +/* + * CXL r4.0 Section 8.2.10.9 - Memory Device Command Sets. See Table 8-308. + * + * The 48h Command Set (Opcodes 4800h - 4803h) defines the device-enabled DCD + * commands. + */ +enum dcd_cmd_enabled_bits { + CXL_DCD_ENABLED_GET_CONFIG, + CXL_DCD_ENABLED_GET_EXTENT_LIST, + CXL_DCD_ENABLED_ADD_RESPONSE, + CXL_DCD_ENABLED_RELEASE, + CXL_DCD_ENABLED_MAX +}; + /* Device enabled poison commands */ enum poison_cmd_enabled_bits { CXL_POISON_ENABLED_LIST, @@ -393,14 +410,13 @@ struct cxl_security_state { struct kernfs_node *sanitize_node; }; -static inline resource_size_t cxl_pmem_size(struct cxl_dev_state *cxlds) +#define CXL_MAX_DC_PARTITIONS 8 + +static inline resource_size_t cxl_part_size(struct cxl_dev_state *cxlds, + enum cxl_partition_mode mode) { - /* - * Static PMEM may be at partition index 0 when there is no static RAM - * capacity. - */ for (int i = 0; i < cxlds->nr_partitions; i++) - if (cxlds->part[i].mode == CXL_PARTMODE_PMEM) + if (cxlds->part[i].mode == mode) return resource_size(&cxlds->part[i].res); return 0; } @@ -427,6 +443,7 @@ static inline struct cxl_dev_state *mbox_to_cxlds(struct cxl_mailbox *cxl_mbox) * @partition_align_bytes: alignment size for partition-able capacity * @active_volatile_bytes: sum of hard + soft volatile * @active_persistent_bytes: sum of hard + soft persistent + * @dcd_supported: all DCD commands are supported * @event: event log driver state * @poison: poison driver state info * @security: security driver state info @@ -445,6 +462,7 @@ struct cxl_memdev_state { u64 partition_align_bytes; u64 active_volatile_bytes; u64 active_persistent_bytes; + bool dcd_supported; struct cxl_event_state event; struct cxl_poison_state poison; @@ -505,6 +523,10 @@ enum cxl_opcode { CXL_MBOX_OP_UNLOCK = 0x4503, CXL_MBOX_OP_FREEZE_SECURITY = 0x4504, CXL_MBOX_OP_PASSPHRASE_SECURE_ERASE = 0x4505, + CXL_MBOX_OP_GET_DC_CONFIG = 0x4800, + CXL_MBOX_OP_GET_DC_EXTENT_LIST = 0x4801, + CXL_MBOX_OP_ADD_DC_RESPONSE = 0x4802, + CXL_MBOX_OP_RELEASE_DC = 0x4803, CXL_MBOX_OP_MAX = 0x10000 }; @@ -669,6 +691,39 @@ struct cxl_mbox_set_shutdown_state_in { u8 state; } __packed; +/* See CXL r4.0 Table 8-345 get dynamic capacity config Input Payload */ +struct cxl_mbox_get_dc_config_in { + u8 partition_count; + u8 start_partition_index; +} __packed; + +/* See CXL r4.0 Table 8-346 get dynamic capacity config Output Payload */ +struct cxl_mbox_get_dc_config_out { + u8 avail_partition_count; + u8 partitions_returned; + u8 rsvd[6]; + /* See CXL r4.0 Table 8-347 */ + struct cxl_dc_partition { + __le64 base; + __le64 decode_length; + __le64 length; + __le64 block_size; + __le32 dsmad_handle; + u8 flags; + u8 rsvd[3]; + } __packed partition[] __counted_by(partitions_returned); + /* Trailing extent/tag count fields unused */ +} __packed; + +/* Trailing counts; cannot be a member after the flex array above */ +struct cxl_mbox_get_dc_config_tail { + __le32 num_extents_supported; + __le32 num_extents_available; + __le32 num_tags_supported; + __le32 num_tags_available; +} __packed; +#define CXL_DCD_BLOCK_LINE_SIZE 0x40 + /* Set Timestamp CXL 3.0 Spec 8.2.9.4.2 */ struct cxl_mbox_set_timestamp_in { __le64 timestamp; @@ -792,9 +847,19 @@ enum { int cxl_internal_send_cmd(struct cxl_mailbox *cxl_mbox, struct cxl_mbox_cmd *cmd); int cxl_dev_state_identify(struct cxl_memdev_state *mds); + +struct cxl_dc_partition_info { + u64 start; + u64 size; + u8 handle; +}; + +int cxl_dev_dc_identify(struct cxl_mailbox *mbox, + struct cxl_dc_partition_info *dc_info); int cxl_await_media_ready(struct cxl_dev_state *cxlds); int cxl_enumerate_cmds(struct cxl_memdev_state *mds); int cxl_mem_dpa_fetch(struct cxl_memdev_state *mds, struct cxl_dpa_info *info); +int cxl_configure_dcd(struct cxl_memdev_state *mds, struct cxl_dpa_info *info); struct cxl_memdev_state *cxl_memdev_state_create(struct device *dev, u64 serial, u16 dvsec); void set_exclusive_cxl_commands(struct cxl_memdev_state *mds, @@ -808,6 +873,17 @@ void cxl_event_trace_record(struct cxl_memdev *cxlmd, const uuid_t *uuid, union cxl_event *evt); int cxl_get_dirty_count(struct cxl_memdev_state *mds, u32 *count); int cxl_arm_dirty_shutdown(struct cxl_memdev_state *mds); + +static inline bool cxl_dcd_supported(struct cxl_memdev_state *mds) +{ + return mds->dcd_supported; +} + +static inline void cxl_disable_dcd(struct cxl_memdev_state *mds) +{ + mds->dcd_supported = false; +} + int cxl_set_timestamp(struct cxl_memdev_state *mds); int cxl_poison_state_init(struct cxl_memdev_state *mds); int cxl_mem_get_poison(struct cxl_memdev *cxlmd, u64 offset, u64 len, diff --git a/drivers/cxl/mem.c b/drivers/cxl/mem.c index 798e5c369cfc..6fba230905c1 100644 --- a/drivers/cxl/mem.c +++ b/drivers/cxl/mem.c @@ -50,8 +50,13 @@ static int cxl_debugfs_poison_inject(void *data, u64 dpa) struct cxl_memdev *cxlmd = data; int rc; - ACQUIRE(device_intr, devlock)(&cxlmd->dev); - if ((rc = ACQUIRE_ERR(device_intr, &devlock))) + /* + * Never wait for this lock: the debugfs proxy holds a file reference + * across the callback and unbind removes the file under the same + * device lock, so waiting here deadlocks against unbind. + */ + ACQUIRE(device_try, devlock)(&cxlmd->dev); + if ((rc = ACQUIRE_ERR(device_try, &devlock))) return rc; return cxl_inject_poison(cxlmd, dpa); @@ -65,8 +70,9 @@ static int cxl_debugfs_poison_clear(void *data, u64 dpa) struct cxl_memdev *cxlmd = data; int rc; - ACQUIRE(device_intr, devlock)(&cxlmd->dev); - if ((rc = ACQUIRE_ERR(device_intr, &devlock))) + /* Never wait, per the inject path above. */ + ACQUIRE(device_try, devlock)(&cxlmd->dev); + if ((rc = ACQUIRE_ERR(device_try, &devlock))) return rc; return cxl_clear_poison(cxlmd, dpa); @@ -133,13 +139,13 @@ static int cxl_mem_probe(struct device *dev) return rc; struct cxl_port *parent_port __free(put_cxl_port) = - cxl_mem_find_port(cxlmd, &dport); + cxl_mem_find_port(cxlmd); if (!parent_port) { dev_err(dev, "CXL port topology not found\n"); return -ENXIO; } - if (cxl_pmem_size(cxlds) && IS_ENABLED(CONFIG_CXL_PMEM)) { + if (cxl_part_size(cxlds, CXL_PARTMODE_PMEM) && IS_ENABLED(CONFIG_CXL_PMEM)) { rc = devm_cxl_add_nvdimm(dev, parent_port, cxlmd); if (rc) { if (rc == -ENODEV) @@ -148,10 +154,7 @@ static int cxl_mem_probe(struct device *dev) } } - if (dport->rch) - endpoint_parent = parent_port->uport_dev; - else - endpoint_parent = &parent_port->dev; + endpoint_parent = cxl_port_dport_host(parent_port); scoped_guard(device, endpoint_parent) { if (!endpoint_parent->driver) { @@ -160,6 +163,13 @@ static int cxl_mem_probe(struct device *dev) return -ENXIO; } + /* @dport is only valid while @endpoint_parent stays bound */ + dport = cxl_mem_find_dport(cxlmd, parent_port); + if (!dport) { + dev_err(dev, "CXL topology dport not found\n"); + return -ENXIO; + } + rc = devm_cxl_add_endpoint(endpoint_parent, cxlmd, dport); if (rc) return rc; diff --git a/drivers/cxl/pci.c b/drivers/cxl/pci.c index b6dcfcf6eb5b..3ed7c3e70042 100644 --- a/drivers/cxl/pci.c +++ b/drivers/cxl/pci.c @@ -510,11 +510,23 @@ static bool cxl_alloc_irq_vectors(struct pci_dev *pdev) return true; } +/* Event logs the driver drains: standard logs when native_cxl */ +static u32 cxl_event_drain_mask(struct pci_host_bridge *host_bridge) +{ + if (host_bridge->native_cxl_error) + return CXLDEV_EVENT_STATUS_STANDARD_LOGS; + return 0; +} + static irqreturn_t cxl_event_thread(int irq, void *id) { struct cxl_dev_id *dev_id = id; struct cxl_dev_state *cxlds = dev_id->cxlds; struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); + struct pci_host_bridge *host_bridge = + pci_find_host_bridge(to_pci_dev(cxlds->dev)->bus); + u32 mask = cxl_event_drain_mask(host_bridge); + bool handled = false; u32 status; do { @@ -523,15 +535,17 @@ static irqreturn_t cxl_event_thread(int irq, void *id) * ignore the reserved upper 32 bits */ status = readl(cxlds->regs.status + CXLDEV_DEV_EVENT_STATUS_OFFSET); - /* Ignore logs unknown to the driver */ - status &= CXLDEV_EVENT_STATUS_ALL; + /* Ignore logs unknown to the driver or owned by BIOS */ + status &= mask; if (!status) break; + + handled = true; cxl_mem_get_event_records(mds, status); cond_resched(); } while (status); - return IRQ_HANDLED; + return handled ? IRQ_HANDLED : IRQ_NONE; } static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting) @@ -551,84 +565,108 @@ static int cxl_event_req_irq(struct cxl_dev_state *cxlds, u8 setting) } static int cxl_event_get_int_policy(struct cxl_memdev_state *mds, - struct cxl_event_interrupt_policy *policy) + struct cxl_event_interrupt_policy *policy, + size_t *policy_size) { struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; struct cxl_mbox_cmd mbox_cmd = { .opcode = CXL_MBOX_OP_GET_EVT_INT_POLICY, .payload_out = policy, .size_out = sizeof(*policy), + /* CXL 2.0 firmware omits dcd_settings; accept the shorter reply */ + .min_out = CXL_EVENT_INT_POLICY_BASE_SIZE, + }; + int rc; + + rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); + if (rc < 0) { + dev_err(mds->cxlds.dev, + "Failed to get event interrupt policy : %d", rc); + return rc; + } + + if (policy_size) + *policy_size = mbox_cmd.size_out; + return rc; +} + +static int cxl_event_set_int_policy(struct cxl_memdev_state *mds, + struct cxl_event_interrupt_policy *policy, + size_t policy_size) +{ + struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; + struct cxl_mbox_cmd mbox_cmd = { + .opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY, + .payload_in = policy, + .size_in = policy_size, }; int rc; rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); if (rc < 0) - dev_err(mds->cxlds.dev, - "Failed to get event interrupt policy : %d", rc); - + dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d", + rc); return rc; } static int cxl_event_config_msgnums(struct cxl_memdev_state *mds, - struct cxl_event_interrupt_policy *policy) + struct cxl_event_interrupt_policy *policy, + bool native_cxl, size_t policy_size) { - struct cxl_mailbox *cxl_mbox = &mds->cxlds.cxl_mbox; - struct cxl_mbox_cmd mbox_cmd; int rc; - *policy = (struct cxl_event_interrupt_policy) { - .info_settings = CXL_INT_MSI_MSIX, - .warn_settings = CXL_INT_MSI_MSIX, - .failure_settings = CXL_INT_MSI_MSIX, - .fatal_settings = CXL_INT_MSI_MSIX, - }; - - mbox_cmd = (struct cxl_mbox_cmd) { - .opcode = CXL_MBOX_OP_SET_EVT_INT_POLICY, - .payload_in = policy, - .size_in = sizeof(*policy), - }; - - rc = cxl_internal_send_cmd(cxl_mbox, &mbox_cmd); - if (rc < 0) { - dev_err(mds->cxlds.dev, "Failed to set event interrupt policy : %d", - rc); - return rc; + /* memory event policy is left if FW has control */ + if (native_cxl) { + *policy = (struct cxl_event_interrupt_policy) { + .info_settings = CXL_INT_MSI_MSIX, + .warn_settings = CXL_INT_MSI_MSIX, + .failure_settings = CXL_INT_MSI_MSIX, + .fatal_settings = CXL_INT_MSI_MSIX, + .dcd_settings = 0, + }; } - /* Retrieve final interrupt settings */ - return cxl_event_get_int_policy(mds, policy); -} + /* + * A CXL 3.0+ device can carry dcd_settings field without DCD command + * support, so size the request by the device's policy_size and only + * enable the DCD interrupt when DCD commands are supported. + */ + if (cxl_dcd_supported(mds)) + policy->dcd_settings = CXL_INT_MSI_MSIX; -static int cxl_event_irqsetup(struct cxl_memdev_state *mds) -{ - struct cxl_dev_state *cxlds = &mds->cxlds; - struct cxl_event_interrupt_policy policy; - int rc; - - rc = cxl_event_config_msgnums(mds, &policy); + rc = cxl_event_set_int_policy(mds, policy, policy_size); if (rc) return rc; - rc = cxl_event_req_irq(cxlds, policy.info_settings); + /* Retrieve final interrupt settings */ + return cxl_event_get_int_policy(mds, policy, NULL); +} + +static int cxl_event_irqsetup(struct cxl_memdev_state *mds, + struct cxl_event_interrupt_policy *policy) +{ + struct cxl_dev_state *cxlds = &mds->cxlds; + int rc; + + rc = cxl_event_req_irq(cxlds, policy->info_settings); if (rc) { dev_err(cxlds->dev, "Failed to get interrupt for event Info log\n"); return rc; } - rc = cxl_event_req_irq(cxlds, policy.warn_settings); + rc = cxl_event_req_irq(cxlds, policy->warn_settings); if (rc) { dev_err(cxlds->dev, "Failed to get interrupt for event Warn log\n"); return rc; } - rc = cxl_event_req_irq(cxlds, policy.failure_settings); + rc = cxl_event_req_irq(cxlds, policy->failure_settings); if (rc) { dev_err(cxlds->dev, "Failed to get interrupt for event Failure log\n"); return rc; } - rc = cxl_event_req_irq(cxlds, policy.fatal_settings); + rc = cxl_event_req_irq(cxlds, policy->fatal_settings); if (rc) { dev_err(cxlds->dev, "Failed to get interrupt for event Fatal log\n"); return rc; @@ -637,6 +675,37 @@ static int cxl_event_irqsetup(struct cxl_memdev_state *mds) return 0; } +static int cxl_irqsetup(struct cxl_memdev_state *mds, + struct cxl_event_interrupt_policy *policy, + bool native_cxl, size_t policy_size) +{ + struct cxl_dev_state *cxlds = &mds->cxlds; + int rc; + + if (native_cxl) { + rc = cxl_event_irqsetup(mds, policy); + if (rc) + return rc; + } + + if (cxl_dcd_supported(mds)) { + rc = cxl_event_req_irq(cxlds, policy->dcd_settings); + if (rc) { + dev_err(cxlds->dev, "Failed to get interrupt for DCD event log\n"); + cxl_disable_dcd(mds); + /* No handler will service the DCD vector; tell the device */ + policy->dcd_settings = CXL_INT_NONE; + rc = cxl_event_set_int_policy(mds, policy, policy_size); + if (rc) + dev_warn(cxlds->dev, + "DCD disabled but its interrupt policy could not be cleared: %d\n", + rc); + } + } + + return 0; +} + static bool cxl_event_int_is_fw(u8 setting) { u8 mode = FIELD_GET(CXLDEV_EVENT_INT_MODE_MASK, setting); @@ -644,48 +713,101 @@ static bool cxl_event_int_is_fw(u8 setting) return mode == CXL_INT_FW; } +static bool cxl_event_fw_owns_mem_logs(struct cxl_event_interrupt_policy *policy) +{ + return cxl_event_int_is_fw(policy->info_settings) || + cxl_event_int_is_fw(policy->warn_settings) || + cxl_event_int_is_fw(policy->failure_settings) || + cxl_event_int_is_fw(policy->fatal_settings); +} + static int cxl_event_config(struct pci_host_bridge *host_bridge, struct cxl_memdev_state *mds, bool irq_avail) { - struct cxl_event_interrupt_policy policy; + struct cxl_event_interrupt_policy policy = { 0 }; + bool native_cxl = host_bridge->native_cxl_error; + size_t policy_size; + u32 mask; int rc; /* * When BIOS maintains CXL error reporting control, it will process * event records. Only one agent can do so. + * + * If BIOS has control of events and DCD is not supported skip event + * configuration. */ - if (!host_bridge->native_cxl_error) + if (!native_cxl && !cxl_dcd_supported(mds)) return 0; if (!irq_avail) { dev_info(mds->cxlds.dev, "No interrupt support, disable event processing.\n"); + if (cxl_dcd_supported(mds)) { + dev_info(mds->cxlds.dev, "DCD requires interrupts, disable DCD\n"); + cxl_disable_dcd(mds); + } return 0; } - rc = cxl_event_get_int_policy(mds, &policy); + rc = cxl_event_get_int_policy(mds, &policy, &policy_size); if (rc) - return rc; + goto err; - if (cxl_event_int_is_fw(policy.info_settings) || - cxl_event_int_is_fw(policy.warn_settings) || - cxl_event_int_is_fw(policy.failure_settings) || - cxl_event_int_is_fw(policy.fatal_settings)) { + /* + * dcd_settings cannot be sent to a device whose policy is too short + * to hold it, so the device would never accept the setting + * cxl_irqsetup() arms the DCD interrupt with. + */ + if (cxl_dcd_supported(mds) && policy_size < sizeof(policy)) { + dev_warn(mds->cxlds.dev, + "DCD supported but interrupt policy is only %zu bytes\n", + policy_size); + cxl_disable_dcd(mds); + /* DCD was the only reason to be here */ + if (!native_cxl) + return 0; + } + + if (native_cxl && cxl_event_fw_owns_mem_logs(&policy)) { dev_err(mds->cxlds.dev, "FW still in control of Event Logs despite _OSC settings\n"); return -EBUSY; } + rc = cxl_event_config_msgnums(mds, &policy, native_cxl, policy_size); + if (rc) + goto err; + rc = cxl_mem_alloc_event_buf(mds); if (rc) return rc; - rc = cxl_event_irqsetup(mds); + rc = cxl_irqsetup(mds, &policy, native_cxl, policy_size); if (rc) return rc; - cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL); + mask = cxl_event_drain_mask(host_bridge); + if (mask) + cxl_mem_get_event_records(mds, mask); + + dev_dbg(mds->cxlds.dev, "Event config : %s DCD %s\n", + native_cxl ? "OS" : "BIOS", + cxl_dcd_supported(mds) ? "supported" : "not supported"); return 0; + +err: + /* + * Failed to Get/Set interrupt policy for DCD while BIOS owns the + * other logs. Disable DCD and continue. + */ + if (!native_cxl) { + dev_warn(mds->cxlds.dev, + "Event interrupt policy failed (%d), disabling DCD\n", rc); + cxl_disable_dcd(mds); + return 0; + } + return rc; } static int cxl_pci_type3_init_mailbox(struct cxl_dev_state *cxlds) @@ -706,22 +828,23 @@ static ssize_t rcd_pcie_cap_emit(struct device *dev, u16 offset, char *buf, size struct device *root_dev; struct cxl_dport *dport; struct cxl_port *root __free(put_cxl_port) = - cxl_mem_find_port(cxlmd, &dport); + cxl_mem_find_port(cxlmd); if (!root) return -ENXIO; - root_dev = root->uport_dev; + root_dev = cxl_port_dport_host(root); if (!root_dev) return -ENXIO; - if (!dport->regs.rcd_pcie_cap) - return -ENXIO; - guard(device)(root_dev); if (!root_dev->driver) return -ENXIO; + dport = cxl_mem_find_dport(cxlmd, root); + if (!dport || !dport->regs.rcd_pcie_cap) + return -ENXIO; + switch (width) { case 2: return sysfs_emit(buf, "%#x\n", @@ -868,6 +991,12 @@ static int cxl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (rc) return rc; + if (cxl_dcd_supported(mds)) { + rc = cxl_configure_dcd(mds, &range_info); + if (rc) + cxl_disable_dcd(mds); + } + rc = cxl_dpa_setup(cxlds, &range_info); if (rc) return rc; diff --git a/drivers/cxl/port.c b/drivers/cxl/port.c index 99cf77b6b699..c12fd0b89883 100644 --- a/drivers/cxl/port.c +++ b/drivers/cxl/port.c @@ -46,6 +46,9 @@ static int discover_region(struct device *dev, void *unused) if (cxled->state != CXL_DECODER_STATE_AUTO) return 0; + if (cxled_empty(cxled)) + return 0; + /* * Region enumeration is opportunistic, if this add-event fails, * continue to the next endpoint decoder. diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 07399cfbb0e2..7c6f2aba8144 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -6030,18 +6030,31 @@ static enum pci_bus_speed to_pcie_link_speed(u16 lnksta) return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)]; } -int pcie_link_speed_mbps(struct pci_dev *pdev) +/** + * pcie_link_bandwidth_mbps - get the current bandwidth of a single PCIe link + * @pdev: PCI device to query + * + * The bandwidth includes all negotiated lanes and does not account for + * encoding overhead. A negotiated width of zero gives zero bandwidth. + * + * Return: Bandwidth in Mb/s, or a negative errno on failure. + */ +int pcie_link_bandwidth_mbps(struct pci_dev *pdev) { u16 lnksta; - int err; + int speed, err; err = pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &lnksta); if (err) - return err; + return pcibios_err_to_errno(err); - return pcie_dev_speed_mbps(to_pcie_link_speed(lnksta)); + speed = pcie_dev_speed_mbps(to_pcie_link_speed(lnksta)); + if (speed < 0) + return speed; + + return speed * FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta); } -EXPORT_SYMBOL(pcie_link_speed_mbps); +EXPORT_SYMBOL(pcie_link_bandwidth_mbps); /** * pcie_bandwidth_available - determine minimum link settings of a PCIe diff --git a/include/cxl/cxl.h b/include/cxl/cxl.h index 802b143de83d..1c04a7823f64 100644 --- a/include/cxl/cxl.h +++ b/include/cxl/cxl.h @@ -130,9 +130,30 @@ struct cxl_dpa_perf { int qos_class; }; +/* + * Partitions share one flat DPA space in a fixed order: + * + * DPA 0 end + * +---------------+---------------+---------------------------+ + * | ram | pmem | dynamic_ram_1 | + * +---------------+---------------+---------------------------+ + * part[0] part[1] part[2] + * + * The static ram and pmem partition order is mandated by CXL r4.0 + * 8.2.10.9.2.1 "Get Partition Info" (opcode 4100h), Table 8-310: volatile + * capacity starts at DPA 0 and persistent capacity starts at the DPA + * immediately following it. A dynamic partition has no such + * requirement -- 8.2.10.9.9.1 "Get Dynamic Capacity Configuration" + * (opcode 4800h), Table 8-347, only requires the DC Region Base to be + * 256MB aligned -- but Linux follows that precedent and requires it to + * start at the DPA immediately following static capacity: after pmem, + * after ram on a device with no pmem, or at DPA 0 on a device with no + * static capacity at all. cxl_configure_dcd() enforces it. + */ enum cxl_partition_mode { CXL_PARTMODE_RAM, CXL_PARTMODE_PMEM, + CXL_PARTMODE_DYNAMIC_RAM, }; /** @@ -140,14 +161,20 @@ enum cxl_partition_mode { * @res: shortcut to the partition in the DPA resource tree (cxlds->dpa_res) * @perf: performance attributes of the partition from CDAT * @mode: operation mode for the DPA capacity, e.g. ram, pmem, dynamic... + * @handle: DSMAS handle that represents this partition + * @shareable: Is the partition shareable (from its CDAT DSMAS entry) + * @read_only: Is the partition read-only (from its CDAT DSMAS entry) */ struct cxl_dpa_partition { struct resource res; struct cxl_dpa_perf perf; enum cxl_partition_mode mode; + u8 handle; + bool shareable; + bool read_only; }; -#define CXL_NR_PARTITIONS_MAX 2 +#define CXL_NR_PARTITIONS_MAX 3 /** * struct cxl_dev_state - The driver device state diff --git a/include/linux/device.h b/include/linux/device.h index 90cdd77458bb..879eb758b5ee 100644 --- a/include/linux/device.h +++ b/include/linux/device.h @@ -1121,6 +1121,7 @@ static inline void device_unlock(struct device *dev) DEFINE_GUARD(device, struct device *, device_lock(_T), device_unlock(_T)) DEFINE_GUARD_COND(device, _intr, device_lock_interruptible(_T), _RET == 0) +DEFINE_GUARD_COND(device, _try, device_trylock(_T)) static inline void device_lock_assert(struct device *dev) { diff --git a/include/linux/pci.h b/include/linux/pci.h index ef6327205520..97a1096d5a8a 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1478,7 +1478,7 @@ int pcie_set_mps(struct pci_dev *dev, int mps); u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev, enum pci_bus_speed *speed, enum pcie_link_width *width); -int pcie_link_speed_mbps(struct pci_dev *pdev); +int pcie_link_bandwidth_mbps(struct pci_dev *pdev); void pcie_print_link_status(struct pci_dev *dev); int pcie_reset_flr(struct pci_dev *dev, bool probe); int pcie_flr(struct pci_dev *dev); diff --git a/tools/testing/cxl/Kbuild b/tools/testing/cxl/Kbuild index 2be1df80fcc9..e49ce644b4b1 100644 --- a/tools/testing/cxl/Kbuild +++ b/tools/testing/cxl/Kbuild @@ -4,16 +4,20 @@ ldflags-y += --wrap=is_acpi_device_node ldflags-y += --wrap=acpi_evaluate_integer ldflags-y += --wrap=acpi_pci_find_root ldflags-y += --wrap=nvdimm_bus_register -ldflags-y += --wrap=cxl_await_media_ready -ldflags-y += --wrap=devm_cxl_add_rch_dport -ldflags-y += --wrap=cxl_endpoint_parse_cdat -ldflags-y += --wrap=devm_cxl_endpoint_decoders_setup ldflags-y += --wrap=hmat_get_extended_linear_cache_size -ldflags-y += --wrap=devm_cxl_add_dport_by_dev -ldflags-y += --wrap=devm_cxl_switch_port_decoders_setup ldflags-y += --wrap=walk_hmem_resources ldflags-y += --wrap=region_intersects ldflags-y += --wrap=region_intersects_soft_reserve +wrap_cxl_core := --wrap=cxl_await_media_ready +wrap_cxl_core += --wrap=devm_cxl_add_rch_dport +wrap_cxl_core += --wrap=devm_cxl_add_dport_by_dev +wrap_cxl_core += --wrap=devm_cxl_switch_port_decoders_setup +wrap_cxl_core += --wrap=devm_cxl_endpoint_decoders_setup +wrap_cxl_core += --wrap=cxl_endpoint_parse_cdat +LDFLAGS_cxl_acpi.o += $(wrap_cxl_core) +LDFLAGS_cxl_pmem.o += $(wrap_cxl_core) +LDFLAGS_cxl_port.o += $(wrap_cxl_core) +LDFLAGS_cxl_mem.o += $(wrap_cxl_core) DRIVERS := ../../../drivers DAX_HMEM_SRC := $(DRIVERS)/dax/hmem diff --git a/tools/testing/cxl/test/cxl.c b/tools/testing/cxl/test/cxl.c index 62bd92b3be45..3eb1051b1ee8 100644 --- a/tools/testing/cxl/test/cxl.c +++ b/tools/testing/cxl/test/cxl.c @@ -18,6 +18,7 @@ static int interleave_arithmetic; static bool extended_linear_cache; static bool fail_autoassemble; static bool type2_test; +static bool mock_zero_size_decoders; #define FAKE_QTG_ID 42 @@ -872,14 +873,13 @@ static int cxld_registry_restore(struct cxl_decoder *cxld, cxld_copy(cxld, &td->cxled.cxld); cxled->state = td->cxled.state; cxled->skip = td->cxled.skip; - if (range_len(&td->dpa_range)) { - rc = devm_cxl_dpa_reserve(cxled, td->dpa_range.start, - range_len(&td->dpa_range), - td->cxled.skip); - if (rc) { - init_disabled_mock_decoder(cxld); - return rc; - } + /* enabled endpoint decoders hold a reservation, sized or not */ + rc = devm_cxl_dpa_reserve(cxled, td->dpa_range.start, + range_len(&td->dpa_range), + td->cxled.skip); + if (rc) { + init_disabled_mock_decoder(cxld); + return rc; } port->commit_end = cxld->id; } @@ -1072,16 +1072,49 @@ static void default_mock_decoder(struct cxl_decoder *cxld) WARN_ON_ONCE(!cxld_registry_new(cxld)); } -static int first_decoder(struct device *dev, const void *data) +static int match_decoder_by_index(struct device *dev, const void *data) { + int target_id = *(const int *)data; struct cxl_decoder *cxld; if (!is_switch_decoder(dev)) return 0; cxld = to_cxl_decoder(dev); - if (cxld->id == 0) - return 1; - return 0; + return cxld->id == target_id; +} + +/* + * Mock a committed, locked, empty decoder + * (CXL r4.0 8.2.4.20.12). Gated by the mock_zero_size_decoders module + * param so the default cxl_test topology, shared by the region test + * suite, is left undisturbed. + */ +static void size_zero_mock_decoder_ep(struct cxl_decoder *cxld, u64 base) +{ + struct cxl_endpoint_decoder *cxled = to_cxl_endpoint_decoder(&cxld->dev); + + cxld->hpa_range = DEFINE_RANGE(base, base - 1); + cxld->interleave_ways = 2; + cxld->interleave_granularity = 4096; + cxld->target_type = CXL_DECODER_HOSTONLYMEM; + cxld->flags = CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; + cxled->state = CXL_DECODER_STATE_AUTO; + /* decoder[0] reserved [0, size/2), empty decoders sit at that watermark */ + devm_cxl_dpa_reserve(cxled, mock_auto_region_size / 2, 0, 0); + cxld->commit = mock_decoder_commit; + cxld->reset = mock_decoder_reset; +} + +static void size_zero_mock_decoder_sw(struct cxl_decoder *cxld, u64 base, + int level) +{ + cxld->flags = CXL_DECODER_F_ENABLE | CXL_DECODER_F_LOCK; + cxld->target_type = CXL_DECODER_HOSTONLYMEM; + cxld->interleave_ways = level == 0 ? 2 : 1; + cxld->interleave_granularity = 4096; + cxld->hpa_range = DEFINE_RANGE(base, base - 1); + cxld->commit = mock_decoder_commit; + cxld->reset = mock_decoder_reset; } enum cxld_init_type { @@ -1114,7 +1147,7 @@ static enum cxld_init_type get_decoder_init_type(struct cxl_decoder *cxld, * See 'cxl list -BMPu -m cxl_mem.0,cxl_mem.4' */ if (!is_endpoint_decoder(&cxld->dev) || !hb0 || pdev->id % 4 || - pdev->id > 4 || cxld->id > 0) + pdev->id > 4 || cxld->id > (mock_zero_size_decoders ? 2 : 0)) return MOCK_DECODER_INIT_DEFAULT; return type2_test ? MOCK_DECODER_INIT_TYPE2_AUTO : @@ -1147,6 +1180,7 @@ static void mock_init_hdm_type2_cxled(struct cxl_endpoint_decoder *cxled, struct cxl_port *root_port; struct device *dev; u64 base; + int id = 0; base = window->base_hpa; cxld->hpa_range = (struct range) { @@ -1172,7 +1206,7 @@ static void mock_init_hdm_type2_cxled(struct cxl_endpoint_decoder *cxled, */ dport = port->parent_dport; root_port = dport->port; - dev = device_find_child(&root_port->dev, NULL, first_decoder); + dev = device_find_child(&root_port->dev, &id, match_decoder_by_index); /* * Ancestor ports are guaranteed to be enumerated before * @port, and all ports have at least one decoder. @@ -1232,6 +1266,20 @@ static void mock_init_hdm_type3_cxled(struct cxl_endpoint_decoder *cxled, base = window->base_hpa; if (extended_linear_cache) base += mock_auto_region_size; + + /* + * With mock_zero_size_decoders, decoders 1 and 2 of the special + * endpoints mock committed, locked, empty decoders above the + * decoder[0] auto-region (CXL r4.0 8.2.4.20.12). commit_end then + * points at a zero-size decoder, exercising the zero-size + * reservation and poison-by-endpoint code paths. + */ + if (cxld->id == 1 || cxld->id == 2) { + size_zero_mock_decoder_ep(cxld, base); + port->commit_end = cxld->id; + WARN_ON_ONCE(!cxld_registry_new(cxld)); + return; + } cxld->hpa_range = (struct range) { .start = base, .end = base + mock_auto_region_size - 1, @@ -1255,9 +1303,11 @@ static void mock_init_hdm_type3_cxled(struct cxl_endpoint_decoder *cxled, */ iter = port; for (i = 0; i < 2; i++) { + int id = 0; + dport = iter->parent_dport; iter = dport->port; - dev = device_find_child(&iter->dev, NULL, first_decoder); + dev = device_find_child(&iter->dev, &id, match_decoder_by_index); /* * Ancestor ports are guaranteed to be enumerated before * @port, and all ports have at least one decoder. @@ -1307,6 +1357,26 @@ static void mock_init_hdm_type3_cxled(struct cxl_endpoint_decoder *cxled, cxld_registry_update(cxld); put_device(dev); + + if (!mock_zero_size_decoders) + continue; + + /* + * Mirror the endpoint: commit the next two switch decoders + * as zero-size + locked so the empty-decoder layout extends + * end-to-end through the switch and host bridge. + */ + for (id = 1; id <= 2; id++) { + dev = device_find_child(&iter->dev, &id, + match_decoder_by_index); + if (WARN_ON(!dev)) + continue; + cxld = to_cxl_decoder(dev); + size_zero_mock_decoder_sw(cxld, base, i); + iter->commit_end = id; + cxld_registry_update(cxld); + put_device(dev); + } } } @@ -1945,7 +2015,7 @@ static int cxl_type2_mem_init(void) goto err_mem; } pdev->dev.parent = &dport->dev; - set_dev_node(&pdev->dev, i % 2); + set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2)); rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]); if (rc) @@ -1974,7 +2044,7 @@ static int cxl_type3_mem_init(void) goto err_mem; } pdev->dev.parent = &dport->dev; - set_dev_node(&pdev->dev, i % 2); + set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2)); rc = cxl_mock_platform_device_add(pdev, &cxl_mem[i]); if (rc) @@ -1991,7 +2061,7 @@ static int cxl_type3_mem_init(void) goto err_single; } pdev->dev.parent = &dport->dev; - set_dev_node(&pdev->dev, i % 2); + set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2)); rc = cxl_mock_platform_device_add(pdev, &cxl_mem_single[i]); if (rc) @@ -2009,7 +2079,7 @@ static int cxl_type3_mem_init(void) goto err_rcd; } pdev->dev.parent = &rch->dev; - set_dev_node(&pdev->dev, i % 2); + set_dev_node(&pdev->dev, numa_map_to_online_node(i % 2)); rc = cxl_mock_platform_device_add(pdev, &cxl_rcd[i]); if (rc) @@ -2076,6 +2146,12 @@ static bool __init have_multiple_modparms(void) count++; if (type2_test) count++; + if (mock_zero_size_decoders) + count++; + + /* Zero-size decoders require decoder[0] to preserve commit order. */ + if (fail_autoassemble && mock_zero_size_decoders) + return true; return count > 1; } @@ -2484,6 +2560,9 @@ module_param(fail_autoassemble, bool, 0444); MODULE_PARM_DESC(fail_autoassemble, "Simulate missing member of an auto-region"); module_param(type2_test, bool, 0444); MODULE_PARM_DESC(type2_test, "Enable type 2 support testing"); +module_param(mock_zero_size_decoders, bool, 0444); +MODULE_PARM_DESC(mock_zero_size_decoders, + "Mock committed, locked, empty decoders under host-bridge0"); module_init(cxl_test_init); module_exit(cxl_test_exit); MODULE_LICENSE("GPL v2"); diff --git a/tools/testing/cxl/test/mem.c b/tools/testing/cxl/test/mem.c index 7b756000a1a6..c740372bdaaa 100644 --- a/tools/testing/cxl/test/mem.c +++ b/tools/testing/cxl/test/mem.c @@ -1818,7 +1818,7 @@ static int cxl_mock_mem_probe(struct platform_device *pdev) if (rc) dev_dbg(dev, "No CXL FWCTL setup\n"); - cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_ALL); + cxl_mem_get_event_records(mds, CXLDEV_EVENT_STATUS_STANDARD_LOGS); cxl_mock_test_feat_init(mdata); return 0;