1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2023 Intel Corporation. All rights reserved. */ 3 #include <linux/acpi.h> 4 #include <linux/xarray.h> 5 #include <linux/fw_table.h> 6 #include <linux/node.h> 7 #include <linux/overflow.h> 8 #include "cxlpci.h" 9 #include "cxlmem.h" 10 #include "core.h" 11 #include "cxl.h" 12 #include "core.h" 13 14 struct dsmas_entry { 15 struct range dpa_range; 16 u8 handle; 17 struct access_coordinate coord; 18 19 int entries; 20 int qos_class; 21 }; 22 23 static int cdat_dsmas_handler(union acpi_subtable_headers *header, void *arg, 24 const unsigned long end) 25 { 26 struct acpi_cdat_header *hdr = &header->cdat; 27 struct acpi_cdat_dsmas *dsmas; 28 int size = sizeof(*hdr) + sizeof(*dsmas); 29 struct xarray *dsmas_xa = arg; 30 struct dsmas_entry *dent; 31 u16 len; 32 int rc; 33 34 len = le16_to_cpu((__force __le16)hdr->length); 35 if (len != size || (unsigned long)hdr + len > end) { 36 pr_warn("Malformed DSMAS table length: (%u:%u)\n", size, len); 37 return -EINVAL; 38 } 39 40 /* Skip common header */ 41 dsmas = (struct acpi_cdat_dsmas *)(hdr + 1); 42 43 dent = kzalloc(sizeof(*dent), GFP_KERNEL); 44 if (!dent) 45 return -ENOMEM; 46 47 dent->handle = dsmas->dsmad_handle; 48 dent->dpa_range.start = le64_to_cpu((__force __le64)dsmas->dpa_base_address); 49 dent->dpa_range.end = le64_to_cpu((__force __le64)dsmas->dpa_base_address) + 50 le64_to_cpu((__force __le64)dsmas->dpa_length) - 1; 51 52 rc = xa_insert(dsmas_xa, dent->handle, dent, GFP_KERNEL); 53 if (rc) { 54 kfree(dent); 55 return rc; 56 } 57 58 return 0; 59 } 60 61 static void cxl_access_coordinate_set(struct access_coordinate *coord, 62 int access, unsigned int val) 63 { 64 switch (access) { 65 case ACPI_HMAT_ACCESS_LATENCY: 66 coord->read_latency = val; 67 coord->write_latency = val; 68 break; 69 case ACPI_HMAT_READ_LATENCY: 70 coord->read_latency = val; 71 break; 72 case ACPI_HMAT_WRITE_LATENCY: 73 coord->write_latency = val; 74 break; 75 case ACPI_HMAT_ACCESS_BANDWIDTH: 76 coord->read_bandwidth = val; 77 coord->write_bandwidth = val; 78 break; 79 case ACPI_HMAT_READ_BANDWIDTH: 80 coord->read_bandwidth = val; 81 break; 82 case ACPI_HMAT_WRITE_BANDWIDTH: 83 coord->write_bandwidth = val; 84 break; 85 } 86 } 87 88 static int cdat_dslbis_handler(union acpi_subtable_headers *header, void *arg, 89 const unsigned long end) 90 { 91 struct acpi_cdat_header *hdr = &header->cdat; 92 struct acpi_cdat_dslbis *dslbis; 93 int size = sizeof(*hdr) + sizeof(*dslbis); 94 struct xarray *dsmas_xa = arg; 95 struct dsmas_entry *dent; 96 __le64 le_base; 97 __le16 le_val; 98 u64 val; 99 u16 len; 100 int rc; 101 102 len = le16_to_cpu((__force __le16)hdr->length); 103 if (len != size || (unsigned long)hdr + len > end) { 104 pr_warn("Malformed DSLBIS table length: (%u:%u)\n", size, len); 105 return -EINVAL; 106 } 107 108 /* Skip common header */ 109 dslbis = (struct acpi_cdat_dslbis *)(hdr + 1); 110 111 /* Skip unrecognized data type */ 112 if (dslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH) 113 return 0; 114 115 /* Not a memory type, skip */ 116 if ((dslbis->flags & ACPI_HMAT_MEMORY_HIERARCHY) != ACPI_HMAT_MEMORY) 117 return 0; 118 119 dent = xa_load(dsmas_xa, dslbis->handle); 120 if (!dent) { 121 pr_warn("No matching DSMAS entry for DSLBIS entry.\n"); 122 return 0; 123 } 124 125 le_base = (__force __le64)dslbis->entry_base_unit; 126 le_val = (__force __le16)dslbis->entry[0]; 127 rc = check_mul_overflow(le64_to_cpu(le_base), 128 le16_to_cpu(le_val), &val); 129 if (rc) 130 pr_warn("DSLBIS value overflowed.\n"); 131 132 cxl_access_coordinate_set(&dent->coord, dslbis->data_type, val); 133 134 return 0; 135 } 136 137 static int cdat_table_parse_output(int rc) 138 { 139 if (rc < 0) 140 return rc; 141 if (rc == 0) 142 return -ENOENT; 143 144 return 0; 145 } 146 147 static int cxl_cdat_endpoint_process(struct cxl_port *port, 148 struct xarray *dsmas_xa) 149 { 150 int rc; 151 152 rc = cdat_table_parse(ACPI_CDAT_TYPE_DSMAS, cdat_dsmas_handler, 153 dsmas_xa, port->cdat.table, port->cdat.length); 154 rc = cdat_table_parse_output(rc); 155 if (rc) 156 return rc; 157 158 rc = cdat_table_parse(ACPI_CDAT_TYPE_DSLBIS, cdat_dslbis_handler, 159 dsmas_xa, port->cdat.table, port->cdat.length); 160 return cdat_table_parse_output(rc); 161 } 162 163 static int cxl_port_perf_data_calculate(struct cxl_port *port, 164 struct xarray *dsmas_xa) 165 { 166 struct access_coordinate ep_c; 167 struct access_coordinate coord[ACCESS_COORDINATE_MAX]; 168 struct dsmas_entry *dent; 169 int valid_entries = 0; 170 unsigned long index; 171 int rc; 172 173 rc = cxl_endpoint_get_perf_coordinates(port, &ep_c); 174 if (rc) { 175 dev_dbg(&port->dev, "Failed to retrieve ep perf coordinates.\n"); 176 return rc; 177 } 178 179 rc = cxl_hb_get_perf_coordinates(port, coord); 180 if (rc) { 181 dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n"); 182 return rc; 183 } 184 185 struct cxl_root *cxl_root __free(put_cxl_root) = find_cxl_root(port); 186 187 if (!cxl_root) 188 return -ENODEV; 189 190 if (!cxl_root->ops || !cxl_root->ops->qos_class) 191 return -EOPNOTSUPP; 192 193 xa_for_each(dsmas_xa, index, dent) { 194 int qos_class; 195 196 cxl_coordinates_combine(&dent->coord, &dent->coord, &ep_c); 197 /* 198 * Keeping the host bridge coordinates separate from the dsmas 199 * coordinates in order to allow calculation of access class 200 * 0 and 1 for region later. 201 */ 202 cxl_coordinates_combine(&coord[ACCESS_COORDINATE_CPU], 203 &coord[ACCESS_COORDINATE_CPU], 204 &dent->coord); 205 dent->entries = 1; 206 rc = cxl_root->ops->qos_class(cxl_root, 207 &coord[ACCESS_COORDINATE_CPU], 208 1, &qos_class); 209 if (rc != 1) 210 continue; 211 212 valid_entries++; 213 dent->qos_class = qos_class; 214 } 215 216 if (!valid_entries) 217 return -ENOENT; 218 219 return 0; 220 } 221 222 static void update_perf_entry(struct device *dev, struct dsmas_entry *dent, 223 struct cxl_dpa_perf *dpa_perf) 224 { 225 dpa_perf->dpa_range = dent->dpa_range; 226 dpa_perf->coord = dent->coord; 227 dpa_perf->qos_class = dent->qos_class; 228 dev_dbg(dev, 229 "DSMAS: dpa: %#llx qos: %d read_bw: %d write_bw %d read_lat: %d write_lat: %d\n", 230 dent->dpa_range.start, dpa_perf->qos_class, 231 dent->coord.read_bandwidth, dent->coord.write_bandwidth, 232 dent->coord.read_latency, dent->coord.write_latency); 233 } 234 235 static void cxl_memdev_set_qos_class(struct cxl_dev_state *cxlds, 236 struct xarray *dsmas_xa) 237 { 238 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 239 struct device *dev = cxlds->dev; 240 struct range pmem_range = { 241 .start = cxlds->pmem_res.start, 242 .end = cxlds->pmem_res.end, 243 }; 244 struct range ram_range = { 245 .start = cxlds->ram_res.start, 246 .end = cxlds->ram_res.end, 247 }; 248 struct dsmas_entry *dent; 249 unsigned long index; 250 251 xa_for_each(dsmas_xa, index, dent) { 252 if (resource_size(&cxlds->ram_res) && 253 range_contains(&ram_range, &dent->dpa_range)) 254 update_perf_entry(dev, dent, &mds->ram_perf); 255 else if (resource_size(&cxlds->pmem_res) && 256 range_contains(&pmem_range, &dent->dpa_range)) 257 update_perf_entry(dev, dent, &mds->pmem_perf); 258 else 259 dev_dbg(dev, "no partition for dsmas dpa: %#llx\n", 260 dent->dpa_range.start); 261 } 262 } 263 264 static int match_cxlrd_qos_class(struct device *dev, void *data) 265 { 266 int dev_qos_class = *(int *)data; 267 struct cxl_root_decoder *cxlrd; 268 269 if (!is_root_decoder(dev)) 270 return 0; 271 272 cxlrd = to_cxl_root_decoder(dev); 273 if (cxlrd->qos_class == CXL_QOS_CLASS_INVALID) 274 return 0; 275 276 if (cxlrd->qos_class == dev_qos_class) 277 return 1; 278 279 return 0; 280 } 281 282 static void reset_dpa_perf(struct cxl_dpa_perf *dpa_perf) 283 { 284 *dpa_perf = (struct cxl_dpa_perf) { 285 .qos_class = CXL_QOS_CLASS_INVALID, 286 }; 287 } 288 289 static bool cxl_qos_match(struct cxl_port *root_port, 290 struct cxl_dpa_perf *dpa_perf) 291 { 292 if (dpa_perf->qos_class == CXL_QOS_CLASS_INVALID) 293 return false; 294 295 if (!device_for_each_child(&root_port->dev, &dpa_perf->qos_class, 296 match_cxlrd_qos_class)) 297 return false; 298 299 return true; 300 } 301 302 static int match_cxlrd_hb(struct device *dev, void *data) 303 { 304 struct device *host_bridge = data; 305 struct cxl_switch_decoder *cxlsd; 306 struct cxl_root_decoder *cxlrd; 307 308 if (!is_root_decoder(dev)) 309 return 0; 310 311 cxlrd = to_cxl_root_decoder(dev); 312 cxlsd = &cxlrd->cxlsd; 313 314 guard(rwsem_read)(&cxl_region_rwsem); 315 for (int i = 0; i < cxlsd->nr_targets; i++) { 316 if (host_bridge == cxlsd->target[i]->dport_dev) 317 return 1; 318 } 319 320 return 0; 321 } 322 323 static int cxl_qos_class_verify(struct cxl_memdev *cxlmd) 324 { 325 struct cxl_dev_state *cxlds = cxlmd->cxlds; 326 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 327 struct cxl_port *root_port; 328 int rc; 329 330 struct cxl_root *cxl_root __free(put_cxl_root) = 331 find_cxl_root(cxlmd->endpoint); 332 333 if (!cxl_root) 334 return -ENODEV; 335 336 root_port = &cxl_root->port; 337 338 /* Check that the QTG IDs are all sane between end device and root decoders */ 339 if (!cxl_qos_match(root_port, &mds->ram_perf)) 340 reset_dpa_perf(&mds->ram_perf); 341 if (!cxl_qos_match(root_port, &mds->pmem_perf)) 342 reset_dpa_perf(&mds->pmem_perf); 343 344 /* Check to make sure that the device's host bridge is under a root decoder */ 345 rc = device_for_each_child(&root_port->dev, 346 cxlmd->endpoint->host_bridge, match_cxlrd_hb); 347 if (!rc) { 348 reset_dpa_perf(&mds->ram_perf); 349 reset_dpa_perf(&mds->pmem_perf); 350 } 351 352 return rc; 353 } 354 355 static void discard_dsmas(struct xarray *xa) 356 { 357 unsigned long index; 358 void *ent; 359 360 xa_for_each(xa, index, ent) { 361 xa_erase(xa, index); 362 kfree(ent); 363 } 364 xa_destroy(xa); 365 } 366 DEFINE_FREE(dsmas, struct xarray *, if (_T) discard_dsmas(_T)) 367 368 void cxl_endpoint_parse_cdat(struct cxl_port *port) 369 { 370 struct cxl_memdev *cxlmd = to_cxl_memdev(port->uport_dev); 371 struct cxl_dev_state *cxlds = cxlmd->cxlds; 372 struct xarray __dsmas_xa; 373 struct xarray *dsmas_xa __free(dsmas) = &__dsmas_xa; 374 int rc; 375 376 xa_init(&__dsmas_xa); 377 if (!port->cdat.table) 378 return; 379 380 rc = cxl_cdat_endpoint_process(port, dsmas_xa); 381 if (rc < 0) { 382 dev_dbg(&port->dev, "Failed to parse CDAT: %d\n", rc); 383 return; 384 } 385 386 rc = cxl_port_perf_data_calculate(port, dsmas_xa); 387 if (rc) { 388 dev_dbg(&port->dev, "Failed to do perf coord calculations.\n"); 389 return; 390 } 391 392 cxl_memdev_set_qos_class(cxlds, dsmas_xa); 393 cxl_qos_class_verify(cxlmd); 394 cxl_memdev_update_perf(cxlmd); 395 } 396 EXPORT_SYMBOL_NS_GPL(cxl_endpoint_parse_cdat, CXL); 397 398 static int cdat_sslbis_handler(union acpi_subtable_headers *header, void *arg, 399 const unsigned long end) 400 { 401 struct acpi_cdat_sslbis_table { 402 struct acpi_cdat_header header; 403 struct acpi_cdat_sslbis sslbis_header; 404 struct acpi_cdat_sslbe entries[]; 405 } *tbl = (struct acpi_cdat_sslbis_table *)header; 406 int size = sizeof(header->cdat) + sizeof(tbl->sslbis_header); 407 struct acpi_cdat_sslbis *sslbis; 408 struct cxl_port *port = arg; 409 struct device *dev = &port->dev; 410 int remain, entries, i; 411 u16 len; 412 413 len = le16_to_cpu((__force __le16)header->cdat.length); 414 remain = len - size; 415 if (!remain || remain % sizeof(tbl->entries[0]) || 416 (unsigned long)header + len > end) { 417 dev_warn(dev, "Malformed SSLBIS table length: (%u)\n", len); 418 return -EINVAL; 419 } 420 421 sslbis = &tbl->sslbis_header; 422 /* Unrecognized data type, we can skip */ 423 if (sslbis->data_type > ACPI_HMAT_WRITE_BANDWIDTH) 424 return 0; 425 426 entries = remain / sizeof(tbl->entries[0]); 427 if (struct_size(tbl, entries, entries) != len) 428 return -EINVAL; 429 430 for (i = 0; i < entries; i++) { 431 u16 x = le16_to_cpu((__force __le16)tbl->entries[i].portx_id); 432 u16 y = le16_to_cpu((__force __le16)tbl->entries[i].porty_id); 433 __le64 le_base; 434 __le16 le_val; 435 struct cxl_dport *dport; 436 unsigned long index; 437 u16 dsp_id; 438 u64 val; 439 440 switch (x) { 441 case ACPI_CDAT_SSLBIS_US_PORT: 442 dsp_id = y; 443 break; 444 case ACPI_CDAT_SSLBIS_ANY_PORT: 445 switch (y) { 446 case ACPI_CDAT_SSLBIS_US_PORT: 447 dsp_id = x; 448 break; 449 case ACPI_CDAT_SSLBIS_ANY_PORT: 450 dsp_id = ACPI_CDAT_SSLBIS_ANY_PORT; 451 break; 452 default: 453 dsp_id = y; 454 break; 455 } 456 break; 457 default: 458 dsp_id = x; 459 break; 460 } 461 462 le_base = (__force __le64)tbl->sslbis_header.entry_base_unit; 463 le_val = (__force __le16)tbl->entries[i].latency_or_bandwidth; 464 465 if (check_mul_overflow(le64_to_cpu(le_base), 466 le16_to_cpu(le_val), &val)) 467 dev_warn(dev, "SSLBIS value overflowed!\n"); 468 469 xa_for_each(&port->dports, index, dport) { 470 if (dsp_id == ACPI_CDAT_SSLBIS_ANY_PORT || 471 dsp_id == dport->port_id) 472 cxl_access_coordinate_set(&dport->sw_coord, 473 sslbis->data_type, 474 val); 475 } 476 } 477 478 return 0; 479 } 480 481 void cxl_switch_parse_cdat(struct cxl_port *port) 482 { 483 int rc; 484 485 if (!port->cdat.table) 486 return; 487 488 rc = cdat_table_parse(ACPI_CDAT_TYPE_SSLBIS, cdat_sslbis_handler, 489 port, port->cdat.table, port->cdat.length); 490 rc = cdat_table_parse_output(rc); 491 if (rc) 492 dev_dbg(&port->dev, "Failed to parse SSLBIS: %d\n", rc); 493 } 494 EXPORT_SYMBOL_NS_GPL(cxl_switch_parse_cdat, CXL); 495 496 /** 497 * cxl_coordinates_combine - Combine the two input coordinates 498 * 499 * @out: Output coordinate of c1 and c2 combined 500 * @c1: input coordinates 501 * @c2: input coordinates 502 */ 503 void cxl_coordinates_combine(struct access_coordinate *out, 504 struct access_coordinate *c1, 505 struct access_coordinate *c2) 506 { 507 if (c1->write_bandwidth && c2->write_bandwidth) 508 out->write_bandwidth = min(c1->write_bandwidth, 509 c2->write_bandwidth); 510 out->write_latency = c1->write_latency + c2->write_latency; 511 512 if (c1->read_bandwidth && c2->read_bandwidth) 513 out->read_bandwidth = min(c1->read_bandwidth, 514 c2->read_bandwidth); 515 out->read_latency = c1->read_latency + c2->read_latency; 516 } 517 518 MODULE_IMPORT_NS(CXL); 519 520 void cxl_region_perf_data_calculate(struct cxl_region *cxlr, 521 struct cxl_endpoint_decoder *cxled) 522 { 523 struct cxl_memdev *cxlmd = cxled_to_memdev(cxled); 524 struct cxl_port *port = cxlmd->endpoint; 525 struct cxl_dev_state *cxlds = cxlmd->cxlds; 526 struct cxl_memdev_state *mds = to_cxl_memdev_state(cxlds); 527 struct access_coordinate hb_coord[ACCESS_COORDINATE_MAX]; 528 struct access_coordinate coord; 529 struct range dpa = { 530 .start = cxled->dpa_res->start, 531 .end = cxled->dpa_res->end, 532 }; 533 struct cxl_dpa_perf *perf; 534 int rc; 535 536 switch (cxlr->mode) { 537 case CXL_DECODER_RAM: 538 perf = &mds->ram_perf; 539 break; 540 case CXL_DECODER_PMEM: 541 perf = &mds->pmem_perf; 542 break; 543 default: 544 return; 545 } 546 547 lockdep_assert_held(&cxl_dpa_rwsem); 548 549 if (!range_contains(&perf->dpa_range, &dpa)) 550 return; 551 552 rc = cxl_hb_get_perf_coordinates(port, hb_coord); 553 if (rc) { 554 dev_dbg(&port->dev, "Failed to retrieve hb perf coordinates.\n"); 555 return; 556 } 557 558 for (int i = 0; i < ACCESS_COORDINATE_MAX; i++) { 559 /* Pickup the host bridge coords */ 560 cxl_coordinates_combine(&coord, &hb_coord[i], &perf->coord); 561 562 /* Get total bandwidth and the worst latency for the cxl region */ 563 cxlr->coord[i].read_latency = max_t(unsigned int, 564 cxlr->coord[i].read_latency, 565 coord.read_latency); 566 cxlr->coord[i].write_latency = max_t(unsigned int, 567 cxlr->coord[i].write_latency, 568 coord.write_latency); 569 cxlr->coord[i].read_bandwidth += coord.read_bandwidth; 570 cxlr->coord[i].write_bandwidth += coord.write_bandwidth; 571 572 /* 573 * Convert latency to nanosec from picosec to be consistent 574 * with the resulting latency coordinates computed by the 575 * HMAT_REPORTING code. 576 */ 577 cxlr->coord[i].read_latency = 578 DIV_ROUND_UP(cxlr->coord[i].read_latency, 1000); 579 cxlr->coord[i].write_latency = 580 DIV_ROUND_UP(cxlr->coord[i].write_latency, 1000); 581 } 582 } 583 584 int cxl_update_hmat_access_coordinates(int nid, struct cxl_region *cxlr, 585 enum access_coordinate_class access) 586 { 587 return hmat_update_target_coordinates(nid, &cxlr->coord[access], access); 588 } 589 590 bool cxl_need_node_perf_attrs_update(int nid) 591 { 592 return !acpi_node_backed_by_real_pxm(nid); 593 } 594