1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * TPH (TLP Processing Hints) support 4 * 5 * Copyright (C) 2024 Advanced Micro Devices, Inc. 6 * Eric Van Tassell <Eric.VanTassell@amd.com> 7 * Wei Huang <wei.huang2@amd.com> 8 */ 9 #include <linux/pci.h> 10 #include <linux/pci-acpi.h> 11 #include <linux/msi.h> 12 #include <linux/bitfield.h> 13 #include <linux/pci-tph.h> 14 15 #include "pci.h" 16 17 /* System-wide TPH disabled */ 18 static bool pci_tph_disabled; 19 20 #ifdef CONFIG_ACPI 21 /* 22 * The st_info struct defines the Steering Tag (ST) info returned by the 23 * firmware PCI ACPI _DSM method (rev=0x7, func=0xF, "_DSM to Query Cache 24 * Locality TPH Features"), as specified in the approved ECN for PCI Firmware 25 * Spec and available at https://members.pcisig.com/wg/PCI-SIG/document/15470. 26 * 27 * @vm_st_valid: 8-bit ST for volatile memory is valid 28 * @vm_xst_valid: 16-bit extended ST for volatile memory is valid 29 * @vm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied 30 * @vm_st: 8-bit ST for volatile mem 31 * @vm_xst: 16-bit extended ST for volatile mem 32 * @pm_st_valid: 8-bit ST for persistent memory is valid 33 * @pm_xst_valid: 16-bit extended ST for persistent memory is valid 34 * @pm_ph_ignore: 1 => PH was and will be ignored, 0 => PH should be supplied 35 * @pm_st: 8-bit ST for persistent mem 36 * @pm_xst: 16-bit extended ST for persistent mem 37 */ 38 union st_info { 39 struct { 40 u64 vm_st_valid : 1; 41 u64 vm_xst_valid : 1; 42 u64 vm_ph_ignore : 1; 43 u64 rsvd1 : 5; 44 u64 vm_st : 8; 45 u64 vm_xst : 16; 46 u64 pm_st_valid : 1; 47 u64 pm_xst_valid : 1; 48 u64 pm_ph_ignore : 1; 49 u64 rsvd2 : 5; 50 u64 pm_st : 8; 51 u64 pm_xst : 16; 52 }; 53 u64 value; 54 }; 55 56 static u16 tph_extract_tag(enum tph_mem_type mem_type, u8 req_type, 57 union st_info *info) 58 { 59 switch (req_type) { 60 case PCI_TPH_REQ_TPH_ONLY: /* 8-bit tag */ 61 switch (mem_type) { 62 case TPH_MEM_TYPE_VM: 63 if (info->vm_st_valid) 64 return info->vm_st; 65 break; 66 case TPH_MEM_TYPE_PM: 67 if (info->pm_st_valid) 68 return info->pm_st; 69 break; 70 } 71 break; 72 case PCI_TPH_REQ_EXT_TPH: /* 16-bit tag */ 73 switch (mem_type) { 74 case TPH_MEM_TYPE_VM: 75 if (info->vm_xst_valid) 76 return info->vm_xst; 77 break; 78 case TPH_MEM_TYPE_PM: 79 if (info->pm_xst_valid) 80 return info->pm_xst; 81 break; 82 } 83 break; 84 default: 85 return 0; 86 } 87 88 return 0; 89 } 90 91 #define TPH_ST_DSM_FUNC_INDEX 0xF 92 static acpi_status tph_invoke_dsm(acpi_handle handle, u32 cpu_uid, 93 union st_info *st_out) 94 { 95 union acpi_object arg3[3], in_obj, *out_obj; 96 97 if (!acpi_check_dsm(handle, &pci_acpi_dsm_guid, 7, 98 BIT(TPH_ST_DSM_FUNC_INDEX))) 99 return AE_ERROR; 100 101 /* DWORD: feature ID (0 for processor cache ST query) */ 102 arg3[0].integer.type = ACPI_TYPE_INTEGER; 103 arg3[0].integer.value = 0; 104 105 /* DWORD: target UID */ 106 arg3[1].integer.type = ACPI_TYPE_INTEGER; 107 arg3[1].integer.value = cpu_uid; 108 109 /* QWORD: properties, all 0's */ 110 arg3[2].integer.type = ACPI_TYPE_INTEGER; 111 arg3[2].integer.value = 0; 112 113 in_obj.type = ACPI_TYPE_PACKAGE; 114 in_obj.package.count = ARRAY_SIZE(arg3); 115 in_obj.package.elements = arg3; 116 117 out_obj = acpi_evaluate_dsm(handle, &pci_acpi_dsm_guid, 7, 118 TPH_ST_DSM_FUNC_INDEX, &in_obj); 119 if (!out_obj) 120 return AE_ERROR; 121 122 if (out_obj->type != ACPI_TYPE_BUFFER) { 123 ACPI_FREE(out_obj); 124 return AE_ERROR; 125 } 126 127 st_out->value = *((u64 *)(out_obj->buffer.pointer)); 128 129 ACPI_FREE(out_obj); 130 131 return AE_OK; 132 } 133 #endif 134 135 /* Update the TPH Requester Enable field of TPH Control Register */ 136 static void set_ctrl_reg_req_en(struct pci_dev *pdev, u8 req_type) 137 { 138 u32 reg; 139 140 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, ®); 141 142 reg &= ~PCI_TPH_CTRL_REQ_EN_MASK; 143 reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, req_type); 144 145 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg); 146 } 147 148 static u8 get_st_modes(struct pci_dev *pdev) 149 { 150 u32 reg; 151 152 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); 153 reg &= PCI_TPH_CAP_ST_NS | PCI_TPH_CAP_ST_IV | PCI_TPH_CAP_ST_DS; 154 155 return reg; 156 } 157 158 static u32 get_st_table_loc(struct pci_dev *pdev) 159 { 160 u32 reg; 161 162 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); 163 164 return FIELD_GET(PCI_TPH_CAP_LOC_MASK, reg); 165 } 166 167 /* 168 * Return the size of ST table. If ST table is not in TPH Requester Extended 169 * Capability space, return 0. Otherwise return the ST Table Size + 1. 170 */ 171 static u16 get_st_table_size(struct pci_dev *pdev) 172 { 173 u32 reg; 174 u32 loc; 175 176 /* Check ST table location first */ 177 loc = get_st_table_loc(pdev); 178 179 /* Convert loc to match with PCI_TPH_LOC_* defined in pci_regs.h */ 180 loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc); 181 if (loc != PCI_TPH_LOC_CAP) 182 return 0; 183 184 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); 185 186 return FIELD_GET(PCI_TPH_CAP_ST_MASK, reg) + 1; 187 } 188 189 /* Return device's Root Port completer capability */ 190 static u8 get_rp_completer_type(struct pci_dev *pdev) 191 { 192 struct pci_dev *rp; 193 u32 reg; 194 int ret; 195 196 rp = pcie_find_root_port(pdev); 197 if (!rp) 198 return 0; 199 200 ret = pcie_capability_read_dword(rp, PCI_EXP_DEVCAP2, ®); 201 if (ret) 202 return 0; 203 204 return FIELD_GET(PCI_EXP_DEVCAP2_TPH_COMP_MASK, reg); 205 } 206 207 /* Write ST to MSI-X vector control reg - Return 0 if OK, otherwise -errno */ 208 static int write_tag_to_msix(struct pci_dev *pdev, int msix_idx, u16 tag) 209 { 210 #ifdef CONFIG_PCI_MSI 211 struct msi_desc *msi_desc = NULL; 212 void __iomem *vec_ctrl; 213 u32 val; 214 int err = 0; 215 216 msi_lock_descs(&pdev->dev); 217 218 /* Find the msi_desc entry with matching msix_idx */ 219 msi_for_each_desc(msi_desc, &pdev->dev, MSI_DESC_ASSOCIATED) { 220 if (msi_desc->msi_index == msix_idx) 221 break; 222 } 223 224 if (!msi_desc) { 225 err = -ENXIO; 226 goto err_out; 227 } 228 229 /* Get the vector control register (offset 0xc) pointed by msix_idx */ 230 vec_ctrl = pdev->msix_base + msix_idx * PCI_MSIX_ENTRY_SIZE; 231 vec_ctrl += PCI_MSIX_ENTRY_VECTOR_CTRL; 232 233 val = readl(vec_ctrl); 234 val &= ~PCI_MSIX_ENTRY_CTRL_ST; 235 val |= FIELD_PREP(PCI_MSIX_ENTRY_CTRL_ST, tag); 236 writel(val, vec_ctrl); 237 238 /* Read back to flush the update */ 239 val = readl(vec_ctrl); 240 241 err_out: 242 msi_unlock_descs(&pdev->dev); 243 return err; 244 #else 245 return -ENODEV; 246 #endif 247 } 248 249 /* Write tag to ST table - Return 0 if OK, otherwise -errno */ 250 static int write_tag_to_st_table(struct pci_dev *pdev, int index, u16 tag) 251 { 252 int st_table_size; 253 int offset; 254 255 /* Check if index is out of bound */ 256 st_table_size = get_st_table_size(pdev); 257 if (index >= st_table_size) 258 return -ENXIO; 259 260 offset = pdev->tph_cap + PCI_TPH_BASE_SIZEOF + index * sizeof(u16); 261 262 return pci_write_config_word(pdev, offset, tag); 263 } 264 265 /** 266 * pcie_tph_get_cpu_st() - Retrieve Steering Tag for a target memory associated 267 * with a specific CPU 268 * @pdev: PCI device 269 * @mem_type: target memory type (volatile or persistent RAM) 270 * @cpu_uid: associated CPU id 271 * @tag: Steering Tag to be returned 272 * 273 * Return the Steering Tag for a target memory that is associated with a 274 * specific CPU as indicated by cpu_uid. 275 * 276 * Return: 0 if success, otherwise negative value (-errno) 277 */ 278 int pcie_tph_get_cpu_st(struct pci_dev *pdev, enum tph_mem_type mem_type, 279 unsigned int cpu_uid, u16 *tag) 280 { 281 #ifdef CONFIG_ACPI 282 struct pci_dev *rp; 283 acpi_handle rp_acpi_handle; 284 union st_info info; 285 286 rp = pcie_find_root_port(pdev); 287 if (!rp || !rp->bus || !rp->bus->bridge) 288 return -ENODEV; 289 290 rp_acpi_handle = ACPI_HANDLE(rp->bus->bridge); 291 292 if (tph_invoke_dsm(rp_acpi_handle, cpu_uid, &info) != AE_OK) { 293 *tag = 0; 294 return -EINVAL; 295 } 296 297 *tag = tph_extract_tag(mem_type, pdev->tph_req_type, &info); 298 299 pci_dbg(pdev, "get steering tag: mem_type=%s, cpu_uid=%d, tag=%#04x\n", 300 (mem_type == TPH_MEM_TYPE_VM) ? "volatile" : "persistent", 301 cpu_uid, *tag); 302 303 return 0; 304 #else 305 return -ENODEV; 306 #endif 307 } 308 EXPORT_SYMBOL(pcie_tph_get_cpu_st); 309 310 /** 311 * pcie_tph_set_st_entry() - Set Steering Tag in the ST table entry 312 * @pdev: PCI device 313 * @index: ST table entry index 314 * @tag: Steering Tag to be written 315 * 316 * Figure out the proper location of ST table, either in the MSI-X table or 317 * in the TPH Extended Capability space, and write the Steering Tag into 318 * the ST entry pointed by index. 319 * 320 * Return: 0 if success, otherwise negative value (-errno) 321 */ 322 int pcie_tph_set_st_entry(struct pci_dev *pdev, unsigned int index, u16 tag) 323 { 324 u32 loc; 325 int err = 0; 326 327 if (!pdev->tph_cap) 328 return -EINVAL; 329 330 if (!pdev->tph_enabled) 331 return -EINVAL; 332 333 /* No need to write tag if device is in "No ST Mode" */ 334 if (pdev->tph_mode == PCI_TPH_ST_NS_MODE) 335 return 0; 336 337 /* 338 * Disable TPH before updating ST to avoid potential instability as 339 * cautioned in PCIe r6.2, sec 6.17.3, "ST Modes of Operation" 340 */ 341 set_ctrl_reg_req_en(pdev, PCI_TPH_REQ_DISABLE); 342 343 loc = get_st_table_loc(pdev); 344 /* Convert loc to match with PCI_TPH_LOC_* */ 345 loc = FIELD_PREP(PCI_TPH_CAP_LOC_MASK, loc); 346 347 switch (loc) { 348 case PCI_TPH_LOC_MSIX: 349 err = write_tag_to_msix(pdev, index, tag); 350 break; 351 case PCI_TPH_LOC_CAP: 352 err = write_tag_to_st_table(pdev, index, tag); 353 break; 354 default: 355 err = -EINVAL; 356 } 357 358 if (err) { 359 pcie_disable_tph(pdev); 360 return err; 361 } 362 363 set_ctrl_reg_req_en(pdev, pdev->tph_mode); 364 365 pci_dbg(pdev, "set steering tag: %s table, index=%d, tag=%#04x\n", 366 (loc == PCI_TPH_LOC_MSIX) ? "MSI-X" : "ST", index, tag); 367 368 return 0; 369 } 370 EXPORT_SYMBOL(pcie_tph_set_st_entry); 371 372 /** 373 * pcie_disable_tph - Turn off TPH support for device 374 * @pdev: PCI device 375 * 376 * Return: none 377 */ 378 void pcie_disable_tph(struct pci_dev *pdev) 379 { 380 if (!pdev->tph_cap) 381 return; 382 383 if (!pdev->tph_enabled) 384 return; 385 386 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, 0); 387 388 pdev->tph_mode = 0; 389 pdev->tph_req_type = 0; 390 pdev->tph_enabled = 0; 391 } 392 EXPORT_SYMBOL(pcie_disable_tph); 393 394 /** 395 * pcie_enable_tph - Enable TPH support for device using a specific ST mode 396 * @pdev: PCI device 397 * @mode: ST mode to enable. Current supported modes include: 398 * 399 * - PCI_TPH_ST_NS_MODE: NO ST Mode 400 * - PCI_TPH_ST_IV_MODE: Interrupt Vector Mode 401 * - PCI_TPH_ST_DS_MODE: Device Specific Mode 402 * 403 * Check whether the mode is actually supported by the device before enabling 404 * and return an error if not. Additionally determine what types of requests, 405 * TPH or extended TPH, can be issued by the device based on its TPH requester 406 * capability and the Root Port's completer capability. 407 * 408 * Return: 0 on success, otherwise negative value (-errno) 409 */ 410 int pcie_enable_tph(struct pci_dev *pdev, int mode) 411 { 412 u32 reg; 413 u8 dev_modes; 414 u8 rp_req_type; 415 416 /* Honor "notph" kernel parameter */ 417 if (pci_tph_disabled) 418 return -EINVAL; 419 420 if (!pdev->tph_cap) 421 return -EINVAL; 422 423 if (pdev->tph_enabled) 424 return -EBUSY; 425 426 /* Sanitize and check ST mode compatibility */ 427 mode &= PCI_TPH_CTRL_MODE_SEL_MASK; 428 dev_modes = get_st_modes(pdev); 429 if (!((1 << mode) & dev_modes)) 430 return -EINVAL; 431 432 pdev->tph_mode = mode; 433 434 /* Get req_type supported by device and its Root Port */ 435 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CAP, ®); 436 if (FIELD_GET(PCI_TPH_CAP_EXT_TPH, reg)) 437 pdev->tph_req_type = PCI_TPH_REQ_EXT_TPH; 438 else 439 pdev->tph_req_type = PCI_TPH_REQ_TPH_ONLY; 440 441 rp_req_type = get_rp_completer_type(pdev); 442 443 /* Final req_type is the smallest value of two */ 444 pdev->tph_req_type = min(pdev->tph_req_type, rp_req_type); 445 446 if (pdev->tph_req_type == PCI_TPH_REQ_DISABLE) 447 return -EINVAL; 448 449 /* Write them into TPH control register */ 450 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, ®); 451 452 reg &= ~PCI_TPH_CTRL_MODE_SEL_MASK; 453 reg |= FIELD_PREP(PCI_TPH_CTRL_MODE_SEL_MASK, pdev->tph_mode); 454 455 reg &= ~PCI_TPH_CTRL_REQ_EN_MASK; 456 reg |= FIELD_PREP(PCI_TPH_CTRL_REQ_EN_MASK, pdev->tph_req_type); 457 458 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, reg); 459 460 pdev->tph_enabled = 1; 461 462 return 0; 463 } 464 EXPORT_SYMBOL(pcie_enable_tph); 465 466 void pci_restore_tph_state(struct pci_dev *pdev) 467 { 468 struct pci_cap_saved_state *save_state; 469 int num_entries, i, offset; 470 u16 *st_entry; 471 u32 *cap; 472 473 if (!pdev->tph_cap) 474 return; 475 476 if (!pdev->tph_enabled) 477 return; 478 479 save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH); 480 if (!save_state) 481 return; 482 483 /* Restore control register and all ST entries */ 484 cap = &save_state->cap.data[0]; 485 pci_write_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, *cap++); 486 st_entry = (u16 *)cap; 487 offset = PCI_TPH_BASE_SIZEOF; 488 num_entries = get_st_table_size(pdev); 489 for (i = 0; i < num_entries; i++) { 490 pci_write_config_word(pdev, pdev->tph_cap + offset, 491 *st_entry++); 492 offset += sizeof(u16); 493 } 494 } 495 496 void pci_save_tph_state(struct pci_dev *pdev) 497 { 498 struct pci_cap_saved_state *save_state; 499 int num_entries, i, offset; 500 u16 *st_entry; 501 u32 *cap; 502 503 if (!pdev->tph_cap) 504 return; 505 506 if (!pdev->tph_enabled) 507 return; 508 509 save_state = pci_find_saved_ext_cap(pdev, PCI_EXT_CAP_ID_TPH); 510 if (!save_state) 511 return; 512 513 /* Save control register */ 514 cap = &save_state->cap.data[0]; 515 pci_read_config_dword(pdev, pdev->tph_cap + PCI_TPH_CTRL, cap++); 516 517 /* Save all ST entries in extended capability structure */ 518 st_entry = (u16 *)cap; 519 offset = PCI_TPH_BASE_SIZEOF; 520 num_entries = get_st_table_size(pdev); 521 for (i = 0; i < num_entries; i++) { 522 pci_read_config_word(pdev, pdev->tph_cap + offset, 523 st_entry++); 524 offset += sizeof(u16); 525 } 526 } 527 528 void pci_no_tph(void) 529 { 530 pci_tph_disabled = true; 531 532 pr_info("PCIe TPH is disabled\n"); 533 } 534 535 void pci_tph_init(struct pci_dev *pdev) 536 { 537 int num_entries; 538 u32 save_size; 539 540 pdev->tph_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_TPH); 541 if (!pdev->tph_cap) 542 return; 543 544 num_entries = get_st_table_size(pdev); 545 save_size = sizeof(u32) + num_entries * sizeof(u16); 546 pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_TPH, save_size); 547 } 548