1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/export.h> 14 #include <linux/pci-ats.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 18 #include "pci.h" 19 20 void pci_ats_init(struct pci_dev *dev) 21 { 22 int pos; 23 24 if (pci_ats_disabled()) 25 return; 26 27 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 28 if (!pos) 29 return; 30 31 dev->ats_cap = pos; 32 } 33 34 /** 35 * pci_ats_supported - check if the device can use ATS 36 * @dev: the PCI device 37 * 38 * Returns true if the device supports ATS and is allowed to use it, false 39 * otherwise. 40 */ 41 bool pci_ats_supported(struct pci_dev *dev) 42 { 43 if (!dev->ats_cap) 44 return false; 45 46 return (dev->untrusted == 0); 47 } 48 EXPORT_SYMBOL_GPL(pci_ats_supported); 49 50 /** 51 * pci_prepare_ats - Setup the PS for ATS 52 * @dev: the PCI device 53 * @ps: the IOMMU page shift 54 * 55 * This must be done by the IOMMU driver on the PF before any VFs are created to 56 * ensure that the VF can have ATS enabled. 57 * 58 * Returns 0 on success, or negative on failure. 59 */ 60 int pci_prepare_ats(struct pci_dev *dev, int ps) 61 { 62 u16 ctrl; 63 64 if (!pci_ats_supported(dev)) 65 return -EINVAL; 66 67 if (WARN_ON(dev->ats_enabled)) 68 return -EBUSY; 69 70 if (ps < PCI_ATS_MIN_STU) 71 return -EINVAL; 72 73 if (dev->is_virtfn) 74 return 0; 75 76 dev->ats_stu = ps; 77 ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 78 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 79 return 0; 80 } 81 EXPORT_SYMBOL_GPL(pci_prepare_ats); 82 83 /** 84 * pci_enable_ats - enable the ATS capability 85 * @dev: the PCI device 86 * @ps: the IOMMU page shift 87 * 88 * Returns 0 on success, or negative on failure. 89 */ 90 int pci_enable_ats(struct pci_dev *dev, int ps) 91 { 92 u16 ctrl; 93 struct pci_dev *pdev; 94 95 if (!pci_ats_supported(dev)) 96 return -EINVAL; 97 98 if (WARN_ON(dev->ats_enabled)) 99 return -EBUSY; 100 101 if (ps < PCI_ATS_MIN_STU) 102 return -EINVAL; 103 104 /* 105 * Note that enabling ATS on a VF fails unless it's already enabled 106 * with the same STU on the PF. 107 */ 108 ctrl = PCI_ATS_CTRL_ENABLE; 109 if (dev->is_virtfn) { 110 pdev = pci_physfn(dev); 111 if (pdev->ats_stu != ps) 112 return -EINVAL; 113 } else { 114 dev->ats_stu = ps; 115 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 116 } 117 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 118 119 dev->ats_enabled = 1; 120 return 0; 121 } 122 EXPORT_SYMBOL_GPL(pci_enable_ats); 123 124 /** 125 * pci_disable_ats - disable the ATS capability 126 * @dev: the PCI device 127 */ 128 void pci_disable_ats(struct pci_dev *dev) 129 { 130 u16 ctrl; 131 132 if (WARN_ON(!dev->ats_enabled)) 133 return; 134 135 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 136 ctrl &= ~PCI_ATS_CTRL_ENABLE; 137 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 138 139 dev->ats_enabled = 0; 140 } 141 EXPORT_SYMBOL_GPL(pci_disable_ats); 142 143 void pci_restore_ats_state(struct pci_dev *dev) 144 { 145 u16 ctrl; 146 147 if (!dev->ats_enabled) 148 return; 149 150 ctrl = PCI_ATS_CTRL_ENABLE; 151 if (!dev->is_virtfn) 152 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 153 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 154 } 155 156 /** 157 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 158 * @dev: the PCI device 159 * 160 * Returns the queue depth on success, or negative on failure. 161 * 162 * The ATS spec uses 0 in the Invalidate Queue Depth field to 163 * indicate that the function can accept 32 Invalidate Request. 164 * But here we use the `real' values (i.e. 1~32) for the Queue 165 * Depth; and 0 indicates the function shares the Queue with 166 * other functions (doesn't exclusively own a Queue). 167 */ 168 int pci_ats_queue_depth(struct pci_dev *dev) 169 { 170 u16 cap; 171 172 if (!dev->ats_cap) 173 return -EINVAL; 174 175 if (dev->is_virtfn) 176 return 0; 177 178 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 179 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 180 } 181 182 /** 183 * pci_ats_page_aligned - Return Page Aligned Request bit status. 184 * @pdev: the PCI device 185 * 186 * Returns 1, if the Untranslated Addresses generated by the device 187 * are always aligned or 0 otherwise. 188 * 189 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit 190 * is set, it indicates the Untranslated Addresses generated by the 191 * device are always aligned to a 4096 byte boundary. 192 */ 193 int pci_ats_page_aligned(struct pci_dev *pdev) 194 { 195 u16 cap; 196 197 if (!pdev->ats_cap) 198 return 0; 199 200 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); 201 202 if (cap & PCI_ATS_CAP_PAGE_ALIGNED) 203 return 1; 204 205 return 0; 206 } 207 208 /* 209 * CXL r4.0, sec 3.2.5.13 Memory Type on CXL.cache notes: to source requests on 210 * CXL.cache, devices need to get the Host Physical Address (HPA) from the Host 211 * by means of an ATS request on CXL.io. 212 * 213 * In other words, CXL.cache devices cannot access host physical memory without 214 * ATS. 215 * 216 * Check Cache_Capable instead of Cache_Enable because CXL.cache may be enabled 217 * after the caller uses this to make its ATS decision. 218 */ 219 static bool pci_cxl_ats_required(struct pci_dev *pdev) 220 { 221 int offset; 222 u16 cap; 223 224 offset = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, 225 PCI_DVSEC_CXL_DEVICE); 226 if (!offset) 227 return false; 228 229 if (pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap)) 230 return false; 231 232 return cap & PCI_DVSEC_CXL_CACHE_CAPABLE; 233 } 234 235 /** 236 * pci_ats_required - Whether the PCI device requires ATS 237 * @pdev: the PCI device 238 * 239 * Returns true, if the PCI device requires ATS for basic functional operation. 240 */ 241 bool pci_ats_required(struct pci_dev *pdev) 242 { 243 if (!pci_ats_supported(pdev)) 244 return false; 245 246 /* A VF inherits its PF's requirement for ATS function */ 247 if (pdev->is_virtfn) 248 pdev = pci_physfn(pdev); 249 250 return pci_cxl_ats_required(pdev) || 251 pci_dev_specific_ats_required(pdev); 252 } 253 EXPORT_SYMBOL_GPL(pci_ats_required); 254 255 #ifdef CONFIG_PCI_PRI 256 void pci_pri_init(struct pci_dev *pdev) 257 { 258 u16 status; 259 260 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 261 262 if (!pdev->pri_cap) 263 return; 264 265 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status); 266 if (status & PCI_PRI_STATUS_PASID) 267 pdev->pasid_required = 1; 268 } 269 270 /** 271 * pci_enable_pri - Enable PRI capability 272 * @pdev: PCI device structure 273 * @reqs: outstanding requests 274 * 275 * Returns 0 on success, negative value on error 276 */ 277 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 278 { 279 u16 control, status; 280 u32 max_requests; 281 int pri = pdev->pri_cap; 282 283 /* 284 * VFs must not implement the PRI Capability. If their PF 285 * implements PRI, it is shared by the VFs, so if the PF PRI is 286 * enabled, it is also enabled for the VF. 287 */ 288 if (pdev->is_virtfn) { 289 if (pci_physfn(pdev)->pri_enabled) 290 return 0; 291 return -EINVAL; 292 } 293 294 if (WARN_ON(pdev->pri_enabled)) 295 return -EBUSY; 296 297 if (!pri) 298 return -EINVAL; 299 300 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status); 301 if (!(status & PCI_PRI_STATUS_STOPPED)) 302 return -EBUSY; 303 304 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests); 305 reqs = min(max_requests, reqs); 306 pdev->pri_reqs_alloc = reqs; 307 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 308 309 control = PCI_PRI_CTRL_ENABLE; 310 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 311 312 pdev->pri_enabled = 1; 313 314 return 0; 315 } 316 317 /** 318 * pci_disable_pri - Disable PRI capability 319 * @pdev: PCI device structure 320 * 321 * Only clears the enabled-bit, regardless of its former value 322 */ 323 void pci_disable_pri(struct pci_dev *pdev) 324 { 325 u16 control; 326 int pri = pdev->pri_cap; 327 328 /* VFs share the PF PRI */ 329 if (pdev->is_virtfn) 330 return; 331 332 if (WARN_ON(!pdev->pri_enabled)) 333 return; 334 335 if (!pri) 336 return; 337 338 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control); 339 control &= ~PCI_PRI_CTRL_ENABLE; 340 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 341 342 pdev->pri_enabled = 0; 343 } 344 EXPORT_SYMBOL_GPL(pci_disable_pri); 345 346 /** 347 * pci_restore_pri_state - Restore PRI 348 * @pdev: PCI device structure 349 */ 350 void pci_restore_pri_state(struct pci_dev *pdev) 351 { 352 u16 control = PCI_PRI_CTRL_ENABLE; 353 u32 reqs = pdev->pri_reqs_alloc; 354 int pri = pdev->pri_cap; 355 356 if (pdev->is_virtfn) 357 return; 358 359 if (!pdev->pri_enabled) 360 return; 361 362 if (!pri) 363 return; 364 365 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 366 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 367 } 368 369 /** 370 * pci_reset_pri - Resets device's PRI state 371 * @pdev: PCI device structure 372 * 373 * The PRI capability must be disabled before this function is called. 374 * Returns 0 on success, negative value on error. 375 */ 376 int pci_reset_pri(struct pci_dev *pdev) 377 { 378 u16 control; 379 int pri = pdev->pri_cap; 380 381 if (pdev->is_virtfn) 382 return 0; 383 384 if (WARN_ON(pdev->pri_enabled)) 385 return -EBUSY; 386 387 if (!pri) 388 return -EINVAL; 389 390 control = PCI_PRI_CTRL_RESET; 391 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 392 393 return 0; 394 } 395 396 /** 397 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit 398 * status. 399 * @pdev: PCI device structure 400 * 401 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. 402 */ 403 int pci_prg_resp_pasid_required(struct pci_dev *pdev) 404 { 405 if (pdev->is_virtfn) 406 pdev = pci_physfn(pdev); 407 408 return pdev->pasid_required; 409 } 410 411 /** 412 * pci_pri_supported - Check if PRI is supported. 413 * @pdev: PCI device structure 414 * 415 * Returns true if PRI capability is present, false otherwise. 416 */ 417 bool pci_pri_supported(struct pci_dev *pdev) 418 { 419 /* VFs share the PF PRI */ 420 if (pci_physfn(pdev)->pri_cap) 421 return true; 422 return false; 423 } 424 EXPORT_SYMBOL_GPL(pci_pri_supported); 425 #endif /* CONFIG_PCI_PRI */ 426 427 #ifdef CONFIG_PCI_PASID 428 void pci_pasid_init(struct pci_dev *pdev) 429 { 430 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 431 } 432 433 /** 434 * pci_enable_pasid - Enable the PASID capability 435 * @pdev: PCI device structure 436 * @features: Features to enable 437 * 438 * Returns 0 on success, negative value on error. This function checks 439 * whether the features are actually supported by the device and returns 440 * an error if not. 441 */ 442 int pci_enable_pasid(struct pci_dev *pdev, int features) 443 { 444 u16 control, supported; 445 int pasid = pdev->pasid_cap; 446 447 /* 448 * VFs must not implement the PASID Capability, but if a PF 449 * supports PASID, its VFs share the PF PASID configuration. 450 */ 451 if (pdev->is_virtfn) { 452 if (pci_physfn(pdev)->pasid_enabled) 453 return 0; 454 return -EINVAL; 455 } 456 457 if (WARN_ON(pdev->pasid_enabled)) 458 return -EBUSY; 459 460 if (!pdev->eetlp_prefix_max && !pdev->pasid_no_tlp) 461 return -EINVAL; 462 463 if (!pasid) 464 return -EINVAL; 465 466 if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF)) 467 return -EINVAL; 468 469 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 470 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 471 472 /* User wants to enable anything unsupported? */ 473 if ((supported & features) != features) 474 return -EINVAL; 475 476 control = PCI_PASID_CTRL_ENABLE | features; 477 pdev->pasid_features = features; 478 479 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 480 481 pdev->pasid_enabled = 1; 482 483 return 0; 484 } 485 EXPORT_SYMBOL_GPL(pci_enable_pasid); 486 487 /** 488 * pci_disable_pasid - Disable the PASID capability 489 * @pdev: PCI device structure 490 */ 491 void pci_disable_pasid(struct pci_dev *pdev) 492 { 493 u16 control = 0; 494 int pasid = pdev->pasid_cap; 495 496 /* VFs share the PF PASID configuration */ 497 if (pdev->is_virtfn) 498 return; 499 500 if (WARN_ON(!pdev->pasid_enabled)) 501 return; 502 503 if (!pasid) 504 return; 505 506 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 507 508 pdev->pasid_enabled = 0; 509 } 510 EXPORT_SYMBOL_GPL(pci_disable_pasid); 511 512 /** 513 * pci_restore_pasid_state - Restore PASID capabilities 514 * @pdev: PCI device structure 515 */ 516 void pci_restore_pasid_state(struct pci_dev *pdev) 517 { 518 u16 control; 519 int pasid = pdev->pasid_cap; 520 521 if (pdev->is_virtfn) 522 return; 523 524 if (!pdev->pasid_enabled) 525 return; 526 527 if (!pasid) 528 return; 529 530 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 531 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 532 } 533 534 /** 535 * pci_pasid_features - Check which PASID features are supported 536 * @pdev: PCI device structure 537 * 538 * Return a negative value when no PASID capability is present. 539 * Otherwise return a bitmask with supported features. Current 540 * features reported are: 541 * PCI_PASID_CAP_EXEC - Execute permission supported 542 * PCI_PASID_CAP_PRIV - Privileged mode supported 543 */ 544 int pci_pasid_features(struct pci_dev *pdev) 545 { 546 u16 supported; 547 int pasid; 548 549 if (pdev->is_virtfn) 550 pdev = pci_physfn(pdev); 551 552 pasid = pdev->pasid_cap; 553 if (!pasid) 554 return -EINVAL; 555 556 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 557 558 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 559 560 return supported; 561 } 562 EXPORT_SYMBOL_GPL(pci_pasid_features); 563 564 /** 565 * pci_max_pasids - Get maximum number of PASIDs supported by device 566 * @pdev: PCI device structure 567 * 568 * Returns negative value when PASID capability is not present. 569 * Otherwise it returns the number of supported PASIDs. 570 */ 571 int pci_max_pasids(struct pci_dev *pdev) 572 { 573 u16 supported; 574 int pasid; 575 576 if (pdev->is_virtfn) 577 pdev = pci_physfn(pdev); 578 579 pasid = pdev->pasid_cap; 580 if (!pasid) 581 return -EINVAL; 582 583 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 584 585 return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported)); 586 } 587 EXPORT_SYMBOL_GPL(pci_max_pasids); 588 589 /** 590 * pci_pasid_status - Check the PASID status 591 * @pdev: PCI device structure 592 * 593 * Returns a negative value when no PASID capability is present. 594 * Otherwise the value of the control register is returned. 595 * Status reported are: 596 * 597 * PCI_PASID_CTRL_ENABLE - PASID enabled 598 * PCI_PASID_CTRL_EXEC - Execute permission enabled 599 * PCI_PASID_CTRL_PRIV - Privileged mode enabled 600 */ 601 int pci_pasid_status(struct pci_dev *pdev) 602 { 603 int pasid; 604 u16 ctrl; 605 606 if (pdev->is_virtfn) 607 pdev = pci_physfn(pdev); 608 609 pasid = pdev->pasid_cap; 610 if (!pasid) 611 return -EINVAL; 612 613 pci_read_config_word(pdev, pasid + PCI_PASID_CTRL, &ctrl); 614 615 ctrl &= PCI_PASID_CTRL_ENABLE | PCI_PASID_CTRL_EXEC | 616 PCI_PASID_CTRL_PRIV; 617 618 return ctrl; 619 } 620 EXPORT_SYMBOL_GPL(pci_pasid_status); 621 #endif /* CONFIG_PCI_PASID */ 622