1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/export.h> 14 #include <linux/pci-ats.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 18 #include "pci.h" 19 20 void pci_ats_init(struct pci_dev *dev) 21 { 22 int pos; 23 24 if (pci_ats_disabled()) 25 return; 26 27 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 28 if (!pos) 29 return; 30 31 dev->ats_cap = pos; 32 } 33 34 /** 35 * pci_ats_supported - check if the device can use ATS 36 * @dev: the PCI device 37 * 38 * Returns true if the device supports ATS and is allowed to use it, false 39 * otherwise. 40 */ 41 bool pci_ats_supported(struct pci_dev *dev) 42 { 43 if (!dev->ats_cap || dev->untrusted) 44 return false; 45 46 if (dev->is_virtfn) 47 return pci_ats_supported(pci_physfn(dev)); 48 49 return true; 50 } 51 EXPORT_SYMBOL_GPL(pci_ats_supported); 52 53 /** 54 * pci_prepare_ats - Setup the PS for ATS 55 * @dev: the PCI device 56 * @ps: the IOMMU page shift 57 * 58 * This must be done by the IOMMU driver on the PF before any VFs are created to 59 * ensure that the VF can have ATS enabled. 60 * 61 * Returns 0 on success, or negative on failure. 62 */ 63 int pci_prepare_ats(struct pci_dev *dev, int ps) 64 { 65 u16 ctrl; 66 67 if (!pci_ats_supported(dev)) 68 return -EINVAL; 69 70 if (WARN_ON(dev->ats_enabled)) 71 return -EBUSY; 72 73 if (ps < PCI_ATS_MIN_STU) 74 return -EINVAL; 75 76 if (dev->is_virtfn) { 77 if (pci_physfn(dev)->ats_stu != ps) 78 return -EINVAL; 79 80 return 0; 81 } 82 83 dev->ats_stu = ps; 84 ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 85 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 86 return 0; 87 } 88 EXPORT_SYMBOL_GPL(pci_prepare_ats); 89 90 /** 91 * pci_enable_ats - enable the ATS capability 92 * @dev: the PCI device 93 * @ps: the IOMMU page shift 94 * 95 * Returns 0 on success, or negative on failure. 96 */ 97 int pci_enable_ats(struct pci_dev *dev, int ps) 98 { 99 u16 ctrl; 100 struct pci_dev *pdev; 101 102 if (!pci_ats_supported(dev)) 103 return -EINVAL; 104 105 if (WARN_ON(dev->ats_enabled)) 106 return -EBUSY; 107 108 if (ps < PCI_ATS_MIN_STU) 109 return -EINVAL; 110 111 /* 112 * Note that enabling ATS on a VF fails unless it's already enabled 113 * with the same STU on the PF. 114 */ 115 ctrl = PCI_ATS_CTRL_ENABLE; 116 if (dev->is_virtfn) { 117 pdev = pci_physfn(dev); 118 if (pdev->ats_stu != ps) 119 return -EINVAL; 120 } else { 121 dev->ats_stu = ps; 122 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 123 } 124 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 125 126 dev->ats_enabled = 1; 127 return 0; 128 } 129 EXPORT_SYMBOL_GPL(pci_enable_ats); 130 131 /** 132 * pci_disable_ats - disable the ATS capability 133 * @dev: the PCI device 134 */ 135 void pci_disable_ats(struct pci_dev *dev) 136 { 137 u16 ctrl; 138 139 if (WARN_ON(!dev->ats_enabled)) 140 return; 141 142 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 143 ctrl &= ~PCI_ATS_CTRL_ENABLE; 144 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 145 146 dev->ats_enabled = 0; 147 } 148 EXPORT_SYMBOL_GPL(pci_disable_ats); 149 150 void pci_restore_ats_state(struct pci_dev *dev) 151 { 152 u16 ctrl; 153 154 if (!dev->ats_enabled) 155 return; 156 157 ctrl = PCI_ATS_CTRL_ENABLE; 158 if (!dev->is_virtfn) 159 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 160 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 161 } 162 163 /** 164 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 165 * @dev: the PCI device 166 * 167 * Returns the queue depth on success, or negative on failure. 168 * 169 * The ATS spec uses 0 in the Invalidate Queue Depth field to 170 * indicate that the function can accept 32 Invalidate Request. 171 * But here we use the `real' values (i.e. 1~32) for the Queue 172 * Depth; and 0 indicates the function shares the Queue with 173 * other functions (doesn't exclusively own a Queue). 174 */ 175 int pci_ats_queue_depth(struct pci_dev *dev) 176 { 177 u16 cap; 178 179 if (!dev->ats_cap) 180 return -EINVAL; 181 182 if (dev->is_virtfn) 183 return 0; 184 185 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 186 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 187 } 188 189 /** 190 * pci_ats_page_aligned - Return Page Aligned Request bit status. 191 * @pdev: the PCI device 192 * 193 * Returns 1, if the Untranslated Addresses generated by the device 194 * are always aligned or 0 otherwise. 195 * 196 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit 197 * is set, it indicates the Untranslated Addresses generated by the 198 * device are always aligned to a 4096 byte boundary. 199 */ 200 int pci_ats_page_aligned(struct pci_dev *pdev) 201 { 202 u16 cap; 203 204 if (!pdev->ats_cap) 205 return 0; 206 207 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); 208 209 if (cap & PCI_ATS_CAP_PAGE_ALIGNED) 210 return 1; 211 212 return 0; 213 } 214 215 /* 216 * CXL r4.0, sec 3.2.5.13 Memory Type on CXL.cache notes: to source requests on 217 * CXL.cache, devices need to get the Host Physical Address (HPA) from the Host 218 * by means of an ATS request on CXL.io. 219 * 220 * In other words, CXL.cache devices cannot access host physical memory without 221 * ATS. 222 * 223 * Check Cache_Capable instead of Cache_Enable because CXL.cache may be enabled 224 * after the caller uses this to make its ATS decision. 225 */ 226 static bool pci_cxl_ats_required(struct pci_dev *pdev) 227 { 228 int offset; 229 u16 cap; 230 231 offset = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL, 232 PCI_DVSEC_CXL_DEVICE); 233 if (!offset) 234 return false; 235 236 if (pci_read_config_word(pdev, offset + PCI_DVSEC_CXL_CAP, &cap)) 237 return false; 238 239 return cap & PCI_DVSEC_CXL_CACHE_CAPABLE; 240 } 241 242 /** 243 * pci_ats_required - Whether the PCI device requires ATS 244 * @pdev: the PCI device 245 * 246 * Returns true, if the PCI device requires ATS for basic functional operation. 247 */ 248 bool pci_ats_required(struct pci_dev *pdev) 249 { 250 if (!pci_ats_supported(pdev)) 251 return false; 252 253 /* A VF inherits its PF's requirement for ATS function */ 254 if (pdev->is_virtfn) 255 pdev = pci_physfn(pdev); 256 257 return pci_cxl_ats_required(pdev) || 258 pci_dev_specific_ats_required(pdev); 259 } 260 EXPORT_SYMBOL_GPL(pci_ats_required); 261 262 #ifdef CONFIG_PCI_PRI 263 void pci_pri_init(struct pci_dev *pdev) 264 { 265 u16 status; 266 267 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 268 269 if (!pdev->pri_cap) 270 return; 271 272 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status); 273 if (status & PCI_PRI_STATUS_PASID) 274 pdev->pasid_required = 1; 275 } 276 277 /** 278 * pci_enable_pri - Enable PRI capability 279 * @pdev: PCI device structure 280 * @reqs: outstanding requests 281 * 282 * Returns 0 on success, negative value on error 283 */ 284 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 285 { 286 u16 control, status; 287 u32 max_requests; 288 int pri = pdev->pri_cap; 289 290 /* 291 * VFs must not implement the PRI Capability. If their PF 292 * implements PRI, it is shared by the VFs, so if the PF PRI is 293 * enabled, it is also enabled for the VF. 294 */ 295 if (pdev->is_virtfn) { 296 if (pci_physfn(pdev)->pri_enabled) 297 return 0; 298 return -EINVAL; 299 } 300 301 if (WARN_ON(pdev->pri_enabled)) 302 return -EBUSY; 303 304 if (!pri) 305 return -EINVAL; 306 307 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status); 308 if (!(status & PCI_PRI_STATUS_STOPPED)) 309 return -EBUSY; 310 311 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests); 312 reqs = min(max_requests, reqs); 313 pdev->pri_reqs_alloc = reqs; 314 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 315 316 control = PCI_PRI_CTRL_ENABLE; 317 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 318 319 pdev->pri_enabled = 1; 320 321 return 0; 322 } 323 324 /** 325 * pci_disable_pri - Disable PRI capability 326 * @pdev: PCI device structure 327 * 328 * Only clears the enabled-bit, regardless of its former value 329 */ 330 void pci_disable_pri(struct pci_dev *pdev) 331 { 332 u16 control; 333 int pri = pdev->pri_cap; 334 335 /* VFs share the PF PRI */ 336 if (pdev->is_virtfn) 337 return; 338 339 if (WARN_ON(!pdev->pri_enabled)) 340 return; 341 342 if (!pri) 343 return; 344 345 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control); 346 control &= ~PCI_PRI_CTRL_ENABLE; 347 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 348 349 pdev->pri_enabled = 0; 350 } 351 EXPORT_SYMBOL_GPL(pci_disable_pri); 352 353 /** 354 * pci_restore_pri_state - Restore PRI 355 * @pdev: PCI device structure 356 */ 357 void pci_restore_pri_state(struct pci_dev *pdev) 358 { 359 u16 control = PCI_PRI_CTRL_ENABLE; 360 u32 reqs = pdev->pri_reqs_alloc; 361 int pri = pdev->pri_cap; 362 363 if (pdev->is_virtfn) 364 return; 365 366 if (!pdev->pri_enabled) 367 return; 368 369 if (!pri) 370 return; 371 372 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 373 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 374 } 375 376 /** 377 * pci_reset_pri - Resets device's PRI state 378 * @pdev: PCI device structure 379 * 380 * The PRI capability must be disabled before this function is called. 381 * Returns 0 on success, negative value on error. 382 */ 383 int pci_reset_pri(struct pci_dev *pdev) 384 { 385 u16 control; 386 int pri = pdev->pri_cap; 387 388 if (pdev->is_virtfn) 389 return 0; 390 391 if (WARN_ON(pdev->pri_enabled)) 392 return -EBUSY; 393 394 if (!pri) 395 return -EINVAL; 396 397 control = PCI_PRI_CTRL_RESET; 398 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 399 400 return 0; 401 } 402 403 /** 404 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit 405 * status. 406 * @pdev: PCI device structure 407 * 408 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. 409 */ 410 int pci_prg_resp_pasid_required(struct pci_dev *pdev) 411 { 412 if (pdev->is_virtfn) 413 pdev = pci_physfn(pdev); 414 415 return pdev->pasid_required; 416 } 417 418 /** 419 * pci_pri_supported - Check if PRI is supported. 420 * @pdev: PCI device structure 421 * 422 * Returns true if PRI capability is present, false otherwise. 423 */ 424 bool pci_pri_supported(struct pci_dev *pdev) 425 { 426 /* VFs share the PF PRI */ 427 if (pci_physfn(pdev)->pri_cap) 428 return true; 429 return false; 430 } 431 EXPORT_SYMBOL_GPL(pci_pri_supported); 432 #endif /* CONFIG_PCI_PRI */ 433 434 #ifdef CONFIG_PCI_PASID 435 void pci_pasid_init(struct pci_dev *pdev) 436 { 437 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 438 } 439 440 /** 441 * pci_enable_pasid - Enable the PASID capability 442 * @pdev: PCI device structure 443 * @features: Features to enable 444 * 445 * Returns 0 on success, negative value on error. This function checks 446 * whether the features are actually supported by the device and returns 447 * an error if not. 448 */ 449 int pci_enable_pasid(struct pci_dev *pdev, int features) 450 { 451 u16 control, supported; 452 int pasid = pdev->pasid_cap; 453 454 /* 455 * VFs must not implement the PASID Capability, but if a PF 456 * supports PASID, its VFs share the PF PASID configuration. 457 */ 458 if (pdev->is_virtfn) { 459 if (pci_physfn(pdev)->pasid_enabled) 460 return 0; 461 return -EINVAL; 462 } 463 464 if (WARN_ON(pdev->pasid_enabled)) 465 return -EBUSY; 466 467 if (!pdev->eetlp_prefix_max && !pdev->pasid_no_tlp) 468 return -EINVAL; 469 470 if (!pasid) 471 return -EINVAL; 472 473 if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF)) 474 return -EINVAL; 475 476 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 477 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 478 479 /* User wants to enable anything unsupported? */ 480 if ((supported & features) != features) 481 return -EINVAL; 482 483 control = PCI_PASID_CTRL_ENABLE | features; 484 pdev->pasid_features = features; 485 486 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 487 488 pdev->pasid_enabled = 1; 489 490 return 0; 491 } 492 EXPORT_SYMBOL_GPL(pci_enable_pasid); 493 494 /** 495 * pci_disable_pasid - Disable the PASID capability 496 * @pdev: PCI device structure 497 */ 498 void pci_disable_pasid(struct pci_dev *pdev) 499 { 500 u16 control = 0; 501 int pasid = pdev->pasid_cap; 502 503 /* VFs share the PF PASID configuration */ 504 if (pdev->is_virtfn) 505 return; 506 507 if (WARN_ON(!pdev->pasid_enabled)) 508 return; 509 510 if (!pasid) 511 return; 512 513 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 514 515 pdev->pasid_enabled = 0; 516 } 517 EXPORT_SYMBOL_GPL(pci_disable_pasid); 518 519 /** 520 * pci_restore_pasid_state - Restore PASID capabilities 521 * @pdev: PCI device structure 522 */ 523 void pci_restore_pasid_state(struct pci_dev *pdev) 524 { 525 u16 control; 526 int pasid = pdev->pasid_cap; 527 528 if (pdev->is_virtfn) 529 return; 530 531 if (!pdev->pasid_enabled) 532 return; 533 534 if (!pasid) 535 return; 536 537 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 538 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 539 } 540 541 /** 542 * pci_pasid_features - Check which PASID features are supported 543 * @pdev: PCI device structure 544 * 545 * Return a negative value when no PASID capability is present. 546 * Otherwise return a bitmask with supported features. Current 547 * features reported are: 548 * PCI_PASID_CAP_EXEC - Execute permission supported 549 * PCI_PASID_CAP_PRIV - Privileged mode supported 550 */ 551 int pci_pasid_features(struct pci_dev *pdev) 552 { 553 u16 supported; 554 int pasid; 555 556 if (pdev->is_virtfn) 557 pdev = pci_physfn(pdev); 558 559 pasid = pdev->pasid_cap; 560 if (!pasid) 561 return -EINVAL; 562 563 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 564 565 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 566 567 return supported; 568 } 569 EXPORT_SYMBOL_GPL(pci_pasid_features); 570 571 /** 572 * pci_max_pasids - Get maximum number of PASIDs supported by device 573 * @pdev: PCI device structure 574 * 575 * Returns negative value when PASID capability is not present. 576 * Otherwise it returns the number of supported PASIDs. 577 */ 578 int pci_max_pasids(struct pci_dev *pdev) 579 { 580 u16 supported; 581 int pasid; 582 583 if (pdev->is_virtfn) 584 pdev = pci_physfn(pdev); 585 586 pasid = pdev->pasid_cap; 587 if (!pasid) 588 return -EINVAL; 589 590 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 591 592 return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported)); 593 } 594 EXPORT_SYMBOL_GPL(pci_max_pasids); 595 596 /** 597 * pci_pasid_status - Check the PASID status 598 * @pdev: PCI device structure 599 * 600 * Returns a negative value when no PASID capability is present. 601 * Otherwise the value of the control register is returned. 602 * Status reported are: 603 * 604 * PCI_PASID_CTRL_ENABLE - PASID enabled 605 * PCI_PASID_CTRL_EXEC - Execute permission enabled 606 * PCI_PASID_CTRL_PRIV - Privileged mode enabled 607 */ 608 int pci_pasid_status(struct pci_dev *pdev) 609 { 610 int pasid; 611 u16 ctrl; 612 613 if (pdev->is_virtfn) 614 pdev = pci_physfn(pdev); 615 616 pasid = pdev->pasid_cap; 617 if (!pasid) 618 return -EINVAL; 619 620 pci_read_config_word(pdev, pasid + PCI_PASID_CTRL, &ctrl); 621 622 ctrl &= PCI_PASID_CTRL_ENABLE | PCI_PASID_CTRL_EXEC | 623 PCI_PASID_CTRL_PRIV; 624 625 return ctrl; 626 } 627 EXPORT_SYMBOL_GPL(pci_pasid_status); 628 #endif /* CONFIG_PCI_PASID */ 629