1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Express I/O Virtualization (IOV) support 4 * Address Translation Service 1.0 5 * Page Request Interface added by Joerg Roedel <joerg.roedel@amd.com> 6 * PASID support added by Joerg Roedel <joerg.roedel@amd.com> 7 * 8 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 9 * Copyright (C) 2011 Advanced Micro Devices, 10 */ 11 12 #include <linux/bitfield.h> 13 #include <linux/export.h> 14 #include <linux/pci-ats.h> 15 #include <linux/pci.h> 16 #include <linux/slab.h> 17 18 #include "pci.h" 19 20 void pci_ats_init(struct pci_dev *dev) 21 { 22 int pos; 23 24 if (pci_ats_disabled()) 25 return; 26 27 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ATS); 28 if (!pos) 29 return; 30 31 dev->ats_cap = pos; 32 } 33 34 /** 35 * pci_ats_supported - check if the device can use ATS 36 * @dev: the PCI device 37 * 38 * Returns true if the device supports ATS and is allowed to use it, false 39 * otherwise. 40 */ 41 bool pci_ats_supported(struct pci_dev *dev) 42 { 43 if (!dev->ats_cap) 44 return false; 45 46 return (dev->untrusted == 0); 47 } 48 EXPORT_SYMBOL_GPL(pci_ats_supported); 49 50 /** 51 * pci_prepare_ats - Setup the PS for ATS 52 * @dev: the PCI device 53 * @ps: the IOMMU page shift 54 * 55 * This must be done by the IOMMU driver on the PF before any VFs are created to 56 * ensure that the VF can have ATS enabled. 57 * 58 * Returns 0 on success, or negative on failure. 59 */ 60 int pci_prepare_ats(struct pci_dev *dev, int ps) 61 { 62 u16 ctrl; 63 64 if (!pci_ats_supported(dev)) 65 return -EINVAL; 66 67 if (WARN_ON(dev->ats_enabled)) 68 return -EBUSY; 69 70 if (ps < PCI_ATS_MIN_STU) 71 return -EINVAL; 72 73 if (dev->is_virtfn) 74 return 0; 75 76 dev->ats_stu = ps; 77 ctrl = PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 78 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 79 return 0; 80 } 81 EXPORT_SYMBOL_GPL(pci_prepare_ats); 82 83 /** 84 * pci_enable_ats - enable the ATS capability 85 * @dev: the PCI device 86 * @ps: the IOMMU page shift 87 * 88 * Returns 0 on success, or negative on failure. 89 */ 90 int pci_enable_ats(struct pci_dev *dev, int ps) 91 { 92 u16 ctrl; 93 struct pci_dev *pdev; 94 95 if (!pci_ats_supported(dev)) 96 return -EINVAL; 97 98 if (WARN_ON(dev->ats_enabled)) 99 return -EBUSY; 100 101 if (ps < PCI_ATS_MIN_STU) 102 return -EINVAL; 103 104 /* 105 * Note that enabling ATS on a VF fails unless it's already enabled 106 * with the same STU on the PF. 107 */ 108 ctrl = PCI_ATS_CTRL_ENABLE; 109 if (dev->is_virtfn) { 110 pdev = pci_physfn(dev); 111 if (pdev->ats_stu != ps) 112 return -EINVAL; 113 } else { 114 dev->ats_stu = ps; 115 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 116 } 117 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 118 119 dev->ats_enabled = 1; 120 return 0; 121 } 122 EXPORT_SYMBOL_GPL(pci_enable_ats); 123 124 /** 125 * pci_disable_ats - disable the ATS capability 126 * @dev: the PCI device 127 */ 128 void pci_disable_ats(struct pci_dev *dev) 129 { 130 u16 ctrl; 131 132 if (WARN_ON(!dev->ats_enabled)) 133 return; 134 135 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, &ctrl); 136 ctrl &= ~PCI_ATS_CTRL_ENABLE; 137 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 138 139 dev->ats_enabled = 0; 140 } 141 EXPORT_SYMBOL_GPL(pci_disable_ats); 142 143 void pci_restore_ats_state(struct pci_dev *dev) 144 { 145 u16 ctrl; 146 147 if (!dev->ats_enabled) 148 return; 149 150 ctrl = PCI_ATS_CTRL_ENABLE; 151 if (!dev->is_virtfn) 152 ctrl |= PCI_ATS_CTRL_STU(dev->ats_stu - PCI_ATS_MIN_STU); 153 pci_write_config_word(dev, dev->ats_cap + PCI_ATS_CTRL, ctrl); 154 } 155 156 /** 157 * pci_ats_queue_depth - query the ATS Invalidate Queue Depth 158 * @dev: the PCI device 159 * 160 * Returns the queue depth on success, or negative on failure. 161 * 162 * The ATS spec uses 0 in the Invalidate Queue Depth field to 163 * indicate that the function can accept 32 Invalidate Request. 164 * But here we use the `real' values (i.e. 1~32) for the Queue 165 * Depth; and 0 indicates the function shares the Queue with 166 * other functions (doesn't exclusively own a Queue). 167 */ 168 int pci_ats_queue_depth(struct pci_dev *dev) 169 { 170 u16 cap; 171 172 if (!dev->ats_cap) 173 return -EINVAL; 174 175 if (dev->is_virtfn) 176 return 0; 177 178 pci_read_config_word(dev, dev->ats_cap + PCI_ATS_CAP, &cap); 179 return PCI_ATS_CAP_QDEP(cap) ? PCI_ATS_CAP_QDEP(cap) : PCI_ATS_MAX_QDEP; 180 } 181 182 /** 183 * pci_ats_page_aligned - Return Page Aligned Request bit status. 184 * @pdev: the PCI device 185 * 186 * Returns 1, if the Untranslated Addresses generated by the device 187 * are always aligned or 0 otherwise. 188 * 189 * Per PCIe spec r4.0, sec 10.5.1.2, if the Page Aligned Request bit 190 * is set, it indicates the Untranslated Addresses generated by the 191 * device are always aligned to a 4096 byte boundary. 192 */ 193 int pci_ats_page_aligned(struct pci_dev *pdev) 194 { 195 u16 cap; 196 197 if (!pdev->ats_cap) 198 return 0; 199 200 pci_read_config_word(pdev, pdev->ats_cap + PCI_ATS_CAP, &cap); 201 202 if (cap & PCI_ATS_CAP_PAGE_ALIGNED) 203 return 1; 204 205 return 0; 206 } 207 208 #ifdef CONFIG_PCI_PRI 209 void pci_pri_init(struct pci_dev *pdev) 210 { 211 u16 status; 212 213 pdev->pri_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PRI); 214 215 if (!pdev->pri_cap) 216 return; 217 218 pci_read_config_word(pdev, pdev->pri_cap + PCI_PRI_STATUS, &status); 219 if (status & PCI_PRI_STATUS_PASID) 220 pdev->pasid_required = 1; 221 } 222 223 /** 224 * pci_enable_pri - Enable PRI capability 225 * @pdev: PCI device structure 226 * @reqs: outstanding requests 227 * 228 * Returns 0 on success, negative value on error 229 */ 230 int pci_enable_pri(struct pci_dev *pdev, u32 reqs) 231 { 232 u16 control, status; 233 u32 max_requests; 234 int pri = pdev->pri_cap; 235 236 /* 237 * VFs must not implement the PRI Capability. If their PF 238 * implements PRI, it is shared by the VFs, so if the PF PRI is 239 * enabled, it is also enabled for the VF. 240 */ 241 if (pdev->is_virtfn) { 242 if (pci_physfn(pdev)->pri_enabled) 243 return 0; 244 return -EINVAL; 245 } 246 247 if (WARN_ON(pdev->pri_enabled)) 248 return -EBUSY; 249 250 if (!pri) 251 return -EINVAL; 252 253 pci_read_config_word(pdev, pri + PCI_PRI_STATUS, &status); 254 if (!(status & PCI_PRI_STATUS_STOPPED)) 255 return -EBUSY; 256 257 pci_read_config_dword(pdev, pri + PCI_PRI_MAX_REQ, &max_requests); 258 reqs = min(max_requests, reqs); 259 pdev->pri_reqs_alloc = reqs; 260 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 261 262 control = PCI_PRI_CTRL_ENABLE; 263 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 264 265 pdev->pri_enabled = 1; 266 267 return 0; 268 } 269 270 /** 271 * pci_disable_pri - Disable PRI capability 272 * @pdev: PCI device structure 273 * 274 * Only clears the enabled-bit, regardless of its former value 275 */ 276 void pci_disable_pri(struct pci_dev *pdev) 277 { 278 u16 control; 279 int pri = pdev->pri_cap; 280 281 /* VFs share the PF PRI */ 282 if (pdev->is_virtfn) 283 return; 284 285 if (WARN_ON(!pdev->pri_enabled)) 286 return; 287 288 if (!pri) 289 return; 290 291 pci_read_config_word(pdev, pri + PCI_PRI_CTRL, &control); 292 control &= ~PCI_PRI_CTRL_ENABLE; 293 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 294 295 pdev->pri_enabled = 0; 296 } 297 EXPORT_SYMBOL_GPL(pci_disable_pri); 298 299 /** 300 * pci_restore_pri_state - Restore PRI 301 * @pdev: PCI device structure 302 */ 303 void pci_restore_pri_state(struct pci_dev *pdev) 304 { 305 u16 control = PCI_PRI_CTRL_ENABLE; 306 u32 reqs = pdev->pri_reqs_alloc; 307 int pri = pdev->pri_cap; 308 309 if (pdev->is_virtfn) 310 return; 311 312 if (!pdev->pri_enabled) 313 return; 314 315 if (!pri) 316 return; 317 318 pci_write_config_dword(pdev, pri + PCI_PRI_ALLOC_REQ, reqs); 319 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 320 } 321 322 /** 323 * pci_reset_pri - Resets device's PRI state 324 * @pdev: PCI device structure 325 * 326 * The PRI capability must be disabled before this function is called. 327 * Returns 0 on success, negative value on error. 328 */ 329 int pci_reset_pri(struct pci_dev *pdev) 330 { 331 u16 control; 332 int pri = pdev->pri_cap; 333 334 if (pdev->is_virtfn) 335 return 0; 336 337 if (WARN_ON(pdev->pri_enabled)) 338 return -EBUSY; 339 340 if (!pri) 341 return -EINVAL; 342 343 control = PCI_PRI_CTRL_RESET; 344 pci_write_config_word(pdev, pri + PCI_PRI_CTRL, control); 345 346 return 0; 347 } 348 349 /** 350 * pci_prg_resp_pasid_required - Return PRG Response PASID Required bit 351 * status. 352 * @pdev: PCI device structure 353 * 354 * Returns 1 if PASID is required in PRG Response Message, 0 otherwise. 355 */ 356 int pci_prg_resp_pasid_required(struct pci_dev *pdev) 357 { 358 if (pdev->is_virtfn) 359 pdev = pci_physfn(pdev); 360 361 return pdev->pasid_required; 362 } 363 364 /** 365 * pci_pri_supported - Check if PRI is supported. 366 * @pdev: PCI device structure 367 * 368 * Returns true if PRI capability is present, false otherwise. 369 */ 370 bool pci_pri_supported(struct pci_dev *pdev) 371 { 372 /* VFs share the PF PRI */ 373 if (pci_physfn(pdev)->pri_cap) 374 return true; 375 return false; 376 } 377 EXPORT_SYMBOL_GPL(pci_pri_supported); 378 #endif /* CONFIG_PCI_PRI */ 379 380 #ifdef CONFIG_PCI_PASID 381 void pci_pasid_init(struct pci_dev *pdev) 382 { 383 pdev->pasid_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_PASID); 384 } 385 386 /** 387 * pci_enable_pasid - Enable the PASID capability 388 * @pdev: PCI device structure 389 * @features: Features to enable 390 * 391 * Returns 0 on success, negative value on error. This function checks 392 * whether the features are actually supported by the device and returns 393 * an error if not. 394 */ 395 int pci_enable_pasid(struct pci_dev *pdev, int features) 396 { 397 u16 control, supported; 398 int pasid = pdev->pasid_cap; 399 400 /* 401 * VFs must not implement the PASID Capability, but if a PF 402 * supports PASID, its VFs share the PF PASID configuration. 403 */ 404 if (pdev->is_virtfn) { 405 if (pci_physfn(pdev)->pasid_enabled) 406 return 0; 407 return -EINVAL; 408 } 409 410 if (WARN_ON(pdev->pasid_enabled)) 411 return -EBUSY; 412 413 if (!pdev->eetlp_prefix_path && !pdev->pasid_no_tlp) 414 return -EINVAL; 415 416 if (!pasid) 417 return -EINVAL; 418 419 if (!pci_acs_path_enabled(pdev, NULL, PCI_ACS_RR | PCI_ACS_UF)) 420 return -EINVAL; 421 422 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 423 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 424 425 /* User wants to enable anything unsupported? */ 426 if ((supported & features) != features) 427 return -EINVAL; 428 429 control = PCI_PASID_CTRL_ENABLE | features; 430 pdev->pasid_features = features; 431 432 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 433 434 pdev->pasid_enabled = 1; 435 436 return 0; 437 } 438 EXPORT_SYMBOL_GPL(pci_enable_pasid); 439 440 /** 441 * pci_disable_pasid - Disable the PASID capability 442 * @pdev: PCI device structure 443 */ 444 void pci_disable_pasid(struct pci_dev *pdev) 445 { 446 u16 control = 0; 447 int pasid = pdev->pasid_cap; 448 449 /* VFs share the PF PASID configuration */ 450 if (pdev->is_virtfn) 451 return; 452 453 if (WARN_ON(!pdev->pasid_enabled)) 454 return; 455 456 if (!pasid) 457 return; 458 459 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 460 461 pdev->pasid_enabled = 0; 462 } 463 EXPORT_SYMBOL_GPL(pci_disable_pasid); 464 465 /** 466 * pci_restore_pasid_state - Restore PASID capabilities 467 * @pdev: PCI device structure 468 */ 469 void pci_restore_pasid_state(struct pci_dev *pdev) 470 { 471 u16 control; 472 int pasid = pdev->pasid_cap; 473 474 if (pdev->is_virtfn) 475 return; 476 477 if (!pdev->pasid_enabled) 478 return; 479 480 if (!pasid) 481 return; 482 483 control = PCI_PASID_CTRL_ENABLE | pdev->pasid_features; 484 pci_write_config_word(pdev, pasid + PCI_PASID_CTRL, control); 485 } 486 487 /** 488 * pci_pasid_features - Check which PASID features are supported 489 * @pdev: PCI device structure 490 * 491 * Returns a negative value when no PASI capability is present. 492 * Otherwise is returns a bitmask with supported features. Current 493 * features reported are: 494 * PCI_PASID_CAP_EXEC - Execute permission supported 495 * PCI_PASID_CAP_PRIV - Privileged mode supported 496 */ 497 int pci_pasid_features(struct pci_dev *pdev) 498 { 499 u16 supported; 500 int pasid; 501 502 if (pdev->is_virtfn) 503 pdev = pci_physfn(pdev); 504 505 pasid = pdev->pasid_cap; 506 if (!pasid) 507 return -EINVAL; 508 509 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 510 511 supported &= PCI_PASID_CAP_EXEC | PCI_PASID_CAP_PRIV; 512 513 return supported; 514 } 515 EXPORT_SYMBOL_GPL(pci_pasid_features); 516 517 /** 518 * pci_max_pasids - Get maximum number of PASIDs supported by device 519 * @pdev: PCI device structure 520 * 521 * Returns negative value when PASID capability is not present. 522 * Otherwise it returns the number of supported PASIDs. 523 */ 524 int pci_max_pasids(struct pci_dev *pdev) 525 { 526 u16 supported; 527 int pasid; 528 529 if (pdev->is_virtfn) 530 pdev = pci_physfn(pdev); 531 532 pasid = pdev->pasid_cap; 533 if (!pasid) 534 return -EINVAL; 535 536 pci_read_config_word(pdev, pasid + PCI_PASID_CAP, &supported); 537 538 return (1 << FIELD_GET(PCI_PASID_CAP_WIDTH, supported)); 539 } 540 EXPORT_SYMBOL_GPL(pci_max_pasids); 541 #endif /* CONFIG_PCI_PASID */ 542