1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * IOMMU API for s390 PCI devices 4 * 5 * Copyright IBM Corp. 2015 6 * Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com> 7 */ 8 9 #include <linux/pci.h> 10 #include <linux/iommu.h> 11 #include <linux/iommu-helper.h> 12 #include <linux/sizes.h> 13 #include <linux/rculist.h> 14 #include <linux/rcupdate.h> 15 #include <asm/pci_dma.h> 16 17 #include "dma-iommu.h" 18 19 static const struct iommu_ops s390_iommu_ops; 20 21 static struct kmem_cache *dma_region_table_cache; 22 static struct kmem_cache *dma_page_table_cache; 23 24 static u64 s390_iommu_aperture; 25 static u32 s390_iommu_aperture_factor = 1; 26 27 struct s390_domain { 28 struct iommu_domain domain; 29 struct list_head devices; 30 struct zpci_iommu_ctrs ctrs; 31 unsigned long *dma_table; 32 spinlock_t list_lock; 33 struct rcu_head rcu; 34 }; 35 36 static struct iommu_domain blocking_domain; 37 38 static inline unsigned int calc_rtx(dma_addr_t ptr) 39 { 40 return ((unsigned long)ptr >> ZPCI_RT_SHIFT) & ZPCI_INDEX_MASK; 41 } 42 43 static inline unsigned int calc_sx(dma_addr_t ptr) 44 { 45 return ((unsigned long)ptr >> ZPCI_ST_SHIFT) & ZPCI_INDEX_MASK; 46 } 47 48 static inline unsigned int calc_px(dma_addr_t ptr) 49 { 50 return ((unsigned long)ptr >> PAGE_SHIFT) & ZPCI_PT_MASK; 51 } 52 53 static inline void set_pt_pfaa(unsigned long *entry, phys_addr_t pfaa) 54 { 55 *entry &= ZPCI_PTE_FLAG_MASK; 56 *entry |= (pfaa & ZPCI_PTE_ADDR_MASK); 57 } 58 59 static inline void set_rt_sto(unsigned long *entry, phys_addr_t sto) 60 { 61 *entry &= ZPCI_RTE_FLAG_MASK; 62 *entry |= (sto & ZPCI_RTE_ADDR_MASK); 63 *entry |= ZPCI_TABLE_TYPE_RTX; 64 } 65 66 static inline void set_st_pto(unsigned long *entry, phys_addr_t pto) 67 { 68 *entry &= ZPCI_STE_FLAG_MASK; 69 *entry |= (pto & ZPCI_STE_ADDR_MASK); 70 *entry |= ZPCI_TABLE_TYPE_SX; 71 } 72 73 static inline void validate_rt_entry(unsigned long *entry) 74 { 75 *entry &= ~ZPCI_TABLE_VALID_MASK; 76 *entry &= ~ZPCI_TABLE_OFFSET_MASK; 77 *entry |= ZPCI_TABLE_VALID; 78 *entry |= ZPCI_TABLE_LEN_RTX; 79 } 80 81 static inline void validate_st_entry(unsigned long *entry) 82 { 83 *entry &= ~ZPCI_TABLE_VALID_MASK; 84 *entry |= ZPCI_TABLE_VALID; 85 } 86 87 static inline void invalidate_pt_entry(unsigned long *entry) 88 { 89 WARN_ON_ONCE((*entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_INVALID); 90 *entry &= ~ZPCI_PTE_VALID_MASK; 91 *entry |= ZPCI_PTE_INVALID; 92 } 93 94 static inline void validate_pt_entry(unsigned long *entry) 95 { 96 WARN_ON_ONCE((*entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID); 97 *entry &= ~ZPCI_PTE_VALID_MASK; 98 *entry |= ZPCI_PTE_VALID; 99 } 100 101 static inline void entry_set_protected(unsigned long *entry) 102 { 103 *entry &= ~ZPCI_TABLE_PROT_MASK; 104 *entry |= ZPCI_TABLE_PROTECTED; 105 } 106 107 static inline void entry_clr_protected(unsigned long *entry) 108 { 109 *entry &= ~ZPCI_TABLE_PROT_MASK; 110 *entry |= ZPCI_TABLE_UNPROTECTED; 111 } 112 113 static inline int reg_entry_isvalid(unsigned long entry) 114 { 115 return (entry & ZPCI_TABLE_VALID_MASK) == ZPCI_TABLE_VALID; 116 } 117 118 static inline int pt_entry_isvalid(unsigned long entry) 119 { 120 return (entry & ZPCI_PTE_VALID_MASK) == ZPCI_PTE_VALID; 121 } 122 123 static inline unsigned long *get_rt_sto(unsigned long entry) 124 { 125 if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_RTX) 126 return phys_to_virt(entry & ZPCI_RTE_ADDR_MASK); 127 else 128 return NULL; 129 } 130 131 static inline unsigned long *get_st_pto(unsigned long entry) 132 { 133 if ((entry & ZPCI_TABLE_TYPE_MASK) == ZPCI_TABLE_TYPE_SX) 134 return phys_to_virt(entry & ZPCI_STE_ADDR_MASK); 135 else 136 return NULL; 137 } 138 139 static int __init dma_alloc_cpu_table_caches(void) 140 { 141 dma_region_table_cache = kmem_cache_create("PCI_DMA_region_tables", 142 ZPCI_TABLE_SIZE, 143 ZPCI_TABLE_ALIGN, 144 0, NULL); 145 if (!dma_region_table_cache) 146 return -ENOMEM; 147 148 dma_page_table_cache = kmem_cache_create("PCI_DMA_page_tables", 149 ZPCI_PT_SIZE, 150 ZPCI_PT_ALIGN, 151 0, NULL); 152 if (!dma_page_table_cache) { 153 kmem_cache_destroy(dma_region_table_cache); 154 return -ENOMEM; 155 } 156 return 0; 157 } 158 159 static unsigned long *dma_alloc_cpu_table(gfp_t gfp) 160 { 161 unsigned long *table, *entry; 162 163 table = kmem_cache_alloc(dma_region_table_cache, gfp); 164 if (!table) 165 return NULL; 166 167 for (entry = table; entry < table + ZPCI_TABLE_ENTRIES; entry++) 168 *entry = ZPCI_TABLE_INVALID; 169 return table; 170 } 171 172 static void dma_free_cpu_table(void *table) 173 { 174 kmem_cache_free(dma_region_table_cache, table); 175 } 176 177 static void dma_free_page_table(void *table) 178 { 179 kmem_cache_free(dma_page_table_cache, table); 180 } 181 182 static void dma_free_seg_table(unsigned long entry) 183 { 184 unsigned long *sto = get_rt_sto(entry); 185 int sx; 186 187 for (sx = 0; sx < ZPCI_TABLE_ENTRIES; sx++) 188 if (reg_entry_isvalid(sto[sx])) 189 dma_free_page_table(get_st_pto(sto[sx])); 190 191 dma_free_cpu_table(sto); 192 } 193 194 static void dma_cleanup_tables(unsigned long *table) 195 { 196 int rtx; 197 198 if (!table) 199 return; 200 201 for (rtx = 0; rtx < ZPCI_TABLE_ENTRIES; rtx++) 202 if (reg_entry_isvalid(table[rtx])) 203 dma_free_seg_table(table[rtx]); 204 205 dma_free_cpu_table(table); 206 } 207 208 static unsigned long *dma_alloc_page_table(gfp_t gfp) 209 { 210 unsigned long *table, *entry; 211 212 table = kmem_cache_alloc(dma_page_table_cache, gfp); 213 if (!table) 214 return NULL; 215 216 for (entry = table; entry < table + ZPCI_PT_ENTRIES; entry++) 217 *entry = ZPCI_PTE_INVALID; 218 return table; 219 } 220 221 static unsigned long *dma_get_seg_table_origin(unsigned long *rtep, gfp_t gfp) 222 { 223 unsigned long old_rte, rte; 224 unsigned long *sto; 225 226 rte = READ_ONCE(*rtep); 227 if (reg_entry_isvalid(rte)) { 228 sto = get_rt_sto(rte); 229 } else { 230 sto = dma_alloc_cpu_table(gfp); 231 if (!sto) 232 return NULL; 233 234 set_rt_sto(&rte, virt_to_phys(sto)); 235 validate_rt_entry(&rte); 236 entry_clr_protected(&rte); 237 238 old_rte = cmpxchg(rtep, ZPCI_TABLE_INVALID, rte); 239 if (old_rte != ZPCI_TABLE_INVALID) { 240 /* Somone else was faster, use theirs */ 241 dma_free_cpu_table(sto); 242 sto = get_rt_sto(old_rte); 243 } 244 } 245 return sto; 246 } 247 248 static unsigned long *dma_get_page_table_origin(unsigned long *step, gfp_t gfp) 249 { 250 unsigned long old_ste, ste; 251 unsigned long *pto; 252 253 ste = READ_ONCE(*step); 254 if (reg_entry_isvalid(ste)) { 255 pto = get_st_pto(ste); 256 } else { 257 pto = dma_alloc_page_table(gfp); 258 if (!pto) 259 return NULL; 260 set_st_pto(&ste, virt_to_phys(pto)); 261 validate_st_entry(&ste); 262 entry_clr_protected(&ste); 263 264 old_ste = cmpxchg(step, ZPCI_TABLE_INVALID, ste); 265 if (old_ste != ZPCI_TABLE_INVALID) { 266 /* Somone else was faster, use theirs */ 267 dma_free_page_table(pto); 268 pto = get_st_pto(old_ste); 269 } 270 } 271 return pto; 272 } 273 274 static unsigned long *dma_walk_cpu_trans(unsigned long *rto, dma_addr_t dma_addr, gfp_t gfp) 275 { 276 unsigned long *sto, *pto; 277 unsigned int rtx, sx, px; 278 279 rtx = calc_rtx(dma_addr); 280 sto = dma_get_seg_table_origin(&rto[rtx], gfp); 281 if (!sto) 282 return NULL; 283 284 sx = calc_sx(dma_addr); 285 pto = dma_get_page_table_origin(&sto[sx], gfp); 286 if (!pto) 287 return NULL; 288 289 px = calc_px(dma_addr); 290 return &pto[px]; 291 } 292 293 static void dma_update_cpu_trans(unsigned long *ptep, phys_addr_t page_addr, int flags) 294 { 295 unsigned long pte; 296 297 pte = READ_ONCE(*ptep); 298 if (flags & ZPCI_PTE_INVALID) { 299 invalidate_pt_entry(&pte); 300 } else { 301 set_pt_pfaa(&pte, page_addr); 302 validate_pt_entry(&pte); 303 } 304 305 if (flags & ZPCI_TABLE_PROTECTED) 306 entry_set_protected(&pte); 307 else 308 entry_clr_protected(&pte); 309 310 xchg(ptep, pte); 311 } 312 313 static struct s390_domain *to_s390_domain(struct iommu_domain *dom) 314 { 315 return container_of(dom, struct s390_domain, domain); 316 } 317 318 static bool s390_iommu_capable(struct device *dev, enum iommu_cap cap) 319 { 320 struct zpci_dev *zdev = to_zpci_dev(dev); 321 322 switch (cap) { 323 case IOMMU_CAP_CACHE_COHERENCY: 324 return true; 325 case IOMMU_CAP_DEFERRED_FLUSH: 326 return zdev->pft != PCI_FUNC_TYPE_ISM; 327 default: 328 return false; 329 } 330 } 331 332 static struct iommu_domain *s390_domain_alloc_paging(struct device *dev) 333 { 334 struct s390_domain *s390_domain; 335 336 s390_domain = kzalloc(sizeof(*s390_domain), GFP_KERNEL); 337 if (!s390_domain) 338 return NULL; 339 340 s390_domain->dma_table = dma_alloc_cpu_table(GFP_KERNEL); 341 if (!s390_domain->dma_table) { 342 kfree(s390_domain); 343 return NULL; 344 } 345 s390_domain->domain.geometry.force_aperture = true; 346 s390_domain->domain.geometry.aperture_start = 0; 347 s390_domain->domain.geometry.aperture_end = ZPCI_TABLE_SIZE_RT - 1; 348 349 spin_lock_init(&s390_domain->list_lock); 350 INIT_LIST_HEAD_RCU(&s390_domain->devices); 351 352 return &s390_domain->domain; 353 } 354 355 static void s390_iommu_rcu_free_domain(struct rcu_head *head) 356 { 357 struct s390_domain *s390_domain = container_of(head, struct s390_domain, rcu); 358 359 dma_cleanup_tables(s390_domain->dma_table); 360 kfree(s390_domain); 361 } 362 363 static void s390_domain_free(struct iommu_domain *domain) 364 { 365 struct s390_domain *s390_domain = to_s390_domain(domain); 366 367 rcu_read_lock(); 368 WARN_ON(!list_empty(&s390_domain->devices)); 369 rcu_read_unlock(); 370 371 call_rcu(&s390_domain->rcu, s390_iommu_rcu_free_domain); 372 } 373 374 static void zdev_s390_domain_update(struct zpci_dev *zdev, 375 struct iommu_domain *domain) 376 { 377 unsigned long flags; 378 379 spin_lock_irqsave(&zdev->dom_lock, flags); 380 zdev->s390_domain = domain; 381 spin_unlock_irqrestore(&zdev->dom_lock, flags); 382 } 383 384 static int blocking_domain_attach_device(struct iommu_domain *domain, 385 struct device *dev) 386 { 387 struct zpci_dev *zdev = to_zpci_dev(dev); 388 struct s390_domain *s390_domain; 389 unsigned long flags; 390 391 if (zdev->s390_domain->type == IOMMU_DOMAIN_BLOCKED) 392 return 0; 393 394 s390_domain = to_s390_domain(zdev->s390_domain); 395 spin_lock_irqsave(&s390_domain->list_lock, flags); 396 list_del_rcu(&zdev->iommu_list); 397 spin_unlock_irqrestore(&s390_domain->list_lock, flags); 398 399 zpci_unregister_ioat(zdev, 0); 400 zdev->dma_table = NULL; 401 zdev_s390_domain_update(zdev, domain); 402 403 return 0; 404 } 405 406 static int s390_iommu_attach_device(struct iommu_domain *domain, 407 struct device *dev) 408 { 409 struct s390_domain *s390_domain = to_s390_domain(domain); 410 struct zpci_dev *zdev = to_zpci_dev(dev); 411 unsigned long flags; 412 u8 status; 413 int cc; 414 415 if (!zdev) 416 return -ENODEV; 417 418 if (WARN_ON(domain->geometry.aperture_start > zdev->end_dma || 419 domain->geometry.aperture_end < zdev->start_dma)) 420 return -EINVAL; 421 422 blocking_domain_attach_device(&blocking_domain, dev); 423 424 /* If we fail now DMA remains blocked via blocking domain */ 425 cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, 426 virt_to_phys(s390_domain->dma_table), &status); 427 if (cc && status != ZPCI_PCI_ST_FUNC_NOT_AVAIL) 428 return -EIO; 429 zdev->dma_table = s390_domain->dma_table; 430 zdev_s390_domain_update(zdev, domain); 431 432 spin_lock_irqsave(&s390_domain->list_lock, flags); 433 list_add_rcu(&zdev->iommu_list, &s390_domain->devices); 434 spin_unlock_irqrestore(&s390_domain->list_lock, flags); 435 436 return 0; 437 } 438 439 static void s390_iommu_get_resv_regions(struct device *dev, 440 struct list_head *list) 441 { 442 struct zpci_dev *zdev = to_zpci_dev(dev); 443 struct iommu_resv_region *region; 444 445 if (zdev->start_dma) { 446 region = iommu_alloc_resv_region(0, zdev->start_dma, 0, 447 IOMMU_RESV_RESERVED, GFP_KERNEL); 448 if (!region) 449 return; 450 list_add_tail(®ion->list, list); 451 } 452 453 if (zdev->end_dma < ZPCI_TABLE_SIZE_RT - 1) { 454 region = iommu_alloc_resv_region(zdev->end_dma + 1, 455 ZPCI_TABLE_SIZE_RT - zdev->end_dma - 1, 456 0, IOMMU_RESV_RESERVED, GFP_KERNEL); 457 if (!region) 458 return; 459 list_add_tail(®ion->list, list); 460 } 461 } 462 463 static struct iommu_device *s390_iommu_probe_device(struct device *dev) 464 { 465 struct zpci_dev *zdev; 466 467 if (!dev_is_pci(dev)) 468 return ERR_PTR(-ENODEV); 469 470 zdev = to_zpci_dev(dev); 471 472 if (zdev->start_dma > zdev->end_dma || 473 zdev->start_dma > ZPCI_TABLE_SIZE_RT - 1) 474 return ERR_PTR(-EINVAL); 475 476 if (zdev->end_dma > ZPCI_TABLE_SIZE_RT - 1) 477 zdev->end_dma = ZPCI_TABLE_SIZE_RT - 1; 478 479 if (zdev->tlb_refresh) 480 dev->iommu->shadow_on_flush = 1; 481 482 /* Start with DMA blocked */ 483 spin_lock_init(&zdev->dom_lock); 484 zdev_s390_domain_update(zdev, &blocking_domain); 485 486 return &zdev->iommu_dev; 487 } 488 489 static int zpci_refresh_all(struct zpci_dev *zdev) 490 { 491 return zpci_refresh_trans((u64)zdev->fh << 32, zdev->start_dma, 492 zdev->end_dma - zdev->start_dma + 1); 493 } 494 495 static void s390_iommu_flush_iotlb_all(struct iommu_domain *domain) 496 { 497 struct s390_domain *s390_domain = to_s390_domain(domain); 498 struct zpci_dev *zdev; 499 500 rcu_read_lock(); 501 list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { 502 atomic64_inc(&s390_domain->ctrs.global_rpcits); 503 zpci_refresh_all(zdev); 504 } 505 rcu_read_unlock(); 506 } 507 508 static void s390_iommu_iotlb_sync(struct iommu_domain *domain, 509 struct iommu_iotlb_gather *gather) 510 { 511 struct s390_domain *s390_domain = to_s390_domain(domain); 512 size_t size = gather->end - gather->start + 1; 513 struct zpci_dev *zdev; 514 515 /* If gather was never added to there is nothing to flush */ 516 if (!gather->end) 517 return; 518 519 rcu_read_lock(); 520 list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { 521 atomic64_inc(&s390_domain->ctrs.sync_rpcits); 522 zpci_refresh_trans((u64)zdev->fh << 32, gather->start, 523 size); 524 } 525 rcu_read_unlock(); 526 } 527 528 static int s390_iommu_iotlb_sync_map(struct iommu_domain *domain, 529 unsigned long iova, size_t size) 530 { 531 struct s390_domain *s390_domain = to_s390_domain(domain); 532 struct zpci_dev *zdev; 533 int ret = 0; 534 535 rcu_read_lock(); 536 list_for_each_entry_rcu(zdev, &s390_domain->devices, iommu_list) { 537 if (!zdev->tlb_refresh) 538 continue; 539 atomic64_inc(&s390_domain->ctrs.sync_map_rpcits); 540 ret = zpci_refresh_trans((u64)zdev->fh << 32, 541 iova, size); 542 /* 543 * let the hypervisor discover invalidated entries 544 * allowing it to free IOVAs and unpin pages 545 */ 546 if (ret == -ENOMEM) { 547 ret = zpci_refresh_all(zdev); 548 if (ret) 549 break; 550 } 551 } 552 rcu_read_unlock(); 553 554 return ret; 555 } 556 557 static int s390_iommu_validate_trans(struct s390_domain *s390_domain, 558 phys_addr_t pa, dma_addr_t dma_addr, 559 unsigned long nr_pages, int flags, 560 gfp_t gfp) 561 { 562 phys_addr_t page_addr = pa & PAGE_MASK; 563 unsigned long *entry; 564 unsigned long i; 565 int rc; 566 567 for (i = 0; i < nr_pages; i++) { 568 entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr, 569 gfp); 570 if (unlikely(!entry)) { 571 rc = -ENOMEM; 572 goto undo_cpu_trans; 573 } 574 dma_update_cpu_trans(entry, page_addr, flags); 575 page_addr += PAGE_SIZE; 576 dma_addr += PAGE_SIZE; 577 } 578 579 return 0; 580 581 undo_cpu_trans: 582 while (i-- > 0) { 583 dma_addr -= PAGE_SIZE; 584 entry = dma_walk_cpu_trans(s390_domain->dma_table, 585 dma_addr, gfp); 586 if (!entry) 587 break; 588 dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID); 589 } 590 591 return rc; 592 } 593 594 static int s390_iommu_invalidate_trans(struct s390_domain *s390_domain, 595 dma_addr_t dma_addr, unsigned long nr_pages) 596 { 597 unsigned long *entry; 598 unsigned long i; 599 int rc = 0; 600 601 for (i = 0; i < nr_pages; i++) { 602 entry = dma_walk_cpu_trans(s390_domain->dma_table, dma_addr, 603 GFP_ATOMIC); 604 if (unlikely(!entry)) { 605 rc = -EINVAL; 606 break; 607 } 608 dma_update_cpu_trans(entry, 0, ZPCI_PTE_INVALID); 609 dma_addr += PAGE_SIZE; 610 } 611 612 return rc; 613 } 614 615 static int s390_iommu_map_pages(struct iommu_domain *domain, 616 unsigned long iova, phys_addr_t paddr, 617 size_t pgsize, size_t pgcount, 618 int prot, gfp_t gfp, size_t *mapped) 619 { 620 struct s390_domain *s390_domain = to_s390_domain(domain); 621 size_t size = pgcount << __ffs(pgsize); 622 int flags = ZPCI_PTE_VALID, rc = 0; 623 624 if (pgsize != SZ_4K) 625 return -EINVAL; 626 627 if (iova < s390_domain->domain.geometry.aperture_start || 628 (iova + size - 1) > s390_domain->domain.geometry.aperture_end) 629 return -EINVAL; 630 631 if (!IS_ALIGNED(iova | paddr, pgsize)) 632 return -EINVAL; 633 634 if (!(prot & IOMMU_WRITE)) 635 flags |= ZPCI_TABLE_PROTECTED; 636 637 rc = s390_iommu_validate_trans(s390_domain, paddr, iova, 638 pgcount, flags, gfp); 639 if (!rc) { 640 *mapped = size; 641 atomic64_add(pgcount, &s390_domain->ctrs.mapped_pages); 642 } 643 644 return rc; 645 } 646 647 static phys_addr_t s390_iommu_iova_to_phys(struct iommu_domain *domain, 648 dma_addr_t iova) 649 { 650 struct s390_domain *s390_domain = to_s390_domain(domain); 651 unsigned long *rto, *sto, *pto; 652 unsigned long ste, pte, rte; 653 unsigned int rtx, sx, px; 654 phys_addr_t phys = 0; 655 656 if (iova < domain->geometry.aperture_start || 657 iova > domain->geometry.aperture_end) 658 return 0; 659 660 rtx = calc_rtx(iova); 661 sx = calc_sx(iova); 662 px = calc_px(iova); 663 rto = s390_domain->dma_table; 664 665 rte = READ_ONCE(rto[rtx]); 666 if (reg_entry_isvalid(rte)) { 667 sto = get_rt_sto(rte); 668 ste = READ_ONCE(sto[sx]); 669 if (reg_entry_isvalid(ste)) { 670 pto = get_st_pto(ste); 671 pte = READ_ONCE(pto[px]); 672 if (pt_entry_isvalid(pte)) 673 phys = pte & ZPCI_PTE_ADDR_MASK; 674 } 675 } 676 677 return phys; 678 } 679 680 static size_t s390_iommu_unmap_pages(struct iommu_domain *domain, 681 unsigned long iova, 682 size_t pgsize, size_t pgcount, 683 struct iommu_iotlb_gather *gather) 684 { 685 struct s390_domain *s390_domain = to_s390_domain(domain); 686 size_t size = pgcount << __ffs(pgsize); 687 int rc; 688 689 if (WARN_ON(iova < s390_domain->domain.geometry.aperture_start || 690 (iova + size - 1) > s390_domain->domain.geometry.aperture_end)) 691 return 0; 692 693 rc = s390_iommu_invalidate_trans(s390_domain, iova, pgcount); 694 if (rc) 695 return 0; 696 697 iommu_iotlb_gather_add_range(gather, iova, size); 698 atomic64_add(pgcount, &s390_domain->ctrs.unmapped_pages); 699 700 return size; 701 } 702 703 struct zpci_iommu_ctrs *zpci_get_iommu_ctrs(struct zpci_dev *zdev) 704 { 705 struct s390_domain *s390_domain; 706 707 lockdep_assert_held(&zdev->dom_lock); 708 709 if (zdev->s390_domain->type == IOMMU_DOMAIN_BLOCKED) 710 return NULL; 711 712 s390_domain = to_s390_domain(zdev->s390_domain); 713 return &s390_domain->ctrs; 714 } 715 716 int zpci_init_iommu(struct zpci_dev *zdev) 717 { 718 u64 aperture_size; 719 int rc = 0; 720 721 rc = iommu_device_sysfs_add(&zdev->iommu_dev, NULL, NULL, 722 "s390-iommu.%08x", zdev->fid); 723 if (rc) 724 goto out_err; 725 726 rc = iommu_device_register(&zdev->iommu_dev, &s390_iommu_ops, NULL); 727 if (rc) 728 goto out_sysfs; 729 730 zdev->start_dma = PAGE_ALIGN(zdev->start_dma); 731 aperture_size = min3(s390_iommu_aperture, 732 ZPCI_TABLE_SIZE_RT - zdev->start_dma, 733 zdev->end_dma - zdev->start_dma + 1); 734 zdev->end_dma = zdev->start_dma + aperture_size - 1; 735 736 return 0; 737 738 out_sysfs: 739 iommu_device_sysfs_remove(&zdev->iommu_dev); 740 741 out_err: 742 return rc; 743 } 744 745 void zpci_destroy_iommu(struct zpci_dev *zdev) 746 { 747 iommu_device_unregister(&zdev->iommu_dev); 748 iommu_device_sysfs_remove(&zdev->iommu_dev); 749 } 750 751 static int __init s390_iommu_setup(char *str) 752 { 753 if (!strcmp(str, "strict")) { 754 pr_warn("s390_iommu=strict deprecated; use iommu.strict=1 instead\n"); 755 iommu_set_dma_strict(); 756 } 757 return 1; 758 } 759 760 __setup("s390_iommu=", s390_iommu_setup); 761 762 static int __init s390_iommu_aperture_setup(char *str) 763 { 764 if (kstrtou32(str, 10, &s390_iommu_aperture_factor)) 765 s390_iommu_aperture_factor = 1; 766 return 1; 767 } 768 769 __setup("s390_iommu_aperture=", s390_iommu_aperture_setup); 770 771 static int __init s390_iommu_init(void) 772 { 773 int rc; 774 775 iommu_dma_forcedac = true; 776 s390_iommu_aperture = (u64)virt_to_phys(high_memory); 777 if (!s390_iommu_aperture_factor) 778 s390_iommu_aperture = ULONG_MAX; 779 else 780 s390_iommu_aperture *= s390_iommu_aperture_factor; 781 782 rc = dma_alloc_cpu_table_caches(); 783 if (rc) 784 return rc; 785 786 return rc; 787 } 788 subsys_initcall(s390_iommu_init); 789 790 static struct iommu_domain blocking_domain = { 791 .type = IOMMU_DOMAIN_BLOCKED, 792 .ops = &(const struct iommu_domain_ops) { 793 .attach_dev = blocking_domain_attach_device, 794 } 795 }; 796 797 static const struct iommu_ops s390_iommu_ops = { 798 .blocked_domain = &blocking_domain, 799 .release_domain = &blocking_domain, 800 .capable = s390_iommu_capable, 801 .domain_alloc_paging = s390_domain_alloc_paging, 802 .probe_device = s390_iommu_probe_device, 803 .device_group = generic_device_group, 804 .pgsize_bitmap = SZ_4K, 805 .get_resv_regions = s390_iommu_get_resv_regions, 806 .default_domain_ops = &(const struct iommu_domain_ops) { 807 .attach_dev = s390_iommu_attach_device, 808 .map_pages = s390_iommu_map_pages, 809 .unmap_pages = s390_iommu_unmap_pages, 810 .flush_iotlb_all = s390_iommu_flush_iotlb_all, 811 .iotlb_sync = s390_iommu_iotlb_sync, 812 .iotlb_sync_map = s390_iommu_iotlb_sync_map, 813 .iova_to_phys = s390_iommu_iova_to_phys, 814 .free = s390_domain_free, 815 } 816 }; 817