1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Apple DART (Device Address Resolution Table) IOMMU driver 4 * 5 * Copyright (C) 2021 The Asahi Linux Contributors 6 * 7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c 8 * Copyright (C) 2013 ARM Limited 9 * Copyright (C) 2015 ARM Limited 10 * and on exynos-iommu.c 11 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. 12 */ 13 14 #include <linux/atomic.h> 15 #include <linux/bitfield.h> 16 #include <linux/clk.h> 17 #include <linux/dev_printk.h> 18 #include <linux/dma-mapping.h> 19 #include <linux/err.h> 20 #include <linux/interrupt.h> 21 #include <linux/io-pgtable.h> 22 #include <linux/iommu.h> 23 #include <linux/iopoll.h> 24 #include <linux/module.h> 25 #include <linux/of.h> 26 #include <linux/of_address.h> 27 #include <linux/of_iommu.h> 28 #include <linux/of_platform.h> 29 #include <linux/pci.h> 30 #include <linux/platform_device.h> 31 #include <linux/slab.h> 32 #include <linux/swab.h> 33 #include <linux/types.h> 34 35 #include "dma-iommu.h" 36 37 #define DART_MAX_STREAMS 256 38 #define DART_MAX_TTBR 4 39 #define MAX_DARTS_PER_DEVICE 3 40 41 /* Common registers */ 42 43 #define DART_PARAMS1 0x00 44 #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24) 45 46 #define DART_PARAMS2 0x04 47 #define DART_PARAMS2_BYPASS_SUPPORT BIT(0) 48 49 /* T8020/T6000 registers */ 50 51 #define DART_T8020_STREAM_COMMAND 0x20 52 #define DART_T8020_STREAM_COMMAND_BUSY BIT(2) 53 #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20) 54 55 #define DART_T8020_STREAM_SELECT 0x34 56 57 #define DART_T8020_ERROR 0x40 58 #define DART_T8020_ERROR_STREAM GENMASK(27, 24) 59 #define DART_T8020_ERROR_CODE GENMASK(11, 0) 60 #define DART_T8020_ERROR_FLAG BIT(31) 61 62 #define DART_T8020_ERROR_READ_FAULT BIT(4) 63 #define DART_T8020_ERROR_WRITE_FAULT BIT(3) 64 #define DART_T8020_ERROR_NO_PTE BIT(2) 65 #define DART_T8020_ERROR_NO_PMD BIT(1) 66 #define DART_T8020_ERROR_NO_TTBR BIT(0) 67 68 #define DART_T8020_CONFIG 0x60 69 #define DART_T8020_CONFIG_LOCK BIT(15) 70 71 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100 72 73 #define DART_T8020_ERROR_ADDR_HI 0x54 74 #define DART_T8020_ERROR_ADDR_LO 0x50 75 76 #define DART_T8020_STREAMS_ENABLE 0xfc 77 78 #define DART_T8020_TCR 0x100 79 #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7) 80 #define DART_T8020_TCR_BYPASS_DART BIT(8) 81 #define DART_T8020_TCR_BYPASS_DAPF BIT(12) 82 83 #define DART_T8020_TTBR 0x200 84 #define DART_T8020_USB4_TTBR 0x400 85 #define DART_T8020_TTBR_VALID BIT(31) 86 #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0 87 #define DART_T8020_TTBR_SHIFT 12 88 89 /* T8110 registers */ 90 91 #define DART_T8110_PARAMS3 0x08 92 #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24) 93 #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16) 94 #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8) 95 #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0) 96 97 #define DART_T8110_PARAMS4 0x0c 98 #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16) 99 #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0) 100 101 #define DART_T8110_TLB_CMD 0x80 102 #define DART_T8110_TLB_CMD_BUSY BIT(31) 103 #define DART_T8110_TLB_CMD_OP GENMASK(10, 8) 104 #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0 105 #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1 106 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) 107 108 #define DART_T8110_ERROR 0x100 109 #define DART_T8110_ERROR_STREAM GENMASK(27, 20) 110 #define DART_T8110_ERROR_CODE GENMASK(14, 0) 111 #define DART_T8110_ERROR_FLAG BIT(31) 112 113 #define DART_T8110_ERROR_MASK 0x104 114 115 #define DART_T8110_ERROR_READ_FAULT BIT(5) 116 #define DART_T8110_ERROR_WRITE_FAULT BIT(4) 117 #define DART_T8110_ERROR_NO_PTE BIT(3) 118 #define DART_T8110_ERROR_NO_PMD BIT(2) 119 #define DART_T8110_ERROR_NO_PGD BIT(1) 120 #define DART_T8110_ERROR_NO_TTBR BIT(0) 121 122 #define DART_T8110_ERROR_ADDR_LO 0x170 123 #define DART_T8110_ERROR_ADDR_HI 0x174 124 125 #define DART_T8110_PROTECT 0x200 126 #define DART_T8110_UNPROTECT 0x204 127 #define DART_T8110_PROTECT_LOCK 0x208 128 #define DART_T8110_PROTECT_TTBR_TCR BIT(0) 129 130 #define DART_T8110_ENABLE_STREAMS 0xc00 131 #define DART_T8110_DISABLE_STREAMS 0xc20 132 133 #define DART_T8110_TCR 0x1000 134 #define DART_T8110_TCR_REMAP GENMASK(11, 8) 135 #define DART_T8110_TCR_REMAP_EN BIT(7) 136 #define DART_T8110_TCR_BYPASS_DAPF BIT(2) 137 #define DART_T8110_TCR_BYPASS_DART BIT(1) 138 #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) 139 140 #define DART_T8110_TTBR 0x1400 141 #define DART_T8110_TTBR_VALID BIT(0) 142 #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2 143 #define DART_T8110_TTBR_SHIFT 14 144 145 #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) 146 147 #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \ 148 (((dart)->hw->ttbr_count * (sid)) << 2) + \ 149 ((idx) << 2)) 150 151 struct apple_dart_stream_map; 152 153 enum dart_type { 154 DART_T8020, 155 DART_T6000, 156 DART_T8110, 157 }; 158 159 struct apple_dart_hw { 160 enum dart_type type; 161 irqreturn_t (*irq_handler)(int irq, void *dev); 162 int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map); 163 164 u32 oas; 165 enum io_pgtable_fmt fmt; 166 167 int max_sid_count; 168 169 u64 lock; 170 u64 lock_bit; 171 172 u64 error; 173 174 u64 enable_streams; 175 176 u64 tcr; 177 u64 tcr_enabled; 178 u64 tcr_disabled; 179 u64 tcr_bypass; 180 181 u64 ttbr; 182 u64 ttbr_valid; 183 u64 ttbr_addr_field_shift; 184 u64 ttbr_shift; 185 int ttbr_count; 186 }; 187 188 /* 189 * Private structure associated with each DART device. 190 * 191 * @dev: device struct 192 * @hw: SoC-specific hardware data 193 * @regs: mapped MMIO region 194 * @irq: interrupt number, can be shared with other DARTs 195 * @clks: clocks associated with this DART 196 * @num_clks: number of @clks 197 * @lock: lock for hardware operations involving this dart 198 * @pgsize: pagesize supported by this DART 199 * @supports_bypass: indicates if this DART supports bypass mode 200 * @sid2group: maps stream ids to iommu_groups 201 * @iommu: iommu core device 202 */ 203 struct apple_dart { 204 struct device *dev; 205 const struct apple_dart_hw *hw; 206 207 void __iomem *regs; 208 209 int irq; 210 struct clk_bulk_data *clks; 211 int num_clks; 212 213 spinlock_t lock; 214 215 u32 ias; 216 u32 oas; 217 u32 pgsize; 218 u32 num_streams; 219 u32 supports_bypass : 1; 220 221 struct iommu_group *sid2group[DART_MAX_STREAMS]; 222 struct iommu_device iommu; 223 224 u32 save_tcr[DART_MAX_STREAMS]; 225 u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; 226 }; 227 228 /* 229 * Convenience struct to identify streams. 230 * 231 * The normal variant is used inside apple_dart_master_cfg which isn't written 232 * to concurrently. 233 * The atomic variant is used inside apple_dart_domain where we have to guard 234 * against races from potential parallel calls to attach/detach_device. 235 * Note that even inside the atomic variant the apple_dart pointer is not 236 * protected: This pointer is initialized once under the domain init mutex 237 * and never changed again afterwards. Devices with different dart pointers 238 * cannot be attached to the same domain. 239 * 240 * @dart dart pointer 241 * @sid stream id bitmap 242 */ 243 struct apple_dart_stream_map { 244 struct apple_dart *dart; 245 DECLARE_BITMAP(sidmap, DART_MAX_STREAMS); 246 }; 247 struct apple_dart_atomic_stream_map { 248 struct apple_dart *dart; 249 atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)]; 250 }; 251 252 /* 253 * This structure is attached to each iommu domain handled by a DART. 254 * 255 * @pgtbl_ops: pagetable ops allocated by io-pgtable 256 * @finalized: true if the domain has been completely initialized 257 * @init_lock: protects domain initialization 258 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only) 259 * @domain: core iommu domain pointer 260 */ 261 struct apple_dart_domain { 262 struct io_pgtable_ops *pgtbl_ops; 263 264 bool finalized; 265 struct mutex init_lock; 266 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; 267 268 struct iommu_domain domain; 269 }; 270 271 /* 272 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate 273 * and contains a list of streams bound to this device. 274 * So far the worst case seen is a single device with two streams 275 * from different darts, such that this simple static array is enough. 276 * 277 * @streams: streams for this device 278 */ 279 struct apple_dart_master_cfg { 280 /* Intersection of DART capabilitles */ 281 u32 supports_bypass : 1; 282 283 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; 284 }; 285 286 /* 287 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and 288 * apple_dart_domain.stream_maps 289 * 290 * @i int used as loop variable 291 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain) 292 * @stream pointer to the apple_dart_streams struct for each loop iteration 293 */ 294 #define for_each_stream_map(i, base, stream_map) \ 295 for (i = 0, stream_map = &(base)->stream_maps[0]; \ 296 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \ 297 stream_map = &(base)->stream_maps[++i]) 298 299 static struct platform_driver apple_dart_driver; 300 static const struct iommu_ops apple_dart_iommu_ops; 301 302 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) 303 { 304 return container_of(dom, struct apple_dart_domain, domain); 305 } 306 307 static void 308 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) 309 { 310 struct apple_dart *dart = stream_map->dart; 311 int sid; 312 313 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 314 writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid)); 315 } 316 317 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) 318 { 319 struct apple_dart *dart = stream_map->dart; 320 int sid; 321 322 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 323 writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid)); 324 } 325 326 static void 327 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) 328 { 329 struct apple_dart *dart = stream_map->dart; 330 int sid; 331 332 WARN_ON(!stream_map->dart->supports_bypass); 333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 334 writel(dart->hw->tcr_bypass, 335 dart->regs + DART_TCR(dart, sid)); 336 } 337 338 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, 339 u8 idx, phys_addr_t paddr) 340 { 341 struct apple_dart *dart = stream_map->dart; 342 int sid; 343 344 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); 345 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 346 writel(dart->hw->ttbr_valid | 347 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift, 348 dart->regs + DART_TTBR(dart, sid, idx)); 349 } 350 351 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, 352 u8 idx) 353 { 354 struct apple_dart *dart = stream_map->dart; 355 int sid; 356 357 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) 358 writel(0, dart->regs + DART_TTBR(dart, sid, idx)); 359 } 360 361 static void 362 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) 363 { 364 int i; 365 366 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) 367 apple_dart_hw_clear_ttbr(stream_map, i); 368 } 369 370 static int 371 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, 372 u32 command) 373 { 374 unsigned long flags; 375 int ret, i; 376 u32 command_reg; 377 378 spin_lock_irqsave(&stream_map->dart->lock, flags); 379 380 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++) 381 writel(stream_map->sidmap[i], 382 stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i); 383 writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND); 384 385 ret = readl_poll_timeout_atomic( 386 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, 387 !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1, 388 DART_STREAM_COMMAND_BUSY_TIMEOUT); 389 390 spin_unlock_irqrestore(&stream_map->dart->lock, flags); 391 392 if (ret) { 393 dev_err(stream_map->dart->dev, 394 "busy bit did not clear after command %x for streams %lx\n", 395 command, stream_map->sidmap[0]); 396 return ret; 397 } 398 399 return 0; 400 } 401 402 static int 403 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, 404 u32 command) 405 { 406 struct apple_dart *dart = stream_map->dart; 407 unsigned long flags; 408 int ret = 0; 409 int sid; 410 411 spin_lock_irqsave(&dart->lock, flags); 412 413 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { 414 u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | 415 FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); 416 writel(val, dart->regs + DART_T8110_TLB_CMD); 417 418 ret = readl_poll_timeout_atomic( 419 dart->regs + DART_T8110_TLB_CMD, val, 420 !(val & DART_T8110_TLB_CMD_BUSY), 1, 421 DART_STREAM_COMMAND_BUSY_TIMEOUT); 422 423 if (ret) 424 break; 425 426 } 427 428 spin_unlock_irqrestore(&dart->lock, flags); 429 430 if (ret) { 431 dev_err(stream_map->dart->dev, 432 "busy bit did not clear after command %x for stream %d\n", 433 command, sid); 434 return ret; 435 } 436 437 return 0; 438 } 439 440 static int 441 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) 442 { 443 return apple_dart_t8020_hw_stream_command( 444 stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE); 445 } 446 447 static int 448 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) 449 { 450 return apple_dart_t8110_hw_tlb_command( 451 stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID); 452 } 453 454 static int apple_dart_hw_reset(struct apple_dart *dart) 455 { 456 u32 config; 457 struct apple_dart_stream_map stream_map; 458 int i; 459 460 config = readl(dart->regs + dart->hw->lock); 461 if (config & dart->hw->lock_bit) { 462 dev_err(dart->dev, "DART is locked down until reboot: %08x\n", 463 config); 464 return -EINVAL; 465 } 466 467 stream_map.dart = dart; 468 bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS); 469 bitmap_set(stream_map.sidmap, 0, dart->num_streams); 470 apple_dart_hw_disable_dma(&stream_map); 471 apple_dart_hw_clear_all_ttbrs(&stream_map); 472 473 /* enable all streams globally since TCR is used to control isolation */ 474 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) 475 writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i); 476 477 /* clear any pending errors before the interrupt is unmasked */ 478 writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error); 479 480 if (dart->hw->type == DART_T8110) 481 writel(0, dart->regs + DART_T8110_ERROR_MASK); 482 483 return dart->hw->invalidate_tlb(&stream_map); 484 } 485 486 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) 487 { 488 int i, j; 489 struct apple_dart_atomic_stream_map *domain_stream_map; 490 struct apple_dart_stream_map stream_map; 491 492 for_each_stream_map(i, domain, domain_stream_map) { 493 stream_map.dart = domain_stream_map->dart; 494 495 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) 496 stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]); 497 498 stream_map.dart->hw->invalidate_tlb(&stream_map); 499 } 500 } 501 502 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain) 503 { 504 apple_dart_domain_flush_tlb(to_dart_domain(domain)); 505 } 506 507 static void apple_dart_iotlb_sync(struct iommu_domain *domain, 508 struct iommu_iotlb_gather *gather) 509 { 510 apple_dart_domain_flush_tlb(to_dart_domain(domain)); 511 } 512 513 static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, 514 unsigned long iova, size_t size) 515 { 516 apple_dart_domain_flush_tlb(to_dart_domain(domain)); 517 return 0; 518 } 519 520 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, 521 dma_addr_t iova) 522 { 523 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 524 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; 525 526 if (!ops) 527 return 0; 528 529 return ops->iova_to_phys(ops, iova); 530 } 531 532 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, 533 phys_addr_t paddr, size_t pgsize, 534 size_t pgcount, int prot, gfp_t gfp, 535 size_t *mapped) 536 { 537 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 538 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; 539 540 if (!ops) 541 return -ENODEV; 542 543 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, 544 mapped); 545 } 546 547 static size_t apple_dart_unmap_pages(struct iommu_domain *domain, 548 unsigned long iova, size_t pgsize, 549 size_t pgcount, 550 struct iommu_iotlb_gather *gather) 551 { 552 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 553 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; 554 555 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); 556 } 557 558 static void 559 apple_dart_setup_translation(struct apple_dart_domain *domain, 560 struct apple_dart_stream_map *stream_map) 561 { 562 int i; 563 struct io_pgtable_cfg *pgtbl_cfg = 564 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; 565 566 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) 567 apple_dart_hw_set_ttbr(stream_map, i, 568 pgtbl_cfg->apple_dart_cfg.ttbr[i]); 569 for (; i < stream_map->dart->hw->ttbr_count; ++i) 570 apple_dart_hw_clear_ttbr(stream_map, i); 571 572 apple_dart_hw_enable_translation(stream_map); 573 stream_map->dart->hw->invalidate_tlb(stream_map); 574 } 575 576 static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, 577 struct apple_dart_master_cfg *cfg) 578 { 579 struct apple_dart *dart = cfg->stream_maps[0].dart; 580 struct io_pgtable_cfg pgtbl_cfg; 581 int ret = 0; 582 int i, j; 583 584 if (dart->pgsize > PAGE_SIZE) 585 return -EINVAL; 586 587 mutex_lock(&dart_domain->init_lock); 588 589 if (dart_domain->finalized) 590 goto done; 591 592 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 593 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; 594 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) 595 atomic_long_set(&dart_domain->stream_maps[i].sidmap[j], 596 cfg->stream_maps[i].sidmap[j]); 597 } 598 599 pgtbl_cfg = (struct io_pgtable_cfg){ 600 .pgsize_bitmap = dart->pgsize, 601 .ias = dart->ias, 602 .oas = dart->oas, 603 .coherent_walk = 1, 604 .iommu_dev = dart->dev, 605 }; 606 607 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, 608 &dart_domain->domain); 609 if (!dart_domain->pgtbl_ops) { 610 ret = -ENOMEM; 611 goto done; 612 } 613 614 dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 615 dart_domain->domain.geometry.aperture_start = 0; 616 dart_domain->domain.geometry.aperture_end = 617 (dma_addr_t)DMA_BIT_MASK(dart->ias); 618 dart_domain->domain.geometry.force_aperture = true; 619 620 dart_domain->finalized = true; 621 622 done: 623 mutex_unlock(&dart_domain->init_lock); 624 return ret; 625 } 626 627 static int 628 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, 629 struct apple_dart_stream_map *master_maps, 630 bool add_streams) 631 { 632 int i, j; 633 634 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 635 if (domain_maps[i].dart != master_maps[i].dart) 636 return -EINVAL; 637 } 638 639 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 640 if (!domain_maps[i].dart) 641 break; 642 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { 643 if (add_streams) 644 atomic_long_or(master_maps[i].sidmap[j], 645 &domain_maps[i].sidmap[j]); 646 else 647 atomic_long_and(~master_maps[i].sidmap[j], 648 &domain_maps[i].sidmap[j]); 649 } 650 } 651 652 return 0; 653 } 654 655 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain, 656 struct apple_dart_master_cfg *cfg) 657 { 658 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps, 659 true); 660 } 661 662 static int apple_dart_attach_dev_paging(struct iommu_domain *domain, 663 struct device *dev) 664 { 665 int ret, i; 666 struct apple_dart_stream_map *stream_map; 667 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 668 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 669 670 ret = apple_dart_finalize_domain(dart_domain, cfg); 671 if (ret) 672 return ret; 673 674 ret = apple_dart_domain_add_streams(dart_domain, cfg); 675 if (ret) 676 return ret; 677 678 for_each_stream_map(i, cfg, stream_map) 679 apple_dart_setup_translation(dart_domain, stream_map); 680 return 0; 681 } 682 683 static int apple_dart_attach_dev_identity(struct iommu_domain *domain, 684 struct device *dev) 685 { 686 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 687 struct apple_dart_stream_map *stream_map; 688 int i; 689 690 if (!cfg->supports_bypass) 691 return -EINVAL; 692 693 for_each_stream_map(i, cfg, stream_map) 694 apple_dart_hw_enable_bypass(stream_map); 695 return 0; 696 } 697 698 static const struct iommu_domain_ops apple_dart_identity_ops = { 699 .attach_dev = apple_dart_attach_dev_identity, 700 }; 701 702 static struct iommu_domain apple_dart_identity_domain = { 703 .type = IOMMU_DOMAIN_IDENTITY, 704 .ops = &apple_dart_identity_ops, 705 }; 706 707 static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, 708 struct device *dev) 709 { 710 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 711 struct apple_dart_stream_map *stream_map; 712 int i; 713 714 for_each_stream_map(i, cfg, stream_map) 715 apple_dart_hw_disable_dma(stream_map); 716 return 0; 717 } 718 719 static const struct iommu_domain_ops apple_dart_blocked_ops = { 720 .attach_dev = apple_dart_attach_dev_blocked, 721 }; 722 723 static struct iommu_domain apple_dart_blocked_domain = { 724 .type = IOMMU_DOMAIN_BLOCKED, 725 .ops = &apple_dart_blocked_ops, 726 }; 727 728 static struct iommu_device *apple_dart_probe_device(struct device *dev) 729 { 730 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 731 struct apple_dart_stream_map *stream_map; 732 int i; 733 734 if (!cfg) 735 return ERR_PTR(-ENODEV); 736 737 for_each_stream_map(i, cfg, stream_map) 738 device_link_add( 739 dev, stream_map->dart->dev, 740 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); 741 742 return &cfg->stream_maps[0].dart->iommu; 743 } 744 745 static void apple_dart_release_device(struct device *dev) 746 { 747 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 748 749 kfree(cfg); 750 } 751 752 static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev) 753 { 754 struct apple_dart_domain *dart_domain; 755 756 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL); 757 if (!dart_domain) 758 return NULL; 759 760 mutex_init(&dart_domain->init_lock); 761 762 if (dev) { 763 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 764 int ret; 765 766 ret = apple_dart_finalize_domain(dart_domain, cfg); 767 if (ret) { 768 kfree(dart_domain); 769 return ERR_PTR(ret); 770 } 771 } 772 return &dart_domain->domain; 773 } 774 775 static void apple_dart_domain_free(struct iommu_domain *domain) 776 { 777 struct apple_dart_domain *dart_domain = to_dart_domain(domain); 778 779 if (dart_domain->pgtbl_ops) 780 free_io_pgtable_ops(dart_domain->pgtbl_ops); 781 782 kfree(dart_domain); 783 } 784 785 static int apple_dart_of_xlate(struct device *dev, 786 const struct of_phandle_args *args) 787 { 788 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 789 struct platform_device *iommu_pdev = of_find_device_by_node(args->np); 790 struct apple_dart *dart = platform_get_drvdata(iommu_pdev); 791 struct apple_dart *cfg_dart; 792 int i, sid; 793 794 if (args->args_count != 1) 795 return -EINVAL; 796 sid = args->args[0]; 797 798 if (!cfg) { 799 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 800 if (!cfg) 801 return -ENOMEM; 802 /* Will be ANDed with DART capabilities */ 803 cfg->supports_bypass = true; 804 } 805 dev_iommu_priv_set(dev, cfg); 806 807 cfg_dart = cfg->stream_maps[0].dart; 808 if (cfg_dart) { 809 if (cfg_dart->pgsize != dart->pgsize) 810 return -EINVAL; 811 } 812 813 cfg->supports_bypass &= dart->supports_bypass; 814 815 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 816 if (cfg->stream_maps[i].dart == dart) { 817 set_bit(sid, cfg->stream_maps[i].sidmap); 818 return 0; 819 } 820 } 821 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { 822 if (!cfg->stream_maps[i].dart) { 823 cfg->stream_maps[i].dart = dart; 824 set_bit(sid, cfg->stream_maps[i].sidmap); 825 return 0; 826 } 827 } 828 829 return -EINVAL; 830 } 831 832 static DEFINE_MUTEX(apple_dart_groups_lock); 833 834 static void apple_dart_release_group(void *iommu_data) 835 { 836 int i, sid; 837 struct apple_dart_stream_map *stream_map; 838 struct apple_dart_master_cfg *group_master_cfg = iommu_data; 839 840 mutex_lock(&apple_dart_groups_lock); 841 842 for_each_stream_map(i, group_master_cfg, stream_map) 843 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) 844 stream_map->dart->sid2group[sid] = NULL; 845 846 kfree(iommu_data); 847 mutex_unlock(&apple_dart_groups_lock); 848 } 849 850 static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst, 851 struct apple_dart_master_cfg *src) 852 { 853 /* 854 * We know that this function is only called for groups returned from 855 * pci_device_group and that all Apple Silicon platforms never spread 856 * PCIe devices from the same bus across multiple DARTs such that we can 857 * just assume that both src and dst only have the same single DART. 858 */ 859 if (src->stream_maps[1].dart) 860 return -EINVAL; 861 if (dst->stream_maps[1].dart) 862 return -EINVAL; 863 if (src->stream_maps[0].dart != dst->stream_maps[0].dart) 864 return -EINVAL; 865 866 bitmap_or(dst->stream_maps[0].sidmap, 867 dst->stream_maps[0].sidmap, 868 src->stream_maps[0].sidmap, 869 dst->stream_maps[0].dart->num_streams); 870 return 0; 871 } 872 873 static struct iommu_group *apple_dart_device_group(struct device *dev) 874 { 875 int i, sid; 876 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 877 struct apple_dart_stream_map *stream_map; 878 struct apple_dart_master_cfg *group_master_cfg; 879 struct iommu_group *group = NULL; 880 struct iommu_group *res = ERR_PTR(-EINVAL); 881 882 mutex_lock(&apple_dart_groups_lock); 883 884 for_each_stream_map(i, cfg, stream_map) { 885 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { 886 struct iommu_group *stream_group = 887 stream_map->dart->sid2group[sid]; 888 889 if (group && group != stream_group) { 890 res = ERR_PTR(-EINVAL); 891 goto out; 892 } 893 894 group = stream_group; 895 } 896 } 897 898 if (group) { 899 res = iommu_group_ref_get(group); 900 goto out; 901 } 902 903 #ifdef CONFIG_PCI 904 if (dev_is_pci(dev)) 905 group = pci_device_group(dev); 906 else 907 #endif 908 group = generic_device_group(dev); 909 910 res = ERR_PTR(-ENOMEM); 911 if (!group) 912 goto out; 913 914 group_master_cfg = iommu_group_get_iommudata(group); 915 if (group_master_cfg) { 916 int ret; 917 918 ret = apple_dart_merge_master_cfg(group_master_cfg, cfg); 919 if (ret) { 920 dev_err(dev, "Failed to merge DART IOMMU groups.\n"); 921 iommu_group_put(group); 922 res = ERR_PTR(ret); 923 goto out; 924 } 925 } else { 926 group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg), 927 GFP_KERNEL); 928 if (!group_master_cfg) { 929 iommu_group_put(group); 930 goto out; 931 } 932 933 iommu_group_set_iommudata(group, group_master_cfg, 934 apple_dart_release_group); 935 } 936 937 for_each_stream_map(i, cfg, stream_map) 938 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) 939 stream_map->dart->sid2group[sid] = group; 940 941 res = group; 942 943 out: 944 mutex_unlock(&apple_dart_groups_lock); 945 return res; 946 } 947 948 static int apple_dart_def_domain_type(struct device *dev) 949 { 950 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); 951 952 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE) 953 return IOMMU_DOMAIN_IDENTITY; 954 if (!cfg->supports_bypass) 955 return IOMMU_DOMAIN_DMA; 956 957 return 0; 958 } 959 960 #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 961 /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */ 962 #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0 963 #endif 964 #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK) 965 966 static void apple_dart_get_resv_regions(struct device *dev, 967 struct list_head *head) 968 { 969 if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) { 970 struct iommu_resv_region *region; 971 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 972 973 region = iommu_alloc_resv_region(DOORBELL_ADDR, 974 PAGE_SIZE, prot, 975 IOMMU_RESV_MSI, GFP_KERNEL); 976 if (!region) 977 return; 978 979 list_add_tail(®ion->list, head); 980 } 981 982 iommu_dma_get_resv_regions(dev, head); 983 } 984 985 static const struct iommu_ops apple_dart_iommu_ops = { 986 .identity_domain = &apple_dart_identity_domain, 987 .blocked_domain = &apple_dart_blocked_domain, 988 .domain_alloc_paging = apple_dart_domain_alloc_paging, 989 .probe_device = apple_dart_probe_device, 990 .release_device = apple_dart_release_device, 991 .device_group = apple_dart_device_group, 992 .of_xlate = apple_dart_of_xlate, 993 .def_domain_type = apple_dart_def_domain_type, 994 .get_resv_regions = apple_dart_get_resv_regions, 995 .pgsize_bitmap = -1UL, /* Restricted during dart probe */ 996 .owner = THIS_MODULE, 997 .default_domain_ops = &(const struct iommu_domain_ops) { 998 .attach_dev = apple_dart_attach_dev_paging, 999 .map_pages = apple_dart_map_pages, 1000 .unmap_pages = apple_dart_unmap_pages, 1001 .flush_iotlb_all = apple_dart_flush_iotlb_all, 1002 .iotlb_sync = apple_dart_iotlb_sync, 1003 .iotlb_sync_map = apple_dart_iotlb_sync_map, 1004 .iova_to_phys = apple_dart_iova_to_phys, 1005 .free = apple_dart_domain_free, 1006 } 1007 }; 1008 1009 static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) 1010 { 1011 struct apple_dart *dart = dev; 1012 const char *fault_name = NULL; 1013 u32 error = readl(dart->regs + DART_T8020_ERROR); 1014 u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error); 1015 u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO); 1016 u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI); 1017 u64 addr = addr_lo | (((u64)addr_hi) << 32); 1018 u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error); 1019 1020 if (!(error & DART_T8020_ERROR_FLAG)) 1021 return IRQ_NONE; 1022 1023 /* there should only be a single bit set but let's use == to be sure */ 1024 if (error_code == DART_T8020_ERROR_READ_FAULT) 1025 fault_name = "READ FAULT"; 1026 else if (error_code == DART_T8020_ERROR_WRITE_FAULT) 1027 fault_name = "WRITE FAULT"; 1028 else if (error_code == DART_T8020_ERROR_NO_PTE) 1029 fault_name = "NO PTE FOR IOVA"; 1030 else if (error_code == DART_T8020_ERROR_NO_PMD) 1031 fault_name = "NO PMD FOR IOVA"; 1032 else if (error_code == DART_T8020_ERROR_NO_TTBR) 1033 fault_name = "NO TTBR FOR IOVA"; 1034 else 1035 fault_name = "unknown"; 1036 1037 dev_err_ratelimited( 1038 dart->dev, 1039 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", 1040 error, stream_idx, error_code, fault_name, addr); 1041 1042 writel(error, dart->regs + DART_T8020_ERROR); 1043 return IRQ_HANDLED; 1044 } 1045 1046 static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) 1047 { 1048 struct apple_dart *dart = dev; 1049 const char *fault_name = NULL; 1050 u32 error = readl(dart->regs + DART_T8110_ERROR); 1051 u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); 1052 u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO); 1053 u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI); 1054 u64 addr = addr_lo | (((u64)addr_hi) << 32); 1055 u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); 1056 1057 if (!(error & DART_T8110_ERROR_FLAG)) 1058 return IRQ_NONE; 1059 1060 /* there should only be a single bit set but let's use == to be sure */ 1061 if (error_code == DART_T8110_ERROR_READ_FAULT) 1062 fault_name = "READ FAULT"; 1063 else if (error_code == DART_T8110_ERROR_WRITE_FAULT) 1064 fault_name = "WRITE FAULT"; 1065 else if (error_code == DART_T8110_ERROR_NO_PTE) 1066 fault_name = "NO PTE FOR IOVA"; 1067 else if (error_code == DART_T8110_ERROR_NO_PMD) 1068 fault_name = "NO PMD FOR IOVA"; 1069 else if (error_code == DART_T8110_ERROR_NO_PGD) 1070 fault_name = "NO PGD FOR IOVA"; 1071 else if (error_code == DART_T8110_ERROR_NO_TTBR) 1072 fault_name = "NO TTBR FOR IOVA"; 1073 else 1074 fault_name = "unknown"; 1075 1076 dev_err_ratelimited( 1077 dart->dev, 1078 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx", 1079 error, stream_idx, error_code, fault_name, addr); 1080 1081 writel(error, dart->regs + DART_T8110_ERROR); 1082 return IRQ_HANDLED; 1083 } 1084 1085 static int apple_dart_probe(struct platform_device *pdev) 1086 { 1087 int ret; 1088 u32 dart_params[4]; 1089 struct resource *res; 1090 struct apple_dart *dart; 1091 struct device *dev = &pdev->dev; 1092 1093 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL); 1094 if (!dart) 1095 return -ENOMEM; 1096 1097 dart->dev = dev; 1098 dart->hw = of_device_get_match_data(dev); 1099 spin_lock_init(&dart->lock); 1100 1101 dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 1102 if (IS_ERR(dart->regs)) 1103 return PTR_ERR(dart->regs); 1104 1105 if (resource_size(res) < 0x4000) { 1106 dev_err(dev, "MMIO region too small (%pr)\n", res); 1107 return -EINVAL; 1108 } 1109 1110 dart->irq = platform_get_irq(pdev, 0); 1111 if (dart->irq < 0) 1112 return -ENODEV; 1113 1114 ret = devm_clk_bulk_get_all(dev, &dart->clks); 1115 if (ret < 0) 1116 return ret; 1117 dart->num_clks = ret; 1118 1119 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks); 1120 if (ret) 1121 return ret; 1122 1123 dart_params[0] = readl(dart->regs + DART_PARAMS1); 1124 dart_params[1] = readl(dart->regs + DART_PARAMS2); 1125 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); 1126 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; 1127 1128 switch (dart->hw->type) { 1129 case DART_T8020: 1130 case DART_T6000: 1131 dart->ias = 32; 1132 dart->oas = dart->hw->oas; 1133 dart->num_streams = dart->hw->max_sid_count; 1134 break; 1135 1136 case DART_T8110: 1137 dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3); 1138 dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4); 1139 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); 1140 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); 1141 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); 1142 break; 1143 } 1144 1145 if (dart->num_streams > DART_MAX_STREAMS) { 1146 dev_err(&pdev->dev, "Too many streams (%d > %d)\n", 1147 dart->num_streams, DART_MAX_STREAMS); 1148 ret = -EINVAL; 1149 goto err_clk_disable; 1150 } 1151 1152 ret = apple_dart_hw_reset(dart); 1153 if (ret) 1154 goto err_clk_disable; 1155 1156 ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED, 1157 "apple-dart fault handler", dart); 1158 if (ret) 1159 goto err_clk_disable; 1160 1161 platform_set_drvdata(pdev, dart); 1162 1163 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s", 1164 dev_name(&pdev->dev)); 1165 if (ret) 1166 goto err_free_irq; 1167 1168 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev); 1169 if (ret) 1170 goto err_sysfs_remove; 1171 1172 dev_info( 1173 &pdev->dev, 1174 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n", 1175 dart->pgsize, dart->num_streams, dart->supports_bypass, 1176 dart->pgsize > PAGE_SIZE); 1177 return 0; 1178 1179 err_sysfs_remove: 1180 iommu_device_sysfs_remove(&dart->iommu); 1181 err_free_irq: 1182 free_irq(dart->irq, dart); 1183 err_clk_disable: 1184 clk_bulk_disable_unprepare(dart->num_clks, dart->clks); 1185 1186 return ret; 1187 } 1188 1189 static void apple_dart_remove(struct platform_device *pdev) 1190 { 1191 struct apple_dart *dart = platform_get_drvdata(pdev); 1192 1193 apple_dart_hw_reset(dart); 1194 free_irq(dart->irq, dart); 1195 1196 iommu_device_unregister(&dart->iommu); 1197 iommu_device_sysfs_remove(&dart->iommu); 1198 1199 clk_bulk_disable_unprepare(dart->num_clks, dart->clks); 1200 } 1201 1202 static const struct apple_dart_hw apple_dart_hw_t8103 = { 1203 .type = DART_T8020, 1204 .irq_handler = apple_dart_t8020_irq, 1205 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1206 .oas = 36, 1207 .fmt = APPLE_DART, 1208 .max_sid_count = 16, 1209 1210 .enable_streams = DART_T8020_STREAMS_ENABLE, 1211 .lock = DART_T8020_CONFIG, 1212 .lock_bit = DART_T8020_CONFIG_LOCK, 1213 1214 .error = DART_T8020_ERROR, 1215 1216 .tcr = DART_T8020_TCR, 1217 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 1218 .tcr_disabled = 0, 1219 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, 1220 1221 .ttbr = DART_T8020_TTBR, 1222 .ttbr_valid = DART_T8020_TTBR_VALID, 1223 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 1224 .ttbr_shift = DART_T8020_TTBR_SHIFT, 1225 .ttbr_count = 4, 1226 }; 1227 1228 static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = { 1229 .type = DART_T8020, 1230 .irq_handler = apple_dart_t8020_irq, 1231 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1232 .oas = 36, 1233 .fmt = APPLE_DART, 1234 .max_sid_count = 64, 1235 1236 .enable_streams = DART_T8020_STREAMS_ENABLE, 1237 .lock = DART_T8020_CONFIG, 1238 .lock_bit = DART_T8020_CONFIG_LOCK, 1239 1240 .error = DART_T8020_ERROR, 1241 1242 .tcr = DART_T8020_TCR, 1243 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 1244 .tcr_disabled = 0, 1245 .tcr_bypass = 0, 1246 1247 .ttbr = DART_T8020_USB4_TTBR, 1248 .ttbr_valid = DART_T8020_TTBR_VALID, 1249 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 1250 .ttbr_shift = DART_T8020_TTBR_SHIFT, 1251 .ttbr_count = 4, 1252 }; 1253 1254 static const struct apple_dart_hw apple_dart_hw_t6000 = { 1255 .type = DART_T6000, 1256 .irq_handler = apple_dart_t8020_irq, 1257 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, 1258 .oas = 42, 1259 .fmt = APPLE_DART2, 1260 .max_sid_count = 16, 1261 1262 .enable_streams = DART_T8020_STREAMS_ENABLE, 1263 .lock = DART_T8020_CONFIG, 1264 .lock_bit = DART_T8020_CONFIG_LOCK, 1265 1266 .error = DART_T8020_ERROR, 1267 1268 .tcr = DART_T8020_TCR, 1269 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, 1270 .tcr_disabled = 0, 1271 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, 1272 1273 .ttbr = DART_T8020_TTBR, 1274 .ttbr_valid = DART_T8020_TTBR_VALID, 1275 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, 1276 .ttbr_shift = DART_T8020_TTBR_SHIFT, 1277 .ttbr_count = 4, 1278 }; 1279 1280 static const struct apple_dart_hw apple_dart_hw_t8110 = { 1281 .type = DART_T8110, 1282 .irq_handler = apple_dart_t8110_irq, 1283 .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb, 1284 .fmt = APPLE_DART2, 1285 .max_sid_count = 256, 1286 1287 .enable_streams = DART_T8110_ENABLE_STREAMS, 1288 .lock = DART_T8110_PROTECT, 1289 .lock_bit = DART_T8110_PROTECT_TTBR_TCR, 1290 1291 .error = DART_T8110_ERROR, 1292 1293 .tcr = DART_T8110_TCR, 1294 .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, 1295 .tcr_disabled = 0, 1296 .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, 1297 1298 .ttbr = DART_T8110_TTBR, 1299 .ttbr_valid = DART_T8110_TTBR_VALID, 1300 .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT, 1301 .ttbr_shift = DART_T8110_TTBR_SHIFT, 1302 .ttbr_count = 1, 1303 }; 1304 1305 static __maybe_unused int apple_dart_suspend(struct device *dev) 1306 { 1307 struct apple_dart *dart = dev_get_drvdata(dev); 1308 unsigned int sid, idx; 1309 1310 for (sid = 0; sid < dart->num_streams; sid++) { 1311 dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid)); 1312 for (idx = 0; idx < dart->hw->ttbr_count; idx++) 1313 dart->save_ttbr[sid][idx] = 1314 readl(dart->regs + DART_TTBR(dart, sid, idx)); 1315 } 1316 1317 return 0; 1318 } 1319 1320 static __maybe_unused int apple_dart_resume(struct device *dev) 1321 { 1322 struct apple_dart *dart = dev_get_drvdata(dev); 1323 unsigned int sid, idx; 1324 int ret; 1325 1326 ret = apple_dart_hw_reset(dart); 1327 if (ret) { 1328 dev_err(dev, "Failed to reset DART on resume\n"); 1329 return ret; 1330 } 1331 1332 for (sid = 0; sid < dart->num_streams; sid++) { 1333 for (idx = 0; idx < dart->hw->ttbr_count; idx++) 1334 writel(dart->save_ttbr[sid][idx], 1335 dart->regs + DART_TTBR(dart, sid, idx)); 1336 writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid)); 1337 } 1338 1339 return 0; 1340 } 1341 1342 static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); 1343 1344 static const struct of_device_id apple_dart_of_match[] = { 1345 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 }, 1346 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 }, 1347 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 }, 1348 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 }, 1349 {}, 1350 }; 1351 MODULE_DEVICE_TABLE(of, apple_dart_of_match); 1352 1353 static struct platform_driver apple_dart_driver = { 1354 .driver = { 1355 .name = "apple-dart", 1356 .of_match_table = apple_dart_of_match, 1357 .suppress_bind_attrs = true, 1358 .pm = pm_sleep_ptr(&apple_dart_pm_ops), 1359 }, 1360 .probe = apple_dart_probe, 1361 .remove = apple_dart_remove, 1362 }; 1363 1364 module_platform_driver(apple_dart_driver); 1365 1366 MODULE_DESCRIPTION("IOMMU API for Apple's DART"); 1367 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>"); 1368 MODULE_LICENSE("GPL v2"); 1369