1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Apple DART (Device Address Resolution Table) IOMMU driver
4 *
5 * Copyright (C) 2021 The Asahi Linux Contributors
6 *
7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
8 * Copyright (C) 2013 ARM Limited
9 * Copyright (C) 2015 ARM Limited
10 * and on exynos-iommu.c
11 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
12 */
13
14 #include <linux/atomic.h>
15 #include <linux/bitfield.h>
16 #include <linux/clk.h>
17 #include <linux/dev_printk.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/module.h>
25 #include <linux/of.h>
26 #include <linux/of_address.h>
27 #include <linux/of_iommu.h>
28 #include <linux/of_platform.h>
29 #include <linux/pci.h>
30 #include <linux/platform_device.h>
31 #include <linux/slab.h>
32 #include <linux/swab.h>
33 #include <linux/types.h>
34
35 #include "dma-iommu.h"
36
37 #define DART_MAX_STREAMS 256
38 #define DART_MAX_TTBR 4
39 #define MAX_DARTS_PER_DEVICE 3
40
41 /* Common registers */
42
43 #define DART_PARAMS1 0x00
44 #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24)
45
46 #define DART_PARAMS2 0x04
47 #define DART_PARAMS2_BYPASS_SUPPORT BIT(0)
48
49 /* T8020/T6000 registers */
50
51 #define DART_T8020_STREAM_COMMAND 0x20
52 #define DART_T8020_STREAM_COMMAND_BUSY BIT(2)
53 #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20)
54
55 #define DART_T8020_STREAM_SELECT 0x34
56
57 #define DART_T8020_ERROR 0x40
58 #define DART_T8020_ERROR_STREAM GENMASK(27, 24)
59 #define DART_T8020_ERROR_CODE GENMASK(11, 0)
60 #define DART_T8020_ERROR_FLAG BIT(31)
61
62 #define DART_T8020_ERROR_READ_FAULT BIT(4)
63 #define DART_T8020_ERROR_WRITE_FAULT BIT(3)
64 #define DART_T8020_ERROR_NO_PTE BIT(2)
65 #define DART_T8020_ERROR_NO_PMD BIT(1)
66 #define DART_T8020_ERROR_NO_TTBR BIT(0)
67
68 #define DART_T8020_CONFIG 0x60
69 #define DART_T8020_CONFIG_LOCK BIT(15)
70
71 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
72
73 #define DART_T8020_ERROR_ADDR_HI 0x54
74 #define DART_T8020_ERROR_ADDR_LO 0x50
75
76 #define DART_T8020_STREAMS_ENABLE 0xfc
77
78 #define DART_T8020_TCR 0x100
79 #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7)
80 #define DART_T8020_TCR_BYPASS_DART BIT(8)
81 #define DART_T8020_TCR_BYPASS_DAPF BIT(12)
82
83 #define DART_T8020_TTBR 0x200
84 #define DART_T8020_USB4_TTBR 0x400
85 #define DART_T8020_TTBR_VALID BIT(31)
86 #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0
87 #define DART_T8020_TTBR_SHIFT 12
88
89 /* T8110 registers */
90
91 #define DART_T8110_PARAMS3 0x08
92 #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24)
93 #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16)
94 #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8)
95 #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0)
96
97 #define DART_T8110_PARAMS4 0x0c
98 #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16)
99 #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0)
100
101 #define DART_T8110_TLB_CMD 0x80
102 #define DART_T8110_TLB_CMD_BUSY BIT(31)
103 #define DART_T8110_TLB_CMD_OP GENMASK(10, 8)
104 #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0
105 #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1
106 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0)
107
108 #define DART_T8110_ERROR 0x100
109 #define DART_T8110_ERROR_STREAM GENMASK(27, 20)
110 #define DART_T8110_ERROR_CODE GENMASK(14, 0)
111 #define DART_T8110_ERROR_FLAG BIT(31)
112
113 #define DART_T8110_ERROR_MASK 0x104
114
115 #define DART_T8110_ERROR_READ_FAULT BIT(5)
116 #define DART_T8110_ERROR_WRITE_FAULT BIT(4)
117 #define DART_T8110_ERROR_NO_PTE BIT(3)
118 #define DART_T8110_ERROR_NO_PMD BIT(2)
119 #define DART_T8110_ERROR_NO_PGD BIT(1)
120 #define DART_T8110_ERROR_NO_TTBR BIT(0)
121
122 #define DART_T8110_ERROR_ADDR_LO 0x170
123 #define DART_T8110_ERROR_ADDR_HI 0x174
124
125 #define DART_T8110_ERROR_STREAMS 0x1c0
126
127 #define DART_T8110_PROTECT 0x200
128 #define DART_T8110_UNPROTECT 0x204
129 #define DART_T8110_PROTECT_LOCK 0x208
130 #define DART_T8110_PROTECT_TTBR_TCR BIT(0)
131
132 #define DART_T8110_ENABLE_STREAMS 0xc00
133 #define DART_T8110_DISABLE_STREAMS 0xc20
134
135 #define DART_T8110_TCR 0x1000
136 #define DART_T8110_TCR_REMAP GENMASK(11, 8)
137 #define DART_T8110_TCR_REMAP_EN BIT(7)
138 #define DART_T8110_TCR_FOUR_LEVEL BIT(3)
139 #define DART_T8110_TCR_BYPASS_DAPF BIT(2)
140 #define DART_T8110_TCR_BYPASS_DART BIT(1)
141 #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
142
143 #define DART_T8110_TTBR 0x1400
144 #define DART_T8110_TTBR_VALID BIT(0)
145 #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2
146 #define DART_T8110_TTBR_SHIFT 14
147
148 #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2))
149
150 #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \
151 (((dart)->hw->ttbr_count * (sid)) << 2) + \
152 ((idx) << 2))
153
154 struct apple_dart_stream_map;
155
156 enum dart_type {
157 DART_T8020,
158 DART_T6000,
159 DART_T8110,
160 };
161
162 struct apple_dart_hw {
163 enum dart_type type;
164 irqreturn_t (*irq_handler)(int irq, void *dev);
165 int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
166
167 u32 oas;
168 enum io_pgtable_fmt fmt;
169
170 int max_sid_count;
171
172 u32 lock;
173 u32 lock_bit;
174
175 u32 error;
176
177 u32 enable_streams;
178
179 u32 tcr;
180 u32 tcr_enabled;
181 u32 tcr_disabled;
182 u32 tcr_bypass;
183 u32 tcr_4level;
184
185 u32 ttbr;
186 u32 ttbr_valid;
187 u32 ttbr_addr_field_shift;
188 u32 ttbr_shift;
189 int ttbr_count;
190 };
191
192 /*
193 * Private structure associated with each DART device.
194 *
195 * @dev: device struct
196 * @hw: SoC-specific hardware data
197 * @regs: mapped MMIO region
198 * @irq: interrupt number, can be shared with other DARTs
199 * @clks: clocks associated with this DART
200 * @num_clks: number of @clks
201 * @lock: lock for hardware operations involving this dart
202 * @pgsize: pagesize supported by this DART
203 * @supports_bypass: indicates if this DART supports bypass mode
204 * @sid2group: maps stream ids to iommu_groups
205 * @iommu: iommu core device
206 */
207 struct apple_dart {
208 struct device *dev;
209 const struct apple_dart_hw *hw;
210
211 void __iomem *regs;
212
213 int irq;
214 struct clk_bulk_data *clks;
215 int num_clks;
216
217 spinlock_t lock;
218
219 u32 ias;
220 u32 oas;
221 u32 pgsize;
222 u32 num_streams;
223 u32 supports_bypass : 1;
224 u32 four_level : 1;
225
226 struct iommu_group *sid2group[DART_MAX_STREAMS];
227 struct iommu_device iommu;
228
229 u32 save_tcr[DART_MAX_STREAMS];
230 u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
231 };
232
233 /*
234 * Convenience struct to identify streams.
235 *
236 * The normal variant is used inside apple_dart_master_cfg which isn't written
237 * to concurrently.
238 * The atomic variant is used inside apple_dart_domain where we have to guard
239 * against races from potential parallel calls to attach/detach_device.
240 * Note that even inside the atomic variant the apple_dart pointer is not
241 * protected: This pointer is initialized once under the domain init mutex
242 * and never changed again afterwards. Devices with different dart pointers
243 * cannot be attached to the same domain.
244 *
245 * @dart dart pointer
246 * @sid stream id bitmap
247 */
248 struct apple_dart_stream_map {
249 struct apple_dart *dart;
250 DECLARE_BITMAP(sidmap, DART_MAX_STREAMS);
251 };
252 struct apple_dart_atomic_stream_map {
253 struct apple_dart *dart;
254 atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)];
255 };
256
257 /*
258 * This structure is attached to each iommu domain handled by a DART.
259 *
260 * @pgtbl_ops: pagetable ops allocated by io-pgtable
261 * @finalized: true if the domain has been completely initialized
262 * @init_lock: protects domain initialization
263 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
264 * @domain: core iommu domain pointer
265 */
266 struct apple_dart_domain {
267 struct io_pgtable_ops *pgtbl_ops;
268
269 bool finalized;
270 struct mutex init_lock;
271 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
272
273 struct iommu_domain domain;
274 };
275
276 /*
277 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
278 * and contains a list of streams bound to this device.
279 * So far the worst case seen is a single device with two streams
280 * from different darts, such that this simple static array is enough.
281 *
282 * @streams: streams for this device
283 */
284 struct apple_dart_master_cfg {
285 /* Intersection of DART capabilitles */
286 u32 supports_bypass : 1;
287
288 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
289 };
290
291 /*
292 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
293 * apple_dart_domain.stream_maps
294 *
295 * @i int used as loop variable
296 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
297 * @stream pointer to the apple_dart_streams struct for each loop iteration
298 */
299 #define for_each_stream_map(i, base, stream_map) \
300 for (i = 0, stream_map = &(base)->stream_maps[0]; \
301 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \
302 stream_map = &(base)->stream_maps[++i])
303
304 static struct platform_driver apple_dart_driver;
305 static const struct iommu_ops apple_dart_iommu_ops;
306
to_dart_domain(struct iommu_domain * dom)307 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
308 {
309 return container_of(dom, struct apple_dart_domain, domain);
310 }
311
312 static void
apple_dart_hw_enable_translation(struct apple_dart_stream_map * stream_map,int levels)313 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int levels)
314 {
315 struct apple_dart *dart = stream_map->dart;
316 u32 tcr = dart->hw->tcr_enabled;
317 int sid;
318
319 if (levels == 4)
320 tcr |= dart->hw->tcr_4level;
321
322 WARN_ON(levels != 3 && levels != 4);
323 WARN_ON(levels == 4 && !dart->four_level);
324 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
325 writel(tcr, dart->regs + DART_TCR(dart, sid));
326 }
327
apple_dart_hw_disable_dma(struct apple_dart_stream_map * stream_map)328 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
329 {
330 struct apple_dart *dart = stream_map->dart;
331 int sid;
332
333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
334 writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid));
335 }
336
337 static void
apple_dart_hw_enable_bypass(struct apple_dart_stream_map * stream_map)338 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
339 {
340 struct apple_dart *dart = stream_map->dart;
341 int sid;
342
343 WARN_ON(!stream_map->dart->supports_bypass);
344 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
345 writel(dart->hw->tcr_bypass,
346 dart->regs + DART_TCR(dart, sid));
347 }
348
apple_dart_hw_set_ttbr(struct apple_dart_stream_map * stream_map,u8 idx,phys_addr_t paddr)349 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
350 u8 idx, phys_addr_t paddr)
351 {
352 struct apple_dart *dart = stream_map->dart;
353 int sid;
354
355 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1));
356 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
357 writel(dart->hw->ttbr_valid |
358 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift,
359 dart->regs + DART_TTBR(dart, sid, idx));
360 }
361
apple_dart_hw_clear_ttbr(struct apple_dart_stream_map * stream_map,u8 idx)362 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
363 u8 idx)
364 {
365 struct apple_dart *dart = stream_map->dart;
366 int sid;
367
368 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
369 writel(0, dart->regs + DART_TTBR(dart, sid, idx));
370 }
371
372 static void
apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map * stream_map)373 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
374 {
375 int i;
376
377 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)
378 apple_dart_hw_clear_ttbr(stream_map, i);
379 }
380
381 static int
apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map * stream_map,u32 command)382 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,
383 u32 command)
384 {
385 unsigned long flags;
386 int ret, i;
387 u32 command_reg;
388
389 spin_lock_irqsave(&stream_map->dart->lock, flags);
390
391 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)
392 writel(stream_map->sidmap[i],
393 stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);
394 writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);
395
396 ret = readl_poll_timeout_atomic(
397 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,
398 !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1,
399 DART_STREAM_COMMAND_BUSY_TIMEOUT);
400
401 spin_unlock_irqrestore(&stream_map->dart->lock, flags);
402
403 if (ret) {
404 dev_err(stream_map->dart->dev,
405 "busy bit did not clear after command %x for streams %lx\n",
406 command, stream_map->sidmap[0]);
407 return ret;
408 }
409
410 return 0;
411 }
412
413 static int
apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map * stream_map,u32 command)414 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,
415 u32 command)
416 {
417 struct apple_dart *dart = stream_map->dart;
418 unsigned long flags;
419 int ret = 0;
420 int sid;
421
422 spin_lock_irqsave(&dart->lock, flags);
423
424 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
425 u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) |
426 FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid);
427 writel(val, dart->regs + DART_T8110_TLB_CMD);
428
429 ret = readl_poll_timeout_atomic(
430 dart->regs + DART_T8110_TLB_CMD, val,
431 !(val & DART_T8110_TLB_CMD_BUSY), 1,
432 DART_STREAM_COMMAND_BUSY_TIMEOUT);
433
434 if (ret)
435 break;
436
437 }
438
439 spin_unlock_irqrestore(&dart->lock, flags);
440
441 if (ret) {
442 dev_err(stream_map->dart->dev,
443 "busy bit did not clear after command %x for stream %d\n",
444 command, sid);
445 return ret;
446 }
447
448 return 0;
449 }
450
451 static int
apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)452 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
453 {
454 return apple_dart_t8020_hw_stream_command(
455 stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);
456 }
457
458 static int
apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)459 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
460 {
461 return apple_dart_t8110_hw_tlb_command(
462 stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);
463 }
464
apple_dart_hw_reset(struct apple_dart * dart)465 static int apple_dart_hw_reset(struct apple_dart *dart)
466 {
467 u32 config;
468 struct apple_dart_stream_map stream_map;
469 int i;
470
471 config = readl(dart->regs + dart->hw->lock);
472 if (config & dart->hw->lock_bit) {
473 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
474 config);
475 return -EINVAL;
476 }
477
478 stream_map.dart = dart;
479 bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS);
480 bitmap_set(stream_map.sidmap, 0, dart->num_streams);
481 apple_dart_hw_disable_dma(&stream_map);
482 apple_dart_hw_clear_all_ttbrs(&stream_map);
483
484 /* enable all streams globally since TCR is used to control isolation */
485 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++)
486 writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i);
487
488 /* clear any pending errors before the interrupt is unmasked */
489 writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error);
490
491 if (dart->hw->type == DART_T8110)
492 writel(0, dart->regs + DART_T8110_ERROR_MASK);
493
494 return dart->hw->invalidate_tlb(&stream_map);
495 }
496
apple_dart_domain_flush_tlb(struct apple_dart_domain * domain)497 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
498 {
499 int i, j;
500 struct apple_dart_atomic_stream_map *domain_stream_map;
501 struct apple_dart_stream_map stream_map;
502
503 for_each_stream_map(i, domain, domain_stream_map) {
504 stream_map.dart = domain_stream_map->dart;
505
506 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)
507 stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);
508
509 stream_map.dart->hw->invalidate_tlb(&stream_map);
510 }
511 }
512
apple_dart_flush_iotlb_all(struct iommu_domain * domain)513 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
514 {
515 apple_dart_domain_flush_tlb(to_dart_domain(domain));
516 }
517
apple_dart_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)518 static void apple_dart_iotlb_sync(struct iommu_domain *domain,
519 struct iommu_iotlb_gather *gather)
520 {
521 apple_dart_domain_flush_tlb(to_dart_domain(domain));
522 }
523
apple_dart_iotlb_sync_map(struct iommu_domain * domain,unsigned long iova,size_t size)524 static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
525 unsigned long iova, size_t size)
526 {
527 apple_dart_domain_flush_tlb(to_dart_domain(domain));
528 return 0;
529 }
530
apple_dart_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)531 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
532 dma_addr_t iova)
533 {
534 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
535 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
536
537 if (!ops)
538 return 0;
539
540 return ops->iova_to_phys(ops, iova);
541 }
542
apple_dart_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)543 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
544 phys_addr_t paddr, size_t pgsize,
545 size_t pgcount, int prot, gfp_t gfp,
546 size_t *mapped)
547 {
548 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
549 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
550
551 if (!ops)
552 return -ENODEV;
553
554 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
555 mapped);
556 }
557
apple_dart_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)558 static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
559 unsigned long iova, size_t pgsize,
560 size_t pgcount,
561 struct iommu_iotlb_gather *gather)
562 {
563 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
564 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
565
566 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
567 }
568
569 static void
apple_dart_setup_translation(struct apple_dart_domain * domain,struct apple_dart_stream_map * stream_map)570 apple_dart_setup_translation(struct apple_dart_domain *domain,
571 struct apple_dart_stream_map *stream_map)
572 {
573 int i;
574 struct io_pgtable_cfg *pgtbl_cfg =
575 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
576
577 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
578 apple_dart_hw_set_ttbr(stream_map, i,
579 pgtbl_cfg->apple_dart_cfg.ttbr[i]);
580 for (; i < stream_map->dart->hw->ttbr_count; ++i)
581 apple_dart_hw_clear_ttbr(stream_map, i);
582
583 apple_dart_hw_enable_translation(stream_map,
584 pgtbl_cfg->apple_dart_cfg.n_levels);
585 stream_map->dart->hw->invalidate_tlb(stream_map);
586 }
587
apple_dart_finalize_domain(struct apple_dart_domain * dart_domain,struct apple_dart_master_cfg * cfg)588 static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
589 struct apple_dart_master_cfg *cfg)
590 {
591 struct apple_dart *dart = cfg->stream_maps[0].dart;
592 struct io_pgtable_cfg pgtbl_cfg;
593 int ret = 0;
594 int i, j;
595
596 if (dart->pgsize > PAGE_SIZE)
597 return -EINVAL;
598
599 mutex_lock(&dart_domain->init_lock);
600
601 if (dart_domain->finalized)
602 goto done;
603
604 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
605 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
606 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++)
607 atomic_long_set(&dart_domain->stream_maps[i].sidmap[j],
608 cfg->stream_maps[i].sidmap[j]);
609 }
610
611 pgtbl_cfg = (struct io_pgtable_cfg){
612 .pgsize_bitmap = dart->pgsize,
613 .ias = dart->ias,
614 .oas = dart->oas,
615 .coherent_walk = 1,
616 .iommu_dev = dart->dev,
617 };
618
619 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
620 &dart_domain->domain);
621 if (!dart_domain->pgtbl_ops) {
622 ret = -ENOMEM;
623 goto done;
624 }
625
626 dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
627 dart_domain->domain.geometry.aperture_start = 0;
628 dart_domain->domain.geometry.aperture_end =
629 (dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias);
630 dart_domain->domain.geometry.force_aperture = true;
631
632 dart_domain->finalized = true;
633
634 done:
635 mutex_unlock(&dart_domain->init_lock);
636 return ret;
637 }
638
639 static int
apple_dart_mod_streams(struct apple_dart_atomic_stream_map * domain_maps,struct apple_dart_stream_map * master_maps,bool add_streams)640 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
641 struct apple_dart_stream_map *master_maps,
642 bool add_streams)
643 {
644 int i, j;
645
646 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
647 if (domain_maps[i].dart != master_maps[i].dart)
648 return -EINVAL;
649 }
650
651 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
652 if (!domain_maps[i].dart)
653 break;
654 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) {
655 if (add_streams)
656 atomic_long_or(master_maps[i].sidmap[j],
657 &domain_maps[i].sidmap[j]);
658 else
659 atomic_long_and(~master_maps[i].sidmap[j],
660 &domain_maps[i].sidmap[j]);
661 }
662 }
663
664 return 0;
665 }
666
apple_dart_domain_add_streams(struct apple_dart_domain * domain,struct apple_dart_master_cfg * cfg)667 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
668 struct apple_dart_master_cfg *cfg)
669 {
670 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
671 true);
672 }
673
apple_dart_attach_dev_paging(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)674 static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
675 struct device *dev,
676 struct iommu_domain *old)
677 {
678 int ret, i;
679 struct apple_dart_stream_map *stream_map;
680 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
681 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
682
683 ret = apple_dart_finalize_domain(dart_domain, cfg);
684 if (ret)
685 return ret;
686
687 ret = apple_dart_domain_add_streams(dart_domain, cfg);
688 if (ret)
689 return ret;
690
691 for_each_stream_map(i, cfg, stream_map)
692 apple_dart_setup_translation(dart_domain, stream_map);
693 return 0;
694 }
695
apple_dart_attach_dev_identity(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)696 static int apple_dart_attach_dev_identity(struct iommu_domain *domain,
697 struct device *dev,
698 struct iommu_domain *old)
699 {
700 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
701 struct apple_dart_stream_map *stream_map;
702 int i;
703
704 if (!cfg->supports_bypass)
705 return -EINVAL;
706
707 for_each_stream_map(i, cfg, stream_map)
708 apple_dart_hw_enable_bypass(stream_map);
709 return 0;
710 }
711
712 static const struct iommu_domain_ops apple_dart_identity_ops = {
713 .attach_dev = apple_dart_attach_dev_identity,
714 };
715
716 static struct iommu_domain apple_dart_identity_domain = {
717 .type = IOMMU_DOMAIN_IDENTITY,
718 .ops = &apple_dart_identity_ops,
719 };
720
apple_dart_attach_dev_blocked(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)721 static int apple_dart_attach_dev_blocked(struct iommu_domain *domain,
722 struct device *dev,
723 struct iommu_domain *old)
724 {
725 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
726 struct apple_dart_stream_map *stream_map;
727 int i;
728
729 for_each_stream_map(i, cfg, stream_map)
730 apple_dart_hw_disable_dma(stream_map);
731 return 0;
732 }
733
734 static const struct iommu_domain_ops apple_dart_blocked_ops = {
735 .attach_dev = apple_dart_attach_dev_blocked,
736 };
737
738 static struct iommu_domain apple_dart_blocked_domain = {
739 .type = IOMMU_DOMAIN_BLOCKED,
740 .ops = &apple_dart_blocked_ops,
741 };
742
apple_dart_probe_device(struct device * dev)743 static struct iommu_device *apple_dart_probe_device(struct device *dev)
744 {
745 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
746 struct apple_dart_stream_map *stream_map;
747 int i;
748
749 if (!cfg)
750 return ERR_PTR(-ENODEV);
751
752 for_each_stream_map(i, cfg, stream_map)
753 device_link_add(
754 dev, stream_map->dart->dev,
755 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
756
757 return &cfg->stream_maps[0].dart->iommu;
758 }
759
apple_dart_release_device(struct device * dev)760 static void apple_dart_release_device(struct device *dev)
761 {
762 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
763
764 kfree(cfg);
765 }
766
apple_dart_domain_alloc_paging(struct device * dev)767 static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev)
768 {
769 struct apple_dart_domain *dart_domain;
770
771 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
772 if (!dart_domain)
773 return NULL;
774
775 mutex_init(&dart_domain->init_lock);
776
777 if (dev) {
778 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
779 int ret;
780
781 ret = apple_dart_finalize_domain(dart_domain, cfg);
782 if (ret) {
783 kfree(dart_domain);
784 return ERR_PTR(ret);
785 }
786 }
787 return &dart_domain->domain;
788 }
789
apple_dart_domain_free(struct iommu_domain * domain)790 static void apple_dart_domain_free(struct iommu_domain *domain)
791 {
792 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
793
794 free_io_pgtable_ops(dart_domain->pgtbl_ops);
795
796 kfree(dart_domain);
797 }
798
apple_dart_of_xlate(struct device * dev,const struct of_phandle_args * args)799 static int apple_dart_of_xlate(struct device *dev,
800 const struct of_phandle_args *args)
801 {
802 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
803 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
804 struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
805 struct apple_dart *cfg_dart;
806 int i, sid;
807
808 put_device(&iommu_pdev->dev);
809
810 if (args->args_count != 1)
811 return -EINVAL;
812 sid = args->args[0];
813
814 if (!cfg) {
815 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
816 if (!cfg)
817 return -ENOMEM;
818 /* Will be ANDed with DART capabilities */
819 cfg->supports_bypass = true;
820 }
821 dev_iommu_priv_set(dev, cfg);
822
823 cfg_dart = cfg->stream_maps[0].dart;
824 if (cfg_dart) {
825 if (cfg_dart->pgsize != dart->pgsize)
826 return -EINVAL;
827 if (cfg_dart->ias != dart->ias)
828 return -EINVAL;
829 }
830
831 cfg->supports_bypass &= dart->supports_bypass;
832
833 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
834 if (cfg->stream_maps[i].dart == dart) {
835 set_bit(sid, cfg->stream_maps[i].sidmap);
836 return 0;
837 }
838 }
839 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
840 if (!cfg->stream_maps[i].dart) {
841 cfg->stream_maps[i].dart = dart;
842 set_bit(sid, cfg->stream_maps[i].sidmap);
843 return 0;
844 }
845 }
846
847 return -EINVAL;
848 }
849
850 static DEFINE_MUTEX(apple_dart_groups_lock);
851
apple_dart_release_group(void * iommu_data)852 static void apple_dart_release_group(void *iommu_data)
853 {
854 int i, sid;
855 struct apple_dart_stream_map *stream_map;
856 struct apple_dart_master_cfg *group_master_cfg = iommu_data;
857
858 mutex_lock(&apple_dart_groups_lock);
859
860 for_each_stream_map(i, group_master_cfg, stream_map)
861 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
862 stream_map->dart->sid2group[sid] = NULL;
863
864 kfree(iommu_data);
865 mutex_unlock(&apple_dart_groups_lock);
866 }
867
apple_dart_merge_master_cfg(struct apple_dart_master_cfg * dst,struct apple_dart_master_cfg * src)868 static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst,
869 struct apple_dart_master_cfg *src)
870 {
871 /*
872 * We know that this function is only called for groups returned from
873 * pci_device_group and that all Apple Silicon platforms never spread
874 * PCIe devices from the same bus across multiple DARTs such that we can
875 * just assume that both src and dst only have the same single DART.
876 */
877 if (src->stream_maps[1].dart)
878 return -EINVAL;
879 if (dst->stream_maps[1].dart)
880 return -EINVAL;
881 if (src->stream_maps[0].dart != dst->stream_maps[0].dart)
882 return -EINVAL;
883
884 bitmap_or(dst->stream_maps[0].sidmap,
885 dst->stream_maps[0].sidmap,
886 src->stream_maps[0].sidmap,
887 dst->stream_maps[0].dart->num_streams);
888 return 0;
889 }
890
apple_dart_device_group(struct device * dev)891 static struct iommu_group *apple_dart_device_group(struct device *dev)
892 {
893 int i, sid;
894 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
895 struct apple_dart_stream_map *stream_map;
896 struct apple_dart_master_cfg *group_master_cfg;
897 struct iommu_group *group = NULL;
898 struct iommu_group *res = ERR_PTR(-EINVAL);
899
900 mutex_lock(&apple_dart_groups_lock);
901
902 for_each_stream_map(i, cfg, stream_map) {
903 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {
904 struct iommu_group *stream_group =
905 stream_map->dart->sid2group[sid];
906
907 if (group && group != stream_group) {
908 res = ERR_PTR(-EINVAL);
909 goto out;
910 }
911
912 group = stream_group;
913 }
914 }
915
916 if (group) {
917 res = iommu_group_ref_get(group);
918 goto out;
919 }
920
921 #ifdef CONFIG_PCI
922 if (dev_is_pci(dev))
923 group = pci_device_group(dev);
924 else
925 #endif
926 group = generic_device_group(dev);
927
928 res = ERR_PTR(-ENOMEM);
929 if (!group)
930 goto out;
931
932 group_master_cfg = iommu_group_get_iommudata(group);
933 if (group_master_cfg) {
934 int ret;
935
936 ret = apple_dart_merge_master_cfg(group_master_cfg, cfg);
937 if (ret) {
938 dev_err(dev, "Failed to merge DART IOMMU groups.\n");
939 iommu_group_put(group);
940 res = ERR_PTR(ret);
941 goto out;
942 }
943 } else {
944 group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg),
945 GFP_KERNEL);
946 if (!group_master_cfg) {
947 iommu_group_put(group);
948 goto out;
949 }
950
951 iommu_group_set_iommudata(group, group_master_cfg,
952 apple_dart_release_group);
953 }
954
955 for_each_stream_map(i, cfg, stream_map)
956 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
957 stream_map->dart->sid2group[sid] = group;
958
959 res = group;
960
961 out:
962 mutex_unlock(&apple_dart_groups_lock);
963 return res;
964 }
965
apple_dart_def_domain_type(struct device * dev)966 static int apple_dart_def_domain_type(struct device *dev)
967 {
968 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
969
970 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE)
971 return IOMMU_DOMAIN_IDENTITY;
972 if (!cfg->supports_bypass)
973 return IOMMU_DOMAIN_DMA;
974
975 return 0;
976 }
977
978 #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
979 /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */
980 #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0
981 #endif
982 #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK)
983
apple_dart_get_resv_regions(struct device * dev,struct list_head * head)984 static void apple_dart_get_resv_regions(struct device *dev,
985 struct list_head *head)
986 {
987 if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) {
988 struct iommu_resv_region *region;
989 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
990
991 region = iommu_alloc_resv_region(DOORBELL_ADDR,
992 PAGE_SIZE, prot,
993 IOMMU_RESV_MSI, GFP_KERNEL);
994 if (!region)
995 return;
996
997 list_add_tail(®ion->list, head);
998 }
999
1000 iommu_dma_get_resv_regions(dev, head);
1001 }
1002
1003 static const struct iommu_ops apple_dart_iommu_ops = {
1004 .identity_domain = &apple_dart_identity_domain,
1005 .blocked_domain = &apple_dart_blocked_domain,
1006 .domain_alloc_paging = apple_dart_domain_alloc_paging,
1007 .probe_device = apple_dart_probe_device,
1008 .release_device = apple_dart_release_device,
1009 .device_group = apple_dart_device_group,
1010 .of_xlate = apple_dart_of_xlate,
1011 .def_domain_type = apple_dart_def_domain_type,
1012 .get_resv_regions = apple_dart_get_resv_regions,
1013 .owner = THIS_MODULE,
1014 .default_domain_ops = &(const struct iommu_domain_ops) {
1015 .attach_dev = apple_dart_attach_dev_paging,
1016 .map_pages = apple_dart_map_pages,
1017 .unmap_pages = apple_dart_unmap_pages,
1018 .flush_iotlb_all = apple_dart_flush_iotlb_all,
1019 .iotlb_sync = apple_dart_iotlb_sync,
1020 .iotlb_sync_map = apple_dart_iotlb_sync_map,
1021 .iova_to_phys = apple_dart_iova_to_phys,
1022 .free = apple_dart_domain_free,
1023 }
1024 };
1025
apple_dart_t8020_irq(int irq,void * dev)1026 static irqreturn_t apple_dart_t8020_irq(int irq, void *dev)
1027 {
1028 struct apple_dart *dart = dev;
1029 const char *fault_name = NULL;
1030 u32 error = readl(dart->regs + DART_T8020_ERROR);
1031 u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error);
1032 u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO);
1033 u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI);
1034 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1035 u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error);
1036
1037 if (!(error & DART_T8020_ERROR_FLAG))
1038 return IRQ_NONE;
1039
1040 /* there should only be a single bit set but let's use == to be sure */
1041 if (error_code == DART_T8020_ERROR_READ_FAULT)
1042 fault_name = "READ FAULT";
1043 else if (error_code == DART_T8020_ERROR_WRITE_FAULT)
1044 fault_name = "WRITE FAULT";
1045 else if (error_code == DART_T8020_ERROR_NO_PTE)
1046 fault_name = "NO PTE FOR IOVA";
1047 else if (error_code == DART_T8020_ERROR_NO_PMD)
1048 fault_name = "NO PMD FOR IOVA";
1049 else if (error_code == DART_T8020_ERROR_NO_TTBR)
1050 fault_name = "NO TTBR FOR IOVA";
1051 else
1052 fault_name = "unknown";
1053
1054 dev_err_ratelimited(
1055 dart->dev,
1056 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1057 error, stream_idx, error_code, fault_name, addr);
1058
1059 writel(error, dart->regs + DART_T8020_ERROR);
1060 return IRQ_HANDLED;
1061 }
1062
apple_dart_t8110_irq(int irq,void * dev)1063 static irqreturn_t apple_dart_t8110_irq(int irq, void *dev)
1064 {
1065 struct apple_dart *dart = dev;
1066 const char *fault_name = NULL;
1067 u32 error = readl(dart->regs + DART_T8110_ERROR);
1068 u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error);
1069 u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO);
1070 u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI);
1071 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1072 u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error);
1073
1074 if (!(error & DART_T8110_ERROR_FLAG))
1075 return IRQ_NONE;
1076
1077 /* there should only be a single bit set but let's use == to be sure */
1078 if (error_code == DART_T8110_ERROR_READ_FAULT)
1079 fault_name = "READ FAULT";
1080 else if (error_code == DART_T8110_ERROR_WRITE_FAULT)
1081 fault_name = "WRITE FAULT";
1082 else if (error_code == DART_T8110_ERROR_NO_PTE)
1083 fault_name = "NO PTE FOR IOVA";
1084 else if (error_code == DART_T8110_ERROR_NO_PMD)
1085 fault_name = "NO PMD FOR IOVA";
1086 else if (error_code == DART_T8110_ERROR_NO_PGD)
1087 fault_name = "NO PGD FOR IOVA";
1088 else if (error_code == DART_T8110_ERROR_NO_TTBR)
1089 fault_name = "NO TTBR FOR IOVA";
1090 else
1091 fault_name = "unknown";
1092
1093 dev_err_ratelimited(
1094 dart->dev,
1095 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1096 error, stream_idx, error_code, fault_name, addr);
1097
1098 writel(error, dart->regs + DART_T8110_ERROR);
1099 for (int i = 0; i < BITS_TO_U32(dart->num_streams); i++)
1100 writel(U32_MAX, dart->regs + DART_T8110_ERROR_STREAMS + 4 * i);
1101
1102 return IRQ_HANDLED;
1103 }
1104
apple_dart_probe(struct platform_device * pdev)1105 static int apple_dart_probe(struct platform_device *pdev)
1106 {
1107 int ret;
1108 u32 dart_params[4];
1109 struct resource *res;
1110 struct apple_dart *dart;
1111 struct device *dev = &pdev->dev;
1112
1113 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
1114 if (!dart)
1115 return -ENOMEM;
1116
1117 dart->dev = dev;
1118 dart->hw = of_device_get_match_data(dev);
1119 spin_lock_init(&dart->lock);
1120
1121 dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1122 if (IS_ERR(dart->regs))
1123 return PTR_ERR(dart->regs);
1124
1125 if (resource_size(res) < 0x4000) {
1126 dev_err(dev, "MMIO region too small (%pr)\n", res);
1127 return -EINVAL;
1128 }
1129
1130 dart->irq = platform_get_irq(pdev, 0);
1131 if (dart->irq < 0)
1132 return -ENODEV;
1133
1134 ret = devm_clk_bulk_get_all(dev, &dart->clks);
1135 if (ret < 0)
1136 return ret;
1137 dart->num_clks = ret;
1138
1139 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
1140 if (ret)
1141 return ret;
1142
1143 dart_params[0] = readl(dart->regs + DART_PARAMS1);
1144 dart_params[1] = readl(dart->regs + DART_PARAMS2);
1145 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]);
1146 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT;
1147
1148 switch (dart->hw->type) {
1149 case DART_T8020:
1150 case DART_T6000:
1151 dart->ias = 32;
1152 dart->oas = dart->hw->oas;
1153 dart->num_streams = dart->hw->max_sid_count;
1154 break;
1155
1156 case DART_T8110:
1157 dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3);
1158 dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4);
1159 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]);
1160 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]);
1161 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]);
1162 dart->four_level = dart->ias > 36;
1163 break;
1164 }
1165
1166 if (dart->num_streams > DART_MAX_STREAMS) {
1167 dev_err(&pdev->dev, "Too many streams (%d > %d)\n",
1168 dart->num_streams, DART_MAX_STREAMS);
1169 ret = -EINVAL;
1170 goto err_clk_disable;
1171 }
1172
1173 ret = apple_dart_hw_reset(dart);
1174 if (ret)
1175 goto err_clk_disable;
1176
1177 ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED,
1178 "apple-dart fault handler", dart);
1179 if (ret)
1180 goto err_clk_disable;
1181
1182 platform_set_drvdata(pdev, dart);
1183
1184 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
1185 dev_name(&pdev->dev));
1186 if (ret)
1187 goto err_free_irq;
1188
1189 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
1190 if (ret)
1191 goto err_sysfs_remove;
1192
1193 dev_info(
1194 &pdev->dev,
1195 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, AS %d -> %d] initialized\n",
1196 dart->pgsize, dart->num_streams, dart->supports_bypass,
1197 dart->pgsize > PAGE_SIZE, dart->ias, dart->oas);
1198 return 0;
1199
1200 err_sysfs_remove:
1201 iommu_device_sysfs_remove(&dart->iommu);
1202 err_free_irq:
1203 free_irq(dart->irq, dart);
1204 err_clk_disable:
1205 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1206
1207 return ret;
1208 }
1209
apple_dart_remove(struct platform_device * pdev)1210 static void apple_dart_remove(struct platform_device *pdev)
1211 {
1212 struct apple_dart *dart = platform_get_drvdata(pdev);
1213
1214 apple_dart_hw_reset(dart);
1215 free_irq(dart->irq, dart);
1216
1217 iommu_device_unregister(&dart->iommu);
1218 iommu_device_sysfs_remove(&dart->iommu);
1219
1220 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1221 }
1222
1223 static const struct apple_dart_hw apple_dart_hw_t8103 = {
1224 .type = DART_T8020,
1225 .irq_handler = apple_dart_t8020_irq,
1226 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1227 .oas = 36,
1228 .fmt = APPLE_DART,
1229 .max_sid_count = 16,
1230
1231 .enable_streams = DART_T8020_STREAMS_ENABLE,
1232 .lock = DART_T8020_CONFIG,
1233 .lock_bit = DART_T8020_CONFIG_LOCK,
1234
1235 .error = DART_T8020_ERROR,
1236
1237 .tcr = DART_T8020_TCR,
1238 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1239 .tcr_disabled = 0,
1240 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1241
1242 .ttbr = DART_T8020_TTBR,
1243 .ttbr_valid = DART_T8020_TTBR_VALID,
1244 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1245 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1246 .ttbr_count = 4,
1247 };
1248
1249 static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = {
1250 .type = DART_T8020,
1251 .irq_handler = apple_dart_t8020_irq,
1252 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1253 .oas = 36,
1254 .fmt = APPLE_DART,
1255 .max_sid_count = 64,
1256
1257 .enable_streams = DART_T8020_STREAMS_ENABLE,
1258 .lock = DART_T8020_CONFIG,
1259 .lock_bit = DART_T8020_CONFIG_LOCK,
1260
1261 .error = DART_T8020_ERROR,
1262
1263 .tcr = DART_T8020_TCR,
1264 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1265 .tcr_disabled = 0,
1266 .tcr_bypass = 0,
1267
1268 .ttbr = DART_T8020_USB4_TTBR,
1269 .ttbr_valid = DART_T8020_TTBR_VALID,
1270 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1271 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1272 .ttbr_count = 4,
1273 };
1274
1275 static const struct apple_dart_hw apple_dart_hw_t6000 = {
1276 .type = DART_T6000,
1277 .irq_handler = apple_dart_t8020_irq,
1278 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1279 .oas = 42,
1280 .fmt = APPLE_DART2,
1281 .max_sid_count = 16,
1282
1283 .enable_streams = DART_T8020_STREAMS_ENABLE,
1284 .lock = DART_T8020_CONFIG,
1285 .lock_bit = DART_T8020_CONFIG_LOCK,
1286
1287 .error = DART_T8020_ERROR,
1288
1289 .tcr = DART_T8020_TCR,
1290 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1291 .tcr_disabled = 0,
1292 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1293
1294 .ttbr = DART_T8020_TTBR,
1295 .ttbr_valid = DART_T8020_TTBR_VALID,
1296 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1297 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1298 .ttbr_count = 4,
1299 };
1300
1301 static const struct apple_dart_hw apple_dart_hw_t8110 = {
1302 .type = DART_T8110,
1303 .irq_handler = apple_dart_t8110_irq,
1304 .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb,
1305 .fmt = APPLE_DART2,
1306 .max_sid_count = 256,
1307
1308 .enable_streams = DART_T8110_ENABLE_STREAMS,
1309 .lock = DART_T8110_PROTECT,
1310 .lock_bit = DART_T8110_PROTECT_TTBR_TCR,
1311
1312 .error = DART_T8110_ERROR,
1313
1314 .tcr = DART_T8110_TCR,
1315 .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE,
1316 .tcr_disabled = 0,
1317 .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART,
1318 .tcr_4level = DART_T8110_TCR_FOUR_LEVEL,
1319
1320 .ttbr = DART_T8110_TTBR,
1321 .ttbr_valid = DART_T8110_TTBR_VALID,
1322 .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT,
1323 .ttbr_shift = DART_T8110_TTBR_SHIFT,
1324 .ttbr_count = 1,
1325 };
1326
apple_dart_suspend(struct device * dev)1327 static __maybe_unused int apple_dart_suspend(struct device *dev)
1328 {
1329 struct apple_dart *dart = dev_get_drvdata(dev);
1330 unsigned int sid, idx;
1331
1332 for (sid = 0; sid < dart->num_streams; sid++) {
1333 dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid));
1334 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1335 dart->save_ttbr[sid][idx] =
1336 readl(dart->regs + DART_TTBR(dart, sid, idx));
1337 }
1338
1339 return 0;
1340 }
1341
apple_dart_resume(struct device * dev)1342 static __maybe_unused int apple_dart_resume(struct device *dev)
1343 {
1344 struct apple_dart *dart = dev_get_drvdata(dev);
1345 unsigned int sid, idx;
1346 int ret;
1347
1348 ret = apple_dart_hw_reset(dart);
1349 if (ret) {
1350 dev_err(dev, "Failed to reset DART on resume\n");
1351 return ret;
1352 }
1353
1354 for (sid = 0; sid < dart->num_streams; sid++) {
1355 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1356 writel(dart->save_ttbr[sid][idx],
1357 dart->regs + DART_TTBR(dart, sid, idx));
1358 writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid));
1359 }
1360
1361 return 0;
1362 }
1363
1364 static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume);
1365
1366 static const struct of_device_id apple_dart_of_match[] = {
1367 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
1368 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
1369 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
1370 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
1371 {},
1372 };
1373 MODULE_DEVICE_TABLE(of, apple_dart_of_match);
1374
1375 static struct platform_driver apple_dart_driver = {
1376 .driver = {
1377 .name = "apple-dart",
1378 .of_match_table = apple_dart_of_match,
1379 .suppress_bind_attrs = true,
1380 .pm = pm_sleep_ptr(&apple_dart_pm_ops),
1381 },
1382 .probe = apple_dart_probe,
1383 .remove = apple_dart_remove,
1384 };
1385
1386 module_platform_driver(apple_dart_driver);
1387
1388 MODULE_DESCRIPTION("IOMMU API for Apple's DART");
1389 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1390 MODULE_LICENSE("GPL v2");
1391