1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Apple DART (Device Address Resolution Table) IOMMU driver
4 *
5 * Copyright (C) 2021 The Asahi Linux Contributors
6 *
7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
8 * Copyright (C) 2013 ARM Limited
9 * Copyright (C) 2015 ARM Limited
10 * and on exynos-iommu.c
11 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
12 */
13
14 #include <linux/atomic.h>
15 #include <linux/bitfield.h>
16 #include <linux/clk.h>
17 #include <linux/dev_printk.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/module.h>
25 #include <linux/of.h>
26 #include <linux/of_address.h>
27 #include <linux/of_iommu.h>
28 #include <linux/of_platform.h>
29 #include <linux/pci.h>
30 #include <linux/platform_device.h>
31 #include <linux/slab.h>
32 #include <linux/swab.h>
33 #include <linux/types.h>
34
35 #include "dma-iommu.h"
36
37 #define DART_MAX_STREAMS 256
38 #define DART_MAX_TTBR 4
39 #define MAX_DARTS_PER_DEVICE 3
40
41 /* Common registers */
42
43 #define DART_PARAMS1 0x00
44 #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24)
45
46 #define DART_PARAMS2 0x04
47 #define DART_PARAMS2_BYPASS_SUPPORT BIT(0)
48
49 /* T8020/T6000 registers */
50
51 #define DART_T8020_STREAM_COMMAND 0x20
52 #define DART_T8020_STREAM_COMMAND_BUSY BIT(2)
53 #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20)
54
55 #define DART_T8020_STREAM_SELECT 0x34
56
57 #define DART_T8020_ERROR 0x40
58 #define DART_T8020_ERROR_STREAM GENMASK(27, 24)
59 #define DART_T8020_ERROR_CODE GENMASK(11, 0)
60 #define DART_T8020_ERROR_FLAG BIT(31)
61
62 #define DART_T8020_ERROR_READ_FAULT BIT(4)
63 #define DART_T8020_ERROR_WRITE_FAULT BIT(3)
64 #define DART_T8020_ERROR_NO_PTE BIT(2)
65 #define DART_T8020_ERROR_NO_PMD BIT(1)
66 #define DART_T8020_ERROR_NO_TTBR BIT(0)
67
68 #define DART_T8020_CONFIG 0x60
69 #define DART_T8020_CONFIG_LOCK BIT(15)
70
71 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
72
73 #define DART_T8020_ERROR_ADDR_HI 0x54
74 #define DART_T8020_ERROR_ADDR_LO 0x50
75
76 #define DART_T8020_STREAMS_ENABLE 0xfc
77
78 #define DART_T8020_TCR 0x100
79 #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7)
80 #define DART_T8020_TCR_BYPASS_DART BIT(8)
81 #define DART_T8020_TCR_BYPASS_DAPF BIT(12)
82
83 #define DART_T8020_TTBR 0x200
84 #define DART_T8020_USB4_TTBR 0x400
85 #define DART_T8020_TTBR_VALID BIT(31)
86 #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0
87 #define DART_T8020_TTBR_SHIFT 12
88
89 /* T8110 registers */
90
91 #define DART_T8110_PARAMS3 0x08
92 #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24)
93 #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16)
94 #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8)
95 #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0)
96
97 #define DART_T8110_PARAMS4 0x0c
98 #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16)
99 #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0)
100
101 #define DART_T8110_TLB_CMD 0x80
102 #define DART_T8110_TLB_CMD_BUSY BIT(31)
103 #define DART_T8110_TLB_CMD_OP GENMASK(10, 8)
104 #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0
105 #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1
106 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0)
107
108 #define DART_T8110_ERROR 0x100
109 #define DART_T8110_ERROR_STREAM GENMASK(27, 20)
110 #define DART_T8110_ERROR_CODE GENMASK(14, 0)
111 #define DART_T8110_ERROR_FLAG BIT(31)
112
113 #define DART_T8110_ERROR_MASK 0x104
114
115 #define DART_T8110_ERROR_READ_FAULT BIT(5)
116 #define DART_T8110_ERROR_WRITE_FAULT BIT(4)
117 #define DART_T8110_ERROR_NO_PTE BIT(3)
118 #define DART_T8110_ERROR_NO_PMD BIT(2)
119 #define DART_T8110_ERROR_NO_PGD BIT(1)
120 #define DART_T8110_ERROR_NO_TTBR BIT(0)
121
122 #define DART_T8110_ERROR_ADDR_LO 0x170
123 #define DART_T8110_ERROR_ADDR_HI 0x174
124
125 #define DART_T8110_PROTECT 0x200
126 #define DART_T8110_UNPROTECT 0x204
127 #define DART_T8110_PROTECT_LOCK 0x208
128 #define DART_T8110_PROTECT_TTBR_TCR BIT(0)
129
130 #define DART_T8110_ENABLE_STREAMS 0xc00
131 #define DART_T8110_DISABLE_STREAMS 0xc20
132
133 #define DART_T8110_TCR 0x1000
134 #define DART_T8110_TCR_REMAP GENMASK(11, 8)
135 #define DART_T8110_TCR_REMAP_EN BIT(7)
136 #define DART_T8110_TCR_BYPASS_DAPF BIT(2)
137 #define DART_T8110_TCR_BYPASS_DART BIT(1)
138 #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
139
140 #define DART_T8110_TTBR 0x1400
141 #define DART_T8110_TTBR_VALID BIT(0)
142 #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2
143 #define DART_T8110_TTBR_SHIFT 14
144
145 #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2))
146
147 #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \
148 (((dart)->hw->ttbr_count * (sid)) << 2) + \
149 ((idx) << 2))
150
151 struct apple_dart_stream_map;
152
153 enum dart_type {
154 DART_T8020,
155 DART_T6000,
156 DART_T8110,
157 };
158
159 struct apple_dart_hw {
160 enum dart_type type;
161 irqreturn_t (*irq_handler)(int irq, void *dev);
162 int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
163
164 u32 oas;
165 enum io_pgtable_fmt fmt;
166
167 int max_sid_count;
168
169 u64 lock;
170 u64 lock_bit;
171
172 u64 error;
173
174 u64 enable_streams;
175
176 u64 tcr;
177 u64 tcr_enabled;
178 u64 tcr_disabled;
179 u64 tcr_bypass;
180
181 u64 ttbr;
182 u64 ttbr_valid;
183 u64 ttbr_addr_field_shift;
184 u64 ttbr_shift;
185 int ttbr_count;
186 };
187
188 /*
189 * Private structure associated with each DART device.
190 *
191 * @dev: device struct
192 * @hw: SoC-specific hardware data
193 * @regs: mapped MMIO region
194 * @irq: interrupt number, can be shared with other DARTs
195 * @clks: clocks associated with this DART
196 * @num_clks: number of @clks
197 * @lock: lock for hardware operations involving this dart
198 * @pgsize: pagesize supported by this DART
199 * @supports_bypass: indicates if this DART supports bypass mode
200 * @sid2group: maps stream ids to iommu_groups
201 * @iommu: iommu core device
202 */
203 struct apple_dart {
204 struct device *dev;
205 const struct apple_dart_hw *hw;
206
207 void __iomem *regs;
208
209 int irq;
210 struct clk_bulk_data *clks;
211 int num_clks;
212
213 spinlock_t lock;
214
215 u32 ias;
216 u32 oas;
217 u32 pgsize;
218 u32 num_streams;
219 u32 supports_bypass : 1;
220
221 struct iommu_group *sid2group[DART_MAX_STREAMS];
222 struct iommu_device iommu;
223
224 u32 save_tcr[DART_MAX_STREAMS];
225 u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
226 };
227
228 /*
229 * Convenience struct to identify streams.
230 *
231 * The normal variant is used inside apple_dart_master_cfg which isn't written
232 * to concurrently.
233 * The atomic variant is used inside apple_dart_domain where we have to guard
234 * against races from potential parallel calls to attach/detach_device.
235 * Note that even inside the atomic variant the apple_dart pointer is not
236 * protected: This pointer is initialized once under the domain init mutex
237 * and never changed again afterwards. Devices with different dart pointers
238 * cannot be attached to the same domain.
239 *
240 * @dart dart pointer
241 * @sid stream id bitmap
242 */
243 struct apple_dart_stream_map {
244 struct apple_dart *dart;
245 DECLARE_BITMAP(sidmap, DART_MAX_STREAMS);
246 };
247 struct apple_dart_atomic_stream_map {
248 struct apple_dart *dart;
249 atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)];
250 };
251
252 /*
253 * This structure is attached to each iommu domain handled by a DART.
254 *
255 * @pgtbl_ops: pagetable ops allocated by io-pgtable
256 * @finalized: true if the domain has been completely initialized
257 * @init_lock: protects domain initialization
258 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
259 * @domain: core iommu domain pointer
260 */
261 struct apple_dart_domain {
262 struct io_pgtable_ops *pgtbl_ops;
263
264 bool finalized;
265 struct mutex init_lock;
266 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
267
268 struct iommu_domain domain;
269 };
270
271 /*
272 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
273 * and contains a list of streams bound to this device.
274 * So far the worst case seen is a single device with two streams
275 * from different darts, such that this simple static array is enough.
276 *
277 * @streams: streams for this device
278 */
279 struct apple_dart_master_cfg {
280 /* Intersection of DART capabilitles */
281 u32 supports_bypass : 1;
282
283 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
284 };
285
286 /*
287 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
288 * apple_dart_domain.stream_maps
289 *
290 * @i int used as loop variable
291 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
292 * @stream pointer to the apple_dart_streams struct for each loop iteration
293 */
294 #define for_each_stream_map(i, base, stream_map) \
295 for (i = 0, stream_map = &(base)->stream_maps[0]; \
296 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \
297 stream_map = &(base)->stream_maps[++i])
298
299 static struct platform_driver apple_dart_driver;
300 static const struct iommu_ops apple_dart_iommu_ops;
301
to_dart_domain(struct iommu_domain * dom)302 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
303 {
304 return container_of(dom, struct apple_dart_domain, domain);
305 }
306
307 static void
apple_dart_hw_enable_translation(struct apple_dart_stream_map * stream_map)308 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)
309 {
310 struct apple_dart *dart = stream_map->dart;
311 int sid;
312
313 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
314 writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid));
315 }
316
apple_dart_hw_disable_dma(struct apple_dart_stream_map * stream_map)317 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
318 {
319 struct apple_dart *dart = stream_map->dart;
320 int sid;
321
322 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
323 writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid));
324 }
325
326 static void
apple_dart_hw_enable_bypass(struct apple_dart_stream_map * stream_map)327 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
328 {
329 struct apple_dart *dart = stream_map->dart;
330 int sid;
331
332 WARN_ON(!stream_map->dart->supports_bypass);
333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
334 writel(dart->hw->tcr_bypass,
335 dart->regs + DART_TCR(dart, sid));
336 }
337
apple_dart_hw_set_ttbr(struct apple_dart_stream_map * stream_map,u8 idx,phys_addr_t paddr)338 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
339 u8 idx, phys_addr_t paddr)
340 {
341 struct apple_dart *dart = stream_map->dart;
342 int sid;
343
344 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1));
345 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
346 writel(dart->hw->ttbr_valid |
347 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift,
348 dart->regs + DART_TTBR(dart, sid, idx));
349 }
350
apple_dart_hw_clear_ttbr(struct apple_dart_stream_map * stream_map,u8 idx)351 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
352 u8 idx)
353 {
354 struct apple_dart *dart = stream_map->dart;
355 int sid;
356
357 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
358 writel(0, dart->regs + DART_TTBR(dart, sid, idx));
359 }
360
361 static void
apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map * stream_map)362 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
363 {
364 int i;
365
366 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)
367 apple_dart_hw_clear_ttbr(stream_map, i);
368 }
369
370 static int
apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map * stream_map,u32 command)371 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,
372 u32 command)
373 {
374 unsigned long flags;
375 int ret, i;
376 u32 command_reg;
377
378 spin_lock_irqsave(&stream_map->dart->lock, flags);
379
380 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)
381 writel(stream_map->sidmap[i],
382 stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);
383 writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);
384
385 ret = readl_poll_timeout_atomic(
386 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,
387 !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1,
388 DART_STREAM_COMMAND_BUSY_TIMEOUT);
389
390 spin_unlock_irqrestore(&stream_map->dart->lock, flags);
391
392 if (ret) {
393 dev_err(stream_map->dart->dev,
394 "busy bit did not clear after command %x for streams %lx\n",
395 command, stream_map->sidmap[0]);
396 return ret;
397 }
398
399 return 0;
400 }
401
402 static int
apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map * stream_map,u32 command)403 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,
404 u32 command)
405 {
406 struct apple_dart *dart = stream_map->dart;
407 unsigned long flags;
408 int ret = 0;
409 int sid;
410
411 spin_lock_irqsave(&dart->lock, flags);
412
413 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
414 u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) |
415 FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid);
416 writel(val, dart->regs + DART_T8110_TLB_CMD);
417
418 ret = readl_poll_timeout_atomic(
419 dart->regs + DART_T8110_TLB_CMD, val,
420 !(val & DART_T8110_TLB_CMD_BUSY), 1,
421 DART_STREAM_COMMAND_BUSY_TIMEOUT);
422
423 if (ret)
424 break;
425
426 }
427
428 spin_unlock_irqrestore(&dart->lock, flags);
429
430 if (ret) {
431 dev_err(stream_map->dart->dev,
432 "busy bit did not clear after command %x for stream %d\n",
433 command, sid);
434 return ret;
435 }
436
437 return 0;
438 }
439
440 static int
apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)441 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
442 {
443 return apple_dart_t8020_hw_stream_command(
444 stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);
445 }
446
447 static int
apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)448 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
449 {
450 return apple_dart_t8110_hw_tlb_command(
451 stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);
452 }
453
apple_dart_hw_reset(struct apple_dart * dart)454 static int apple_dart_hw_reset(struct apple_dart *dart)
455 {
456 u32 config;
457 struct apple_dart_stream_map stream_map;
458 int i;
459
460 config = readl(dart->regs + dart->hw->lock);
461 if (config & dart->hw->lock_bit) {
462 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
463 config);
464 return -EINVAL;
465 }
466
467 stream_map.dart = dart;
468 bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS);
469 bitmap_set(stream_map.sidmap, 0, dart->num_streams);
470 apple_dart_hw_disable_dma(&stream_map);
471 apple_dart_hw_clear_all_ttbrs(&stream_map);
472
473 /* enable all streams globally since TCR is used to control isolation */
474 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++)
475 writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i);
476
477 /* clear any pending errors before the interrupt is unmasked */
478 writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error);
479
480 if (dart->hw->type == DART_T8110)
481 writel(0, dart->regs + DART_T8110_ERROR_MASK);
482
483 return dart->hw->invalidate_tlb(&stream_map);
484 }
485
apple_dart_domain_flush_tlb(struct apple_dart_domain * domain)486 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
487 {
488 int i, j;
489 struct apple_dart_atomic_stream_map *domain_stream_map;
490 struct apple_dart_stream_map stream_map;
491
492 for_each_stream_map(i, domain, domain_stream_map) {
493 stream_map.dart = domain_stream_map->dart;
494
495 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)
496 stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);
497
498 stream_map.dart->hw->invalidate_tlb(&stream_map);
499 }
500 }
501
apple_dart_flush_iotlb_all(struct iommu_domain * domain)502 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
503 {
504 apple_dart_domain_flush_tlb(to_dart_domain(domain));
505 }
506
apple_dart_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)507 static void apple_dart_iotlb_sync(struct iommu_domain *domain,
508 struct iommu_iotlb_gather *gather)
509 {
510 apple_dart_domain_flush_tlb(to_dart_domain(domain));
511 }
512
apple_dart_iotlb_sync_map(struct iommu_domain * domain,unsigned long iova,size_t size)513 static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
514 unsigned long iova, size_t size)
515 {
516 apple_dart_domain_flush_tlb(to_dart_domain(domain));
517 return 0;
518 }
519
apple_dart_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)520 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
521 dma_addr_t iova)
522 {
523 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
524 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
525
526 if (!ops)
527 return 0;
528
529 return ops->iova_to_phys(ops, iova);
530 }
531
apple_dart_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)532 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
533 phys_addr_t paddr, size_t pgsize,
534 size_t pgcount, int prot, gfp_t gfp,
535 size_t *mapped)
536 {
537 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
538 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
539
540 if (!ops)
541 return -ENODEV;
542
543 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
544 mapped);
545 }
546
apple_dart_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)547 static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
548 unsigned long iova, size_t pgsize,
549 size_t pgcount,
550 struct iommu_iotlb_gather *gather)
551 {
552 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
553 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
554
555 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
556 }
557
558 static void
apple_dart_setup_translation(struct apple_dart_domain * domain,struct apple_dart_stream_map * stream_map)559 apple_dart_setup_translation(struct apple_dart_domain *domain,
560 struct apple_dart_stream_map *stream_map)
561 {
562 int i;
563 struct io_pgtable_cfg *pgtbl_cfg =
564 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
565
566 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
567 apple_dart_hw_set_ttbr(stream_map, i,
568 pgtbl_cfg->apple_dart_cfg.ttbr[i]);
569 for (; i < stream_map->dart->hw->ttbr_count; ++i)
570 apple_dart_hw_clear_ttbr(stream_map, i);
571
572 apple_dart_hw_enable_translation(stream_map);
573 stream_map->dart->hw->invalidate_tlb(stream_map);
574 }
575
apple_dart_finalize_domain(struct apple_dart_domain * dart_domain,struct apple_dart_master_cfg * cfg)576 static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
577 struct apple_dart_master_cfg *cfg)
578 {
579 struct apple_dart *dart = cfg->stream_maps[0].dart;
580 struct io_pgtable_cfg pgtbl_cfg;
581 int ret = 0;
582 int i, j;
583
584 if (dart->pgsize > PAGE_SIZE)
585 return -EINVAL;
586
587 mutex_lock(&dart_domain->init_lock);
588
589 if (dart_domain->finalized)
590 goto done;
591
592 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
593 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
594 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++)
595 atomic_long_set(&dart_domain->stream_maps[i].sidmap[j],
596 cfg->stream_maps[i].sidmap[j]);
597 }
598
599 pgtbl_cfg = (struct io_pgtable_cfg){
600 .pgsize_bitmap = dart->pgsize,
601 .ias = dart->ias,
602 .oas = dart->oas,
603 .coherent_walk = 1,
604 .iommu_dev = dart->dev,
605 };
606
607 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
608 &dart_domain->domain);
609 if (!dart_domain->pgtbl_ops) {
610 ret = -ENOMEM;
611 goto done;
612 }
613
614 dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
615 dart_domain->domain.geometry.aperture_start = 0;
616 dart_domain->domain.geometry.aperture_end =
617 (dma_addr_t)DMA_BIT_MASK(dart->ias);
618 dart_domain->domain.geometry.force_aperture = true;
619
620 dart_domain->finalized = true;
621
622 done:
623 mutex_unlock(&dart_domain->init_lock);
624 return ret;
625 }
626
627 static int
apple_dart_mod_streams(struct apple_dart_atomic_stream_map * domain_maps,struct apple_dart_stream_map * master_maps,bool add_streams)628 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
629 struct apple_dart_stream_map *master_maps,
630 bool add_streams)
631 {
632 int i, j;
633
634 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
635 if (domain_maps[i].dart != master_maps[i].dart)
636 return -EINVAL;
637 }
638
639 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
640 if (!domain_maps[i].dart)
641 break;
642 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) {
643 if (add_streams)
644 atomic_long_or(master_maps[i].sidmap[j],
645 &domain_maps[i].sidmap[j]);
646 else
647 atomic_long_and(~master_maps[i].sidmap[j],
648 &domain_maps[i].sidmap[j]);
649 }
650 }
651
652 return 0;
653 }
654
apple_dart_domain_add_streams(struct apple_dart_domain * domain,struct apple_dart_master_cfg * cfg)655 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
656 struct apple_dart_master_cfg *cfg)
657 {
658 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
659 true);
660 }
661
apple_dart_attach_dev_paging(struct iommu_domain * domain,struct device * dev)662 static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
663 struct device *dev)
664 {
665 int ret, i;
666 struct apple_dart_stream_map *stream_map;
667 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
668 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
669
670 ret = apple_dart_finalize_domain(dart_domain, cfg);
671 if (ret)
672 return ret;
673
674 ret = apple_dart_domain_add_streams(dart_domain, cfg);
675 if (ret)
676 return ret;
677
678 for_each_stream_map(i, cfg, stream_map)
679 apple_dart_setup_translation(dart_domain, stream_map);
680 return 0;
681 }
682
apple_dart_attach_dev_identity(struct iommu_domain * domain,struct device * dev)683 static int apple_dart_attach_dev_identity(struct iommu_domain *domain,
684 struct device *dev)
685 {
686 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
687 struct apple_dart_stream_map *stream_map;
688 int i;
689
690 if (!cfg->supports_bypass)
691 return -EINVAL;
692
693 for_each_stream_map(i, cfg, stream_map)
694 apple_dart_hw_enable_bypass(stream_map);
695 return 0;
696 }
697
698 static const struct iommu_domain_ops apple_dart_identity_ops = {
699 .attach_dev = apple_dart_attach_dev_identity,
700 };
701
702 static struct iommu_domain apple_dart_identity_domain = {
703 .type = IOMMU_DOMAIN_IDENTITY,
704 .ops = &apple_dart_identity_ops,
705 };
706
apple_dart_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)707 static int apple_dart_attach_dev_blocked(struct iommu_domain *domain,
708 struct device *dev)
709 {
710 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
711 struct apple_dart_stream_map *stream_map;
712 int i;
713
714 for_each_stream_map(i, cfg, stream_map)
715 apple_dart_hw_disable_dma(stream_map);
716 return 0;
717 }
718
719 static const struct iommu_domain_ops apple_dart_blocked_ops = {
720 .attach_dev = apple_dart_attach_dev_blocked,
721 };
722
723 static struct iommu_domain apple_dart_blocked_domain = {
724 .type = IOMMU_DOMAIN_BLOCKED,
725 .ops = &apple_dart_blocked_ops,
726 };
727
apple_dart_probe_device(struct device * dev)728 static struct iommu_device *apple_dart_probe_device(struct device *dev)
729 {
730 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
731 struct apple_dart_stream_map *stream_map;
732 int i;
733
734 if (!cfg)
735 return ERR_PTR(-ENODEV);
736
737 for_each_stream_map(i, cfg, stream_map)
738 device_link_add(
739 dev, stream_map->dart->dev,
740 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
741
742 return &cfg->stream_maps[0].dart->iommu;
743 }
744
apple_dart_release_device(struct device * dev)745 static void apple_dart_release_device(struct device *dev)
746 {
747 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
748
749 kfree(cfg);
750 }
751
apple_dart_domain_alloc_paging(struct device * dev)752 static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev)
753 {
754 struct apple_dart_domain *dart_domain;
755
756 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
757 if (!dart_domain)
758 return NULL;
759
760 mutex_init(&dart_domain->init_lock);
761
762 if (dev) {
763 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
764 int ret;
765
766 ret = apple_dart_finalize_domain(dart_domain, cfg);
767 if (ret) {
768 kfree(dart_domain);
769 return ERR_PTR(ret);
770 }
771 }
772 return &dart_domain->domain;
773 }
774
apple_dart_domain_free(struct iommu_domain * domain)775 static void apple_dart_domain_free(struct iommu_domain *domain)
776 {
777 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
778
779 free_io_pgtable_ops(dart_domain->pgtbl_ops);
780
781 kfree(dart_domain);
782 }
783
apple_dart_of_xlate(struct device * dev,const struct of_phandle_args * args)784 static int apple_dart_of_xlate(struct device *dev,
785 const struct of_phandle_args *args)
786 {
787 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
788 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
789 struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
790 struct apple_dart *cfg_dart;
791 int i, sid;
792
793 if (args->args_count != 1)
794 return -EINVAL;
795 sid = args->args[0];
796
797 if (!cfg) {
798 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
799 if (!cfg)
800 return -ENOMEM;
801 /* Will be ANDed with DART capabilities */
802 cfg->supports_bypass = true;
803 }
804 dev_iommu_priv_set(dev, cfg);
805
806 cfg_dart = cfg->stream_maps[0].dart;
807 if (cfg_dart) {
808 if (cfg_dart->pgsize != dart->pgsize)
809 return -EINVAL;
810 }
811
812 cfg->supports_bypass &= dart->supports_bypass;
813
814 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
815 if (cfg->stream_maps[i].dart == dart) {
816 set_bit(sid, cfg->stream_maps[i].sidmap);
817 return 0;
818 }
819 }
820 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
821 if (!cfg->stream_maps[i].dart) {
822 cfg->stream_maps[i].dart = dart;
823 set_bit(sid, cfg->stream_maps[i].sidmap);
824 return 0;
825 }
826 }
827
828 return -EINVAL;
829 }
830
831 static DEFINE_MUTEX(apple_dart_groups_lock);
832
apple_dart_release_group(void * iommu_data)833 static void apple_dart_release_group(void *iommu_data)
834 {
835 int i, sid;
836 struct apple_dart_stream_map *stream_map;
837 struct apple_dart_master_cfg *group_master_cfg = iommu_data;
838
839 mutex_lock(&apple_dart_groups_lock);
840
841 for_each_stream_map(i, group_master_cfg, stream_map)
842 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
843 stream_map->dart->sid2group[sid] = NULL;
844
845 kfree(iommu_data);
846 mutex_unlock(&apple_dart_groups_lock);
847 }
848
apple_dart_merge_master_cfg(struct apple_dart_master_cfg * dst,struct apple_dart_master_cfg * src)849 static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst,
850 struct apple_dart_master_cfg *src)
851 {
852 /*
853 * We know that this function is only called for groups returned from
854 * pci_device_group and that all Apple Silicon platforms never spread
855 * PCIe devices from the same bus across multiple DARTs such that we can
856 * just assume that both src and dst only have the same single DART.
857 */
858 if (src->stream_maps[1].dart)
859 return -EINVAL;
860 if (dst->stream_maps[1].dart)
861 return -EINVAL;
862 if (src->stream_maps[0].dart != dst->stream_maps[0].dart)
863 return -EINVAL;
864
865 bitmap_or(dst->stream_maps[0].sidmap,
866 dst->stream_maps[0].sidmap,
867 src->stream_maps[0].sidmap,
868 dst->stream_maps[0].dart->num_streams);
869 return 0;
870 }
871
apple_dart_device_group(struct device * dev)872 static struct iommu_group *apple_dart_device_group(struct device *dev)
873 {
874 int i, sid;
875 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
876 struct apple_dart_stream_map *stream_map;
877 struct apple_dart_master_cfg *group_master_cfg;
878 struct iommu_group *group = NULL;
879 struct iommu_group *res = ERR_PTR(-EINVAL);
880
881 mutex_lock(&apple_dart_groups_lock);
882
883 for_each_stream_map(i, cfg, stream_map) {
884 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {
885 struct iommu_group *stream_group =
886 stream_map->dart->sid2group[sid];
887
888 if (group && group != stream_group) {
889 res = ERR_PTR(-EINVAL);
890 goto out;
891 }
892
893 group = stream_group;
894 }
895 }
896
897 if (group) {
898 res = iommu_group_ref_get(group);
899 goto out;
900 }
901
902 #ifdef CONFIG_PCI
903 if (dev_is_pci(dev))
904 group = pci_device_group(dev);
905 else
906 #endif
907 group = generic_device_group(dev);
908
909 res = ERR_PTR(-ENOMEM);
910 if (!group)
911 goto out;
912
913 group_master_cfg = iommu_group_get_iommudata(group);
914 if (group_master_cfg) {
915 int ret;
916
917 ret = apple_dart_merge_master_cfg(group_master_cfg, cfg);
918 if (ret) {
919 dev_err(dev, "Failed to merge DART IOMMU groups.\n");
920 iommu_group_put(group);
921 res = ERR_PTR(ret);
922 goto out;
923 }
924 } else {
925 group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg),
926 GFP_KERNEL);
927 if (!group_master_cfg) {
928 iommu_group_put(group);
929 goto out;
930 }
931
932 iommu_group_set_iommudata(group, group_master_cfg,
933 apple_dart_release_group);
934 }
935
936 for_each_stream_map(i, cfg, stream_map)
937 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
938 stream_map->dart->sid2group[sid] = group;
939
940 res = group;
941
942 out:
943 mutex_unlock(&apple_dart_groups_lock);
944 return res;
945 }
946
apple_dart_def_domain_type(struct device * dev)947 static int apple_dart_def_domain_type(struct device *dev)
948 {
949 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
950
951 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE)
952 return IOMMU_DOMAIN_IDENTITY;
953 if (!cfg->supports_bypass)
954 return IOMMU_DOMAIN_DMA;
955
956 return 0;
957 }
958
959 #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
960 /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */
961 #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0
962 #endif
963 #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK)
964
apple_dart_get_resv_regions(struct device * dev,struct list_head * head)965 static void apple_dart_get_resv_regions(struct device *dev,
966 struct list_head *head)
967 {
968 if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) {
969 struct iommu_resv_region *region;
970 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
971
972 region = iommu_alloc_resv_region(DOORBELL_ADDR,
973 PAGE_SIZE, prot,
974 IOMMU_RESV_MSI, GFP_KERNEL);
975 if (!region)
976 return;
977
978 list_add_tail(®ion->list, head);
979 }
980
981 iommu_dma_get_resv_regions(dev, head);
982 }
983
984 static const struct iommu_ops apple_dart_iommu_ops = {
985 .identity_domain = &apple_dart_identity_domain,
986 .blocked_domain = &apple_dart_blocked_domain,
987 .domain_alloc_paging = apple_dart_domain_alloc_paging,
988 .probe_device = apple_dart_probe_device,
989 .release_device = apple_dart_release_device,
990 .device_group = apple_dart_device_group,
991 .of_xlate = apple_dart_of_xlate,
992 .def_domain_type = apple_dart_def_domain_type,
993 .get_resv_regions = apple_dart_get_resv_regions,
994 .pgsize_bitmap = -1UL, /* Restricted during dart probe */
995 .owner = THIS_MODULE,
996 .default_domain_ops = &(const struct iommu_domain_ops) {
997 .attach_dev = apple_dart_attach_dev_paging,
998 .map_pages = apple_dart_map_pages,
999 .unmap_pages = apple_dart_unmap_pages,
1000 .flush_iotlb_all = apple_dart_flush_iotlb_all,
1001 .iotlb_sync = apple_dart_iotlb_sync,
1002 .iotlb_sync_map = apple_dart_iotlb_sync_map,
1003 .iova_to_phys = apple_dart_iova_to_phys,
1004 .free = apple_dart_domain_free,
1005 }
1006 };
1007
apple_dart_t8020_irq(int irq,void * dev)1008 static irqreturn_t apple_dart_t8020_irq(int irq, void *dev)
1009 {
1010 struct apple_dart *dart = dev;
1011 const char *fault_name = NULL;
1012 u32 error = readl(dart->regs + DART_T8020_ERROR);
1013 u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error);
1014 u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO);
1015 u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI);
1016 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1017 u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error);
1018
1019 if (!(error & DART_T8020_ERROR_FLAG))
1020 return IRQ_NONE;
1021
1022 /* there should only be a single bit set but let's use == to be sure */
1023 if (error_code == DART_T8020_ERROR_READ_FAULT)
1024 fault_name = "READ FAULT";
1025 else if (error_code == DART_T8020_ERROR_WRITE_FAULT)
1026 fault_name = "WRITE FAULT";
1027 else if (error_code == DART_T8020_ERROR_NO_PTE)
1028 fault_name = "NO PTE FOR IOVA";
1029 else if (error_code == DART_T8020_ERROR_NO_PMD)
1030 fault_name = "NO PMD FOR IOVA";
1031 else if (error_code == DART_T8020_ERROR_NO_TTBR)
1032 fault_name = "NO TTBR FOR IOVA";
1033 else
1034 fault_name = "unknown";
1035
1036 dev_err_ratelimited(
1037 dart->dev,
1038 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1039 error, stream_idx, error_code, fault_name, addr);
1040
1041 writel(error, dart->regs + DART_T8020_ERROR);
1042 return IRQ_HANDLED;
1043 }
1044
apple_dart_t8110_irq(int irq,void * dev)1045 static irqreturn_t apple_dart_t8110_irq(int irq, void *dev)
1046 {
1047 struct apple_dart *dart = dev;
1048 const char *fault_name = NULL;
1049 u32 error = readl(dart->regs + DART_T8110_ERROR);
1050 u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error);
1051 u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO);
1052 u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI);
1053 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1054 u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error);
1055
1056 if (!(error & DART_T8110_ERROR_FLAG))
1057 return IRQ_NONE;
1058
1059 /* there should only be a single bit set but let's use == to be sure */
1060 if (error_code == DART_T8110_ERROR_READ_FAULT)
1061 fault_name = "READ FAULT";
1062 else if (error_code == DART_T8110_ERROR_WRITE_FAULT)
1063 fault_name = "WRITE FAULT";
1064 else if (error_code == DART_T8110_ERROR_NO_PTE)
1065 fault_name = "NO PTE FOR IOVA";
1066 else if (error_code == DART_T8110_ERROR_NO_PMD)
1067 fault_name = "NO PMD FOR IOVA";
1068 else if (error_code == DART_T8110_ERROR_NO_PGD)
1069 fault_name = "NO PGD FOR IOVA";
1070 else if (error_code == DART_T8110_ERROR_NO_TTBR)
1071 fault_name = "NO TTBR FOR IOVA";
1072 else
1073 fault_name = "unknown";
1074
1075 dev_err_ratelimited(
1076 dart->dev,
1077 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1078 error, stream_idx, error_code, fault_name, addr);
1079
1080 writel(error, dart->regs + DART_T8110_ERROR);
1081 return IRQ_HANDLED;
1082 }
1083
apple_dart_probe(struct platform_device * pdev)1084 static int apple_dart_probe(struct platform_device *pdev)
1085 {
1086 int ret;
1087 u32 dart_params[4];
1088 struct resource *res;
1089 struct apple_dart *dart;
1090 struct device *dev = &pdev->dev;
1091
1092 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
1093 if (!dart)
1094 return -ENOMEM;
1095
1096 dart->dev = dev;
1097 dart->hw = of_device_get_match_data(dev);
1098 spin_lock_init(&dart->lock);
1099
1100 dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1101 if (IS_ERR(dart->regs))
1102 return PTR_ERR(dart->regs);
1103
1104 if (resource_size(res) < 0x4000) {
1105 dev_err(dev, "MMIO region too small (%pr)\n", res);
1106 return -EINVAL;
1107 }
1108
1109 dart->irq = platform_get_irq(pdev, 0);
1110 if (dart->irq < 0)
1111 return -ENODEV;
1112
1113 ret = devm_clk_bulk_get_all(dev, &dart->clks);
1114 if (ret < 0)
1115 return ret;
1116 dart->num_clks = ret;
1117
1118 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
1119 if (ret)
1120 return ret;
1121
1122 dart_params[0] = readl(dart->regs + DART_PARAMS1);
1123 dart_params[1] = readl(dart->regs + DART_PARAMS2);
1124 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]);
1125 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT;
1126
1127 switch (dart->hw->type) {
1128 case DART_T8020:
1129 case DART_T6000:
1130 dart->ias = 32;
1131 dart->oas = dart->hw->oas;
1132 dart->num_streams = dart->hw->max_sid_count;
1133 break;
1134
1135 case DART_T8110:
1136 dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3);
1137 dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4);
1138 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]);
1139 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]);
1140 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]);
1141 break;
1142 }
1143
1144 if (dart->num_streams > DART_MAX_STREAMS) {
1145 dev_err(&pdev->dev, "Too many streams (%d > %d)\n",
1146 dart->num_streams, DART_MAX_STREAMS);
1147 ret = -EINVAL;
1148 goto err_clk_disable;
1149 }
1150
1151 ret = apple_dart_hw_reset(dart);
1152 if (ret)
1153 goto err_clk_disable;
1154
1155 ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED,
1156 "apple-dart fault handler", dart);
1157 if (ret)
1158 goto err_clk_disable;
1159
1160 platform_set_drvdata(pdev, dart);
1161
1162 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
1163 dev_name(&pdev->dev));
1164 if (ret)
1165 goto err_free_irq;
1166
1167 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
1168 if (ret)
1169 goto err_sysfs_remove;
1170
1171 dev_info(
1172 &pdev->dev,
1173 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n",
1174 dart->pgsize, dart->num_streams, dart->supports_bypass,
1175 dart->pgsize > PAGE_SIZE);
1176 return 0;
1177
1178 err_sysfs_remove:
1179 iommu_device_sysfs_remove(&dart->iommu);
1180 err_free_irq:
1181 free_irq(dart->irq, dart);
1182 err_clk_disable:
1183 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1184
1185 return ret;
1186 }
1187
apple_dart_remove(struct platform_device * pdev)1188 static void apple_dart_remove(struct platform_device *pdev)
1189 {
1190 struct apple_dart *dart = platform_get_drvdata(pdev);
1191
1192 apple_dart_hw_reset(dart);
1193 free_irq(dart->irq, dart);
1194
1195 iommu_device_unregister(&dart->iommu);
1196 iommu_device_sysfs_remove(&dart->iommu);
1197
1198 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1199 }
1200
1201 static const struct apple_dart_hw apple_dart_hw_t8103 = {
1202 .type = DART_T8020,
1203 .irq_handler = apple_dart_t8020_irq,
1204 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1205 .oas = 36,
1206 .fmt = APPLE_DART,
1207 .max_sid_count = 16,
1208
1209 .enable_streams = DART_T8020_STREAMS_ENABLE,
1210 .lock = DART_T8020_CONFIG,
1211 .lock_bit = DART_T8020_CONFIG_LOCK,
1212
1213 .error = DART_T8020_ERROR,
1214
1215 .tcr = DART_T8020_TCR,
1216 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1217 .tcr_disabled = 0,
1218 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1219
1220 .ttbr = DART_T8020_TTBR,
1221 .ttbr_valid = DART_T8020_TTBR_VALID,
1222 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1223 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1224 .ttbr_count = 4,
1225 };
1226
1227 static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = {
1228 .type = DART_T8020,
1229 .irq_handler = apple_dart_t8020_irq,
1230 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1231 .oas = 36,
1232 .fmt = APPLE_DART,
1233 .max_sid_count = 64,
1234
1235 .enable_streams = DART_T8020_STREAMS_ENABLE,
1236 .lock = DART_T8020_CONFIG,
1237 .lock_bit = DART_T8020_CONFIG_LOCK,
1238
1239 .error = DART_T8020_ERROR,
1240
1241 .tcr = DART_T8020_TCR,
1242 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1243 .tcr_disabled = 0,
1244 .tcr_bypass = 0,
1245
1246 .ttbr = DART_T8020_USB4_TTBR,
1247 .ttbr_valid = DART_T8020_TTBR_VALID,
1248 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1249 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1250 .ttbr_count = 4,
1251 };
1252
1253 static const struct apple_dart_hw apple_dart_hw_t6000 = {
1254 .type = DART_T6000,
1255 .irq_handler = apple_dart_t8020_irq,
1256 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1257 .oas = 42,
1258 .fmt = APPLE_DART2,
1259 .max_sid_count = 16,
1260
1261 .enable_streams = DART_T8020_STREAMS_ENABLE,
1262 .lock = DART_T8020_CONFIG,
1263 .lock_bit = DART_T8020_CONFIG_LOCK,
1264
1265 .error = DART_T8020_ERROR,
1266
1267 .tcr = DART_T8020_TCR,
1268 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1269 .tcr_disabled = 0,
1270 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1271
1272 .ttbr = DART_T8020_TTBR,
1273 .ttbr_valid = DART_T8020_TTBR_VALID,
1274 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1275 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1276 .ttbr_count = 4,
1277 };
1278
1279 static const struct apple_dart_hw apple_dart_hw_t8110 = {
1280 .type = DART_T8110,
1281 .irq_handler = apple_dart_t8110_irq,
1282 .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb,
1283 .fmt = APPLE_DART2,
1284 .max_sid_count = 256,
1285
1286 .enable_streams = DART_T8110_ENABLE_STREAMS,
1287 .lock = DART_T8110_PROTECT,
1288 .lock_bit = DART_T8110_PROTECT_TTBR_TCR,
1289
1290 .error = DART_T8110_ERROR,
1291
1292 .tcr = DART_T8110_TCR,
1293 .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE,
1294 .tcr_disabled = 0,
1295 .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART,
1296
1297 .ttbr = DART_T8110_TTBR,
1298 .ttbr_valid = DART_T8110_TTBR_VALID,
1299 .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT,
1300 .ttbr_shift = DART_T8110_TTBR_SHIFT,
1301 .ttbr_count = 1,
1302 };
1303
apple_dart_suspend(struct device * dev)1304 static __maybe_unused int apple_dart_suspend(struct device *dev)
1305 {
1306 struct apple_dart *dart = dev_get_drvdata(dev);
1307 unsigned int sid, idx;
1308
1309 for (sid = 0; sid < dart->num_streams; sid++) {
1310 dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid));
1311 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1312 dart->save_ttbr[sid][idx] =
1313 readl(dart->regs + DART_TTBR(dart, sid, idx));
1314 }
1315
1316 return 0;
1317 }
1318
apple_dart_resume(struct device * dev)1319 static __maybe_unused int apple_dart_resume(struct device *dev)
1320 {
1321 struct apple_dart *dart = dev_get_drvdata(dev);
1322 unsigned int sid, idx;
1323 int ret;
1324
1325 ret = apple_dart_hw_reset(dart);
1326 if (ret) {
1327 dev_err(dev, "Failed to reset DART on resume\n");
1328 return ret;
1329 }
1330
1331 for (sid = 0; sid < dart->num_streams; sid++) {
1332 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1333 writel(dart->save_ttbr[sid][idx],
1334 dart->regs + DART_TTBR(dart, sid, idx));
1335 writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid));
1336 }
1337
1338 return 0;
1339 }
1340
1341 static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume);
1342
1343 static const struct of_device_id apple_dart_of_match[] = {
1344 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
1345 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
1346 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
1347 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
1348 {},
1349 };
1350 MODULE_DEVICE_TABLE(of, apple_dart_of_match);
1351
1352 static struct platform_driver apple_dart_driver = {
1353 .driver = {
1354 .name = "apple-dart",
1355 .of_match_table = apple_dart_of_match,
1356 .suppress_bind_attrs = true,
1357 .pm = pm_sleep_ptr(&apple_dart_pm_ops),
1358 },
1359 .probe = apple_dart_probe,
1360 .remove = apple_dart_remove,
1361 };
1362
1363 module_platform_driver(apple_dart_driver);
1364
1365 MODULE_DESCRIPTION("IOMMU API for Apple's DART");
1366 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1367 MODULE_LICENSE("GPL v2");
1368