1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Apple DART (Device Address Resolution Table) IOMMU driver
4 *
5 * Copyright (C) 2021 The Asahi Linux Contributors
6 *
7 * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
8 * Copyright (C) 2013 ARM Limited
9 * Copyright (C) 2015 ARM Limited
10 * and on exynos-iommu.c
11 * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd.
12 */
13
14 #include <linux/atomic.h>
15 #include <linux/bitfield.h>
16 #include <linux/clk.h>
17 #include <linux/dev_printk.h>
18 #include <linux/dma-mapping.h>
19 #include <linux/err.h>
20 #include <linux/interrupt.h>
21 #include <linux/io-pgtable.h>
22 #include <linux/iommu.h>
23 #include <linux/iopoll.h>
24 #include <linux/module.h>
25 #include <linux/of.h>
26 #include <linux/of_address.h>
27 #include <linux/of_iommu.h>
28 #include <linux/of_platform.h>
29 #include <linux/pci.h>
30 #include <linux/platform_device.h>
31 #include <linux/slab.h>
32 #include <linux/swab.h>
33 #include <linux/types.h>
34
35 #include "dma-iommu.h"
36
37 #define DART_MAX_STREAMS 256
38 #define DART_MAX_TTBR 4
39 #define MAX_DARTS_PER_DEVICE 3
40
41 /* Common registers */
42
43 #define DART_PARAMS1 0x00
44 #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24)
45
46 #define DART_PARAMS2 0x04
47 #define DART_PARAMS2_BYPASS_SUPPORT BIT(0)
48
49 /* T8020/T6000 registers */
50
51 #define DART_T8020_STREAM_COMMAND 0x20
52 #define DART_T8020_STREAM_COMMAND_BUSY BIT(2)
53 #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20)
54
55 #define DART_T8020_STREAM_SELECT 0x34
56
57 #define DART_T8020_ERROR 0x40
58 #define DART_T8020_ERROR_STREAM GENMASK(27, 24)
59 #define DART_T8020_ERROR_CODE GENMASK(11, 0)
60 #define DART_T8020_ERROR_FLAG BIT(31)
61
62 #define DART_T8020_ERROR_READ_FAULT BIT(4)
63 #define DART_T8020_ERROR_WRITE_FAULT BIT(3)
64 #define DART_T8020_ERROR_NO_PTE BIT(2)
65 #define DART_T8020_ERROR_NO_PMD BIT(1)
66 #define DART_T8020_ERROR_NO_TTBR BIT(0)
67
68 #define DART_T8020_CONFIG 0x60
69 #define DART_T8020_CONFIG_LOCK BIT(15)
70
71 #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100
72
73 #define DART_T8020_ERROR_ADDR_HI 0x54
74 #define DART_T8020_ERROR_ADDR_LO 0x50
75
76 #define DART_T8020_STREAMS_ENABLE 0xfc
77
78 #define DART_T8020_TCR 0x100
79 #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7)
80 #define DART_T8020_TCR_BYPASS_DART BIT(8)
81 #define DART_T8020_TCR_BYPASS_DAPF BIT(12)
82
83 #define DART_T8020_TTBR 0x200
84 #define DART_T8020_USB4_TTBR 0x400
85 #define DART_T8020_TTBR_VALID BIT(31)
86 #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0
87 #define DART_T8020_TTBR_SHIFT 12
88
89 /* T8110 registers */
90
91 #define DART_T8110_PARAMS3 0x08
92 #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24)
93 #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16)
94 #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8)
95 #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0)
96
97 #define DART_T8110_PARAMS4 0x0c
98 #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16)
99 #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0)
100
101 #define DART_T8110_TLB_CMD 0x80
102 #define DART_T8110_TLB_CMD_BUSY BIT(31)
103 #define DART_T8110_TLB_CMD_OP GENMASK(10, 8)
104 #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0
105 #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1
106 #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0)
107
108 #define DART_T8110_ERROR 0x100
109 #define DART_T8110_ERROR_STREAM GENMASK(27, 20)
110 #define DART_T8110_ERROR_CODE GENMASK(14, 0)
111 #define DART_T8110_ERROR_FLAG BIT(31)
112
113 #define DART_T8110_ERROR_MASK 0x104
114
115 #define DART_T8110_ERROR_READ_FAULT BIT(5)
116 #define DART_T8110_ERROR_WRITE_FAULT BIT(4)
117 #define DART_T8110_ERROR_NO_PTE BIT(3)
118 #define DART_T8110_ERROR_NO_PMD BIT(2)
119 #define DART_T8110_ERROR_NO_PGD BIT(1)
120 #define DART_T8110_ERROR_NO_TTBR BIT(0)
121
122 #define DART_T8110_ERROR_ADDR_LO 0x170
123 #define DART_T8110_ERROR_ADDR_HI 0x174
124
125 #define DART_T8110_PROTECT 0x200
126 #define DART_T8110_UNPROTECT 0x204
127 #define DART_T8110_PROTECT_LOCK 0x208
128 #define DART_T8110_PROTECT_TTBR_TCR BIT(0)
129
130 #define DART_T8110_ENABLE_STREAMS 0xc00
131 #define DART_T8110_DISABLE_STREAMS 0xc20
132
133 #define DART_T8110_TCR 0x1000
134 #define DART_T8110_TCR_REMAP GENMASK(11, 8)
135 #define DART_T8110_TCR_REMAP_EN BIT(7)
136 #define DART_T8110_TCR_BYPASS_DAPF BIT(2)
137 #define DART_T8110_TCR_BYPASS_DART BIT(1)
138 #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
139
140 #define DART_T8110_TTBR 0x1400
141 #define DART_T8110_TTBR_VALID BIT(0)
142 #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2
143 #define DART_T8110_TTBR_SHIFT 14
144
145 #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2))
146
147 #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \
148 (((dart)->hw->ttbr_count * (sid)) << 2) + \
149 ((idx) << 2))
150
151 struct apple_dart_stream_map;
152
153 enum dart_type {
154 DART_T8020,
155 DART_T6000,
156 DART_T8110,
157 };
158
159 struct apple_dart_hw {
160 enum dart_type type;
161 irqreturn_t (*irq_handler)(int irq, void *dev);
162 int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
163
164 u32 oas;
165 enum io_pgtable_fmt fmt;
166
167 int max_sid_count;
168
169 u64 lock;
170 u64 lock_bit;
171
172 u64 error;
173
174 u64 enable_streams;
175
176 u64 tcr;
177 u64 tcr_enabled;
178 u64 tcr_disabled;
179 u64 tcr_bypass;
180
181 u64 ttbr;
182 u64 ttbr_valid;
183 u64 ttbr_addr_field_shift;
184 u64 ttbr_shift;
185 int ttbr_count;
186 };
187
188 /*
189 * Private structure associated with each DART device.
190 *
191 * @dev: device struct
192 * @hw: SoC-specific hardware data
193 * @regs: mapped MMIO region
194 * @irq: interrupt number, can be shared with other DARTs
195 * @clks: clocks associated with this DART
196 * @num_clks: number of @clks
197 * @lock: lock for hardware operations involving this dart
198 * @pgsize: pagesize supported by this DART
199 * @supports_bypass: indicates if this DART supports bypass mode
200 * @sid2group: maps stream ids to iommu_groups
201 * @iommu: iommu core device
202 */
203 struct apple_dart {
204 struct device *dev;
205 const struct apple_dart_hw *hw;
206
207 void __iomem *regs;
208
209 int irq;
210 struct clk_bulk_data *clks;
211 int num_clks;
212
213 spinlock_t lock;
214
215 u32 ias;
216 u32 oas;
217 u32 pgsize;
218 u32 num_streams;
219 u32 supports_bypass : 1;
220
221 struct iommu_group *sid2group[DART_MAX_STREAMS];
222 struct iommu_device iommu;
223
224 u32 save_tcr[DART_MAX_STREAMS];
225 u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
226 };
227
228 /*
229 * Convenience struct to identify streams.
230 *
231 * The normal variant is used inside apple_dart_master_cfg which isn't written
232 * to concurrently.
233 * The atomic variant is used inside apple_dart_domain where we have to guard
234 * against races from potential parallel calls to attach/detach_device.
235 * Note that even inside the atomic variant the apple_dart pointer is not
236 * protected: This pointer is initialized once under the domain init mutex
237 * and never changed again afterwards. Devices with different dart pointers
238 * cannot be attached to the same domain.
239 *
240 * @dart dart pointer
241 * @sid stream id bitmap
242 */
243 struct apple_dart_stream_map {
244 struct apple_dart *dart;
245 DECLARE_BITMAP(sidmap, DART_MAX_STREAMS);
246 };
247 struct apple_dart_atomic_stream_map {
248 struct apple_dart *dart;
249 atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)];
250 };
251
252 /*
253 * This structure is attached to each iommu domain handled by a DART.
254 *
255 * @pgtbl_ops: pagetable ops allocated by io-pgtable
256 * @finalized: true if the domain has been completely initialized
257 * @init_lock: protects domain initialization
258 * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only)
259 * @domain: core iommu domain pointer
260 */
261 struct apple_dart_domain {
262 struct io_pgtable_ops *pgtbl_ops;
263
264 bool finalized;
265 struct mutex init_lock;
266 struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
267
268 struct iommu_domain domain;
269 };
270
271 /*
272 * This structure is attached to devices with dev_iommu_priv_set() on of_xlate
273 * and contains a list of streams bound to this device.
274 * So far the worst case seen is a single device with two streams
275 * from different darts, such that this simple static array is enough.
276 *
277 * @streams: streams for this device
278 */
279 struct apple_dart_master_cfg {
280 /* Intersection of DART capabilitles */
281 u32 supports_bypass : 1;
282
283 struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE];
284 };
285
286 /*
287 * Helper macro to iterate over apple_dart_master_cfg.stream_maps and
288 * apple_dart_domain.stream_maps
289 *
290 * @i int used as loop variable
291 * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain)
292 * @stream pointer to the apple_dart_streams struct for each loop iteration
293 */
294 #define for_each_stream_map(i, base, stream_map) \
295 for (i = 0, stream_map = &(base)->stream_maps[0]; \
296 i < MAX_DARTS_PER_DEVICE && stream_map->dart; \
297 stream_map = &(base)->stream_maps[++i])
298
299 static struct platform_driver apple_dart_driver;
300 static const struct iommu_ops apple_dart_iommu_ops;
301
to_dart_domain(struct iommu_domain * dom)302 static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
303 {
304 return container_of(dom, struct apple_dart_domain, domain);
305 }
306
307 static void
apple_dart_hw_enable_translation(struct apple_dart_stream_map * stream_map)308 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)
309 {
310 struct apple_dart *dart = stream_map->dart;
311 int sid;
312
313 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
314 writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid));
315 }
316
apple_dart_hw_disable_dma(struct apple_dart_stream_map * stream_map)317 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
318 {
319 struct apple_dart *dart = stream_map->dart;
320 int sid;
321
322 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
323 writel(dart->hw->tcr_disabled, dart->regs + DART_TCR(dart, sid));
324 }
325
326 static void
apple_dart_hw_enable_bypass(struct apple_dart_stream_map * stream_map)327 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)
328 {
329 struct apple_dart *dart = stream_map->dart;
330 int sid;
331
332 WARN_ON(!stream_map->dart->supports_bypass);
333 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
334 writel(dart->hw->tcr_bypass,
335 dart->regs + DART_TCR(dart, sid));
336 }
337
apple_dart_hw_set_ttbr(struct apple_dart_stream_map * stream_map,u8 idx,phys_addr_t paddr)338 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,
339 u8 idx, phys_addr_t paddr)
340 {
341 struct apple_dart *dart = stream_map->dart;
342 int sid;
343
344 WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1));
345 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
346 writel(dart->hw->ttbr_valid |
347 (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift,
348 dart->regs + DART_TTBR(dart, sid, idx));
349 }
350
apple_dart_hw_clear_ttbr(struct apple_dart_stream_map * stream_map,u8 idx)351 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,
352 u8 idx)
353 {
354 struct apple_dart *dart = stream_map->dart;
355 int sid;
356
357 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
358 writel(0, dart->regs + DART_TTBR(dart, sid, idx));
359 }
360
361 static void
apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map * stream_map)362 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
363 {
364 int i;
365
366 for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)
367 apple_dart_hw_clear_ttbr(stream_map, i);
368 }
369
370 static int
apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map * stream_map,u32 command)371 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,
372 u32 command)
373 {
374 unsigned long flags;
375 int ret, i;
376 u32 command_reg;
377
378 spin_lock_irqsave(&stream_map->dart->lock, flags);
379
380 for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++)
381 writel(stream_map->sidmap[i],
382 stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i);
383 writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);
384
385 ret = readl_poll_timeout_atomic(
386 stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,
387 !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1,
388 DART_STREAM_COMMAND_BUSY_TIMEOUT);
389
390 spin_unlock_irqrestore(&stream_map->dart->lock, flags);
391
392 if (ret) {
393 dev_err(stream_map->dart->dev,
394 "busy bit did not clear after command %x for streams %lx\n",
395 command, stream_map->sidmap[0]);
396 return ret;
397 }
398
399 return 0;
400 }
401
402 static int
apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map * stream_map,u32 command)403 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,
404 u32 command)
405 {
406 struct apple_dart *dart = stream_map->dart;
407 unsigned long flags;
408 int ret = 0;
409 int sid;
410
411 spin_lock_irqsave(&dart->lock, flags);
412
413 for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
414 u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) |
415 FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid);
416 writel(val, dart->regs + DART_T8110_TLB_CMD);
417
418 ret = readl_poll_timeout_atomic(
419 dart->regs + DART_T8110_TLB_CMD, val,
420 !(val & DART_T8110_TLB_CMD_BUSY), 1,
421 DART_STREAM_COMMAND_BUSY_TIMEOUT);
422
423 if (ret)
424 break;
425
426 }
427
428 spin_unlock_irqrestore(&dart->lock, flags);
429
430 if (ret) {
431 dev_err(stream_map->dart->dev,
432 "busy bit did not clear after command %x for stream %d\n",
433 command, sid);
434 return ret;
435 }
436
437 return 0;
438 }
439
440 static int
apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)441 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
442 {
443 return apple_dart_t8020_hw_stream_command(
444 stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);
445 }
446
447 static int
apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map * stream_map)448 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)
449 {
450 return apple_dart_t8110_hw_tlb_command(
451 stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);
452 }
453
apple_dart_hw_reset(struct apple_dart * dart)454 static int apple_dart_hw_reset(struct apple_dart *dart)
455 {
456 u32 config;
457 struct apple_dart_stream_map stream_map;
458 int i;
459
460 config = readl(dart->regs + dart->hw->lock);
461 if (config & dart->hw->lock_bit) {
462 dev_err(dart->dev, "DART is locked down until reboot: %08x\n",
463 config);
464 return -EINVAL;
465 }
466
467 stream_map.dart = dart;
468 bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS);
469 bitmap_set(stream_map.sidmap, 0, dart->num_streams);
470 apple_dart_hw_disable_dma(&stream_map);
471 apple_dart_hw_clear_all_ttbrs(&stream_map);
472
473 /* enable all streams globally since TCR is used to control isolation */
474 for (i = 0; i < BITS_TO_U32(dart->num_streams); i++)
475 writel(U32_MAX, dart->regs + dart->hw->enable_streams + 4 * i);
476
477 /* clear any pending errors before the interrupt is unmasked */
478 writel(readl(dart->regs + dart->hw->error), dart->regs + dart->hw->error);
479
480 if (dart->hw->type == DART_T8110)
481 writel(0, dart->regs + DART_T8110_ERROR_MASK);
482
483 return dart->hw->invalidate_tlb(&stream_map);
484 }
485
apple_dart_domain_flush_tlb(struct apple_dart_domain * domain)486 static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
487 {
488 int i, j;
489 struct apple_dart_atomic_stream_map *domain_stream_map;
490 struct apple_dart_stream_map stream_map;
491
492 for_each_stream_map(i, domain, domain_stream_map) {
493 stream_map.dart = domain_stream_map->dart;
494
495 for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)
496 stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);
497
498 stream_map.dart->hw->invalidate_tlb(&stream_map);
499 }
500 }
501
apple_dart_flush_iotlb_all(struct iommu_domain * domain)502 static void apple_dart_flush_iotlb_all(struct iommu_domain *domain)
503 {
504 apple_dart_domain_flush_tlb(to_dart_domain(domain));
505 }
506
apple_dart_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)507 static void apple_dart_iotlb_sync(struct iommu_domain *domain,
508 struct iommu_iotlb_gather *gather)
509 {
510 apple_dart_domain_flush_tlb(to_dart_domain(domain));
511 }
512
apple_dart_iotlb_sync_map(struct iommu_domain * domain,unsigned long iova,size_t size)513 static int apple_dart_iotlb_sync_map(struct iommu_domain *domain,
514 unsigned long iova, size_t size)
515 {
516 apple_dart_domain_flush_tlb(to_dart_domain(domain));
517 return 0;
518 }
519
apple_dart_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)520 static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain,
521 dma_addr_t iova)
522 {
523 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
524 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
525
526 if (!ops)
527 return 0;
528
529 return ops->iova_to_phys(ops, iova);
530 }
531
apple_dart_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)532 static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova,
533 phys_addr_t paddr, size_t pgsize,
534 size_t pgcount, int prot, gfp_t gfp,
535 size_t *mapped)
536 {
537 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
538 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
539
540 if (!ops)
541 return -ENODEV;
542
543 return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,
544 mapped);
545 }
546
apple_dart_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)547 static size_t apple_dart_unmap_pages(struct iommu_domain *domain,
548 unsigned long iova, size_t pgsize,
549 size_t pgcount,
550 struct iommu_iotlb_gather *gather)
551 {
552 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
553 struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;
554
555 return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
556 }
557
558 static void
apple_dart_setup_translation(struct apple_dart_domain * domain,struct apple_dart_stream_map * stream_map)559 apple_dart_setup_translation(struct apple_dart_domain *domain,
560 struct apple_dart_stream_map *stream_map)
561 {
562 int i;
563 struct io_pgtable_cfg *pgtbl_cfg =
564 &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
565
566 for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
567 apple_dart_hw_set_ttbr(stream_map, i,
568 pgtbl_cfg->apple_dart_cfg.ttbr[i]);
569 for (; i < stream_map->dart->hw->ttbr_count; ++i)
570 apple_dart_hw_clear_ttbr(stream_map, i);
571
572 apple_dart_hw_enable_translation(stream_map);
573 stream_map->dart->hw->invalidate_tlb(stream_map);
574 }
575
apple_dart_finalize_domain(struct apple_dart_domain * dart_domain,struct apple_dart_master_cfg * cfg)576 static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
577 struct apple_dart_master_cfg *cfg)
578 {
579 struct apple_dart *dart = cfg->stream_maps[0].dart;
580 struct io_pgtable_cfg pgtbl_cfg;
581 int ret = 0;
582 int i, j;
583
584 if (dart->pgsize > PAGE_SIZE)
585 return -EINVAL;
586
587 mutex_lock(&dart_domain->init_lock);
588
589 if (dart_domain->finalized)
590 goto done;
591
592 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
593 dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;
594 for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++)
595 atomic_long_set(&dart_domain->stream_maps[i].sidmap[j],
596 cfg->stream_maps[i].sidmap[j]);
597 }
598
599 pgtbl_cfg = (struct io_pgtable_cfg){
600 .pgsize_bitmap = dart->pgsize,
601 .ias = dart->ias,
602 .oas = dart->oas,
603 .coherent_walk = 1,
604 .iommu_dev = dart->dev,
605 };
606
607 dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
608 &dart_domain->domain);
609 if (!dart_domain->pgtbl_ops) {
610 ret = -ENOMEM;
611 goto done;
612 }
613
614 dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
615 dart_domain->domain.geometry.aperture_start = 0;
616 dart_domain->domain.geometry.aperture_end =
617 (dma_addr_t)DMA_BIT_MASK(dart->ias);
618 dart_domain->domain.geometry.force_aperture = true;
619
620 dart_domain->finalized = true;
621
622 done:
623 mutex_unlock(&dart_domain->init_lock);
624 return ret;
625 }
626
627 static int
apple_dart_mod_streams(struct apple_dart_atomic_stream_map * domain_maps,struct apple_dart_stream_map * master_maps,bool add_streams)628 apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps,
629 struct apple_dart_stream_map *master_maps,
630 bool add_streams)
631 {
632 int i, j;
633
634 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
635 if (domain_maps[i].dart != master_maps[i].dart)
636 return -EINVAL;
637 }
638
639 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
640 if (!domain_maps[i].dart)
641 break;
642 for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) {
643 if (add_streams)
644 atomic_long_or(master_maps[i].sidmap[j],
645 &domain_maps[i].sidmap[j]);
646 else
647 atomic_long_and(~master_maps[i].sidmap[j],
648 &domain_maps[i].sidmap[j]);
649 }
650 }
651
652 return 0;
653 }
654
apple_dart_domain_add_streams(struct apple_dart_domain * domain,struct apple_dart_master_cfg * cfg)655 static int apple_dart_domain_add_streams(struct apple_dart_domain *domain,
656 struct apple_dart_master_cfg *cfg)
657 {
658 return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,
659 true);
660 }
661
apple_dart_attach_dev_paging(struct iommu_domain * domain,struct device * dev)662 static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
663 struct device *dev)
664 {
665 int ret, i;
666 struct apple_dart_stream_map *stream_map;
667 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
668 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
669
670 ret = apple_dart_finalize_domain(dart_domain, cfg);
671 if (ret)
672 return ret;
673
674 ret = apple_dart_domain_add_streams(dart_domain, cfg);
675 if (ret)
676 return ret;
677
678 for_each_stream_map(i, cfg, stream_map)
679 apple_dart_setup_translation(dart_domain, stream_map);
680 return 0;
681 }
682
apple_dart_attach_dev_identity(struct iommu_domain * domain,struct device * dev)683 static int apple_dart_attach_dev_identity(struct iommu_domain *domain,
684 struct device *dev)
685 {
686 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
687 struct apple_dart_stream_map *stream_map;
688 int i;
689
690 if (!cfg->supports_bypass)
691 return -EINVAL;
692
693 for_each_stream_map(i, cfg, stream_map)
694 apple_dart_hw_enable_bypass(stream_map);
695 return 0;
696 }
697
698 static const struct iommu_domain_ops apple_dart_identity_ops = {
699 .attach_dev = apple_dart_attach_dev_identity,
700 };
701
702 static struct iommu_domain apple_dart_identity_domain = {
703 .type = IOMMU_DOMAIN_IDENTITY,
704 .ops = &apple_dart_identity_ops,
705 };
706
apple_dart_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)707 static int apple_dart_attach_dev_blocked(struct iommu_domain *domain,
708 struct device *dev)
709 {
710 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
711 struct apple_dart_stream_map *stream_map;
712 int i;
713
714 for_each_stream_map(i, cfg, stream_map)
715 apple_dart_hw_disable_dma(stream_map);
716 return 0;
717 }
718
719 static const struct iommu_domain_ops apple_dart_blocked_ops = {
720 .attach_dev = apple_dart_attach_dev_blocked,
721 };
722
723 static struct iommu_domain apple_dart_blocked_domain = {
724 .type = IOMMU_DOMAIN_BLOCKED,
725 .ops = &apple_dart_blocked_ops,
726 };
727
apple_dart_probe_device(struct device * dev)728 static struct iommu_device *apple_dart_probe_device(struct device *dev)
729 {
730 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
731 struct apple_dart_stream_map *stream_map;
732 int i;
733
734 if (!cfg)
735 return ERR_PTR(-ENODEV);
736
737 for_each_stream_map(i, cfg, stream_map)
738 device_link_add(
739 dev, stream_map->dart->dev,
740 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
741
742 return &cfg->stream_maps[0].dart->iommu;
743 }
744
apple_dart_release_device(struct device * dev)745 static void apple_dart_release_device(struct device *dev)
746 {
747 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
748
749 kfree(cfg);
750 }
751
apple_dart_domain_alloc_paging(struct device * dev)752 static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev)
753 {
754 struct apple_dart_domain *dart_domain;
755
756 dart_domain = kzalloc(sizeof(*dart_domain), GFP_KERNEL);
757 if (!dart_domain)
758 return NULL;
759
760 mutex_init(&dart_domain->init_lock);
761
762 if (dev) {
763 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
764 int ret;
765
766 ret = apple_dart_finalize_domain(dart_domain, cfg);
767 if (ret) {
768 kfree(dart_domain);
769 return ERR_PTR(ret);
770 }
771 }
772 return &dart_domain->domain;
773 }
774
apple_dart_domain_free(struct iommu_domain * domain)775 static void apple_dart_domain_free(struct iommu_domain *domain)
776 {
777 struct apple_dart_domain *dart_domain = to_dart_domain(domain);
778
779 if (dart_domain->pgtbl_ops)
780 free_io_pgtable_ops(dart_domain->pgtbl_ops);
781
782 kfree(dart_domain);
783 }
784
apple_dart_of_xlate(struct device * dev,const struct of_phandle_args * args)785 static int apple_dart_of_xlate(struct device *dev,
786 const struct of_phandle_args *args)
787 {
788 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
789 struct platform_device *iommu_pdev = of_find_device_by_node(args->np);
790 struct apple_dart *dart = platform_get_drvdata(iommu_pdev);
791 struct apple_dart *cfg_dart;
792 int i, sid;
793
794 if (args->args_count != 1)
795 return -EINVAL;
796 sid = args->args[0];
797
798 if (!cfg) {
799 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL);
800 if (!cfg)
801 return -ENOMEM;
802 /* Will be ANDed with DART capabilities */
803 cfg->supports_bypass = true;
804 }
805 dev_iommu_priv_set(dev, cfg);
806
807 cfg_dart = cfg->stream_maps[0].dart;
808 if (cfg_dart) {
809 if (cfg_dart->pgsize != dart->pgsize)
810 return -EINVAL;
811 }
812
813 cfg->supports_bypass &= dart->supports_bypass;
814
815 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
816 if (cfg->stream_maps[i].dart == dart) {
817 set_bit(sid, cfg->stream_maps[i].sidmap);
818 return 0;
819 }
820 }
821 for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) {
822 if (!cfg->stream_maps[i].dart) {
823 cfg->stream_maps[i].dart = dart;
824 set_bit(sid, cfg->stream_maps[i].sidmap);
825 return 0;
826 }
827 }
828
829 return -EINVAL;
830 }
831
832 static DEFINE_MUTEX(apple_dart_groups_lock);
833
apple_dart_release_group(void * iommu_data)834 static void apple_dart_release_group(void *iommu_data)
835 {
836 int i, sid;
837 struct apple_dart_stream_map *stream_map;
838 struct apple_dart_master_cfg *group_master_cfg = iommu_data;
839
840 mutex_lock(&apple_dart_groups_lock);
841
842 for_each_stream_map(i, group_master_cfg, stream_map)
843 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
844 stream_map->dart->sid2group[sid] = NULL;
845
846 kfree(iommu_data);
847 mutex_unlock(&apple_dart_groups_lock);
848 }
849
apple_dart_merge_master_cfg(struct apple_dart_master_cfg * dst,struct apple_dart_master_cfg * src)850 static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst,
851 struct apple_dart_master_cfg *src)
852 {
853 /*
854 * We know that this function is only called for groups returned from
855 * pci_device_group and that all Apple Silicon platforms never spread
856 * PCIe devices from the same bus across multiple DARTs such that we can
857 * just assume that both src and dst only have the same single DART.
858 */
859 if (src->stream_maps[1].dart)
860 return -EINVAL;
861 if (dst->stream_maps[1].dart)
862 return -EINVAL;
863 if (src->stream_maps[0].dart != dst->stream_maps[0].dart)
864 return -EINVAL;
865
866 bitmap_or(dst->stream_maps[0].sidmap,
867 dst->stream_maps[0].sidmap,
868 src->stream_maps[0].sidmap,
869 dst->stream_maps[0].dart->num_streams);
870 return 0;
871 }
872
apple_dart_device_group(struct device * dev)873 static struct iommu_group *apple_dart_device_group(struct device *dev)
874 {
875 int i, sid;
876 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
877 struct apple_dart_stream_map *stream_map;
878 struct apple_dart_master_cfg *group_master_cfg;
879 struct iommu_group *group = NULL;
880 struct iommu_group *res = ERR_PTR(-EINVAL);
881
882 mutex_lock(&apple_dart_groups_lock);
883
884 for_each_stream_map(i, cfg, stream_map) {
885 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {
886 struct iommu_group *stream_group =
887 stream_map->dart->sid2group[sid];
888
889 if (group && group != stream_group) {
890 res = ERR_PTR(-EINVAL);
891 goto out;
892 }
893
894 group = stream_group;
895 }
896 }
897
898 if (group) {
899 res = iommu_group_ref_get(group);
900 goto out;
901 }
902
903 #ifdef CONFIG_PCI
904 if (dev_is_pci(dev))
905 group = pci_device_group(dev);
906 else
907 #endif
908 group = generic_device_group(dev);
909
910 res = ERR_PTR(-ENOMEM);
911 if (!group)
912 goto out;
913
914 group_master_cfg = iommu_group_get_iommudata(group);
915 if (group_master_cfg) {
916 int ret;
917
918 ret = apple_dart_merge_master_cfg(group_master_cfg, cfg);
919 if (ret) {
920 dev_err(dev, "Failed to merge DART IOMMU groups.\n");
921 iommu_group_put(group);
922 res = ERR_PTR(ret);
923 goto out;
924 }
925 } else {
926 group_master_cfg = kmemdup(cfg, sizeof(*group_master_cfg),
927 GFP_KERNEL);
928 if (!group_master_cfg) {
929 iommu_group_put(group);
930 goto out;
931 }
932
933 iommu_group_set_iommudata(group, group_master_cfg,
934 apple_dart_release_group);
935 }
936
937 for_each_stream_map(i, cfg, stream_map)
938 for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)
939 stream_map->dart->sid2group[sid] = group;
940
941 res = group;
942
943 out:
944 mutex_unlock(&apple_dart_groups_lock);
945 return res;
946 }
947
apple_dart_def_domain_type(struct device * dev)948 static int apple_dart_def_domain_type(struct device *dev)
949 {
950 struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
951
952 if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE)
953 return IOMMU_DOMAIN_IDENTITY;
954 if (!cfg->supports_bypass)
955 return IOMMU_DOMAIN_DMA;
956
957 return 0;
958 }
959
960 #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR
961 /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */
962 #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0
963 #endif
964 #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK)
965
apple_dart_get_resv_regions(struct device * dev,struct list_head * head)966 static void apple_dart_get_resv_regions(struct device *dev,
967 struct list_head *head)
968 {
969 if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) {
970 struct iommu_resv_region *region;
971 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
972
973 region = iommu_alloc_resv_region(DOORBELL_ADDR,
974 PAGE_SIZE, prot,
975 IOMMU_RESV_MSI, GFP_KERNEL);
976 if (!region)
977 return;
978
979 list_add_tail(®ion->list, head);
980 }
981
982 iommu_dma_get_resv_regions(dev, head);
983 }
984
985 static const struct iommu_ops apple_dart_iommu_ops = {
986 .identity_domain = &apple_dart_identity_domain,
987 .blocked_domain = &apple_dart_blocked_domain,
988 .domain_alloc_paging = apple_dart_domain_alloc_paging,
989 .probe_device = apple_dart_probe_device,
990 .release_device = apple_dart_release_device,
991 .device_group = apple_dart_device_group,
992 .of_xlate = apple_dart_of_xlate,
993 .def_domain_type = apple_dart_def_domain_type,
994 .get_resv_regions = apple_dart_get_resv_regions,
995 .pgsize_bitmap = -1UL, /* Restricted during dart probe */
996 .owner = THIS_MODULE,
997 .default_domain_ops = &(const struct iommu_domain_ops) {
998 .attach_dev = apple_dart_attach_dev_paging,
999 .map_pages = apple_dart_map_pages,
1000 .unmap_pages = apple_dart_unmap_pages,
1001 .flush_iotlb_all = apple_dart_flush_iotlb_all,
1002 .iotlb_sync = apple_dart_iotlb_sync,
1003 .iotlb_sync_map = apple_dart_iotlb_sync_map,
1004 .iova_to_phys = apple_dart_iova_to_phys,
1005 .free = apple_dart_domain_free,
1006 }
1007 };
1008
apple_dart_t8020_irq(int irq,void * dev)1009 static irqreturn_t apple_dart_t8020_irq(int irq, void *dev)
1010 {
1011 struct apple_dart *dart = dev;
1012 const char *fault_name = NULL;
1013 u32 error = readl(dart->regs + DART_T8020_ERROR);
1014 u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error);
1015 u32 addr_lo = readl(dart->regs + DART_T8020_ERROR_ADDR_LO);
1016 u32 addr_hi = readl(dart->regs + DART_T8020_ERROR_ADDR_HI);
1017 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1018 u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error);
1019
1020 if (!(error & DART_T8020_ERROR_FLAG))
1021 return IRQ_NONE;
1022
1023 /* there should only be a single bit set but let's use == to be sure */
1024 if (error_code == DART_T8020_ERROR_READ_FAULT)
1025 fault_name = "READ FAULT";
1026 else if (error_code == DART_T8020_ERROR_WRITE_FAULT)
1027 fault_name = "WRITE FAULT";
1028 else if (error_code == DART_T8020_ERROR_NO_PTE)
1029 fault_name = "NO PTE FOR IOVA";
1030 else if (error_code == DART_T8020_ERROR_NO_PMD)
1031 fault_name = "NO PMD FOR IOVA";
1032 else if (error_code == DART_T8020_ERROR_NO_TTBR)
1033 fault_name = "NO TTBR FOR IOVA";
1034 else
1035 fault_name = "unknown";
1036
1037 dev_err_ratelimited(
1038 dart->dev,
1039 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1040 error, stream_idx, error_code, fault_name, addr);
1041
1042 writel(error, dart->regs + DART_T8020_ERROR);
1043 return IRQ_HANDLED;
1044 }
1045
apple_dart_t8110_irq(int irq,void * dev)1046 static irqreturn_t apple_dart_t8110_irq(int irq, void *dev)
1047 {
1048 struct apple_dart *dart = dev;
1049 const char *fault_name = NULL;
1050 u32 error = readl(dart->regs + DART_T8110_ERROR);
1051 u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error);
1052 u32 addr_lo = readl(dart->regs + DART_T8110_ERROR_ADDR_LO);
1053 u32 addr_hi = readl(dart->regs + DART_T8110_ERROR_ADDR_HI);
1054 u64 addr = addr_lo | (((u64)addr_hi) << 32);
1055 u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error);
1056
1057 if (!(error & DART_T8110_ERROR_FLAG))
1058 return IRQ_NONE;
1059
1060 /* there should only be a single bit set but let's use == to be sure */
1061 if (error_code == DART_T8110_ERROR_READ_FAULT)
1062 fault_name = "READ FAULT";
1063 else if (error_code == DART_T8110_ERROR_WRITE_FAULT)
1064 fault_name = "WRITE FAULT";
1065 else if (error_code == DART_T8110_ERROR_NO_PTE)
1066 fault_name = "NO PTE FOR IOVA";
1067 else if (error_code == DART_T8110_ERROR_NO_PMD)
1068 fault_name = "NO PMD FOR IOVA";
1069 else if (error_code == DART_T8110_ERROR_NO_PGD)
1070 fault_name = "NO PGD FOR IOVA";
1071 else if (error_code == DART_T8110_ERROR_NO_TTBR)
1072 fault_name = "NO TTBR FOR IOVA";
1073 else
1074 fault_name = "unknown";
1075
1076 dev_err_ratelimited(
1077 dart->dev,
1078 "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx",
1079 error, stream_idx, error_code, fault_name, addr);
1080
1081 writel(error, dart->regs + DART_T8110_ERROR);
1082 return IRQ_HANDLED;
1083 }
1084
apple_dart_probe(struct platform_device * pdev)1085 static int apple_dart_probe(struct platform_device *pdev)
1086 {
1087 int ret;
1088 u32 dart_params[4];
1089 struct resource *res;
1090 struct apple_dart *dart;
1091 struct device *dev = &pdev->dev;
1092
1093 dart = devm_kzalloc(dev, sizeof(*dart), GFP_KERNEL);
1094 if (!dart)
1095 return -ENOMEM;
1096
1097 dart->dev = dev;
1098 dart->hw = of_device_get_match_data(dev);
1099 spin_lock_init(&dart->lock);
1100
1101 dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
1102 if (IS_ERR(dart->regs))
1103 return PTR_ERR(dart->regs);
1104
1105 if (resource_size(res) < 0x4000) {
1106 dev_err(dev, "MMIO region too small (%pr)\n", res);
1107 return -EINVAL;
1108 }
1109
1110 dart->irq = platform_get_irq(pdev, 0);
1111 if (dart->irq < 0)
1112 return -ENODEV;
1113
1114 ret = devm_clk_bulk_get_all(dev, &dart->clks);
1115 if (ret < 0)
1116 return ret;
1117 dart->num_clks = ret;
1118
1119 ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);
1120 if (ret)
1121 return ret;
1122
1123 dart_params[0] = readl(dart->regs + DART_PARAMS1);
1124 dart_params[1] = readl(dart->regs + DART_PARAMS2);
1125 dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]);
1126 dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT;
1127
1128 switch (dart->hw->type) {
1129 case DART_T8020:
1130 case DART_T6000:
1131 dart->ias = 32;
1132 dart->oas = dart->hw->oas;
1133 dart->num_streams = dart->hw->max_sid_count;
1134 break;
1135
1136 case DART_T8110:
1137 dart_params[2] = readl(dart->regs + DART_T8110_PARAMS3);
1138 dart_params[3] = readl(dart->regs + DART_T8110_PARAMS4);
1139 dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]);
1140 dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]);
1141 dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]);
1142 break;
1143 }
1144
1145 if (dart->num_streams > DART_MAX_STREAMS) {
1146 dev_err(&pdev->dev, "Too many streams (%d > %d)\n",
1147 dart->num_streams, DART_MAX_STREAMS);
1148 ret = -EINVAL;
1149 goto err_clk_disable;
1150 }
1151
1152 ret = apple_dart_hw_reset(dart);
1153 if (ret)
1154 goto err_clk_disable;
1155
1156 ret = request_irq(dart->irq, dart->hw->irq_handler, IRQF_SHARED,
1157 "apple-dart fault handler", dart);
1158 if (ret)
1159 goto err_clk_disable;
1160
1161 platform_set_drvdata(pdev, dart);
1162
1163 ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",
1164 dev_name(&pdev->dev));
1165 if (ret)
1166 goto err_free_irq;
1167
1168 ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);
1169 if (ret)
1170 goto err_sysfs_remove;
1171
1172 dev_info(
1173 &pdev->dev,
1174 "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n",
1175 dart->pgsize, dart->num_streams, dart->supports_bypass,
1176 dart->pgsize > PAGE_SIZE);
1177 return 0;
1178
1179 err_sysfs_remove:
1180 iommu_device_sysfs_remove(&dart->iommu);
1181 err_free_irq:
1182 free_irq(dart->irq, dart);
1183 err_clk_disable:
1184 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1185
1186 return ret;
1187 }
1188
apple_dart_remove(struct platform_device * pdev)1189 static void apple_dart_remove(struct platform_device *pdev)
1190 {
1191 struct apple_dart *dart = platform_get_drvdata(pdev);
1192
1193 apple_dart_hw_reset(dart);
1194 free_irq(dart->irq, dart);
1195
1196 iommu_device_unregister(&dart->iommu);
1197 iommu_device_sysfs_remove(&dart->iommu);
1198
1199 clk_bulk_disable_unprepare(dart->num_clks, dart->clks);
1200 }
1201
1202 static const struct apple_dart_hw apple_dart_hw_t8103 = {
1203 .type = DART_T8020,
1204 .irq_handler = apple_dart_t8020_irq,
1205 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1206 .oas = 36,
1207 .fmt = APPLE_DART,
1208 .max_sid_count = 16,
1209
1210 .enable_streams = DART_T8020_STREAMS_ENABLE,
1211 .lock = DART_T8020_CONFIG,
1212 .lock_bit = DART_T8020_CONFIG_LOCK,
1213
1214 .error = DART_T8020_ERROR,
1215
1216 .tcr = DART_T8020_TCR,
1217 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1218 .tcr_disabled = 0,
1219 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1220
1221 .ttbr = DART_T8020_TTBR,
1222 .ttbr_valid = DART_T8020_TTBR_VALID,
1223 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1224 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1225 .ttbr_count = 4,
1226 };
1227
1228 static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = {
1229 .type = DART_T8020,
1230 .irq_handler = apple_dart_t8020_irq,
1231 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1232 .oas = 36,
1233 .fmt = APPLE_DART,
1234 .max_sid_count = 64,
1235
1236 .enable_streams = DART_T8020_STREAMS_ENABLE,
1237 .lock = DART_T8020_CONFIG,
1238 .lock_bit = DART_T8020_CONFIG_LOCK,
1239
1240 .error = DART_T8020_ERROR,
1241
1242 .tcr = DART_T8020_TCR,
1243 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1244 .tcr_disabled = 0,
1245 .tcr_bypass = 0,
1246
1247 .ttbr = DART_T8020_USB4_TTBR,
1248 .ttbr_valid = DART_T8020_TTBR_VALID,
1249 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1250 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1251 .ttbr_count = 4,
1252 };
1253
1254 static const struct apple_dart_hw apple_dart_hw_t6000 = {
1255 .type = DART_T6000,
1256 .irq_handler = apple_dart_t8020_irq,
1257 .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb,
1258 .oas = 42,
1259 .fmt = APPLE_DART2,
1260 .max_sid_count = 16,
1261
1262 .enable_streams = DART_T8020_STREAMS_ENABLE,
1263 .lock = DART_T8020_CONFIG,
1264 .lock_bit = DART_T8020_CONFIG_LOCK,
1265
1266 .error = DART_T8020_ERROR,
1267
1268 .tcr = DART_T8020_TCR,
1269 .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE,
1270 .tcr_disabled = 0,
1271 .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART,
1272
1273 .ttbr = DART_T8020_TTBR,
1274 .ttbr_valid = DART_T8020_TTBR_VALID,
1275 .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT,
1276 .ttbr_shift = DART_T8020_TTBR_SHIFT,
1277 .ttbr_count = 4,
1278 };
1279
1280 static const struct apple_dart_hw apple_dart_hw_t8110 = {
1281 .type = DART_T8110,
1282 .irq_handler = apple_dart_t8110_irq,
1283 .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb,
1284 .fmt = APPLE_DART2,
1285 .max_sid_count = 256,
1286
1287 .enable_streams = DART_T8110_ENABLE_STREAMS,
1288 .lock = DART_T8110_PROTECT,
1289 .lock_bit = DART_T8110_PROTECT_TTBR_TCR,
1290
1291 .error = DART_T8110_ERROR,
1292
1293 .tcr = DART_T8110_TCR,
1294 .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE,
1295 .tcr_disabled = 0,
1296 .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART,
1297
1298 .ttbr = DART_T8110_TTBR,
1299 .ttbr_valid = DART_T8110_TTBR_VALID,
1300 .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT,
1301 .ttbr_shift = DART_T8110_TTBR_SHIFT,
1302 .ttbr_count = 1,
1303 };
1304
apple_dart_suspend(struct device * dev)1305 static __maybe_unused int apple_dart_suspend(struct device *dev)
1306 {
1307 struct apple_dart *dart = dev_get_drvdata(dev);
1308 unsigned int sid, idx;
1309
1310 for (sid = 0; sid < dart->num_streams; sid++) {
1311 dart->save_tcr[sid] = readl(dart->regs + DART_TCR(dart, sid));
1312 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1313 dart->save_ttbr[sid][idx] =
1314 readl(dart->regs + DART_TTBR(dart, sid, idx));
1315 }
1316
1317 return 0;
1318 }
1319
apple_dart_resume(struct device * dev)1320 static __maybe_unused int apple_dart_resume(struct device *dev)
1321 {
1322 struct apple_dart *dart = dev_get_drvdata(dev);
1323 unsigned int sid, idx;
1324 int ret;
1325
1326 ret = apple_dart_hw_reset(dart);
1327 if (ret) {
1328 dev_err(dev, "Failed to reset DART on resume\n");
1329 return ret;
1330 }
1331
1332 for (sid = 0; sid < dart->num_streams; sid++) {
1333 for (idx = 0; idx < dart->hw->ttbr_count; idx++)
1334 writel(dart->save_ttbr[sid][idx],
1335 dart->regs + DART_TTBR(dart, sid, idx));
1336 writel(dart->save_tcr[sid], dart->regs + DART_TCR(dart, sid));
1337 }
1338
1339 return 0;
1340 }
1341
1342 static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume);
1343
1344 static const struct of_device_id apple_dart_of_match[] = {
1345 { .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
1346 { .compatible = "apple,t8103-usb4-dart", .data = &apple_dart_hw_t8103_usb4 },
1347 { .compatible = "apple,t8110-dart", .data = &apple_dart_hw_t8110 },
1348 { .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
1349 {},
1350 };
1351 MODULE_DEVICE_TABLE(of, apple_dart_of_match);
1352
1353 static struct platform_driver apple_dart_driver = {
1354 .driver = {
1355 .name = "apple-dart",
1356 .of_match_table = apple_dart_of_match,
1357 .suppress_bind_attrs = true,
1358 .pm = pm_sleep_ptr(&apple_dart_pm_ops),
1359 },
1360 .probe = apple_dart_probe,
1361 .remove = apple_dart_remove,
1362 };
1363
1364 module_platform_driver(apple_dart_driver);
1365
1366 MODULE_DESCRIPTION("IOMMU API for Apple's DART");
1367 MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
1368 MODULE_LICENSE("GPL v2");
1369