1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2025 Collabora Ltd.
3 *
4 * IOMMU API for Verisilicon
5 *
6 * Module Authors: Yandong Lin <yandong.lin@rock-chips.com>
7 * Simon Xue <xxm@rock-chips.com>
8 * Benjamin Gaignard <benjamin.gaignard@collabora.com>
9 *
10 * This hardware block is using a 2 pages tables allocation structure.
11 * That make very similar to Rockhip iommu hardware blocks but it has
12 * it own driver because the registers offset and configuration bits
13 * are completely different. An additional reason is that this hardware
14 * has been developed by Verisilicon to be used by their hardware video
15 * decoders and not for a general purpose like Rockchip iommus.
16 */
17
18 #include <linux/clk.h>
19 #include <linux/compiler.h>
20 #include <linux/delay.h>
21 #include <linux/device.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/errno.h>
24 #include <linux/init.h>
25 #include <linux/interrupt.h>
26 #include <linux/io.h>
27 #include <linux/iommu.h>
28 #include <linux/list.h>
29 #include <linux/mm.h>
30 #include <linux/module.h>
31 #include <linux/of.h>
32 #include <linux/of_iommu.h>
33 #include <linux/of_platform.h>
34 #include <linux/platform_device.h>
35 #include <linux/pm_runtime.h>
36 #include <linux/slab.h>
37 #include <linux/spinlock.h>
38
39 #include "iommu-pages.h"
40
41 struct vsi_iommu {
42 struct device *dev;
43 void __iomem *regs;
44 struct clk_bulk_data *clocks;
45 int num_clocks;
46 struct iommu_device iommu;
47 struct list_head node; /* entry in vsi_iommu_domain.iommus */
48 struct iommu_domain *domain; /* domain to which iommu is attached */
49 spinlock_t lock; /* lock to protect vsi_iommu fields */
50 int irq;
51 bool enable;
52 };
53
54 struct vsi_iommu_domain {
55 struct list_head iommus;
56 struct device *dev;
57 u32 *dt;
58 dma_addr_t dt_dma;
59 struct iommu_domain domain;
60 u64 *pta;
61 dma_addr_t pta_dma;
62 spinlock_t lock; /* lock to protect vsi_iommu_domain fields */
63 };
64
65 static struct iommu_domain vsi_identity_domain;
66
67 #define NUM_DT_ENTRIES 1024
68 #define NUM_PT_ENTRIES 1024
69
70 #define SPAGE_SIZE BIT(12)
71
72 /* vsi iommu regs address */
73 #define VSI_MMU_CONFIG1_BASE 0x1ac
74 #define VSI_MMU_AHB_EXCEPTION_BASE 0x380
75 #define VSI_MMU_AHB_CONTROL_BASE 0x388
76 #define VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE 0x38C
77
78 /* MMU register offsets */
79 #define VSI_MMU_FLUSH_BASE 0x184
80 #define VSI_MMU_BIT_FLUSH BIT(4)
81
82 #define VSI_MMU_PAGE_FAULT_ADDR 0x380
83 #define VSI_MMU_STATUS_BASE 0x384 /* IRQ status */
84
85 #define VSI_MMU_BIT_ENABLE BIT(0)
86
87 #define VSI_MMU_OUT_OF_BOUND BIT(28)
88 /* Irq mask */
89 #define VSI_MMU_IRQ_MASK 0x7
90
91 #define VSI_DTE_PT_ADDRESS_MASK 0xffffffc0
92 #define VSI_DTE_PT_VALID BIT(0)
93
94 #define VSI_PAGE_DESC_LO_MASK 0xfffff000
95 #define VSI_PAGE_DESC_HI_MASK GENMASK_ULL(39, 32)
96 #define VSI_PAGE_DESC_HI_SHIFT (32 - 4)
97
vsi_dte_pt_address(u32 dte)98 static inline phys_addr_t vsi_dte_pt_address(u32 dte)
99 {
100 return (phys_addr_t)dte & VSI_DTE_PT_ADDRESS_MASK;
101 }
102
vsi_mk_dte(u32 dte)103 static inline u32 vsi_mk_dte(u32 dte)
104 {
105 return (phys_addr_t)dte | VSI_DTE_PT_VALID;
106 }
107
108 #define VSI_PTE_PAGE_WRITABLE BIT(2)
109 #define VSI_PTE_PAGE_VALID BIT(0)
110
vsi_pte_page_address(u64 pte)111 static inline phys_addr_t vsi_pte_page_address(u64 pte)
112 {
113 return ((pte << VSI_PAGE_DESC_HI_SHIFT) & VSI_PAGE_DESC_HI_MASK) |
114 (pte & VSI_PAGE_DESC_LO_MASK);
115 }
116
vsi_mk_pte(phys_addr_t page,int prot)117 static u32 vsi_mk_pte(phys_addr_t page, int prot)
118 {
119 u32 flags = 0;
120
121 flags |= (prot & IOMMU_WRITE) ? VSI_PTE_PAGE_WRITABLE : 0;
122
123 page = (page & VSI_PAGE_DESC_LO_MASK) |
124 ((page & VSI_PAGE_DESC_HI_MASK) >> VSI_PAGE_DESC_HI_SHIFT);
125
126 return page | flags | VSI_PTE_PAGE_VALID;
127 }
128
129 #define VSI_DTE_PT_VALID BIT(0)
130
vsi_dte_is_pt_valid(u32 dte)131 static inline bool vsi_dte_is_pt_valid(u32 dte)
132 {
133 return dte & VSI_DTE_PT_VALID;
134 }
135
vsi_pte_is_page_valid(u32 pte)136 static inline bool vsi_pte_is_page_valid(u32 pte)
137 {
138 return pte & VSI_PTE_PAGE_VALID;
139 }
140
vsi_mk_pte_invalid(u32 pte)141 static u32 vsi_mk_pte_invalid(u32 pte)
142 {
143 return pte & ~VSI_PTE_PAGE_VALID;
144 }
145
146 #define VSI_MASTER_TLB_MASK GENMASK_ULL(31, 10)
147 /* mode 0 : 4k */
148 #define VSI_PTA_4K_MODE 0
149
vsi_mk_pta(dma_addr_t dt_dma)150 static u64 vsi_mk_pta(dma_addr_t dt_dma)
151 {
152 u64 val = (dt_dma & VSI_MASTER_TLB_MASK) | VSI_PTA_4K_MODE;
153
154 return val;
155 }
156
to_vsi_domain(struct iommu_domain * dom)157 static struct vsi_iommu_domain *to_vsi_domain(struct iommu_domain *dom)
158 {
159 return container_of(dom, struct vsi_iommu_domain, domain);
160 }
161
vsi_table_flush(struct vsi_iommu_domain * vsi_domain,dma_addr_t dma,unsigned int count)162 static inline void vsi_table_flush(struct vsi_iommu_domain *vsi_domain, dma_addr_t dma,
163 unsigned int count)
164 {
165 size_t size = count * sizeof(u32); /* count of u32 entry */
166
167 dma_sync_single_for_device(vsi_domain->dev, dma, size, DMA_TO_DEVICE);
168 }
169
170 #define VSI_IOVA_DTE_MASK 0xffc00000
171 #define VSI_IOVA_DTE_SHIFT 22
172 #define VSI_IOVA_PTE_MASK 0x003ff000
173 #define VSI_IOVA_PTE_SHIFT 12
174 #define VSI_IOVA_PAGE_MASK 0x00000fff
175 #define VSI_IOVA_PAGE_SHIFT 0
176
vsi_iova_dte_index(u32 iova)177 static u32 vsi_iova_dte_index(u32 iova)
178 {
179 return (iova & VSI_IOVA_DTE_MASK) >> VSI_IOVA_DTE_SHIFT;
180 }
181
vsi_iova_pte_index(u32 iova)182 static u32 vsi_iova_pte_index(u32 iova)
183 {
184 return (iova & VSI_IOVA_PTE_MASK) >> VSI_IOVA_PTE_SHIFT;
185 }
186
vsi_iova_page_offset(u32 iova)187 static u32 vsi_iova_page_offset(u32 iova)
188 {
189 return (iova & VSI_IOVA_PAGE_MASK) >> VSI_IOVA_PAGE_SHIFT;
190 }
191
vsi_iommu_irq(int irq,void * dev_id)192 static irqreturn_t vsi_iommu_irq(int irq, void *dev_id)
193 {
194 struct vsi_iommu *iommu = dev_id;
195 unsigned long flags;
196 dma_addr_t iova;
197 u32 status;
198
199 if (pm_runtime_resume_and_get(iommu->dev) < 0)
200 return IRQ_NONE;
201
202 spin_lock_irqsave(&iommu->lock, flags);
203
204 status = readl(iommu->regs + VSI_MMU_STATUS_BASE);
205 if (status & VSI_MMU_IRQ_MASK) {
206 dev_err(iommu->dev, "unexpected int_status=%08x\n", status);
207 iova = readl(iommu->regs + VSI_MMU_PAGE_FAULT_ADDR);
208 report_iommu_fault(iommu->domain, iommu->dev, iova, status);
209 }
210 writel(0, iommu->regs + VSI_MMU_STATUS_BASE);
211
212 spin_unlock_irqrestore(&iommu->lock, flags);
213 pm_runtime_put_autosuspend(iommu->dev);
214
215 return IRQ_HANDLED;
216 }
217
vsi_iommu_get_from_dev(struct device * dev)218 static struct vsi_iommu *vsi_iommu_get_from_dev(struct device *dev)
219 {
220 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
221 struct device *iommu_dev = bus_find_device_by_fwnode(&platform_bus_type,
222 fwspec->iommu_fwnode);
223
224 put_device(iommu_dev);
225
226 return iommu_dev ? dev_get_drvdata(iommu_dev) : NULL;
227 }
228
vsi_iommu_domain_alloc_paging(struct device * dev)229 static struct iommu_domain *vsi_iommu_domain_alloc_paging(struct device *dev)
230 {
231 struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
232 struct vsi_iommu_domain *vsi_domain;
233
234 vsi_domain = kzalloc_obj(*vsi_domain);
235 if (!vsi_domain)
236 return NULL;
237
238 vsi_domain->dev = iommu->dev;
239 spin_lock_init(&vsi_domain->lock);
240
241 /*
242 * iommu use a 2 level pagetable.
243 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
244 * Allocate one 4 KiB page for each table.
245 */
246 vsi_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
247 SPAGE_SIZE);
248 if (!vsi_domain->dt)
249 goto err_free_domain;
250
251 vsi_domain->dt_dma = dma_map_single(vsi_domain->dev, vsi_domain->dt,
252 SPAGE_SIZE, DMA_TO_DEVICE);
253 if (dma_mapping_error(vsi_domain->dev, vsi_domain->dt_dma)) {
254 dev_err(dev, "DMA map error for DT\n");
255 goto err_free_dt;
256 }
257
258 vsi_domain->pta = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
259 SPAGE_SIZE);
260 if (!vsi_domain->pta)
261 goto err_unmap_dt;
262
263 vsi_domain->pta[0] = vsi_mk_pta(vsi_domain->dt_dma);
264 vsi_domain->pta_dma = dma_map_single(vsi_domain->dev, vsi_domain->pta,
265 SPAGE_SIZE, DMA_TO_DEVICE);
266 if (dma_mapping_error(vsi_domain->dev, vsi_domain->pta_dma)) {
267 dev_err(dev, "DMA map error for PTA\n");
268 goto err_free_pta;
269 }
270
271 INIT_LIST_HEAD(&vsi_domain->iommus);
272
273 vsi_domain->domain.geometry.aperture_start = 0;
274 vsi_domain->domain.geometry.aperture_end = DMA_BIT_MASK(32);
275 vsi_domain->domain.geometry.force_aperture = true;
276 vsi_domain->domain.pgsize_bitmap = SZ_4K;
277
278 return &vsi_domain->domain;
279
280 err_free_pta:
281 iommu_free_pages(vsi_domain->pta);
282 err_unmap_dt:
283 dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
284 SPAGE_SIZE, DMA_TO_DEVICE);
285 err_free_dt:
286 iommu_free_pages(vsi_domain->dt);
287 err_free_domain:
288 kfree(vsi_domain);
289
290 return NULL;
291 }
292
vsi_iommu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)293 static phys_addr_t vsi_iommu_iova_to_phys(struct iommu_domain *domain,
294 dma_addr_t iova)
295 {
296 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
297 phys_addr_t pt_phys, phys = 0;
298 unsigned long flags;
299 u32 dte, pte;
300 u32 *page_table;
301
302 spin_lock_irqsave(&vsi_domain->lock, flags);
303 dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
304 if (!vsi_dte_is_pt_valid(dte))
305 goto unlock;
306
307 pt_phys = vsi_dte_pt_address(dte);
308 page_table = (u32 *)phys_to_virt(pt_phys);
309 pte = page_table[vsi_iova_pte_index(iova)];
310 if (!vsi_pte_is_page_valid(pte))
311 goto unlock;
312
313 phys = vsi_pte_page_address(pte) + vsi_iova_page_offset(iova);
314
315 unlock:
316 spin_unlock_irqrestore(&vsi_domain->lock, flags);
317 return phys;
318 }
319
vsi_iommu_unmap_iova(struct vsi_iommu_domain * vsi_domain,u32 * pte_addr,dma_addr_t pte_dma,size_t size)320 static size_t vsi_iommu_unmap_iova(struct vsi_iommu_domain *vsi_domain,
321 u32 *pte_addr, dma_addr_t pte_dma,
322 size_t size)
323 {
324 unsigned int pte_count;
325 unsigned int pte_total = size / SPAGE_SIZE;
326
327 for (pte_count = 0;
328 pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
329 u32 pte = pte_addr[pte_count];
330
331 if (!vsi_pte_is_page_valid(pte))
332 break;
333
334 pte_addr[pte_count] = vsi_mk_pte_invalid(pte);
335 }
336
337 vsi_table_flush(vsi_domain, pte_dma, pte_total);
338
339 return pte_count * SPAGE_SIZE;
340 }
341
vsi_iommu_map_iova(struct vsi_iommu_domain * vsi_domain,u32 * pte_addr,dma_addr_t pte_dma,dma_addr_t iova,phys_addr_t paddr,size_t size,int prot)342 static int vsi_iommu_map_iova(struct vsi_iommu_domain *vsi_domain, u32 *pte_addr,
343 dma_addr_t pte_dma, dma_addr_t iova,
344 phys_addr_t paddr, size_t size, int prot)
345 {
346 unsigned int pte_count;
347 unsigned int pte_total = size / SPAGE_SIZE;
348
349 for (pte_count = 0;
350 pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
351 u32 pte = pte_addr[pte_count];
352
353 if (vsi_pte_is_page_valid(pte))
354 return (pte_count - 1) * SPAGE_SIZE;
355
356 pte_addr[pte_count] = vsi_mk_pte(paddr, prot);
357
358 paddr += SPAGE_SIZE;
359 }
360
361 vsi_table_flush(vsi_domain, pte_dma, pte_total);
362
363 return 0;
364 }
365
vsi_iommu_flush_tlb(struct iommu_domain * domain)366 static void vsi_iommu_flush_tlb(struct iommu_domain *domain)
367 {
368 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
369 struct vsi_iommu *iommu;
370
371 list_for_each_entry(iommu, &vsi_domain->iommus, node) {
372 if (pm_runtime_get(iommu->dev) < 0)
373 continue;
374
375 spin_lock(&iommu->lock);
376
377 if (iommu->enable) {
378 writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
379 writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
380 }
381
382 spin_unlock(&iommu->lock);
383
384 pm_runtime_put_autosuspend(iommu->dev);
385 }
386 }
387
vsi_iommu_unmap(struct iommu_domain * domain,unsigned long _iova,size_t size,size_t count,struct iommu_iotlb_gather * gather)388 static size_t vsi_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
389 size_t size, size_t count, struct iommu_iotlb_gather *gather)
390 {
391 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
392 dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
393 unsigned long flags;
394 phys_addr_t pt_phys;
395 u32 dte;
396 u32 *pte_addr;
397 size_t unmap_size = 0;
398
399 spin_lock_irqsave(&vsi_domain->lock, flags);
400
401 dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
402 /* Just return 0 if iova is unmapped */
403 if (!vsi_dte_is_pt_valid(dte))
404 goto unlock;
405
406 pt_phys = vsi_dte_pt_address(dte);
407 pte_addr = (u32 *)phys_to_virt(pt_phys) + vsi_iova_pte_index(iova);
408 pte_dma = pt_phys + vsi_iova_pte_index(iova) * sizeof(u32);
409 unmap_size = vsi_iommu_unmap_iova(vsi_domain, pte_addr, pte_dma, size);
410 if (!unmap_size)
411 goto unlock;
412
413 vsi_iommu_flush_tlb(domain);
414 unlock:
415 spin_unlock_irqrestore(&vsi_domain->lock, flags);
416
417 return unmap_size;
418 }
419
vsi_dte_get_page_table(struct vsi_iommu_domain * vsi_domain,dma_addr_t iova,gfp_t gfp)420 static u32 *vsi_dte_get_page_table(struct vsi_iommu_domain *vsi_domain,
421 dma_addr_t iova, gfp_t gfp)
422 {
423 u32 *page_table, *dte_addr;
424 u32 dte_index, dte;
425 phys_addr_t pt_phys;
426 dma_addr_t pt_dma;
427 gfp_t flags;
428
429 dte_index = vsi_iova_dte_index(iova);
430 dte_addr = &vsi_domain->dt[dte_index];
431 dte = *dte_addr;
432 if (vsi_dte_is_pt_valid(dte))
433 goto done;
434
435 /* Do not allow to sleep while allocating the buffer */
436 flags = (gfp & ~GFP_KERNEL) | GFP_ATOMIC | GFP_DMA32;
437 page_table = iommu_alloc_pages_sz(flags, PAGE_SIZE);
438 if (!page_table)
439 return ERR_PTR(-ENOMEM);
440
441 pt_dma = dma_map_single(vsi_domain->dev, page_table, PAGE_SIZE, DMA_TO_DEVICE);
442 if (dma_mapping_error(vsi_domain->dev, pt_dma)) {
443 dev_err(vsi_domain->dev, "DMA mapping error while allocating page table\n");
444 iommu_free_pages(page_table);
445 return ERR_PTR(-ENOMEM);
446 }
447
448 dte = vsi_mk_dte(pt_dma);
449 *dte_addr = dte;
450
451 vsi_table_flush(vsi_domain,
452 vsi_domain->dt_dma + dte_index * sizeof(u32), 1);
453 done:
454 pt_phys = vsi_dte_pt_address(dte);
455 return (u32 *)phys_to_virt(pt_phys);
456 }
457
vsi_iommu_map(struct iommu_domain * domain,unsigned long _iova,phys_addr_t paddr,size_t size,size_t count,int prot,gfp_t gfp,size_t * mapped)458 static int vsi_iommu_map(struct iommu_domain *domain, unsigned long _iova,
459 phys_addr_t paddr, size_t size, size_t count,
460 int prot, gfp_t gfp, size_t *mapped)
461 {
462 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
463 dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
464 u32 *page_table, *pte_addr;
465 u32 dte, pte_index;
466 unsigned long flags;
467 int ret;
468
469 spin_lock_irqsave(&vsi_domain->lock, flags);
470
471 page_table = vsi_dte_get_page_table(vsi_domain, iova, gfp);
472 if (IS_ERR(page_table)) {
473 spin_unlock_irqrestore(&vsi_domain->lock, flags);
474 return PTR_ERR(page_table);
475 }
476
477 dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
478 pte_index = vsi_iova_pte_index(iova);
479 pte_addr = &page_table[pte_index];
480 pte_dma = vsi_dte_pt_address(dte) + pte_index * sizeof(u32);
481 ret = vsi_iommu_map_iova(vsi_domain, pte_addr, pte_dma, iova,
482 paddr, size, prot);
483 if (!ret)
484 *mapped = size;
485
486 vsi_iommu_flush_tlb(domain);
487
488 spin_unlock_irqrestore(&vsi_domain->lock, flags);
489
490 return ret;
491 }
492
vsi_iommu_disable(struct vsi_iommu * iommu)493 static void vsi_iommu_disable(struct vsi_iommu *iommu)
494 {
495 writel(0, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
496 iommu->enable = false;
497 }
498
vsi_iommu_identity_attach(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)499 static int vsi_iommu_identity_attach(struct iommu_domain *domain,
500 struct device *dev, struct iommu_domain *old)
501 {
502 struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
503 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
504 unsigned long flags;
505 int ret;
506
507 ret = pm_runtime_resume_and_get(iommu->dev);
508 if (ret < 0)
509 return ret;
510
511 spin_lock_irqsave(&vsi_domain->lock, flags);
512 spin_lock(&iommu->lock);
513 if (iommu->domain == domain)
514 goto unlock;
515
516 vsi_iommu_disable(iommu);
517 list_del_init(&iommu->node);
518
519 iommu->domain = domain;
520
521 unlock:
522 spin_unlock(&iommu->lock);
523 spin_unlock_irqrestore(&vsi_domain->lock, flags);
524 pm_runtime_put_autosuspend(iommu->dev);
525 return 0;
526 }
527
528 static const struct iommu_domain_ops vsi_identity_ops = {
529 .attach_dev = vsi_iommu_identity_attach,
530 };
531
532 static struct iommu_domain vsi_identity_domain = {
533 .type = IOMMU_DOMAIN_IDENTITY,
534 .ops = &vsi_identity_ops,
535 };
536
vsi_iommu_enable(struct vsi_iommu * iommu,struct iommu_domain * domain)537 static void vsi_iommu_enable(struct vsi_iommu *iommu, struct iommu_domain *domain)
538 {
539 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
540
541 if (domain == &vsi_identity_domain)
542 return;
543
544 writel(vsi_domain->pta_dma, iommu->regs + VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE);
545 writel(VSI_MMU_OUT_OF_BOUND, iommu->regs + VSI_MMU_CONFIG1_BASE);
546 writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_EXCEPTION_BASE);
547 writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
548 iommu->enable = true;
549 }
550
vsi_iommu_attach_device(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)551 static int vsi_iommu_attach_device(struct iommu_domain *domain,
552 struct device *dev, struct iommu_domain *old)
553 {
554 struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
555 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
556 unsigned long flags;
557 int ret = 0;
558
559 ret = pm_runtime_resume_and_get(iommu->dev);
560 if (ret < 0)
561 return ret;
562
563 spin_lock_irqsave(&vsi_domain->lock, flags);
564 spin_lock(&iommu->lock);
565
566 vsi_iommu_enable(iommu, domain);
567 writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
568 writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
569
570 list_del_init(&iommu->node);
571 list_add_tail(&iommu->node, &vsi_domain->iommus);
572
573 iommu->domain = domain;
574
575 spin_unlock(&iommu->lock);
576 spin_unlock_irqrestore(&vsi_domain->lock, flags);
577 pm_runtime_put_autosuspend(iommu->dev);
578 return ret;
579 }
580
vsi_iommu_domain_free(struct iommu_domain * domain)581 static void vsi_iommu_domain_free(struct iommu_domain *domain)
582 {
583 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
584 unsigned long flags;
585 int i;
586
587 spin_lock_irqsave(&vsi_domain->lock, flags);
588
589 WARN_ON(!list_empty(&vsi_domain->iommus));
590
591 for (i = 0; i < NUM_DT_ENTRIES; i++) {
592 u32 dte = vsi_domain->dt[i];
593
594 if (vsi_dte_is_pt_valid(dte)) {
595 phys_addr_t pt_phys = vsi_dte_pt_address(dte);
596 u32 *page_table = phys_to_virt(pt_phys);
597
598 dma_unmap_single(vsi_domain->dev, pt_phys,
599 SPAGE_SIZE, DMA_TO_DEVICE);
600 iommu_free_pages(page_table);
601 }
602 }
603
604 dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
605 SPAGE_SIZE, DMA_TO_DEVICE);
606 iommu_free_pages(vsi_domain->dt);
607
608 dma_unmap_single(vsi_domain->dev, vsi_domain->pta_dma,
609 SPAGE_SIZE, DMA_TO_DEVICE);
610 iommu_free_pages(vsi_domain->pta);
611
612 spin_unlock_irqrestore(&vsi_domain->lock, flags);
613
614 kfree(vsi_domain);
615 }
616
vsi_iommu_probe_device(struct device * dev)617 static struct iommu_device *vsi_iommu_probe_device(struct device *dev)
618 {
619 struct vsi_iommu *iommu = vsi_iommu_get_from_dev(dev);
620 struct device_link *link;
621
622 link = device_link_add(dev, iommu->dev,
623 DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
624 if (!link)
625 dev_err(dev, "Unable to link %s\n", dev_name(iommu->dev));
626
627 dev_iommu_priv_set(dev, iommu);
628 return &iommu->iommu;
629 }
630
vsi_iommu_release_device(struct device * dev)631 static void vsi_iommu_release_device(struct device *dev)
632 {
633 struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
634
635 device_link_remove(dev, iommu->dev);
636 }
637
vsi_iommu_of_xlate(struct device * dev,const struct of_phandle_args * args)638 static int vsi_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
639 {
640 return iommu_fwspec_add_ids(dev, args->args, 1);
641 }
642
643 static const struct iommu_ops vsi_iommu_ops = {
644 .identity_domain = &vsi_identity_domain,
645 .release_domain = &vsi_identity_domain,
646 .domain_alloc_paging = vsi_iommu_domain_alloc_paging,
647 .of_xlate = vsi_iommu_of_xlate,
648 .probe_device = vsi_iommu_probe_device,
649 .release_device = vsi_iommu_release_device,
650 .device_group = generic_single_device_group,
651 .owner = THIS_MODULE,
652 .default_domain_ops = &(const struct iommu_domain_ops) {
653 .attach_dev = vsi_iommu_attach_device,
654 .map_pages = vsi_iommu_map,
655 .unmap_pages = vsi_iommu_unmap,
656 .iova_to_phys = vsi_iommu_iova_to_phys,
657 .free = vsi_iommu_domain_free,
658 }
659 };
660
661 static const struct of_device_id vsi_iommu_dt_ids[] = {
662 {
663 .compatible = "verisilicon,iommu-1.2",
664 },
665 { /* sentinel */ }
666 };
667 MODULE_DEVICE_TABLE(of, vsi_iommu_dt_ids);
668
vsi_iommu_probe(struct platform_device * pdev)669 static int vsi_iommu_probe(struct platform_device *pdev)
670 {
671 struct device *dev = &pdev->dev;
672 struct vsi_iommu *iommu;
673 int err;
674
675 iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
676 if (!iommu)
677 return -ENOMEM;
678
679 iommu->dev = dev;
680 spin_lock_init(&iommu->lock);
681 INIT_LIST_HEAD(&iommu->node);
682
683 iommu->regs = devm_platform_ioremap_resource(pdev, 0);
684 if (IS_ERR(iommu->regs))
685 return -ENOMEM;
686
687 iommu->num_clocks = devm_clk_bulk_get_all(dev, &iommu->clocks);
688 if (iommu->num_clocks < 0)
689 return iommu->num_clocks;
690
691 err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);
692 if (err)
693 return err;
694
695 iommu->irq = platform_get_irq(pdev, 0);
696 if (iommu->irq < 0)
697 return iommu->irq;
698
699 err = devm_request_irq(iommu->dev, iommu->irq, vsi_iommu_irq,
700 IRQF_SHARED, dev_name(dev), iommu);
701 if (err)
702 goto err_unprepare_clocks;
703
704 dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
705 platform_set_drvdata(pdev, iommu);
706
707 pm_runtime_set_autosuspend_delay(dev, 100);
708 pm_runtime_use_autosuspend(dev);
709 pm_runtime_enable(dev);
710
711 err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, "%s",
712 dev_name(dev));
713 if (err)
714 goto err_runtime_disable;
715
716 err = iommu_device_register(&iommu->iommu, &vsi_iommu_ops, dev);
717 if (err)
718 goto err_remove_sysfs;
719
720 return 0;
721
722 err_remove_sysfs:
723 iommu_device_sysfs_remove(&iommu->iommu);
724 err_runtime_disable:
725 pm_runtime_disable(dev);
726 err_unprepare_clocks:
727 clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);
728 return err;
729 }
730
vsi_iommu_shutdown(struct platform_device * pdev)731 static void vsi_iommu_shutdown(struct platform_device *pdev)
732 {
733 struct vsi_iommu *iommu = platform_get_drvdata(pdev);
734
735 disable_irq(iommu->irq);
736 pm_runtime_force_suspend(&pdev->dev);
737 }
738
vsi_iommu_suspend(struct device * dev)739 static int __maybe_unused vsi_iommu_suspend(struct device *dev)
740 {
741 struct vsi_iommu *iommu = dev_get_drvdata(dev);
742
743 vsi_iommu_disable(iommu);
744
745 clk_bulk_disable(iommu->num_clocks, iommu->clocks);
746
747 return 0;
748 }
749
vsi_iommu_resume(struct device * dev)750 static int __maybe_unused vsi_iommu_resume(struct device *dev)
751 {
752 struct vsi_iommu *iommu = dev_get_drvdata(dev);
753 unsigned long flags;
754 int ret;
755
756 ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
757 if (ret)
758 return ret;
759
760 if (iommu->domain) {
761 struct vsi_iommu_domain *vsi_domain = to_vsi_domain(iommu->domain);
762
763 spin_lock_irqsave(&vsi_domain->lock, flags);
764 spin_lock(&iommu->lock);
765 vsi_iommu_enable(iommu, iommu->domain);
766 spin_unlock(&iommu->lock);
767 spin_unlock_irqrestore(&vsi_domain->lock, flags);
768 }
769
770 return 0;
771 }
772
773 static DEFINE_RUNTIME_DEV_PM_OPS(vsi_iommu_pm_ops,
774 vsi_iommu_suspend, vsi_iommu_resume,
775 NULL);
776
777 static struct platform_driver rockchip_vsi_iommu_driver = {
778 .probe = vsi_iommu_probe,
779 .shutdown = vsi_iommu_shutdown,
780 .driver = {
781 .name = "vsi_iommu",
782 .of_match_table = vsi_iommu_dt_ids,
783 .pm = pm_sleep_ptr(&vsi_iommu_pm_ops),
784 .suppress_bind_attrs = true,
785 },
786 };
787 module_platform_driver(rockchip_vsi_iommu_driver);
788
789 MODULE_LICENSE("GPL");
790 MODULE_AUTHOR("Benjamin Gaignard <benjamin.gaignard@collabora.com>");
791 MODULE_DESCRIPTION("Verisilicon IOMMU driver");
792