xref: /linux/drivers/iommu/vsi-iommu.c (revision 00599d4841790d05401820a96cf7edb193888b00) 
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2025 Collabora Ltd.
3  *
4  * IOMMU API for Verisilicon
5  *
6  * Module Authors:	Yandong Lin <yandong.lin@rock-chips.com>
7  *			Simon Xue <xxm@rock-chips.com>
8  *			Benjamin Gaignard <benjamin.gaignard@collabora.com>
9  *
10  * This hardware block is using a 2 pages tables allocation structure.
11  * That make very similar to Rockhip iommu hardware blocks but it has
12  * it own driver because the registers offset and configuration bits
13  * are completely different. An additional reason is that this hardware
14  * has been developed by Verisilicon to be used by their hardware video
15  * decoders and not for a general purpose like Rockchip iommus.
16  */
17 
18 #include <linux/clk.h>
19 #include <linux/compiler.h>
20 #include <linux/delay.h>
21 #include <linux/device.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/errno.h>
24 #include <linux/init.h>
25 #include <linux/interrupt.h>
26 #include <linux/io.h>
27 #include <linux/iommu.h>
28 #include <linux/list.h>
29 #include <linux/mm.h>
30 #include <linux/module.h>
31 #include <linux/of.h>
32 #include <linux/of_iommu.h>
33 #include <linux/of_platform.h>
34 #include <linux/platform_device.h>
35 #include <linux/pm_runtime.h>
36 #include <linux/slab.h>
37 #include <linux/spinlock.h>
38 
39 #include "iommu-pages.h"
40 
41 struct vsi_iommu {
42 	struct device *dev;
43 	void __iomem *regs;
44 	struct clk_bulk_data *clocks;
45 	int num_clocks;
46 	struct iommu_device iommu;
47 	struct list_head node; /* entry in vsi_iommu_domain.iommus */
48 	struct iommu_domain *domain; /* domain to which iommu is attached */
49 	spinlock_t lock; /* lock to protect vsi_iommu fields */
50 	int irq;
51 	bool enable;
52 };
53 
54 struct vsi_iommu_domain {
55 	struct list_head iommus;
56 	struct device *dev;
57 	u32 *dt;
58 	dma_addr_t dt_dma;
59 	struct iommu_domain domain;
60 	u64 *pta;
61 	dma_addr_t pta_dma;
62 	spinlock_t lock; /* lock to protect vsi_iommu_domain fields */
63 };
64 
65 static struct iommu_domain vsi_identity_domain;
66 
67 #define NUM_DT_ENTRIES	1024
68 #define NUM_PT_ENTRIES	1024
69 
70 #define SPAGE_SIZE	BIT(12)
71 
72 /* vsi iommu regs address */
73 #define VSI_MMU_CONFIG1_BASE			0x1ac
74 #define VSI_MMU_AHB_EXCEPTION_BASE		0x380
75 #define VSI_MMU_AHB_CONTROL_BASE		0x388
76 #define VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE	0x38C
77 
78 /* MMU register offsets */
79 #define VSI_MMU_FLUSH_BASE		0x184
80 #define VSI_MMU_BIT_FLUSH		BIT(4)
81 
82 #define VSI_MMU_PAGE_FAULT_ADDR		0x380
83 #define VSI_MMU_STATUS_BASE		0x384	/* IRQ status */
84 
85 #define VSI_MMU_BIT_ENABLE		BIT(0)
86 
87 #define VSI_MMU_OUT_OF_BOUND		BIT(28)
88 /* Irq mask */
89 #define VSI_MMU_IRQ_MASK		0x7
90 
91 #define VSI_DTE_PT_ADDRESS_MASK		0xffffffc0
92 #define VSI_DTE_PT_VALID		BIT(0)
93 
94 #define VSI_PAGE_DESC_LO_MASK		0xfffff000
95 #define VSI_PAGE_DESC_HI_MASK		GENMASK_ULL(39, 32)
96 #define VSI_PAGE_DESC_HI_SHIFT		(32 - 4)
97 
98 static inline phys_addr_t vsi_dte_pt_address(u32 dte)
99 {
100 	return (phys_addr_t)dte & VSI_DTE_PT_ADDRESS_MASK;
101 }
102 
103 static inline u32 vsi_mk_dte(u32 dte)
104 {
105 	return (phys_addr_t)dte | VSI_DTE_PT_VALID;
106 }
107 
108 #define VSI_PTE_PAGE_WRITABLE		BIT(2)
109 #define VSI_PTE_PAGE_VALID		BIT(0)
110 
111 static inline phys_addr_t vsi_pte_page_address(u64 pte)
112 {
113 	return ((pte << VSI_PAGE_DESC_HI_SHIFT) & VSI_PAGE_DESC_HI_MASK) |
114 	       (pte & VSI_PAGE_DESC_LO_MASK);
115 }
116 
117 static u32 vsi_mk_pte(phys_addr_t page, int prot)
118 {
119 	u32 flags = 0;
120 
121 	flags |= (prot & IOMMU_WRITE) ? VSI_PTE_PAGE_WRITABLE : 0;
122 
123 	page = (page & VSI_PAGE_DESC_LO_MASK) |
124 	       ((page & VSI_PAGE_DESC_HI_MASK) >> VSI_PAGE_DESC_HI_SHIFT);
125 
126 	return page | flags | VSI_PTE_PAGE_VALID;
127 }
128 
129 #define VSI_DTE_PT_VALID	BIT(0)
130 
131 static inline bool vsi_dte_is_pt_valid(u32 dte)
132 {
133 	return dte & VSI_DTE_PT_VALID;
134 }
135 
136 static inline bool vsi_pte_is_page_valid(u32 pte)
137 {
138 	return pte & VSI_PTE_PAGE_VALID;
139 }
140 
141 static u32 vsi_mk_pte_invalid(u32 pte)
142 {
143 	return pte & ~VSI_PTE_PAGE_VALID;
144 }
145 
146 #define VSI_MASTER_TLB_MASK	GENMASK_ULL(31, 10)
147 /* mode 0 : 4k */
148 #define VSI_PTA_4K_MODE	0
149 
150 static u64 vsi_mk_pta(dma_addr_t dt_dma)
151 {
152 	u64 val = (dt_dma & VSI_MASTER_TLB_MASK) | VSI_PTA_4K_MODE;
153 
154 	return val;
155 }
156 
157 static struct vsi_iommu_domain *to_vsi_domain(struct iommu_domain *dom)
158 {
159 	return container_of(dom, struct vsi_iommu_domain, domain);
160 }
161 
162 static inline void vsi_table_flush(struct vsi_iommu_domain *vsi_domain, dma_addr_t dma,
163 				   unsigned int count)
164 {
165 	size_t size = count * sizeof(u32); /* count of u32 entry */
166 
167 	dma_sync_single_for_device(vsi_domain->dev, dma, size, DMA_TO_DEVICE);
168 }
169 
170 #define VSI_IOVA_DTE_MASK	0xffc00000
171 #define VSI_IOVA_DTE_SHIFT	22
172 #define VSI_IOVA_PTE_MASK	0x003ff000
173 #define VSI_IOVA_PTE_SHIFT	12
174 #define VSI_IOVA_PAGE_MASK	0x00000fff
175 #define VSI_IOVA_PAGE_SHIFT	0
176 
177 static u32 vsi_iova_dte_index(u32 iova)
178 {
179 	return (iova & VSI_IOVA_DTE_MASK) >> VSI_IOVA_DTE_SHIFT;
180 }
181 
182 static u32 vsi_iova_pte_index(u32 iova)
183 {
184 	return (iova & VSI_IOVA_PTE_MASK) >> VSI_IOVA_PTE_SHIFT;
185 }
186 
187 static u32 vsi_iova_page_offset(u32 iova)
188 {
189 	return (iova & VSI_IOVA_PAGE_MASK) >> VSI_IOVA_PAGE_SHIFT;
190 }
191 
192 static irqreturn_t vsi_iommu_irq(int irq, void *dev_id)
193 {
194 	struct vsi_iommu *iommu = dev_id;
195 	unsigned long flags;
196 	dma_addr_t iova;
197 	u32 status;
198 
199 	if (pm_runtime_resume_and_get(iommu->dev) < 0)
200 		return IRQ_NONE;
201 
202 	spin_lock_irqsave(&iommu->lock, flags);
203 
204 	status = readl(iommu->regs + VSI_MMU_STATUS_BASE);
205 	if (status & VSI_MMU_IRQ_MASK) {
206 		dev_err(iommu->dev, "unexpected int_status=%08x\n", status);
207 		iova = readl(iommu->regs + VSI_MMU_PAGE_FAULT_ADDR);
208 		report_iommu_fault(iommu->domain, iommu->dev, iova, status);
209 	}
210 	writel(0, iommu->regs + VSI_MMU_STATUS_BASE);
211 
212 	spin_unlock_irqrestore(&iommu->lock, flags);
213 	pm_runtime_put_autosuspend(iommu->dev);
214 
215 	return IRQ_HANDLED;
216 }
217 
218 static struct vsi_iommu *vsi_iommu_get_from_dev(struct device *dev)
219 {
220 	struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
221 	struct device *iommu_dev = bus_find_device_by_fwnode(&platform_bus_type,
222 							     fwspec->iommu_fwnode);
223 
224 	put_device(iommu_dev);
225 
226 	return iommu_dev ? dev_get_drvdata(iommu_dev) : NULL;
227 }
228 
229 static struct iommu_domain *vsi_iommu_domain_alloc_paging(struct device *dev)
230 {
231 	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
232 	struct vsi_iommu_domain *vsi_domain;
233 
234 	vsi_domain = kzalloc(sizeof(*vsi_domain), GFP_KERNEL);
235 	if (!vsi_domain)
236 		return NULL;
237 
238 	vsi_domain->dev = iommu->dev;
239 	spin_lock_init(&vsi_domain->lock);
240 
241 	/*
242 	 * iommu use a 2 level pagetable.
243 	 * Each level1 (dt) and level2 (pt) table has 1024 4-byte entries.
244 	 * Allocate one 4 KiB page for each table.
245 	 */
246 	vsi_domain->dt = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
247 					      SPAGE_SIZE);
248 	if (!vsi_domain->dt)
249 		goto err_free_domain;
250 
251 	vsi_domain->dt_dma = dma_map_single(vsi_domain->dev, vsi_domain->dt,
252 					    SPAGE_SIZE, DMA_TO_DEVICE);
253 	if (dma_mapping_error(vsi_domain->dev, vsi_domain->dt_dma)) {
254 		dev_err(dev, "DMA map error for DT\n");
255 		goto err_free_dt;
256 	}
257 
258 	vsi_domain->pta = iommu_alloc_pages_sz(GFP_KERNEL | GFP_DMA32,
259 					       SPAGE_SIZE);
260 	if (!vsi_domain->pta)
261 		goto err_unmap_dt;
262 
263 	vsi_domain->pta[0] = vsi_mk_pta(vsi_domain->dt_dma);
264 	vsi_domain->pta_dma = dma_map_single(vsi_domain->dev, vsi_domain->pta,
265 					     SPAGE_SIZE, DMA_TO_DEVICE);
266 	if (dma_mapping_error(vsi_domain->dev, vsi_domain->pta_dma)) {
267 		dev_err(dev, "DMA map error for PTA\n");
268 		goto err_free_pta;
269 	}
270 
271 	INIT_LIST_HEAD(&vsi_domain->iommus);
272 
273 	vsi_domain->domain.geometry.aperture_start = 0;
274 	vsi_domain->domain.geometry.aperture_end   = DMA_BIT_MASK(32);
275 	vsi_domain->domain.geometry.force_aperture = true;
276 	vsi_domain->domain.pgsize_bitmap	   = SZ_4K;
277 
278 	return &vsi_domain->domain;
279 
280 err_free_pta:
281 	iommu_free_pages(vsi_domain->pta);
282 err_unmap_dt:
283 	dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
284 			 SPAGE_SIZE, DMA_TO_DEVICE);
285 err_free_dt:
286 	iommu_free_pages(vsi_domain->dt);
287 err_free_domain:
288 	kfree(vsi_domain);
289 
290 	return NULL;
291 }
292 
293 static phys_addr_t vsi_iommu_iova_to_phys(struct iommu_domain *domain,
294 					  dma_addr_t iova)
295 {
296 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
297 	phys_addr_t pt_phys, phys = 0;
298 	unsigned long flags;
299 	u32 dte, pte;
300 	u32 *page_table;
301 
302 	spin_lock_irqsave(&vsi_domain->lock, flags);
303 	dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
304 	if (!vsi_dte_is_pt_valid(dte))
305 		goto unlock;
306 
307 	pt_phys = vsi_dte_pt_address(dte);
308 	page_table = (u32 *)phys_to_virt(pt_phys);
309 	pte = page_table[vsi_iova_pte_index(iova)];
310 	if (!vsi_pte_is_page_valid(pte))
311 		goto unlock;
312 
313 	phys = vsi_pte_page_address(pte) + vsi_iova_page_offset(iova);
314 
315 unlock:
316 	spin_unlock_irqrestore(&vsi_domain->lock, flags);
317 	return phys;
318 }
319 
320 static size_t vsi_iommu_unmap_iova(struct vsi_iommu_domain *vsi_domain,
321 				   u32 *pte_addr, dma_addr_t pte_dma,
322 				   size_t size)
323 {
324 	unsigned int pte_count;
325 	unsigned int pte_total = size / SPAGE_SIZE;
326 
327 	for (pte_count = 0;
328 	     pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
329 		u32 pte = pte_addr[pte_count];
330 
331 		if (!vsi_pte_is_page_valid(pte))
332 			break;
333 
334 		pte_addr[pte_count] = vsi_mk_pte_invalid(pte);
335 	}
336 
337 	vsi_table_flush(vsi_domain, pte_dma, pte_total);
338 
339 	return pte_count * SPAGE_SIZE;
340 }
341 
342 static int vsi_iommu_map_iova(struct vsi_iommu_domain *vsi_domain, u32 *pte_addr,
343 			      dma_addr_t pte_dma, dma_addr_t iova,
344 			      phys_addr_t paddr, size_t size, int prot)
345 {
346 	unsigned int pte_count;
347 	unsigned int pte_total = size / SPAGE_SIZE;
348 
349 	for (pte_count = 0;
350 	     pte_count < pte_total && pte_count < NUM_PT_ENTRIES; pte_count++) {
351 		u32 pte = pte_addr[pte_count];
352 
353 		if (vsi_pte_is_page_valid(pte))
354 			return (pte_count - 1) * SPAGE_SIZE;
355 
356 		pte_addr[pte_count] = vsi_mk_pte(paddr, prot);
357 
358 		paddr += SPAGE_SIZE;
359 	}
360 
361 	vsi_table_flush(vsi_domain, pte_dma, pte_total);
362 
363 	return 0;
364 }
365 
366 static void vsi_iommu_flush_tlb(struct iommu_domain *domain)
367 {
368 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
369 	struct vsi_iommu *iommu;
370 
371 	list_for_each_entry(iommu, &vsi_domain->iommus, node) {
372 		if (pm_runtime_get(iommu->dev) < 0)
373 			continue;
374 
375 		spin_lock(&iommu->lock);
376 
377 		if (iommu->enable) {
378 			writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
379 			writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
380 		}
381 
382 		spin_unlock(&iommu->lock);
383 
384 		pm_runtime_put_autosuspend(iommu->dev);
385 	}
386 }
387 
388 static size_t vsi_iommu_unmap(struct iommu_domain *domain, unsigned long _iova,
389 			      size_t size, size_t count, struct iommu_iotlb_gather *gather)
390 {
391 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
392 	dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
393 	unsigned long flags;
394 	phys_addr_t pt_phys;
395 	u32 dte;
396 	u32 *pte_addr;
397 	size_t unmap_size = 0;
398 
399 	spin_lock_irqsave(&vsi_domain->lock, flags);
400 
401 	dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
402 	/* Just return 0 if iova is unmapped */
403 	if (!vsi_dte_is_pt_valid(dte))
404 		goto unlock;
405 
406 	pt_phys = vsi_dte_pt_address(dte);
407 	pte_addr = (u32 *)phys_to_virt(pt_phys) + vsi_iova_pte_index(iova);
408 	pte_dma = pt_phys + vsi_iova_pte_index(iova) * sizeof(u32);
409 	unmap_size = vsi_iommu_unmap_iova(vsi_domain, pte_addr, pte_dma, size);
410 	if (!unmap_size)
411 		goto unlock;
412 
413 	vsi_iommu_flush_tlb(domain);
414 unlock:
415 	spin_unlock_irqrestore(&vsi_domain->lock, flags);
416 
417 	return unmap_size;
418 }
419 
420 static u32 *vsi_dte_get_page_table(struct vsi_iommu_domain *vsi_domain,
421 				   dma_addr_t iova, gfp_t gfp)
422 {
423 	u32 *page_table, *dte_addr;
424 	u32 dte_index, dte;
425 	phys_addr_t pt_phys;
426 	dma_addr_t pt_dma;
427 	gfp_t flags;
428 
429 	dte_index = vsi_iova_dte_index(iova);
430 	dte_addr = &vsi_domain->dt[dte_index];
431 	dte = *dte_addr;
432 	if (vsi_dte_is_pt_valid(dte))
433 		goto done;
434 
435 	/* Do not allow to sleep while allocating the buffer */
436 	flags = (gfp & ~GFP_KERNEL) | GFP_ATOMIC | GFP_DMA32;
437 	page_table = iommu_alloc_pages_sz(flags, PAGE_SIZE);
438 	if (!page_table)
439 		return ERR_PTR(-ENOMEM);
440 
441 	pt_dma = dma_map_single(vsi_domain->dev, page_table, PAGE_SIZE, DMA_TO_DEVICE);
442 	if (dma_mapping_error(vsi_domain->dev, pt_dma)) {
443 		dev_err(vsi_domain->dev, "DMA mapping error while allocating page table\n");
444 		iommu_free_pages(page_table);
445 		return ERR_PTR(-ENOMEM);
446 	}
447 
448 	dte = vsi_mk_dte(pt_dma);
449 	*dte_addr = dte;
450 
451 	vsi_table_flush(vsi_domain,
452 			vsi_domain->dt_dma + dte_index * sizeof(u32), 1);
453 done:
454 	pt_phys = vsi_dte_pt_address(dte);
455 	return (u32 *)phys_to_virt(pt_phys);
456 }
457 
458 static int vsi_iommu_map(struct iommu_domain *domain, unsigned long _iova,
459 			 phys_addr_t paddr, size_t size, size_t count,
460 			 int prot, gfp_t gfp, size_t *mapped)
461 {
462 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
463 	dma_addr_t pte_dma, iova = (dma_addr_t)_iova;
464 	u32 *page_table, *pte_addr;
465 	u32 dte, pte_index;
466 	unsigned long flags;
467 	int ret;
468 
469 	spin_lock_irqsave(&vsi_domain->lock, flags);
470 
471 	page_table = vsi_dte_get_page_table(vsi_domain, iova, gfp);
472 	if (IS_ERR(page_table)) {
473 		spin_unlock_irqrestore(&vsi_domain->lock, flags);
474 		return PTR_ERR(page_table);
475 	}
476 
477 	dte = vsi_domain->dt[vsi_iova_dte_index(iova)];
478 	pte_index = vsi_iova_pte_index(iova);
479 	pte_addr = &page_table[pte_index];
480 	pte_dma = vsi_dte_pt_address(dte) + pte_index * sizeof(u32);
481 	ret = vsi_iommu_map_iova(vsi_domain, pte_addr, pte_dma, iova,
482 				 paddr, size, prot);
483 	if (!ret)
484 		*mapped = size;
485 
486 	vsi_iommu_flush_tlb(domain);
487 
488 	spin_unlock_irqrestore(&vsi_domain->lock, flags);
489 
490 	return ret;
491 }
492 
493 static void vsi_iommu_disable(struct vsi_iommu *iommu)
494 {
495 	writel(0, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
496 	iommu->enable = false;
497 }
498 
499 static int vsi_iommu_identity_attach(struct iommu_domain *domain,
500 				     struct device *dev, struct iommu_domain *old)
501 {
502 	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
503 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
504 	unsigned long flags;
505 	int ret;
506 
507 	ret = pm_runtime_resume_and_get(iommu->dev);
508 	if (ret < 0)
509 		return ret;
510 
511 	spin_lock_irqsave(&vsi_domain->lock, flags);
512 	spin_lock(&iommu->lock);
513 	if (iommu->domain == domain)
514 		goto unlock;
515 
516 	vsi_iommu_disable(iommu);
517 	list_del_init(&iommu->node);
518 
519 	iommu->domain = domain;
520 
521 unlock:
522 	spin_unlock(&iommu->lock);
523 	spin_unlock_irqrestore(&vsi_domain->lock, flags);
524 	pm_runtime_put_autosuspend(iommu->dev);
525 	return 0;
526 }
527 
528 static const struct iommu_domain_ops vsi_identity_ops = {
529 	.attach_dev = vsi_iommu_identity_attach,
530 };
531 
532 static struct iommu_domain vsi_identity_domain = {
533 	.type = IOMMU_DOMAIN_IDENTITY,
534 	.ops = &vsi_identity_ops,
535 };
536 
537 static void vsi_iommu_enable(struct vsi_iommu *iommu, struct iommu_domain *domain)
538 {
539 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
540 
541 	if (domain == &vsi_identity_domain)
542 		return;
543 
544 	writel(vsi_domain->pta_dma, iommu->regs + VSI_MMU_AHB_TLB_ARRAY_BASE_L_BASE);
545 	writel(VSI_MMU_OUT_OF_BOUND, iommu->regs + VSI_MMU_CONFIG1_BASE);
546 	writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_EXCEPTION_BASE);
547 	writel(VSI_MMU_BIT_ENABLE, iommu->regs + VSI_MMU_AHB_CONTROL_BASE);
548 	iommu->enable = true;
549 }
550 
551 static int vsi_iommu_attach_device(struct iommu_domain *domain,
552 				   struct device *dev, struct iommu_domain *old)
553 {
554 	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
555 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
556 	unsigned long flags;
557 	int ret = 0;
558 
559 	ret = pm_runtime_resume_and_get(iommu->dev);
560 	if (ret < 0)
561 		return ret;
562 
563 	spin_lock_irqsave(&vsi_domain->lock, flags);
564 	spin_lock(&iommu->lock);
565 
566 	vsi_iommu_enable(iommu, domain);
567 	writel(VSI_MMU_BIT_FLUSH, iommu->regs + VSI_MMU_FLUSH_BASE);
568 	writel(0, iommu->regs + VSI_MMU_FLUSH_BASE);
569 
570 	list_del_init(&iommu->node);
571 	list_add_tail(&iommu->node, &vsi_domain->iommus);
572 
573 	iommu->domain = domain;
574 
575 	spin_unlock(&iommu->lock);
576 	spin_unlock_irqrestore(&vsi_domain->lock, flags);
577 	pm_runtime_put_autosuspend(iommu->dev);
578 	return ret;
579 }
580 
581 static void vsi_iommu_domain_free(struct iommu_domain *domain)
582 {
583 	struct vsi_iommu_domain *vsi_domain = to_vsi_domain(domain);
584 	unsigned long flags;
585 	int i;
586 
587 	spin_lock_irqsave(&vsi_domain->lock, flags);
588 
589 	WARN_ON(!list_empty(&vsi_domain->iommus));
590 
591 	for (i = 0; i < NUM_DT_ENTRIES; i++) {
592 		u32 dte = vsi_domain->dt[i];
593 
594 		if (vsi_dte_is_pt_valid(dte)) {
595 			phys_addr_t pt_phys = vsi_dte_pt_address(dte);
596 			u32 *page_table = phys_to_virt(pt_phys);
597 
598 			dma_unmap_single(vsi_domain->dev, pt_phys,
599 					 SPAGE_SIZE, DMA_TO_DEVICE);
600 			iommu_free_pages(page_table);
601 		}
602 	}
603 
604 	dma_unmap_single(vsi_domain->dev, vsi_domain->dt_dma,
605 			 SPAGE_SIZE, DMA_TO_DEVICE);
606 	iommu_free_pages(vsi_domain->dt);
607 
608 	dma_unmap_single(vsi_domain->dev, vsi_domain->pta_dma,
609 			 SPAGE_SIZE, DMA_TO_DEVICE);
610 	iommu_free_pages(vsi_domain->pta);
611 
612 	spin_unlock_irqrestore(&vsi_domain->lock, flags);
613 
614 	kfree(vsi_domain);
615 }
616 
617 static struct iommu_device *vsi_iommu_probe_device(struct device *dev)
618 {
619 	struct vsi_iommu *iommu = vsi_iommu_get_from_dev(dev);
620 	struct device_link *link;
621 
622 	link = device_link_add(dev, iommu->dev,
623 			       DL_FLAG_STATELESS | DL_FLAG_PM_RUNTIME);
624 	if (!link)
625 		dev_err(dev, "Unable to link %s\n", dev_name(iommu->dev));
626 
627 	dev_iommu_priv_set(dev, iommu);
628 	return &iommu->iommu;
629 }
630 
631 static void vsi_iommu_release_device(struct device *dev)
632 {
633 	struct vsi_iommu *iommu = dev_iommu_priv_get(dev);
634 
635 	device_link_remove(dev, iommu->dev);
636 }
637 
638 static int vsi_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
639 {
640 	return iommu_fwspec_add_ids(dev, args->args, 1);
641 }
642 
643 static const struct iommu_ops vsi_iommu_ops = {
644 	.identity_domain = &vsi_identity_domain,
645 	.release_domain = &vsi_identity_domain,
646 	.domain_alloc_paging = vsi_iommu_domain_alloc_paging,
647 	.of_xlate = vsi_iommu_of_xlate,
648 	.probe_device = vsi_iommu_probe_device,
649 	.release_device = vsi_iommu_release_device,
650 	.device_group = generic_single_device_group,
651 	.owner = THIS_MODULE,
652 	.default_domain_ops = &(const struct iommu_domain_ops) {
653 		.attach_dev		= vsi_iommu_attach_device,
654 		.map_pages		= vsi_iommu_map,
655 		.unmap_pages		= vsi_iommu_unmap,
656 		.iova_to_phys		= vsi_iommu_iova_to_phys,
657 		.free			= vsi_iommu_domain_free,
658 	}
659 };
660 
661 static const struct of_device_id vsi_iommu_dt_ids[] = {
662 	{
663 		.compatible = "verisilicon,iommu-1.2",
664 	},
665 	{ /* sentinel */ }
666 };
667 MODULE_DEVICE_TABLE(of, vsi_iommu_dt_ids);
668 
669 static int vsi_iommu_probe(struct platform_device *pdev)
670 {
671 	struct device *dev = &pdev->dev;
672 	struct vsi_iommu *iommu;
673 	int err;
674 
675 	iommu = devm_kzalloc(dev, sizeof(*iommu), GFP_KERNEL);
676 	if (!iommu)
677 		return -ENOMEM;
678 
679 	iommu->dev = dev;
680 	spin_lock_init(&iommu->lock);
681 	INIT_LIST_HEAD(&iommu->node);
682 
683 	iommu->regs = devm_platform_ioremap_resource(pdev, 0);
684 	if (IS_ERR(iommu->regs))
685 		return -ENOMEM;
686 
687 	iommu->num_clocks = devm_clk_bulk_get_all(dev, &iommu->clocks);
688 	if  (iommu->num_clocks < 0)
689 		return iommu->num_clocks;
690 
691 	err = clk_bulk_prepare(iommu->num_clocks, iommu->clocks);
692 	if (err)
693 		return err;
694 
695 	iommu->irq = platform_get_irq(pdev, 0);
696 	if (iommu->irq < 0)
697 		return iommu->irq;
698 
699 	err = devm_request_irq(iommu->dev, iommu->irq, vsi_iommu_irq,
700 			       IRQF_SHARED, dev_name(dev), iommu);
701 	if (err)
702 		goto err_unprepare_clocks;
703 
704 	dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
705 	platform_set_drvdata(pdev, iommu);
706 
707 	pm_runtime_set_autosuspend_delay(dev, 100);
708 	pm_runtime_use_autosuspend(dev);
709 	pm_runtime_enable(dev);
710 
711 	err = iommu_device_sysfs_add(&iommu->iommu, dev, NULL, "%s",
712 				     dev_name(dev));
713 	if (err)
714 		goto err_runtime_disable;
715 
716 	err = iommu_device_register(&iommu->iommu, &vsi_iommu_ops, dev);
717 	if (err)
718 		goto err_remove_sysfs;
719 
720 	return 0;
721 
722 err_remove_sysfs:
723 	iommu_device_sysfs_remove(&iommu->iommu);
724 err_runtime_disable:
725 	pm_runtime_disable(dev);
726 err_unprepare_clocks:
727 	clk_bulk_unprepare(iommu->num_clocks, iommu->clocks);
728 	return err;
729 }
730 
731 static void vsi_iommu_shutdown(struct platform_device *pdev)
732 {
733 	struct vsi_iommu *iommu = platform_get_drvdata(pdev);
734 
735 	disable_irq(iommu->irq);
736 	pm_runtime_force_suspend(&pdev->dev);
737 }
738 
739 static int __maybe_unused vsi_iommu_suspend(struct device *dev)
740 {
741 	struct vsi_iommu *iommu = dev_get_drvdata(dev);
742 
743 	vsi_iommu_disable(iommu);
744 
745 	clk_bulk_disable(iommu->num_clocks, iommu->clocks);
746 
747 	return 0;
748 }
749 
750 static int __maybe_unused vsi_iommu_resume(struct device *dev)
751 {
752 	struct vsi_iommu *iommu = dev_get_drvdata(dev);
753 	unsigned long flags;
754 	int ret;
755 
756 	ret = clk_bulk_enable(iommu->num_clocks, iommu->clocks);
757 	if (ret)
758 		return ret;
759 
760 	if (iommu->domain) {
761 		struct vsi_iommu_domain *vsi_domain = to_vsi_domain(iommu->domain);
762 
763 		spin_lock_irqsave(&vsi_domain->lock, flags);
764 		spin_lock(&iommu->lock);
765 		vsi_iommu_enable(iommu, iommu->domain);
766 		spin_unlock(&iommu->lock);
767 		spin_unlock_irqrestore(&vsi_domain->lock, flags);
768 	}
769 
770 	return 0;
771 }
772 
773 static DEFINE_RUNTIME_DEV_PM_OPS(vsi_iommu_pm_ops,
774 				 vsi_iommu_suspend, vsi_iommu_resume,
775 				 NULL);
776 
777 static struct platform_driver rockchip_vsi_iommu_driver = {
778 	.probe = vsi_iommu_probe,
779 	.shutdown = vsi_iommu_shutdown,
780 	.driver = {
781 		   .name = "vsi_iommu",
782 		   .of_match_table = vsi_iommu_dt_ids,
783 		   .pm = pm_sleep_ptr(&vsi_iommu_pm_ops),
784 		   .suppress_bind_attrs = true,
785 	},
786 };
787 module_platform_driver(rockchip_vsi_iommu_driver);
788 
789 MODULE_LICENSE("GPL");
790 MODULE_AUTHOR("Benjamin Gaignard <benjamin.gaignard@collabora.com>");
791 MODULE_DESCRIPTION("Verisilicon IOMMU driver");
792