xref: /linux/drivers/vfio/vfio_iommu_spapr_tce.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * VFIO: IOMMU DMA mapping support for TCE on POWER
4  *
5  * Copyright (C) 2013 IBM Corp.  All rights reserved.
6  *     Author: Alexey Kardashevskiy <aik@ozlabs.ru>
7  * Copyright Gavin Shan, IBM Corporation 2014.
8  *
9  * Derived from original vfio_iommu_type1.c:
10  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
11  *     Author: Alex Williamson <alex.williamson@redhat.com>
12  */
13 
14 #include <linux/module.h>
15 #include <linux/pci.h>
16 #include <linux/slab.h>
17 #include <linux/uaccess.h>
18 #include <linux/err.h>
19 #include <linux/vfio.h>
20 #include <linux/vmalloc.h>
21 #include <linux/sched/mm.h>
22 #include <linux/sched/signal.h>
23 #include <linux/mm.h>
24 #include "vfio.h"
25 
26 #include <asm/iommu.h>
27 #include <asm/tce.h>
28 #include <asm/mmu_context.h>
29 
30 #define DRIVER_VERSION  "0.1"
31 #define DRIVER_AUTHOR   "aik@ozlabs.ru"
32 #define DRIVER_DESC     "VFIO IOMMU SPAPR TCE"
33 
34 static void tce_iommu_detach_group(void *iommu_data,
35 		struct iommu_group *iommu_group);
36 
37 /*
38  * VFIO IOMMU fd for SPAPR_TCE IOMMU implementation
39  *
40  * This code handles mapping and unmapping of user data buffers
41  * into DMA'ble space using the IOMMU
42  */
43 
44 struct tce_iommu_group {
45 	struct list_head next;
46 	struct iommu_group *grp;
47 };
48 
49 /*
50  * A container needs to remember which preregistered region  it has
51  * referenced to do proper cleanup at the userspace process exit.
52  */
53 struct tce_iommu_prereg {
54 	struct list_head next;
55 	struct mm_iommu_table_group_mem_t *mem;
56 };
57 
58 /*
59  * The container descriptor supports only a single group per container.
60  * Required by the API as the container is not supplied with the IOMMU group
61  * at the moment of initialization.
62  */
63 struct tce_container {
64 	struct mutex lock;
65 	bool enabled;
66 	bool v2;
67 	bool def_window_pending;
68 	unsigned long locked_pages;
69 	struct mm_struct *mm;
70 	struct iommu_table *tables[IOMMU_TABLE_GROUP_MAX_TABLES];
71 	struct list_head group_list;
72 	struct list_head prereg_list;
73 };
74 
tce_iommu_mm_set(struct tce_container * container)75 static long tce_iommu_mm_set(struct tce_container *container)
76 {
77 	if (container->mm) {
78 		if (container->mm == current->mm)
79 			return 0;
80 		return -EPERM;
81 	}
82 	BUG_ON(!current->mm);
83 	container->mm = current->mm;
84 	mmgrab(container->mm);
85 
86 	return 0;
87 }
88 
tce_iommu_prereg_free(struct tce_container * container,struct tce_iommu_prereg * tcemem)89 static long tce_iommu_prereg_free(struct tce_container *container,
90 		struct tce_iommu_prereg *tcemem)
91 {
92 	long ret;
93 
94 	ret = mm_iommu_put(container->mm, tcemem->mem);
95 	if (ret)
96 		return ret;
97 
98 	list_del(&tcemem->next);
99 	kfree(tcemem);
100 
101 	return 0;
102 }
103 
tce_iommu_unregister_pages(struct tce_container * container,__u64 vaddr,__u64 size)104 static long tce_iommu_unregister_pages(struct tce_container *container,
105 		__u64 vaddr, __u64 size)
106 {
107 	struct mm_iommu_table_group_mem_t *mem;
108 	struct tce_iommu_prereg *tcemem;
109 	bool found = false;
110 	long ret;
111 
112 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK))
113 		return -EINVAL;
114 
115 	mem = mm_iommu_get(container->mm, vaddr, size >> PAGE_SHIFT);
116 	if (!mem)
117 		return -ENOENT;
118 
119 	list_for_each_entry(tcemem, &container->prereg_list, next) {
120 		if (tcemem->mem == mem) {
121 			found = true;
122 			break;
123 		}
124 	}
125 
126 	if (!found)
127 		ret = -ENOENT;
128 	else
129 		ret = tce_iommu_prereg_free(container, tcemem);
130 
131 	mm_iommu_put(container->mm, mem);
132 
133 	return ret;
134 }
135 
tce_iommu_register_pages(struct tce_container * container,__u64 vaddr,__u64 size)136 static long tce_iommu_register_pages(struct tce_container *container,
137 		__u64 vaddr, __u64 size)
138 {
139 	long ret = 0;
140 	struct mm_iommu_table_group_mem_t *mem = NULL;
141 	struct tce_iommu_prereg *tcemem;
142 	unsigned long entries = size >> PAGE_SHIFT;
143 
144 	if ((vaddr & ~PAGE_MASK) || (size & ~PAGE_MASK) ||
145 			((vaddr + size) < vaddr))
146 		return -EINVAL;
147 
148 	mem = mm_iommu_get(container->mm, vaddr, entries);
149 	if (mem) {
150 		list_for_each_entry(tcemem, &container->prereg_list, next) {
151 			if (tcemem->mem == mem) {
152 				ret = -EBUSY;
153 				goto put_exit;
154 			}
155 		}
156 	} else {
157 		ret = mm_iommu_new(container->mm, vaddr, entries, &mem);
158 		if (ret)
159 			return ret;
160 	}
161 
162 	tcemem = kzalloc(sizeof(*tcemem), GFP_KERNEL);
163 	if (!tcemem) {
164 		ret = -ENOMEM;
165 		goto put_exit;
166 	}
167 
168 	tcemem->mem = mem;
169 	list_add(&tcemem->next, &container->prereg_list);
170 
171 	container->enabled = true;
172 
173 	return 0;
174 
175 put_exit:
176 	mm_iommu_put(container->mm, mem);
177 	return ret;
178 }
179 
tce_page_is_contained(struct mm_struct * mm,unsigned long hpa,unsigned int it_page_shift)180 static bool tce_page_is_contained(struct mm_struct *mm, unsigned long hpa,
181 		unsigned int it_page_shift)
182 {
183 	struct page *page;
184 	unsigned long size = 0;
185 
186 	if (mm_iommu_is_devmem(mm, hpa, it_page_shift, &size))
187 		return size == (1UL << it_page_shift);
188 
189 	page = pfn_to_page(hpa >> PAGE_SHIFT);
190 	/*
191 	 * Check that the TCE table granularity is not bigger than the size of
192 	 * a page we just found. Otherwise the hardware can get access to
193 	 * a bigger memory chunk that it should.
194 	 */
195 	return page_shift(compound_head(page)) >= it_page_shift;
196 }
197 
tce_groups_attached(struct tce_container * container)198 static inline bool tce_groups_attached(struct tce_container *container)
199 {
200 	return !list_empty(&container->group_list);
201 }
202 
tce_iommu_find_table(struct tce_container * container,phys_addr_t ioba,struct iommu_table ** ptbl)203 static long tce_iommu_find_table(struct tce_container *container,
204 		phys_addr_t ioba, struct iommu_table **ptbl)
205 {
206 	long i;
207 
208 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
209 		struct iommu_table *tbl = container->tables[i];
210 
211 		if (tbl) {
212 			unsigned long entry = ioba >> tbl->it_page_shift;
213 			unsigned long start = tbl->it_offset;
214 			unsigned long end = start + tbl->it_size;
215 
216 			if ((start <= entry) && (entry < end)) {
217 				*ptbl = tbl;
218 				return i;
219 			}
220 		}
221 	}
222 
223 	return -1;
224 }
225 
tce_iommu_find_free_table(struct tce_container * container)226 static int tce_iommu_find_free_table(struct tce_container *container)
227 {
228 	int i;
229 
230 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
231 		if (!container->tables[i])
232 			return i;
233 	}
234 
235 	return -ENOSPC;
236 }
237 
tce_iommu_enable(struct tce_container * container)238 static int tce_iommu_enable(struct tce_container *container)
239 {
240 	int ret = 0;
241 	unsigned long locked;
242 	struct iommu_table_group *table_group;
243 	struct tce_iommu_group *tcegrp;
244 
245 	if (container->enabled)
246 		return -EBUSY;
247 
248 	/*
249 	 * When userspace pages are mapped into the IOMMU, they are effectively
250 	 * locked memory, so, theoretically, we need to update the accounting
251 	 * of locked pages on each map and unmap.  For powerpc, the map unmap
252 	 * paths can be very hot, though, and the accounting would kill
253 	 * performance, especially since it would be difficult to impossible
254 	 * to handle the accounting in real mode only.
255 	 *
256 	 * To address that, rather than precisely accounting every page, we
257 	 * instead account for a worst case on locked memory when the iommu is
258 	 * enabled and disabled.  The worst case upper bound on locked memory
259 	 * is the size of the whole iommu window, which is usually relatively
260 	 * small (compared to total memory sizes) on POWER hardware.
261 	 *
262 	 * Also we don't have a nice way to fail on H_PUT_TCE due to ulimits,
263 	 * that would effectively kill the guest at random points, much better
264 	 * enforcing the limit based on the max that the guest can map.
265 	 *
266 	 * Unfortunately at the moment it counts whole tables, no matter how
267 	 * much memory the guest has. I.e. for 4GB guest and 4 IOMMU groups
268 	 * each with 2GB DMA window, 8GB will be counted here. The reason for
269 	 * this is that we cannot tell here the amount of RAM used by the guest
270 	 * as this information is only available from KVM and VFIO is
271 	 * KVM agnostic.
272 	 *
273 	 * So we do not allow enabling a container without a group attached
274 	 * as there is no way to know how much we should increment
275 	 * the locked_vm counter.
276 	 */
277 	if (!tce_groups_attached(container))
278 		return -ENODEV;
279 
280 	tcegrp = list_first_entry(&container->group_list,
281 			struct tce_iommu_group, next);
282 	table_group = iommu_group_get_iommudata(tcegrp->grp);
283 	if (!table_group)
284 		return -ENODEV;
285 
286 	if (!table_group->tce32_size)
287 		return -EPERM;
288 
289 	ret = tce_iommu_mm_set(container);
290 	if (ret)
291 		return ret;
292 
293 	locked = table_group->tce32_size >> PAGE_SHIFT;
294 	ret = account_locked_vm(container->mm, locked, true);
295 	if (ret)
296 		return ret;
297 
298 	container->locked_pages = locked;
299 
300 	container->enabled = true;
301 
302 	return ret;
303 }
304 
tce_iommu_disable(struct tce_container * container)305 static void tce_iommu_disable(struct tce_container *container)
306 {
307 	if (!container->enabled)
308 		return;
309 
310 	container->enabled = false;
311 
312 	BUG_ON(!container->mm);
313 	account_locked_vm(container->mm, container->locked_pages, false);
314 }
315 
tce_iommu_open(unsigned long arg)316 static void *tce_iommu_open(unsigned long arg)
317 {
318 	struct tce_container *container;
319 
320 	if ((arg != VFIO_SPAPR_TCE_IOMMU) && (arg != VFIO_SPAPR_TCE_v2_IOMMU)) {
321 		pr_err("tce_vfio: Wrong IOMMU type\n");
322 		return ERR_PTR(-EINVAL);
323 	}
324 
325 	container = kzalloc(sizeof(*container), GFP_KERNEL);
326 	if (!container)
327 		return ERR_PTR(-ENOMEM);
328 
329 	mutex_init(&container->lock);
330 	INIT_LIST_HEAD_RCU(&container->group_list);
331 	INIT_LIST_HEAD_RCU(&container->prereg_list);
332 
333 	container->v2 = arg == VFIO_SPAPR_TCE_v2_IOMMU;
334 
335 	return container;
336 }
337 
338 static int tce_iommu_clear(struct tce_container *container,
339 		struct iommu_table *tbl,
340 		unsigned long entry, unsigned long pages);
341 static void tce_iommu_free_table(struct tce_container *container,
342 		struct iommu_table *tbl);
343 
tce_iommu_release(void * iommu_data)344 static void tce_iommu_release(void *iommu_data)
345 {
346 	struct tce_container *container = iommu_data;
347 	struct tce_iommu_group *tcegrp;
348 	struct tce_iommu_prereg *tcemem, *tmtmp;
349 	long i;
350 
351 	while (tce_groups_attached(container)) {
352 		tcegrp = list_first_entry(&container->group_list,
353 				struct tce_iommu_group, next);
354 		tce_iommu_detach_group(iommu_data, tcegrp->grp);
355 	}
356 
357 	/*
358 	 * If VFIO created a table, it was not disposed
359 	 * by tce_iommu_detach_group() so do it now.
360 	 */
361 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
362 		struct iommu_table *tbl = container->tables[i];
363 
364 		if (!tbl)
365 			continue;
366 
367 		tce_iommu_free_table(container, tbl);
368 	}
369 
370 	list_for_each_entry_safe(tcemem, tmtmp, &container->prereg_list, next)
371 		WARN_ON(tce_iommu_prereg_free(container, tcemem));
372 
373 	tce_iommu_disable(container);
374 	if (container->mm)
375 		mmdrop(container->mm);
376 	mutex_destroy(&container->lock);
377 
378 	kfree(container);
379 }
380 
tce_iommu_unuse_page(unsigned long hpa)381 static void tce_iommu_unuse_page(unsigned long hpa)
382 {
383 	struct page *page;
384 
385 	page = pfn_to_page(hpa >> PAGE_SHIFT);
386 	unpin_user_page(page);
387 }
388 
tce_iommu_prereg_ua_to_hpa(struct tce_container * container,unsigned long tce,unsigned long shift,unsigned long * phpa,struct mm_iommu_table_group_mem_t ** pmem)389 static int tce_iommu_prereg_ua_to_hpa(struct tce_container *container,
390 		unsigned long tce, unsigned long shift,
391 		unsigned long *phpa, struct mm_iommu_table_group_mem_t **pmem)
392 {
393 	long ret = 0;
394 	struct mm_iommu_table_group_mem_t *mem;
395 
396 	mem = mm_iommu_lookup(container->mm, tce, 1ULL << shift);
397 	if (!mem)
398 		return -EINVAL;
399 
400 	ret = mm_iommu_ua_to_hpa(mem, tce, shift, phpa);
401 	if (ret)
402 		return -EINVAL;
403 
404 	*pmem = mem;
405 
406 	return 0;
407 }
408 
tce_iommu_unuse_page_v2(struct tce_container * container,struct iommu_table * tbl,unsigned long entry)409 static void tce_iommu_unuse_page_v2(struct tce_container *container,
410 		struct iommu_table *tbl, unsigned long entry)
411 {
412 	struct mm_iommu_table_group_mem_t *mem = NULL;
413 	int ret;
414 	unsigned long hpa = 0;
415 	__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry);
416 
417 	if (!pua)
418 		return;
419 
420 	ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua),
421 			tbl->it_page_shift, &hpa, &mem);
422 	if (ret)
423 		pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n",
424 				__func__, be64_to_cpu(*pua), entry, ret);
425 	if (mem)
426 		mm_iommu_mapped_dec(mem);
427 
428 	*pua = cpu_to_be64(0);
429 }
430 
tce_iommu_clear(struct tce_container * container,struct iommu_table * tbl,unsigned long entry,unsigned long pages)431 static int tce_iommu_clear(struct tce_container *container,
432 		struct iommu_table *tbl,
433 		unsigned long entry, unsigned long pages)
434 {
435 	unsigned long oldhpa;
436 	long ret;
437 	enum dma_data_direction direction;
438 	unsigned long lastentry = entry + pages, firstentry = entry;
439 
440 	for ( ; entry < lastentry; ++entry) {
441 		if (tbl->it_indirect_levels && tbl->it_userspace) {
442 			/*
443 			 * For multilevel tables, we can take a shortcut here
444 			 * and skip some TCEs as we know that the userspace
445 			 * addresses cache is a mirror of the real TCE table
446 			 * and if it is missing some indirect levels, then
447 			 * the hardware table does not have them allocated
448 			 * either and therefore does not require updating.
449 			 */
450 			__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl,
451 					entry);
452 			if (!pua) {
453 				/* align to level_size which is power of two */
454 				entry |= tbl->it_level_size - 1;
455 				continue;
456 			}
457 		}
458 
459 		cond_resched();
460 
461 		direction = DMA_NONE;
462 		oldhpa = 0;
463 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry, &oldhpa,
464 				&direction);
465 		if (ret)
466 			continue;
467 
468 		if (direction == DMA_NONE)
469 			continue;
470 
471 		if (container->v2) {
472 			tce_iommu_unuse_page_v2(container, tbl, entry);
473 			continue;
474 		}
475 
476 		tce_iommu_unuse_page(oldhpa);
477 	}
478 
479 	iommu_tce_kill(tbl, firstentry, pages);
480 
481 	return 0;
482 }
483 
tce_iommu_use_page(unsigned long tce,unsigned long * hpa)484 static int tce_iommu_use_page(unsigned long tce, unsigned long *hpa)
485 {
486 	struct page *page = NULL;
487 	enum dma_data_direction direction = iommu_tce_direction(tce);
488 
489 	if (pin_user_pages_fast(tce & PAGE_MASK, 1,
490 			direction != DMA_TO_DEVICE ? FOLL_WRITE : 0,
491 			&page) != 1)
492 		return -EFAULT;
493 
494 	*hpa = __pa((unsigned long) page_address(page));
495 
496 	return 0;
497 }
498 
tce_iommu_build(struct tce_container * container,struct iommu_table * tbl,unsigned long entry,unsigned long tce,unsigned long pages,enum dma_data_direction direction)499 static long tce_iommu_build(struct tce_container *container,
500 		struct iommu_table *tbl,
501 		unsigned long entry, unsigned long tce, unsigned long pages,
502 		enum dma_data_direction direction)
503 {
504 	long i, ret = 0;
505 	unsigned long hpa;
506 	enum dma_data_direction dirtmp;
507 
508 	for (i = 0; i < pages; ++i) {
509 		unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
510 
511 		ret = tce_iommu_use_page(tce, &hpa);
512 		if (ret)
513 			break;
514 
515 		if (!tce_page_is_contained(container->mm, hpa,
516 				tbl->it_page_shift)) {
517 			ret = -EPERM;
518 			break;
519 		}
520 
521 		hpa |= offset;
522 		dirtmp = direction;
523 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
524 				&hpa, &dirtmp);
525 		if (ret) {
526 			tce_iommu_unuse_page(hpa);
527 			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
528 					__func__, entry << tbl->it_page_shift,
529 					tce, ret);
530 			break;
531 		}
532 
533 		if (dirtmp != DMA_NONE)
534 			tce_iommu_unuse_page(hpa);
535 
536 		tce += IOMMU_PAGE_SIZE(tbl);
537 	}
538 
539 	if (ret)
540 		tce_iommu_clear(container, tbl, entry, i);
541 	else
542 		iommu_tce_kill(tbl, entry, pages);
543 
544 	return ret;
545 }
546 
tce_iommu_build_v2(struct tce_container * container,struct iommu_table * tbl,unsigned long entry,unsigned long tce,unsigned long pages,enum dma_data_direction direction)547 static long tce_iommu_build_v2(struct tce_container *container,
548 		struct iommu_table *tbl,
549 		unsigned long entry, unsigned long tce, unsigned long pages,
550 		enum dma_data_direction direction)
551 {
552 	long i, ret = 0;
553 	unsigned long hpa;
554 	enum dma_data_direction dirtmp;
555 
556 	for (i = 0; i < pages; ++i) {
557 		struct mm_iommu_table_group_mem_t *mem = NULL;
558 		__be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i);
559 
560 		ret = tce_iommu_prereg_ua_to_hpa(container,
561 				tce, tbl->it_page_shift, &hpa, &mem);
562 		if (ret)
563 			break;
564 
565 		if (!tce_page_is_contained(container->mm, hpa,
566 				tbl->it_page_shift)) {
567 			ret = -EPERM;
568 			break;
569 		}
570 
571 		/* Preserve offset within IOMMU page */
572 		hpa |= tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
573 		dirtmp = direction;
574 
575 		/* The registered region is being unregistered */
576 		if (mm_iommu_mapped_inc(mem))
577 			break;
578 
579 		ret = iommu_tce_xchg_no_kill(container->mm, tbl, entry + i,
580 				&hpa, &dirtmp);
581 		if (ret) {
582 			/* dirtmp cannot be DMA_NONE here */
583 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
584 			pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
585 					__func__, entry << tbl->it_page_shift,
586 					tce, ret);
587 			break;
588 		}
589 
590 		if (dirtmp != DMA_NONE)
591 			tce_iommu_unuse_page_v2(container, tbl, entry + i);
592 
593 		*pua = cpu_to_be64(tce);
594 
595 		tce += IOMMU_PAGE_SIZE(tbl);
596 	}
597 
598 	if (ret)
599 		tce_iommu_clear(container, tbl, entry, i);
600 	else
601 		iommu_tce_kill(tbl, entry, pages);
602 
603 	return ret;
604 }
605 
tce_iommu_create_table(struct tce_container * container,struct iommu_table_group * table_group,int num,__u32 page_shift,__u64 window_size,__u32 levels,struct iommu_table ** ptbl)606 static long tce_iommu_create_table(struct tce_container *container,
607 			struct iommu_table_group *table_group,
608 			int num,
609 			__u32 page_shift,
610 			__u64 window_size,
611 			__u32 levels,
612 			struct iommu_table **ptbl)
613 {
614 	long ret, table_size;
615 
616 	table_size = table_group->ops->get_table_size(page_shift, window_size,
617 			levels);
618 	if (!table_size)
619 		return -EINVAL;
620 
621 	ret = account_locked_vm(container->mm, table_size >> PAGE_SHIFT, true);
622 	if (ret)
623 		return ret;
624 
625 	ret = table_group->ops->create_table(table_group, num,
626 			page_shift, window_size, levels, ptbl);
627 
628 	WARN_ON(!ret && !(*ptbl)->it_ops->free);
629 	WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size));
630 
631 	return ret;
632 }
633 
tce_iommu_free_table(struct tce_container * container,struct iommu_table * tbl)634 static void tce_iommu_free_table(struct tce_container *container,
635 		struct iommu_table *tbl)
636 {
637 	unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT;
638 
639 	iommu_tce_table_put(tbl);
640 	account_locked_vm(container->mm, pages, false);
641 }
642 
tce_iommu_create_window(struct tce_container * container,__u32 page_shift,__u64 window_size,__u32 levels,__u64 * start_addr)643 static long tce_iommu_create_window(struct tce_container *container,
644 		__u32 page_shift, __u64 window_size, __u32 levels,
645 		__u64 *start_addr)
646 {
647 	struct tce_iommu_group *tcegrp;
648 	struct iommu_table_group *table_group;
649 	struct iommu_table *tbl = NULL;
650 	long ret, num;
651 
652 	num = tce_iommu_find_free_table(container);
653 	if (num < 0)
654 		return num;
655 
656 	/* Get the first group for ops::create_table */
657 	tcegrp = list_first_entry(&container->group_list,
658 			struct tce_iommu_group, next);
659 	table_group = iommu_group_get_iommudata(tcegrp->grp);
660 	if (!table_group)
661 		return -EFAULT;
662 
663 	if (!(table_group->pgsizes & (1ULL << page_shift)))
664 		return -EINVAL;
665 
666 	if (!table_group->ops->set_window || !table_group->ops->unset_window ||
667 			!table_group->ops->get_table_size ||
668 			!table_group->ops->create_table)
669 		return -EPERM;
670 
671 	/* Create TCE table */
672 	ret = tce_iommu_create_table(container, table_group, num,
673 			page_shift, window_size, levels, &tbl);
674 	if (ret)
675 		return ret;
676 
677 	BUG_ON(!tbl->it_ops->free);
678 
679 	/*
680 	 * Program the table to every group.
681 	 * Groups have been tested for compatibility at the attach time.
682 	 */
683 	list_for_each_entry(tcegrp, &container->group_list, next) {
684 		table_group = iommu_group_get_iommudata(tcegrp->grp);
685 
686 		ret = table_group->ops->set_window(table_group, num, tbl);
687 		if (ret)
688 			goto unset_exit;
689 	}
690 
691 	container->tables[num] = tbl;
692 
693 	/* Return start address assigned by platform in create_table() */
694 	*start_addr = tbl->it_offset << tbl->it_page_shift;
695 
696 	return 0;
697 
698 unset_exit:
699 	list_for_each_entry(tcegrp, &container->group_list, next) {
700 		table_group = iommu_group_get_iommudata(tcegrp->grp);
701 		table_group->ops->unset_window(table_group, num);
702 	}
703 	tce_iommu_free_table(container, tbl);
704 
705 	return ret;
706 }
707 
tce_iommu_remove_window(struct tce_container * container,__u64 start_addr)708 static long tce_iommu_remove_window(struct tce_container *container,
709 		__u64 start_addr)
710 {
711 	struct iommu_table_group *table_group = NULL;
712 	struct iommu_table *tbl;
713 	struct tce_iommu_group *tcegrp;
714 	int num;
715 
716 	num = tce_iommu_find_table(container, start_addr, &tbl);
717 	if (num < 0)
718 		return -EINVAL;
719 
720 	BUG_ON(!tbl->it_size);
721 
722 	tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
723 
724 	/* Detach groups from IOMMUs */
725 	list_for_each_entry(tcegrp, &container->group_list, next) {
726 		table_group = iommu_group_get_iommudata(tcegrp->grp);
727 
728 		/*
729 		 * SPAPR TCE IOMMU exposes the default DMA window to
730 		 * the guest via dma32_window_start/size of
731 		 * VFIO_IOMMU_SPAPR_TCE_GET_INFO. Some platforms allow
732 		 * the userspace to remove this window, some do not so
733 		 * here we check for the platform capability.
734 		 */
735 		if (!table_group->ops || !table_group->ops->unset_window)
736 			return -EPERM;
737 
738 		table_group->ops->unset_window(table_group, num);
739 	}
740 
741 	/* Free table */
742 	tce_iommu_free_table(container, tbl);
743 	container->tables[num] = NULL;
744 
745 	return 0;
746 }
747 
tce_iommu_create_default_window(struct tce_container * container)748 static long tce_iommu_create_default_window(struct tce_container *container)
749 {
750 	long ret;
751 	__u64 start_addr = 0;
752 	struct tce_iommu_group *tcegrp;
753 	struct iommu_table_group *table_group;
754 
755 	if (!container->def_window_pending)
756 		return 0;
757 
758 	if (!tce_groups_attached(container))
759 		return -ENODEV;
760 
761 	tcegrp = list_first_entry(&container->group_list,
762 			struct tce_iommu_group, next);
763 	table_group = iommu_group_get_iommudata(tcegrp->grp);
764 	if (!table_group)
765 		return -ENODEV;
766 
767 	ret = tce_iommu_create_window(container, IOMMU_PAGE_SHIFT_4K,
768 			table_group->tce32_size, 1, &start_addr);
769 	WARN_ON_ONCE(!ret && start_addr);
770 
771 	if (!ret)
772 		container->def_window_pending = false;
773 
774 	return ret;
775 }
776 
vfio_spapr_ioctl_eeh_pe_op(struct iommu_group * group,unsigned long arg)777 static long vfio_spapr_ioctl_eeh_pe_op(struct iommu_group *group,
778 				       unsigned long arg)
779 {
780 	struct eeh_pe *pe;
781 	struct vfio_eeh_pe_op op;
782 	unsigned long minsz;
783 
784 	pe = eeh_iommu_group_to_pe(group);
785 	if (!pe)
786 		return -ENODEV;
787 
788 	minsz = offsetofend(struct vfio_eeh_pe_op, op);
789 	if (copy_from_user(&op, (void __user *)arg, minsz))
790 		return -EFAULT;
791 	if (op.argsz < minsz || op.flags)
792 		return -EINVAL;
793 
794 	switch (op.op) {
795 	case VFIO_EEH_PE_DISABLE:
796 		return eeh_pe_set_option(pe, EEH_OPT_DISABLE);
797 	case VFIO_EEH_PE_ENABLE:
798 		return eeh_pe_set_option(pe, EEH_OPT_ENABLE);
799 	case VFIO_EEH_PE_UNFREEZE_IO:
800 		return eeh_pe_set_option(pe, EEH_OPT_THAW_MMIO);
801 	case VFIO_EEH_PE_UNFREEZE_DMA:
802 		return eeh_pe_set_option(pe, EEH_OPT_THAW_DMA);
803 	case VFIO_EEH_PE_GET_STATE:
804 		return eeh_pe_get_state(pe);
805 		break;
806 	case VFIO_EEH_PE_RESET_DEACTIVATE:
807 		return eeh_pe_reset(pe, EEH_RESET_DEACTIVATE, true);
808 	case VFIO_EEH_PE_RESET_HOT:
809 		return eeh_pe_reset(pe, EEH_RESET_HOT, true);
810 	case VFIO_EEH_PE_RESET_FUNDAMENTAL:
811 		return eeh_pe_reset(pe, EEH_RESET_FUNDAMENTAL, true);
812 	case VFIO_EEH_PE_CONFIGURE:
813 		return eeh_pe_configure(pe);
814 	case VFIO_EEH_PE_INJECT_ERR:
815 		minsz = offsetofend(struct vfio_eeh_pe_op, err.mask);
816 		if (op.argsz < minsz)
817 			return -EINVAL;
818 		if (copy_from_user(&op, (void __user *)arg, minsz))
819 			return -EFAULT;
820 
821 		return eeh_pe_inject_err(pe, op.err.type, op.err.func,
822 					 op.err.addr, op.err.mask);
823 	default:
824 		return -EINVAL;
825 	}
826 }
827 
tce_iommu_ioctl(void * iommu_data,unsigned int cmd,unsigned long arg)828 static long tce_iommu_ioctl(void *iommu_data,
829 				 unsigned int cmd, unsigned long arg)
830 {
831 	struct tce_container *container = iommu_data;
832 	unsigned long minsz, ddwsz;
833 	long ret;
834 
835 	switch (cmd) {
836 	case VFIO_CHECK_EXTENSION:
837 		switch (arg) {
838 		case VFIO_SPAPR_TCE_IOMMU:
839 		case VFIO_SPAPR_TCE_v2_IOMMU:
840 			return 1;
841 		case VFIO_EEH:
842 			return eeh_enabled();
843 		default:
844 			return 0;
845 		}
846 	}
847 
848 	/*
849 	 * Sanity check to prevent one userspace from manipulating
850 	 * another userspace mm.
851 	 */
852 	BUG_ON(!container);
853 	if (container->mm && container->mm != current->mm)
854 		return -EPERM;
855 
856 	switch (cmd) {
857 	case VFIO_IOMMU_SPAPR_TCE_GET_INFO: {
858 		struct vfio_iommu_spapr_tce_info info;
859 		struct tce_iommu_group *tcegrp;
860 		struct iommu_table_group *table_group;
861 
862 		if (!tce_groups_attached(container))
863 			return -ENXIO;
864 
865 		tcegrp = list_first_entry(&container->group_list,
866 				struct tce_iommu_group, next);
867 		table_group = iommu_group_get_iommudata(tcegrp->grp);
868 
869 		if (!table_group)
870 			return -ENXIO;
871 
872 		minsz = offsetofend(struct vfio_iommu_spapr_tce_info,
873 				dma32_window_size);
874 
875 		if (copy_from_user(&info, (void __user *)arg, minsz))
876 			return -EFAULT;
877 
878 		if (info.argsz < minsz)
879 			return -EINVAL;
880 
881 		info.dma32_window_start = table_group->tce32_start;
882 		info.dma32_window_size = table_group->tce32_size;
883 		info.flags = 0;
884 		memset(&info.ddw, 0, sizeof(info.ddw));
885 
886 		if (table_group->max_dynamic_windows_supported &&
887 				container->v2) {
888 			info.flags |= VFIO_IOMMU_SPAPR_INFO_DDW;
889 			info.ddw.pgsizes = table_group->pgsizes;
890 			info.ddw.max_dynamic_windows_supported =
891 				table_group->max_dynamic_windows_supported;
892 			info.ddw.levels = table_group->max_levels;
893 		}
894 
895 		ddwsz = offsetofend(struct vfio_iommu_spapr_tce_info, ddw);
896 
897 		if (info.argsz >= ddwsz)
898 			minsz = ddwsz;
899 
900 		if (copy_to_user((void __user *)arg, &info, minsz))
901 			return -EFAULT;
902 
903 		return 0;
904 	}
905 	case VFIO_IOMMU_MAP_DMA: {
906 		struct vfio_iommu_type1_dma_map param;
907 		struct iommu_table *tbl = NULL;
908 		long num;
909 		enum dma_data_direction direction;
910 
911 		if (!container->enabled)
912 			return -EPERM;
913 
914 		minsz = offsetofend(struct vfio_iommu_type1_dma_map, size);
915 
916 		if (copy_from_user(&param, (void __user *)arg, minsz))
917 			return -EFAULT;
918 
919 		if (param.argsz < minsz)
920 			return -EINVAL;
921 
922 		if (param.flags & ~(VFIO_DMA_MAP_FLAG_READ |
923 				VFIO_DMA_MAP_FLAG_WRITE))
924 			return -EINVAL;
925 
926 		ret = tce_iommu_create_default_window(container);
927 		if (ret)
928 			return ret;
929 
930 		num = tce_iommu_find_table(container, param.iova, &tbl);
931 		if (num < 0)
932 			return -ENXIO;
933 
934 		if ((param.size & ~IOMMU_PAGE_MASK(tbl)) ||
935 				(param.vaddr & ~IOMMU_PAGE_MASK(tbl)))
936 			return -EINVAL;
937 
938 		/* iova is checked by the IOMMU API */
939 		if (param.flags & VFIO_DMA_MAP_FLAG_READ) {
940 			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
941 				direction = DMA_BIDIRECTIONAL;
942 			else
943 				direction = DMA_TO_DEVICE;
944 		} else {
945 			if (param.flags & VFIO_DMA_MAP_FLAG_WRITE)
946 				direction = DMA_FROM_DEVICE;
947 			else
948 				return -EINVAL;
949 		}
950 
951 		ret = iommu_tce_put_param_check(tbl, param.iova, param.vaddr);
952 		if (ret)
953 			return ret;
954 
955 		if (container->v2)
956 			ret = tce_iommu_build_v2(container, tbl,
957 					param.iova >> tbl->it_page_shift,
958 					param.vaddr,
959 					param.size >> tbl->it_page_shift,
960 					direction);
961 		else
962 			ret = tce_iommu_build(container, tbl,
963 					param.iova >> tbl->it_page_shift,
964 					param.vaddr,
965 					param.size >> tbl->it_page_shift,
966 					direction);
967 
968 		iommu_flush_tce(tbl);
969 
970 		return ret;
971 	}
972 	case VFIO_IOMMU_UNMAP_DMA: {
973 		struct vfio_iommu_type1_dma_unmap param;
974 		struct iommu_table *tbl = NULL;
975 		long num;
976 
977 		if (!container->enabled)
978 			return -EPERM;
979 
980 		minsz = offsetofend(struct vfio_iommu_type1_dma_unmap,
981 				size);
982 
983 		if (copy_from_user(&param, (void __user *)arg, minsz))
984 			return -EFAULT;
985 
986 		if (param.argsz < minsz)
987 			return -EINVAL;
988 
989 		/* No flag is supported now */
990 		if (param.flags)
991 			return -EINVAL;
992 
993 		ret = tce_iommu_create_default_window(container);
994 		if (ret)
995 			return ret;
996 
997 		num = tce_iommu_find_table(container, param.iova, &tbl);
998 		if (num < 0)
999 			return -ENXIO;
1000 
1001 		if (param.size & ~IOMMU_PAGE_MASK(tbl))
1002 			return -EINVAL;
1003 
1004 		ret = iommu_tce_clear_param_check(tbl, param.iova, 0,
1005 				param.size >> tbl->it_page_shift);
1006 		if (ret)
1007 			return ret;
1008 
1009 		ret = tce_iommu_clear(container, tbl,
1010 				param.iova >> tbl->it_page_shift,
1011 				param.size >> tbl->it_page_shift);
1012 		iommu_flush_tce(tbl);
1013 
1014 		return ret;
1015 	}
1016 	case VFIO_IOMMU_SPAPR_REGISTER_MEMORY: {
1017 		struct vfio_iommu_spapr_register_memory param;
1018 
1019 		if (!container->v2)
1020 			break;
1021 
1022 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1023 				size);
1024 
1025 		ret = tce_iommu_mm_set(container);
1026 		if (ret)
1027 			return ret;
1028 
1029 		if (copy_from_user(&param, (void __user *)arg, minsz))
1030 			return -EFAULT;
1031 
1032 		if (param.argsz < minsz)
1033 			return -EINVAL;
1034 
1035 		/* No flag is supported now */
1036 		if (param.flags)
1037 			return -EINVAL;
1038 
1039 		mutex_lock(&container->lock);
1040 		ret = tce_iommu_register_pages(container, param.vaddr,
1041 				param.size);
1042 		mutex_unlock(&container->lock);
1043 
1044 		return ret;
1045 	}
1046 	case VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY: {
1047 		struct vfio_iommu_spapr_register_memory param;
1048 
1049 		if (!container->v2)
1050 			break;
1051 
1052 		if (!container->mm)
1053 			return -EPERM;
1054 
1055 		minsz = offsetofend(struct vfio_iommu_spapr_register_memory,
1056 				size);
1057 
1058 		if (copy_from_user(&param, (void __user *)arg, minsz))
1059 			return -EFAULT;
1060 
1061 		if (param.argsz < minsz)
1062 			return -EINVAL;
1063 
1064 		/* No flag is supported now */
1065 		if (param.flags)
1066 			return -EINVAL;
1067 
1068 		mutex_lock(&container->lock);
1069 		ret = tce_iommu_unregister_pages(container, param.vaddr,
1070 				param.size);
1071 		mutex_unlock(&container->lock);
1072 
1073 		return ret;
1074 	}
1075 	case VFIO_IOMMU_ENABLE:
1076 		if (container->v2)
1077 			break;
1078 
1079 		mutex_lock(&container->lock);
1080 		ret = tce_iommu_enable(container);
1081 		mutex_unlock(&container->lock);
1082 		return ret;
1083 
1084 
1085 	case VFIO_IOMMU_DISABLE:
1086 		if (container->v2)
1087 			break;
1088 
1089 		mutex_lock(&container->lock);
1090 		tce_iommu_disable(container);
1091 		mutex_unlock(&container->lock);
1092 		return 0;
1093 
1094 	case VFIO_EEH_PE_OP: {
1095 		struct tce_iommu_group *tcegrp;
1096 
1097 		ret = 0;
1098 		list_for_each_entry(tcegrp, &container->group_list, next) {
1099 			ret = vfio_spapr_ioctl_eeh_pe_op(tcegrp->grp, arg);
1100 			if (ret)
1101 				return ret;
1102 		}
1103 		return ret;
1104 	}
1105 
1106 	case VFIO_IOMMU_SPAPR_TCE_CREATE: {
1107 		struct vfio_iommu_spapr_tce_create create;
1108 
1109 		if (!container->v2)
1110 			break;
1111 
1112 		ret = tce_iommu_mm_set(container);
1113 		if (ret)
1114 			return ret;
1115 
1116 		if (!tce_groups_attached(container))
1117 			return -ENXIO;
1118 
1119 		minsz = offsetofend(struct vfio_iommu_spapr_tce_create,
1120 				start_addr);
1121 
1122 		if (copy_from_user(&create, (void __user *)arg, minsz))
1123 			return -EFAULT;
1124 
1125 		if (create.argsz < minsz)
1126 			return -EINVAL;
1127 
1128 		if (create.flags)
1129 			return -EINVAL;
1130 
1131 		mutex_lock(&container->lock);
1132 
1133 		ret = tce_iommu_create_default_window(container);
1134 		if (!ret)
1135 			ret = tce_iommu_create_window(container,
1136 					create.page_shift,
1137 					create.window_size, create.levels,
1138 					&create.start_addr);
1139 
1140 		mutex_unlock(&container->lock);
1141 
1142 		if (!ret && copy_to_user((void __user *)arg, &create, minsz))
1143 			ret = -EFAULT;
1144 
1145 		return ret;
1146 	}
1147 	case VFIO_IOMMU_SPAPR_TCE_REMOVE: {
1148 		struct vfio_iommu_spapr_tce_remove remove;
1149 
1150 		if (!container->v2)
1151 			break;
1152 
1153 		ret = tce_iommu_mm_set(container);
1154 		if (ret)
1155 			return ret;
1156 
1157 		if (!tce_groups_attached(container))
1158 			return -ENXIO;
1159 
1160 		minsz = offsetofend(struct vfio_iommu_spapr_tce_remove,
1161 				start_addr);
1162 
1163 		if (copy_from_user(&remove, (void __user *)arg, minsz))
1164 			return -EFAULT;
1165 
1166 		if (remove.argsz < minsz)
1167 			return -EINVAL;
1168 
1169 		if (remove.flags)
1170 			return -EINVAL;
1171 
1172 		if (container->def_window_pending && !remove.start_addr) {
1173 			container->def_window_pending = false;
1174 			return 0;
1175 		}
1176 
1177 		mutex_lock(&container->lock);
1178 
1179 		ret = tce_iommu_remove_window(container, remove.start_addr);
1180 
1181 		mutex_unlock(&container->lock);
1182 
1183 		return ret;
1184 	}
1185 	}
1186 
1187 	return -ENOTTY;
1188 }
1189 
tce_iommu_release_ownership(struct tce_container * container,struct iommu_table_group * table_group)1190 static void tce_iommu_release_ownership(struct tce_container *container,
1191 		struct iommu_table_group *table_group)
1192 {
1193 	long i;
1194 
1195 	if (!table_group->ops->unset_window) {
1196 		WARN_ON_ONCE(1);
1197 		return;
1198 	}
1199 
1200 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1201 		if (container->tables[i]) {
1202 			tce_iommu_clear(container, container->tables[i],
1203 					container->tables[i]->it_offset,
1204 					container->tables[i]->it_size);
1205 			table_group->ops->unset_window(table_group, i);
1206 		}
1207 	}
1208 }
1209 
tce_iommu_take_ownership(struct tce_container * container,struct iommu_table_group * table_group)1210 static long tce_iommu_take_ownership(struct tce_container *container,
1211 		struct iommu_table_group *table_group)
1212 {
1213 	long i, ret = 0;
1214 
1215 	/* Set all windows to the new group */
1216 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) {
1217 		struct iommu_table *tbl = container->tables[i];
1218 
1219 		if (!tbl)
1220 			continue;
1221 
1222 		ret = table_group->ops->set_window(table_group, i, tbl);
1223 		if (ret)
1224 			goto release_exit;
1225 	}
1226 
1227 	return 0;
1228 
1229 release_exit:
1230 	for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i)
1231 		table_group->ops->unset_window(table_group, i);
1232 
1233 	return ret;
1234 }
1235 
tce_iommu_attach_group(void * iommu_data,struct iommu_group * iommu_group,enum vfio_group_type type)1236 static int tce_iommu_attach_group(void *iommu_data,
1237 		struct iommu_group *iommu_group, enum vfio_group_type type)
1238 {
1239 	int ret = 0;
1240 	struct tce_container *container = iommu_data;
1241 	struct iommu_table_group *table_group;
1242 	struct tce_iommu_group *tcegrp = NULL;
1243 
1244 	if (type == VFIO_EMULATED_IOMMU)
1245 		return -EINVAL;
1246 
1247 	mutex_lock(&container->lock);
1248 
1249 	/* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n",
1250 			iommu_group_id(iommu_group), iommu_group); */
1251 	table_group = iommu_group_get_iommudata(iommu_group);
1252 	if (!table_group) {
1253 		ret = -ENODEV;
1254 		goto unlock_exit;
1255 	}
1256 
1257 	/* v2 requires full support of dynamic DMA windows */
1258 	if (container->v2 && table_group->max_dynamic_windows_supported == 0) {
1259 		ret = -EINVAL;
1260 		goto unlock_exit;
1261 	}
1262 
1263 	/* v1 reuses TCE tables and does not share them among PEs */
1264 	if (!container->v2 && tce_groups_attached(container)) {
1265 		ret = -EBUSY;
1266 		goto unlock_exit;
1267 	}
1268 
1269 	/*
1270 	 * Check if new group has the same iommu_table_group_ops
1271 	 * (i.e. compatible)
1272 	 */
1273 	list_for_each_entry(tcegrp, &container->group_list, next) {
1274 		struct iommu_table_group *table_group_tmp;
1275 
1276 		if (tcegrp->grp == iommu_group) {
1277 			pr_warn("tce_vfio: Group %d is already attached\n",
1278 					iommu_group_id(iommu_group));
1279 			ret = -EBUSY;
1280 			goto unlock_exit;
1281 		}
1282 		table_group_tmp = iommu_group_get_iommudata(tcegrp->grp);
1283 		if (table_group_tmp->ops->create_table !=
1284 				table_group->ops->create_table) {
1285 			pr_warn("tce_vfio: Group %d is incompatible with group %d\n",
1286 					iommu_group_id(iommu_group),
1287 					iommu_group_id(tcegrp->grp));
1288 			ret = -EPERM;
1289 			goto unlock_exit;
1290 		}
1291 	}
1292 
1293 	tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL);
1294 	if (!tcegrp) {
1295 		ret = -ENOMEM;
1296 		goto unlock_exit;
1297 	}
1298 
1299 	ret = tce_iommu_take_ownership(container, table_group);
1300 	if (!tce_groups_attached(container) && !container->tables[0])
1301 		container->def_window_pending = true;
1302 
1303 	if (!ret) {
1304 		tcegrp->grp = iommu_group;
1305 		list_add(&tcegrp->next, &container->group_list);
1306 	}
1307 
1308 	if (ret && tcegrp)
1309 		kfree(tcegrp);
1310 
1311 unlock_exit:
1312 	mutex_unlock(&container->lock);
1313 
1314 	return ret;
1315 }
1316 
tce_iommu_detach_group(void * iommu_data,struct iommu_group * iommu_group)1317 static void tce_iommu_detach_group(void *iommu_data,
1318 		struct iommu_group *iommu_group)
1319 {
1320 	struct tce_container *container = iommu_data;
1321 	struct iommu_table_group *table_group;
1322 	bool found = false;
1323 	struct tce_iommu_group *tcegrp;
1324 
1325 	mutex_lock(&container->lock);
1326 
1327 	list_for_each_entry(tcegrp, &container->group_list, next) {
1328 		if (tcegrp->grp == iommu_group) {
1329 			found = true;
1330 			break;
1331 		}
1332 	}
1333 
1334 	if (!found) {
1335 		pr_warn("tce_vfio: detaching unattached group #%u\n",
1336 				iommu_group_id(iommu_group));
1337 		goto unlock_exit;
1338 	}
1339 
1340 	list_del(&tcegrp->next);
1341 	kfree(tcegrp);
1342 
1343 	table_group = iommu_group_get_iommudata(iommu_group);
1344 	BUG_ON(!table_group);
1345 
1346 	tce_iommu_release_ownership(container, table_group);
1347 
1348 unlock_exit:
1349 	mutex_unlock(&container->lock);
1350 }
1351 
1352 static const struct vfio_iommu_driver_ops tce_iommu_driver_ops = {
1353 	.name		= "iommu-vfio-powerpc",
1354 	.owner		= THIS_MODULE,
1355 	.open		= tce_iommu_open,
1356 	.release	= tce_iommu_release,
1357 	.ioctl		= tce_iommu_ioctl,
1358 	.attach_group	= tce_iommu_attach_group,
1359 	.detach_group	= tce_iommu_detach_group,
1360 };
1361 
tce_iommu_init(void)1362 static int __init tce_iommu_init(void)
1363 {
1364 	return vfio_register_iommu_driver(&tce_iommu_driver_ops);
1365 }
1366 
tce_iommu_cleanup(void)1367 static void __exit tce_iommu_cleanup(void)
1368 {
1369 	vfio_unregister_iommu_driver(&tce_iommu_driver_ops);
1370 }
1371 
1372 module_init(tce_iommu_init);
1373 module_exit(tce_iommu_cleanup);
1374 
1375 MODULE_VERSION(DRIVER_VERSION);
1376 MODULE_LICENSE("GPL v2");
1377 MODULE_AUTHOR(DRIVER_AUTHOR);
1378 MODULE_DESCRIPTION(DRIVER_DESC);
1379 
1380