304 static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)  in add_mem_pool()  argument
308 	list_add_rcu(&pool->node, &mem->pools);  in add_mem_pool()
309 	mem->nslabs += pool->nslabs;  in add_mem_pool()
312 	mem->nslabs = pool->nslabs;  in add_mem_pool()
685 	struct io_tlb_pool *pool;  in swiotlb_alloc_pool()  local
696 	pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);  in swiotlb_alloc_pool()
697 	pool = kzalloc(pool_size, gfp);  in swiotlb_alloc_pool()
698 	if (!pool)  in swiotlb_alloc_pool()
700 	pool->areas = (void *)pool + sizeof(*pool);  in swiotlb_alloc_pool()
711 	slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));  in swiotlb_alloc_pool()
712 	pool->slots = (struct io_tlb_slot *)  in swiotlb_alloc_pool()
714 	if (!pool->slots)  in swiotlb_alloc_pool()
717 	swiotlb_init_io_tlb_pool(pool, page_to_phys(tlb), nslabs, true, nareas);  in swiotlb_alloc_pool()
718 	return pool;  in swiotlb_alloc_pool()
723 	kfree(pool);  in swiotlb_alloc_pool()
736 	struct io_tlb_pool *pool;  in swiotlb_dyn_alloc()  local
738 	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, default_nslabs,  in swiotlb_dyn_alloc()
740 	if (!pool) {  in swiotlb_dyn_alloc()
745 	add_mem_pool(mem, pool);  in swiotlb_dyn_alloc()
754 	struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);  in swiotlb_dyn_free()  local
755 	size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);  in swiotlb_dyn_free()
756 	size_t tlb_size = pool->end - pool->start;  in swiotlb_dyn_free()
758 	free_pages((unsigned long)pool->slots, get_order(slots_size));  in swiotlb_dyn_free()
759 	swiotlb_free_tlb(pool->vaddr, tlb_size);  in swiotlb_dyn_free()
760 	kfree(pool);  in swiotlb_dyn_free()
778 	struct io_tlb_pool *pool;  in __swiotlb_find_pool()  local
781 	list_for_each_entry_rcu(pool, &mem->pools, node) {  in __swiotlb_find_pool()
782 		if (paddr >= pool->start && paddr < pool->end)  in __swiotlb_find_pool()
786 	list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {  in __swiotlb_find_pool()
787 		if (paddr >= pool->start && paddr < pool->end)  in __swiotlb_find_pool()
790 	pool = NULL;  in __swiotlb_find_pool()
793 	return pool;  in __swiotlb_find_pool()
801 static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)  in swiotlb_del_pool()  argument
806 	list_del_rcu(&pool->node);  in swiotlb_del_pool()
809 	call_rcu(&pool->rcu, swiotlb_dyn_free);  in swiotlb_del_pool()
1014 static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool,  in swiotlb_search_pool_area()  argument
1018 	struct io_tlb_area *area = pool->areas + area_index;  in swiotlb_search_pool_area()
1021 		phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;  in swiotlb_search_pool_area()
1032 	BUG_ON(area_index >= pool->nareas);  in swiotlb_search_pool_area()
1060 	if (unlikely(nslots > pool->area_nslabs - area->used))  in swiotlb_search_pool_area()
1063 	slot_base = area_index * pool->area_nslabs;  in swiotlb_search_pool_area()
1066 	for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {  in swiotlb_search_pool_area()
1075 			index = wrap_area_index(pool, index + 1);  in swiotlb_search_pool_area()
1083 			if (pool->slots[slot_index].list >= nslots)  in swiotlb_search_pool_area()
1086 		index = wrap_area_index(pool, index + stride);  in swiotlb_search_pool_area()
1101 		pool->slots[i].list = 0;  in swiotlb_search_pool_area()
1102 		pool->slots[i].alloc_size = alloc_size - (offset +  in swiotlb_search_pool_area()
1107 	     pool->slots[i].list; i--)  in swiotlb_search_pool_area()
1108 		pool->slots[i].list = ++count;  in swiotlb_search_pool_area()
1113 	area->index = wrap_area_index(pool, index + nslots);  in swiotlb_search_pool_area()
1144 	struct io_tlb_pool *pool;  in swiotlb_search_area()  local
1149 	list_for_each_entry_rcu(pool, &mem->pools, node) {  in swiotlb_search_area()
1150 		if (cpu_offset >= pool->nareas)  in swiotlb_search_area()
1152 		area_index = (start_cpu + cpu_offset) & (pool->nareas - 1);  in swiotlb_search_area()
1153 		index = swiotlb_search_pool_area(dev, pool, area_index,  in swiotlb_search_area()
1157 			*retpool = pool;  in swiotlb_search_area()
1184 	struct io_tlb_pool *pool;  in swiotlb_find_slots()  local
1197 					    alloc_align_mask, &pool);  in swiotlb_find_slots()
1209 	pool = swiotlb_alloc_pool(dev, nslabs, nslabs, 1, phys_limit,  in swiotlb_find_slots()
1211 	if (!pool)  in swiotlb_find_slots()
1214 	index = swiotlb_search_pool_area(dev, pool, 0, orig_addr,  in swiotlb_find_slots()
1217 		swiotlb_dyn_free(&pool->rcu);  in swiotlb_find_slots()
1221 	pool->transient = true;  in swiotlb_find_slots()
1223 	list_add_rcu(&pool->node, &dev->dma_io_tlb_pools);  in swiotlb_find_slots()
1225 	inc_transient_used(mem, pool->nslabs);  in swiotlb_find_slots()
1249 	*retpool = pool;  in swiotlb_find_slots()
1259 	struct io_tlb_pool *pool;  in swiotlb_find_slots()  local
1263 	*retpool = pool = &dev->dma_io_tlb_mem->defpool;  in swiotlb_find_slots()
1264 	i = start = raw_smp_processor_id() & (pool->nareas - 1);  in swiotlb_find_slots()
1266 		index = swiotlb_search_pool_area(dev, pool, i, orig_addr,  in swiotlb_find_slots()
1270 		if (++i >= pool->nareas)  in swiotlb_find_slots()
1304 static unsigned long mem_pool_used(struct io_tlb_pool *pool)  in mem_pool_used()  argument
1309 	for (i = 0; i < pool->nareas; i++)  in mem_pool_used()
1310 		used += pool->areas[i].used;  in mem_pool_used()
1326 	struct io_tlb_pool *pool;  in mem_used()  local
1330 	list_for_each_entry_rcu(pool, &mem->pools, node)  in mem_used()
1331 		used += mem_pool_used(pool);  in mem_used()
1373 	struct io_tlb_pool *pool;  in swiotlb_tbl_map_single()  local
1401 	index = swiotlb_find_slots(dev, orig_addr, size, alloc_align_mask, &pool);  in swiotlb_tbl_map_single()
1424 	pool->slots[index].pad_slots = pad_slots;  in swiotlb_tbl_map_single()
1426 		pool->slots[index + i].orig_addr = slot_addr(orig_addr, i);  in swiotlb_tbl_map_single()
1427 	tlb_addr = slot_addr(pool->start, index) + offset;  in swiotlb_tbl_map_single()
1437 	swiotlb_bounce(dev, tlb_addr, mapping_size, DMA_TO_DEVICE, pool);  in swiotlb_tbl_map_single()
1509 		struct io_tlb_pool *pool)  in swiotlb_del_transient()  argument
1511 	if (!pool->transient)  in swiotlb_del_transient()
1514 	dec_used(dev->dma_io_tlb_mem, pool->nslabs);  in swiotlb_del_transient()
1515 	swiotlb_del_pool(dev, pool);  in swiotlb_del_transient()
1516 	dec_transient_used(dev->dma_io_tlb_mem, pool->nslabs);  in swiotlb_del_transient()
1523 		phys_addr_t tlb_addr, struct io_tlb_pool *pool)  in swiotlb_del_transient()  argument
1535 		unsigned long attrs, struct io_tlb_pool *pool)  in __swiotlb_tbl_unmap_single()  argument
1543 						DMA_FROM_DEVICE, pool);  in __swiotlb_tbl_unmap_single()
1545 	if (swiotlb_del_transient(dev, tlb_addr, pool))  in __swiotlb_tbl_unmap_single()
1547 	swiotlb_release_slots(dev, tlb_addr, pool);  in __swiotlb_tbl_unmap_single()
1552 		struct io_tlb_pool *pool)  in __swiotlb_sync_single_for_device()  argument
1555 		swiotlb_bounce(dev, tlb_addr, size, DMA_TO_DEVICE, pool);  in __swiotlb_sync_single_for_device()
1562 		struct io_tlb_pool *pool)  in __swiotlb_sync_single_for_cpu()  argument
1565 		swiotlb_bounce(dev, tlb_addr, size, DMA_FROM_DEVICE, pool);  in __swiotlb_sync_single_for_cpu()
1752 	struct io_tlb_pool *pool;  in swiotlb_alloc()  local
1761 	index = swiotlb_find_slots(dev, 0, size, align, &pool);  in swiotlb_alloc()
1765 	tlb_addr = slot_addr(pool->start, index);  in swiotlb_alloc()
1769 		swiotlb_release_slots(dev, tlb_addr, pool);  in swiotlb_alloc()
1779 	struct io_tlb_pool *pool;  in swiotlb_free()  local
1781 	pool = swiotlb_find_pool(dev, tlb_addr);  in swiotlb_free()
1782 	if (!pool)  in swiotlb_free()
1785 	swiotlb_release_slots(dev, tlb_addr, pool);  in swiotlb_free()
1810 		struct io_tlb_pool *pool;  in rmem_swiotlb_device_init()  local
1815 		pool = &mem->defpool;  in rmem_swiotlb_device_init()
1817 		pool->slots = kcalloc(nslabs, sizeof(*pool->slots), GFP_KERNEL);  in rmem_swiotlb_device_init()
1818 		if (!pool->slots) {  in rmem_swiotlb_device_init()
1823 		pool->areas = kcalloc(nareas, sizeof(*pool->areas),  in rmem_swiotlb_device_init()
1825 		if (!pool->areas) {  in rmem_swiotlb_device_init()
1826 			kfree(pool->slots);  in rmem_swiotlb_device_init()
1833 		swiotlb_init_io_tlb_pool(pool, rmem->base, nslabs,  in rmem_swiotlb_device_init()
1841 		add_mem_pool(mem, pool);  in rmem_swiotlb_device_init()