44  * @pa: page_array on which to perform the operation
55  *   -EINVAL if pa->pa_nr is not initially zero, or pa->pa_iova is not NULL
58 static int page_array_alloc(struct page_array *pa, unsigned int len)  in page_array_alloc()  argument
60 	if (pa->pa_nr || pa->pa_iova)  in page_array_alloc()
66 	pa->pa_nr = len;  in page_array_alloc()
68 	pa->pa_iova = kcalloc(len, sizeof(*pa->pa_iova), GFP_KERNEL);  in page_array_alloc()
69 	if (!pa->pa_iova)  in page_array_alloc()
72 	pa->pa_page = kcalloc(len, sizeof(*pa->pa_page), GFP_KERNEL);  in page_array_alloc()
73 	if (!pa->pa_page) {  in page_array_alloc()
74 		kfree(pa->pa_iova);  in page_array_alloc()
83  * @pa: page_array on which to perform the operation
89  * otherwise only clear pa->pa_nr
91 static void page_array_unpin(struct page_array *pa,  in page_array_unpin()  argument
97 		dma_addr_t *first = &pa->pa_iova[unpinned];  in page_array_unpin()
112 	pa->pa_nr = 0;  in page_array_unpin()
117  * @pa: page_array on which to perform the operation
131 static int page_array_pin(struct page_array *pa, struct vfio_device *vdev, bool unaligned)  in page_array_pin()  argument
136 	while (pinned < pa->pa_nr) {  in page_array_pin()
137 		dma_addr_t *first = &pa->pa_iova[pinned];  in page_array_pin()
140 		if (pinned + npage < pa->pa_nr &&  in page_array_pin()
149 				     &pa->pa_page[pinned]);  in page_array_pin()
164 	page_array_unpin(pa, vdev, pinned, unaligned);  in page_array_pin()
169 static void page_array_unpin_free(struct page_array *pa, struct vfio_device *vdev, bool unaligned)  in page_array_unpin_free()  argument
171 	page_array_unpin(pa, vdev, pa->pa_nr, unaligned);  in page_array_unpin_free()
172 	kfree(pa->pa_page);  in page_array_unpin_free()
173 	kfree(pa->pa_iova);  in page_array_unpin_free()
176 static bool page_array_iova_pinned(struct page_array *pa, u64 iova, u64 length)  in page_array_iova_pinned()  argument
183 	for (i = 0; i < pa->pa_nr; i++) {  in page_array_iova_pinned()
184 		pfn = pa->pa_iova[i] >> PAGE_SHIFT;  in page_array_iova_pinned()
192 static inline void page_array_idal_create_words(struct page_array *pa,  in page_array_idal_create_words()  argument
205 	for (i = 0; i < pa->pa_nr; i++) {  in page_array_idal_create_words()
206 		idaws[i] = virt_to_dma64(page_to_virt(pa->pa_page[i]));  in page_array_idal_create_words()
209 		idaws[i] = dma64_add(idaws[i], pa->pa_iova[i] & ~PAGE_MASK);  in page_array_idal_create_words()
608 			      struct page_array *pa,  in ccwchain_fetch_ccw()  argument
637 	ret = page_array_alloc(pa, idaw_nr);  in ccwchain_fetch_ccw()
648 			pa->pa_iova[i] = dma64_to_u64(idaws[i]);  in ccwchain_fetch_ccw()
650 			pa->pa_iova[i] = dma32_to_u32(idaws_f1[i]);  in ccwchain_fetch_ccw()
654 		ret = page_array_pin(pa, vdev, idal_is_2k(cp));  in ccwchain_fetch_ccw()
658 		pa->pa_nr = 0;  in ccwchain_fetch_ccw()
665 	page_array_idal_create_words(pa, idaws);  in ccwchain_fetch_ccw()
670 	page_array_unpin_free(pa, vdev, idal_is_2k(cp));  in ccwchain_fetch_ccw()
685 			      struct page_array *pa,  in ccwchain_fetch_one()  argument
692 	return ccwchain_fetch_ccw(ccw, pa, cp);  in ccwchain_fetch_one()
816 	struct page_array *pa;  in cp_prefetch()  local
827 			pa = &chain->ch_pa[idx];  in cp_prefetch()
829 			ret = ccwchain_fetch_one(ccw, pa, cp);  in cp_prefetch()