1 // SPDX-License-Identifier: GPL-2.0
43  * page_array_alloc() - alloc memory for page array
55  *   -EINVAL if pa->pa_nr is not initially zero, or pa->pa_iova is not NULL
56  *   -ENOMEM if alloc failed
60 	if (pa->pa_nr || pa->pa_iova)  in page_array_alloc()
61 		return -EINVAL;  in page_array_alloc()
64 		return -EINVAL;  in page_array_alloc()
66 	pa->pa_nr = len;  in page_array_alloc()
68 	pa->pa_iova = kcalloc(len, sizeof(*pa->pa_iova), GFP_KERNEL);  in page_array_alloc()
69 	if (!pa->pa_iova)  in page_array_alloc()
70 		return -ENOMEM;  in page_array_alloc()
72 	pa->pa_page = kcalloc(len, sizeof(*pa->pa_page), GFP_KERNEL);  in page_array_alloc()
73 	if (!pa->pa_page) {  in page_array_alloc()
74 		kfree(pa->pa_iova);  in page_array_alloc()
75 		return -ENOMEM;  in page_array_alloc()
82  * page_array_unpin() - Unpin user pages in memory
89  * otherwise only clear pa->pa_nr
97 		dma_addr_t *first = &pa->pa_iova[unpinned];  in page_array_unpin()
112 	pa->pa_nr = 0;  in page_array_unpin()
116  * page_array_pin() - Pin user pages in memory
136 	while (pinned < pa->pa_nr) {  in page_array_pin()
137 		dma_addr_t *first = &pa->pa_iova[pinned];  in page_array_pin()
140 		if (pinned + npage < pa->pa_nr &&  in page_array_pin()
149 				     &pa->pa_page[pinned]);  in page_array_pin()
154 			ret = -EINVAL;  in page_array_pin()
171 	page_array_unpin(pa, vdev, pa->pa_nr, unaligned);  in page_array_unpin_free()
172 	kfree(pa->pa_page);  in page_array_unpin_free()
173 	kfree(pa->pa_iova);  in page_array_unpin_free()
179 	u64 iova_pfn_end = (iova + length - 1) >> PAGE_SHIFT;  in page_array_iova_pinned()
183 	for (i = 0; i < pa->pa_nr; i++) {  in page_array_iova_pinned()
184 		pfn = pa->pa_iova[i] >> PAGE_SHIFT;  in page_array_iova_pinned()
205 	for (i = 0; i < pa->pa_nr; i++) {  in page_array_idal_create_words()
206 		idaws[i] = virt_to_dma64(page_to_virt(pa->pa_page[i]));  in page_array_idal_create_words()
209 		idaws[i] = dma64_add(idaws[i], pa->pa_iova[i] & ~PAGE_MASK);  in page_array_idal_create_words()
221 		if ((pccw1->cmd_code & 0x0f) == CCW_CMD_TIC) {  in convert_ccw0_to_ccw1()
222 			pccw1->cmd_code = CCW_CMD_TIC;  in convert_ccw0_to_ccw1()
223 			pccw1->flags = 0;  in convert_ccw0_to_ccw1()
224 			pccw1->count = 0;  in convert_ccw0_to_ccw1()
226 			pccw1->cmd_code = ccw0.cmd_code;  in convert_ccw0_to_ccw1()
227 			pccw1->flags = ccw0.flags;  in convert_ccw0_to_ccw1()
228 			pccw1->count = ccw0.count;  in convert_ccw0_to_ccw1()
230 		pccw1->cda = u32_to_dma32(ccw0.cda);  in convert_ccw0_to_ccw1()
235 #define idal_is_2k(_cp) (!(_cp)->orb.cmd.c64 || (_cp)->orb.cmd.i2k)
240 #define ccw_is_read(_ccw) (((_ccw)->cmd_code & 0x03) == 0x02)
241 #define ccw_is_read_backward(_ccw) (((_ccw)->cmd_code & 0x0F) == 0x0C)
242 #define ccw_is_sense(_ccw) (((_ccw)->cmd_code & 0x0F) == CCW_CMD_BASIC_SENSE)
244 #define ccw_is_noop(_ccw) ((_ccw)->cmd_code == CCW_CMD_NOOP)
246 #define ccw_is_tic(_ccw) ((_ccw)->cmd_code == CCW_CMD_TIC)
248 #define ccw_is_idal(_ccw) ((_ccw)->flags & CCW_FLAG_IDA)
249 #define ccw_is_skip(_ccw) ((_ccw)->flags & CCW_FLAG_SKIP)
251 #define ccw_is_chain(_ccw) ((_ccw)->flags & (CCW_FLAG_CC | CCW_FLAG_DC))
264 	if (ccw->count == 0)  in ccw_does_data_transfer()
304 	u32 tail = head + (len - 1) * sizeof(struct ccw1);  in is_cpa_within_range()
315 	return is_cpa_within_range(ccw->cda, head, len);  in is_tic_within_range()
326 	chain->ch_ccw = kcalloc(len, sizeof(*chain->ch_ccw), GFP_DMA | GFP_KERNEL);  in ccwchain_alloc()
327 	if (!chain->ch_ccw)  in ccwchain_alloc()
330 	chain->ch_pa = kcalloc(len, sizeof(*chain->ch_pa), GFP_KERNEL);  in ccwchain_alloc()
331 	if (!chain->ch_pa)  in ccwchain_alloc()
334 	list_add_tail(&chain->next, &cp->ccwchain_list);  in ccwchain_alloc()
339 	kfree(chain->ch_ccw);  in ccwchain_alloc()
346 	list_del(&chain->next);  in ccwchain_free()
347 	kfree(chain->ch_pa);  in ccwchain_free()
348 	kfree(chain->ch_ccw);  in ccwchain_free()
355 	struct ccw1 *ccw = &chain->ch_ccw[idx];  in ccwchain_cda_free()
360 	kfree(dma32_to_virt(ccw->cda));  in ccwchain_cda_free()
364  * ccwchain_calc_length - calculate the length of the ccw chain.
374  * Returns: the length of the ccw chain or -errno.
378 	struct ccw1 *ccw = cp->guest_cp;  in ccwchain_calc_length()
386 		 * command-chaining flag enabled, or if it is a TIC CCW  in ccwchain_calc_length()
399 		cnt = -EINVAL;  in ccwchain_calc_length()
409 	list_for_each_entry(chain, &cp->ccwchain_list, next) {  in tic_target_chain_exists()
410 		ccw_head = chain->ch_iova;  in tic_target_chain_exists()
411 		if (is_cpa_within_range(tic->cda, ccw_head, chain->ch_len))  in tic_target_chain_exists()
424 		&container_of(cp, struct vfio_ccw_private, cp)->vdev;  in ccwchain_handle_ccw()
431 	ret = vfio_dma_rw(vdev, gcda, cp->guest_cp, CCWCHAIN_LEN_MAX * sizeof(struct ccw1), false);  in ccwchain_handle_ccw()
435 	/* Convert any Format-0 CCWs to Format-1 */  in ccwchain_handle_ccw()
436 	if (!cp->orb.cmd.fmt)  in ccwchain_handle_ccw()
437 		convert_ccw0_to_ccw1(cp->guest_cp, CCWCHAIN_LEN_MAX);  in ccwchain_handle_ccw()
447 		return -ENOMEM;  in ccwchain_handle_ccw()
449 	chain->ch_len = len;  in ccwchain_handle_ccw()
450 	chain->ch_iova = gcda;  in ccwchain_handle_ccw()
453 	memcpy(chain->ch_ccw, cp->guest_cp, len * sizeof(struct ccw1));  in ccwchain_handle_ccw()
470 	for (i = 0; i < chain->ch_len; i++) {  in ccwchain_loop_tic()
471 		tic = &chain->ch_ccw[i];  in ccwchain_loop_tic()
481 		ret = ccwchain_handle_ccw(tic->cda, cp);  in ccwchain_loop_tic()
495 	list_for_each_entry(iter, &cp->ccwchain_list, next) {  in ccwchain_fetch_tic()
496 		ccw_head = iter->ch_iova;  in ccwchain_fetch_tic()
497 		if (is_cpa_within_range(ccw->cda, ccw_head, iter->ch_len)) {  in ccwchain_fetch_tic()
499 			offset = dma32_to_u32(ccw->cda) - ccw_head;  in ccwchain_fetch_tic()
500 			ccw->cda = virt_to_dma32((void *)iter->ch_ccw + offset);  in ccwchain_fetch_tic()
505 	return -EFAULT;  in ccwchain_fetch_tic()
511 		&container_of(cp, struct vfio_ccw_private, cp)->vdev;  in get_guest_idal()
516 	int idaw_mask = ~(idaw_size - 1);  in get_guest_idal()
521 		return ERR_PTR(-ENOMEM);  in get_guest_idal()
525 		ret = vfio_dma_rw(vdev, dma32_to_u32(ccw->cda), idaws, idal_len, false);  in get_guest_idal()
532 		if (cp->orb.cmd.c64) {  in get_guest_idal()
533 			idaws[0] = u64_to_dma64(dma32_to_u32(ccw->cda));  in get_guest_idal()
535 				idaws[i] = dma64_add(idaws[i - 1], idaw_size);  in get_guest_idal()
540 			idaws_f1[0] = ccw->cda;  in get_guest_idal()
542 				idaws_f1[i] = dma32_add(idaws_f1[i - 1], idaw_size);  in get_guest_idal()
552  * ccw_count_idaws() - Calculate the number of IDAWs needed to transfer
560  * the CCW actually does. An ORB that does not specify Format-2-IDAW
562  * Format-1 and thus only move 2K with each IDAW. Thus all CCWs within
569 		&container_of(cp, struct vfio_ccw_private, cp)->vdev;  in ccw_count_idaws()
571 	int size = cp->orb.cmd.c64 ? sizeof(u64) : sizeof(u32);  in ccw_count_idaws()
575 	if (ccw->count)  in ccw_count_idaws()
576 		bytes = ccw->count;  in ccw_count_idaws()
580 		/* All subsequent IDAWs will be 2K- or 4K-aligned. */  in ccw_count_idaws()
581 		ret = vfio_dma_rw(vdev, dma32_to_u32(ccw->cda), &iova, size, false);  in ccw_count_idaws()
586 		 * Format-1 IDAWs only occupy the first 32 bits,  in ccw_count_idaws()
589 		if (!cp->orb.cmd.c64)  in ccw_count_idaws()
592 		iova = dma32_to_u32(ccw->cda);  in ccw_count_idaws()
595 	/* Format-1 IDAWs operate on 2K each */  in ccw_count_idaws()
596 	if (!cp->orb.cmd.c64)  in ccw_count_idaws()
599 	/* Using the 2K variant of Format-2 IDAWs? */  in ccw_count_idaws()
600 	if (cp->orb.cmd.i2k)  in ccw_count_idaws()
603 	/* The 'usual' case is 4K Format-2 IDAWs */  in ccw_count_idaws()
612 		&container_of(cp, struct vfio_ccw_private, cp)->vdev;  in ccwchain_fetch_ccw()
647 		if (cp->orb.cmd.c64)  in ccwchain_fetch_ccw()
648 			pa->pa_iova[i] = dma64_to_u64(idaws[i]);  in ccwchain_fetch_ccw()
650 			pa->pa_iova[i] = dma32_to_u32(idaws_f1[i]);  in ccwchain_fetch_ccw()
658 		pa->pa_nr = 0;  in ccwchain_fetch_ccw()
661 	ccw->cda = virt_to_dma32(idaws);  in ccwchain_fetch_ccw()
662 	ccw->flags |= CCW_FLAG_IDA;  in ccwchain_fetch_ccw()
674 	ccw->cda = 0;  in ccwchain_fetch_ccw()
696  * cp_init() - allocate ccwchains for a channel program.
701  * the target channel program from @orb->cmd.iova to the new ccwchain(s).
713 		&container_of(cp, struct vfio_ccw_private, cp)->vdev;  in cp_init()
714 	/* custom ratelimit used to avoid flood during guest IPL */  in cp_init()
719 	if (cp->initialized)  in cp_init()
720 		return -EBUSY;  in cp_init()
729 	if (!orb->cmd.pfch && __ratelimit(&ratelimit_state))  in cp_init()
731 			vdev->dev,  in cp_init()
734 	INIT_LIST_HEAD(&cp->ccwchain_list);  in cp_init()
735 	memcpy(&cp->orb, orb, sizeof(*orb));  in cp_init()
738 	ret = ccwchain_handle_ccw(orb->cmd.cpa, cp);  in cp_init()
741 		cp->initialized = true;  in cp_init()
748  * cp_free() - free resources for channel program.
758 		&container_of(cp, struct vfio_ccw_private, cp)->vdev;  in cp_free()
759 	struct ccwchain *chain, *temp;  in cp_free()  local
762 	if (!cp->initialized)  in cp_free()
765 	cp->initialized = false;  in cp_free()
766 	list_for_each_entry_safe(chain, temp, &cp->ccwchain_list, next) {  in cp_free()
767 		for (i = 0; i < chain->ch_len; i++) {  in cp_free()
768 			page_array_unpin_free(&chain->ch_pa[i], vdev, idal_is_2k(cp));  in cp_free()
776  * cp_prefetch() - translate a guest physical address channel program to
777  *                 a real-device runnable channel program.
780  * This function translates the guest-physical-address channel program
785  * - On entry ch_len holds the count of CCWs to be translated.
786  * - On exit ch_len is adjusted to the count of successfully translated CCWs.
792  * translate the channel program to a real-device runnable channel
795  * These APIs will copy the ccws into kernel-space buffers, and update
820 	if (!cp->initialized)  in cp_prefetch()
821 		return -EINVAL;  in cp_prefetch()
823 	list_for_each_entry(chain, &cp->ccwchain_list, next) {  in cp_prefetch()
824 		len = chain->ch_len;  in cp_prefetch()
826 			ccw = &chain->ch_ccw[idx];  in cp_prefetch()
827 			pa = &chain->ch_pa[idx];  in cp_prefetch()
838 	chain->ch_len = idx;  in cp_prefetch()
839 	list_for_each_entry_continue(chain, &cp->ccwchain_list, next) {  in cp_prefetch()
840 		chain->ch_len = 0;  in cp_prefetch()
846  * cp_get_orb() - get the orb of the channel program
861 	if (!cp->initialized)  in cp_get_orb()
864 	orb = &cp->orb;  in cp_get_orb()
866 	orb->cmd.intparm = (u32)virt_to_phys(sch);  in cp_get_orb()
867 	orb->cmd.fmt = 1;  in cp_get_orb()
870 	 * Everything built by vfio-ccw is a Format-2 IDAL.  in cp_get_orb()
871 	 * If the input was a Format-1 IDAL, indicate that  in cp_get_orb()
872 	 * 2K Format-2 IDAWs were created here.  in cp_get_orb()
874 	if (!orb->cmd.c64)  in cp_get_orb()
875 		orb->cmd.i2k = 1;  in cp_get_orb()
876 	orb->cmd.c64 = 1;  in cp_get_orb()
878 	if (orb->cmd.lpm == 0)  in cp_get_orb()
879 		orb->cmd.lpm = sch->lpm;  in cp_get_orb()
881 	chain = list_first_entry(&cp->ccwchain_list, struct ccwchain, next);  in cp_get_orb()
882 	cpa = chain->ch_ccw;  in cp_get_orb()
883 	orb->cmd.cpa = virt_to_dma32(cpa);  in cp_get_orb()
889  * cp_update_scsw() - update scsw for a channel program.
895  * to by @cp. However what @scsw->cpa stores is a host physical
899  * This function updates @scsw->cpa to its coressponding guest physical
905 	dma32_t cpa = scsw->cmd.cpa;  in cp_update_scsw()
908 	if (!cp->initialized)  in cp_update_scsw()
917 	list_for_each_entry(chain, &cp->ccwchain_list, next) {  in cp_update_scsw()
918 		ccw_head = dma32_to_u32(virt_to_dma32(chain->ch_ccw));  in cp_update_scsw()
923 		if (is_cpa_within_range(cpa, ccw_head, chain->ch_len + 1)) {  in cp_update_scsw()
925 			 * (cpa - ccw_head) is the offset value of the host  in cp_update_scsw()
929 			 * cpa = chain->ch_iova + (cpa - ccw_head)  in cp_update_scsw()
931 			cpa = dma32_add(cpa, chain->ch_iova - ccw_head);  in cp_update_scsw()
936 	scsw->cmd.cpa = cpa;  in cp_update_scsw()
940  * cp_iova_pinned() - check if an iova is pinned for a ccw chain.
953 	if (!cp->initialized)  in cp_iova_pinned()
956 	list_for_each_entry(chain, &cp->ccwchain_list, next) {  in cp_iova_pinned()
957 		for (i = 0; i < chain->ch_len; i++)  in cp_iova_pinned()
958 			if (page_array_iova_pinned(&chain->ch_pa[i], iova, length))  in cp_iova_pinned()