204 		struct vring_virtqueue_packed packed;  member
1339 	head = vq->packed.next_avail_idx;  in virtqueue_add_indirect_packed()
1355 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_indirect_packed()
1386 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);  in virtqueue_add_indirect_packed()
1387 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *  in virtqueue_add_indirect_packed()
1389 	vq->packed.vring.desc[head].id = cpu_to_le16(id);  in virtqueue_add_indirect_packed()
1392 		vq->packed.desc_extra[id].addr = addr;  in virtqueue_add_indirect_packed()
1393 		vq->packed.desc_extra[id].len = total_sg *  in virtqueue_add_indirect_packed()
1395 		vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1396 						  vq->packed.avail_used_flags;  in virtqueue_add_indirect_packed()
1405 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |  in virtqueue_add_indirect_packed()
1406 						vq->packed.avail_used_flags);  in virtqueue_add_indirect_packed()
1413 	if (n >= vq->packed.vring.num) {  in virtqueue_add_indirect_packed()
1415 		vq->packed.avail_wrap_counter ^= 1;  in virtqueue_add_indirect_packed()
1416 		vq->packed.avail_used_flags ^=  in virtqueue_add_indirect_packed()
1420 	vq->packed.next_avail_idx = n;  in virtqueue_add_indirect_packed()
1421 	vq->free_head = vq->packed.desc_extra[id].next;  in virtqueue_add_indirect_packed()
1424 	vq->packed.desc_state[id].num = 1;  in virtqueue_add_indirect_packed()
1425 	vq->packed.desc_state[id].data = data;  in virtqueue_add_indirect_packed()
1426 	vq->packed.desc_state[id].indir_desc = desc;  in virtqueue_add_indirect_packed()
1427 	vq->packed.desc_state[id].last = id;  in virtqueue_add_indirect_packed()
1491 	head = vq->packed.next_avail_idx;  in virtqueue_add_packed()
1492 	avail_used_flags = vq->packed.avail_used_flags;  in virtqueue_add_packed()
1494 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);  in virtqueue_add_packed()
1496 	desc = vq->packed.vring.desc;  in virtqueue_add_packed()
1508 	BUG_ON(id == vq->packed.vring.num);  in virtqueue_add_packed()
1521 			flags = cpu_to_le16(vq->packed.avail_used_flags |  in virtqueue_add_packed()
1534 				vq->packed.desc_extra[curr].addr = premapped ?  in virtqueue_add_packed()
1536 				vq->packed.desc_extra[curr].len = len;  in virtqueue_add_packed()
1537 				vq->packed.desc_extra[curr].flags =  in virtqueue_add_packed()
1541 			curr = vq->packed.desc_extra[curr].next;  in virtqueue_add_packed()
1543 			if ((unlikely(++i >= vq->packed.vring.num))) {  in virtqueue_add_packed()
1545 				vq->packed.avail_used_flags ^=  in virtqueue_add_packed()
1553 		vq->packed.avail_wrap_counter ^= 1;  in virtqueue_add_packed()
1559 	vq->packed.next_avail_idx = i;  in virtqueue_add_packed()
1563 	vq->packed.desc_state[id].num = descs_used;  in virtqueue_add_packed()
1564 	vq->packed.desc_state[id].data = data;  in virtqueue_add_packed()
1565 	vq->packed.desc_state[id].indir_desc = ctx;  in virtqueue_add_packed()
1566 	vq->packed.desc_state[id].last = prev;  in virtqueue_add_packed()
1574 	vq->packed.vring.desc[head].flags = head_flags;  in virtqueue_add_packed()
1587 	vq->packed.avail_used_flags = avail_used_flags;  in virtqueue_add_packed()
1592 		vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);  in virtqueue_add_packed()
1593 		curr = vq->packed.desc_extra[curr].next;  in virtqueue_add_packed()
1595 		if (i >= vq->packed.vring.num)  in virtqueue_add_packed()
1624 	old = vq->packed.next_avail_idx - vq->num_added;  in virtqueue_kick_prepare_packed()
1625 	new = vq->packed.next_avail_idx;  in virtqueue_kick_prepare_packed()
1628 	snapshot.u32 = *(u32 *)vq->packed.vring.device;  in virtqueue_kick_prepare_packed()
1643 	if (wrap_counter != vq->packed.avail_wrap_counter)  in virtqueue_kick_prepare_packed()
1644 		event_idx -= vq->packed.vring.num;  in virtqueue_kick_prepare_packed()
1659 	state = &vq->packed.desc_state[id];  in detach_buf_packed()
1664 	vq->packed.desc_extra[state->last].next = vq->free_head;  in detach_buf_packed()
1672 						 &vq->packed.desc_extra[curr]);  in detach_buf_packed()
1673 			curr = vq->packed.desc_extra[curr].next;  in detach_buf_packed()
1687 			len = vq->packed.desc_extra[id].len;  in detach_buf_packed()
1708 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);  in is_used_desc_packed()
1755 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);  in virtqueue_get_buf_ctx_packed()
1756 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);  in virtqueue_get_buf_ctx_packed()
1758 	if (unlikely(id >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1762 	if (unlikely(!vq->packed.desc_state[id].data)) {  in virtqueue_get_buf_ctx_packed()
1768 	ret = vq->packed.desc_state[id].data;  in virtqueue_get_buf_ctx_packed()
1771 	last_used += vq->packed.desc_state[id].num;  in virtqueue_get_buf_ctx_packed()
1772 	if (unlikely(last_used >= vq->packed.vring.num)) {  in virtqueue_get_buf_ctx_packed()
1773 		last_used -= vq->packed.vring.num;  in virtqueue_get_buf_ctx_packed()
1785 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)  in virtqueue_get_buf_ctx_packed()
1787 				&vq->packed.vring.driver->off_wrap,  in virtqueue_get_buf_ctx_packed()
1800 	if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {  in virtqueue_disable_cb_packed()
1801 		vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;  in virtqueue_disable_cb_packed()
1810 		vq->packed.vring.driver->flags =  in virtqueue_disable_cb_packed()
1811 			cpu_to_le16(vq->packed.event_flags_shadow);  in virtqueue_disable_cb_packed()
1827 		vq->packed.vring.driver->off_wrap =  in virtqueue_enable_cb_prepare_packed()
1836 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {  in virtqueue_enable_cb_prepare_packed()
1837 		vq->packed.event_flags_shadow = vq->event ?  in virtqueue_enable_cb_prepare_packed()
1840 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_prepare_packed()
1841 				cpu_to_le16(vq->packed.event_flags_shadow);  in virtqueue_enable_cb_prepare_packed()
1875 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;  in virtqueue_enable_cb_delayed_packed()
1880 		if (used_idx >= vq->packed.vring.num) {  in virtqueue_enable_cb_delayed_packed()
1881 			used_idx -= vq->packed.vring.num;  in virtqueue_enable_cb_delayed_packed()
1885 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |  in virtqueue_enable_cb_delayed_packed()
1895 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {  in virtqueue_enable_cb_delayed_packed()
1896 		vq->packed.event_flags_shadow = vq->event ?  in virtqueue_enable_cb_delayed_packed()
1899 		vq->packed.vring.driver->flags =  in virtqueue_enable_cb_delayed_packed()
1900 				cpu_to_le16(vq->packed.event_flags_shadow);  in virtqueue_enable_cb_delayed_packed()
1929 	for (i = 0; i < vq->packed.vring.num; i++) {  in virtqueue_detach_unused_buf_packed()
1930 		if (!vq->packed.desc_state[i].data)  in virtqueue_detach_unused_buf_packed()
1933 		buf = vq->packed.desc_state[i].data;  in virtqueue_detach_unused_buf_packed()
1939 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);  in virtqueue_detach_unused_buf_packed()
2089 	vq->packed = *vring_packed;  in virtqueue_vring_attach_packed()
2097 	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
2098 	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);  in virtqueue_reinit_packed()
2101 	memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);  in virtqueue_reinit_packed()
2103 	virtqueue_init(vq, vq->packed.vring.num);  in virtqueue_reinit_packed()
2104 	virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);  in virtqueue_reinit_packed()
2808 	if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)  in virtqueue_resize()
2901 					 vq->packed.ring_size_in_bytes,  in vring_free()
2902 					 vq->packed.vring.desc,  in vring_free()
2903 					 vq->packed.ring_dma_addr,  in vring_free()
2907 					 vq->packed.event_size_in_bytes,  in vring_free()
2908 					 vq->packed.vring.driver,  in vring_free()
2909 					 vq->packed.driver_event_dma_addr,  in vring_free()
2913 					 vq->packed.event_size_in_bytes,  in vring_free()
2914 					 vq->packed.vring.device,  in vring_free()
2915 					 vq->packed.device_event_dma_addr,  in vring_free()
2918 			kfree(vq->packed.desc_state);  in vring_free()
2919 			kfree(vq->packed.desc_extra);  in vring_free()
2954 		next = (vq->packed.next_avail_idx &  in vring_notification_data()
2956 			vq->packed.avail_wrap_counter <<  in vring_notification_data()
3006 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;  in virtqueue_get_vring_size()
3090 		return vq->packed.ring_dma_addr;  in virtqueue_get_desc_addr()
3103 		return vq->packed.driver_event_dma_addr;  in virtqueue_get_avail_addr()
3117 		return vq->packed.device_event_dma_addr;  in virtqueue_get_used_addr()