Lines Matching defs:packed
235 * for packed ring:
253 /* Available for packed ring */
254 struct vring_virtqueue_packed packed;
1504 head = vq->packed.next_avail_idx;
1551 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1552 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1554 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1557 vq->packed.desc_extra[id].addr = addr;
1558 vq->packed.desc_extra[id].len = total_sg *
1560 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1561 vq->packed.avail_used_flags;
1570 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1571 vq->packed.avail_used_flags);
1578 if (n >= vq->packed.vring.num) {
1580 vq->packed.avail_wrap_counter ^= 1;
1581 vq->packed.avail_used_flags ^=
1585 vq->packed.next_avail_idx = n;
1587 vq->free_head = vq->packed.desc_extra[id].next;
1590 vq->packed.desc_state[id].num = 1;
1591 vq->packed.desc_state[id].data = data;
1592 vq->packed.desc_state[id].indir_desc = desc;
1593 vq->packed.desc_state[id].last = id;
1594 vq->packed.desc_state[id].total_in_len = total_in_len;
1649 BUG_ON(id == vq->packed.vring.num);
1661 head = vq->packed.next_avail_idx;
1662 avail_used_flags = vq->packed.avail_used_flags;
1664 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1666 desc = vq->packed.vring.desc;
1678 BUG_ON(id == vq->packed.vring.num);
1691 flags = cpu_to_le16(vq->packed.avail_used_flags |
1704 vq->packed.desc_extra[curr].addr = premapped ?
1706 vq->packed.desc_extra[curr].len = len;
1707 vq->packed.desc_extra[curr].flags =
1711 curr = vq->packed.desc_extra[curr].next;
1713 if ((unlikely(++i >= vq->packed.vring.num))) {
1715 vq->packed.avail_used_flags ^=
1723 vq->packed.avail_wrap_counter ^= 1;
1729 vq->packed.next_avail_idx = i;
1733 vq->packed.desc_state[id].num = descs_used;
1734 vq->packed.desc_state[id].data = data;
1735 vq->packed.desc_state[id].indir_desc = ctx;
1736 vq->packed.desc_state[id].last = prev;
1744 vq->packed.vring.desc[head].flags = head_flags;
1757 vq->packed.avail_used_flags = avail_used_flags;
1762 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1763 curr = vq->packed.desc_extra[curr].next;
1765 if (i >= vq->packed.vring.num)
1809 vq->packed.next_avail_idx,
1819 head = vq->packed.next_avail_idx;
1820 avail_used_flags = vq->packed.avail_used_flags;
1821 avail_wrap_counter = vq->packed.avail_wrap_counter;
1823 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1825 desc = vq->packed.vring.desc;
1852 flags |= cpu_to_le16(vq->packed.avail_used_flags);
1864 vq->packed.desc_extra[i].addr = premapped ?
1866 vq->packed.desc_extra[i].len = len;
1867 vq->packed.desc_extra[i].flags =
1871 if ((unlikely(++i >= vq->packed.vring.num))) {
1873 vq->packed.avail_used_flags ^=
1876 vq->packed.avail_wrap_counter ^= 1;
1888 vq->packed.next_avail_idx = i;
1891 vq->packed.desc_state[head].num = total_sg;
1892 vq->packed.desc_state[head].data = data;
1893 vq->packed.desc_state[head].indir_desc = ctx;
1894 vq->packed.desc_state[head].total_in_len = total_in_len;
1902 vq->packed.vring.desc[head].flags = head_flags;
1913 vq->packed.avail_used_flags = avail_used_flags;
1914 vq->packed.avail_wrap_counter = avail_wrap_counter;
1919 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]);
1921 if (i >= vq->packed.vring.num)
1949 old = vq->packed.next_avail_idx - vq->num_added;
1950 new = vq->packed.next_avail_idx;
1953 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1968 if (wrap_counter != vq->packed.avail_wrap_counter)
1969 event_idx -= vq->packed.vring.num;
1984 state = &vq->packed.desc_state[id];
1995 &vq->packed.desc_extra[curr]);
1996 curr = vq->packed.desc_extra[curr].next;
2010 len = vq->packed.desc_extra[id].len;
2028 struct vring_desc_state_packed *state = &vq->packed.desc_state[id];
2030 vq->packed.desc_extra[state->last].next = vq->free_head;
2042 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
2070 last_used += vq->packed.desc_state[id].num;
2071 if (unlikely(last_used >= vq->packed.vring.num)) {
2072 last_used -= vq->packed.vring.num;
2084 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
2086 &vq->packed.vring.driver->off_wrap,
2102 unsigned int num = vq->packed.vring.num;
2127 le16_to_cpu(vq->packed.vring.desc[last_used].id);
2129 le32_to_cpu(vq->packed.vring.desc[last_used].len);
2136 *len = vq->packed.desc_state[last_used].total_in_len;
2143 if (unlikely(!vq->packed.desc_state[last_used].data)) {
2149 ret = vq->packed.desc_state[last_used].data;
2165 unsigned int num = vq->packed.vring.num;
2189 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
2190 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
2196 if (unlikely(!vq->packed.desc_state[id].data)) {
2202 ret = vq->packed.desc_state[id].data;
2215 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
2216 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
2225 vq->packed.vring.driver->flags =
2226 cpu_to_le16(vq->packed.event_flags_shadow);
2240 vq->packed.vring.driver->off_wrap =
2249 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
2250 vq->packed.event_flags_shadow = vq->event ?
2253 vq->packed.vring.driver->flags =
2254 cpu_to_le16(vq->packed.event_flags_shadow);
2275 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
2280 if (used_idx >= vq->packed.vring.num) {
2281 used_idx -= vq->packed.vring.num;
2285 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
2295 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
2296 vq->packed.event_flags_shadow = vq->event ?
2299 vq->packed.vring.driver->flags =
2300 cpu_to_le16(vq->packed.event_flags_shadow);
2328 for (i = 0; i < vq->packed.vring.num; i++) {
2329 if (!vq->packed.desc_state[i].data)
2332 buf = vq->packed.desc_state[i].data;
2341 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
2492 vq->packed = *vring_packed;
2507 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
2508 memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
2511 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
2512 virtqueue_init(vq, vq->packed.vring.num);
2513 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
3440 vq->packed.ring_size_in_bytes,
3441 vq->packed.vring.desc,
3442 vq->packed.ring_dma_addr,
3446 vq->packed.event_size_in_bytes,
3447 vq->packed.vring.driver,
3448 vq->packed.driver_event_dma_addr,
3452 vq->packed.event_size_in_bytes,
3453 vq->packed.vring.device,
3454 vq->packed.device_event_dma_addr,
3457 kfree(vq->packed.desc_state);
3458 kfree(vq->packed.desc_extra);
3493 next = (vq->packed.next_avail_idx &
3495 vq->packed.avail_wrap_counter <<
3547 return virtqueue_is_packed(vq) ? vq->packed.vring.num :
3632 return vq->packed.ring_dma_addr;
3645 return vq->packed.driver_event_dma_addr;
3659 return vq->packed.device_event_dma_addr;