1 /* SPDX-License-Identifier: (GPL-2.0 WITH Linux-syscall-note) OR MIT */ 2 /* 3 * Header file for the io_uring zerocopy receive (zcrx) interface. 4 * 5 * Copyright (C) 2026 Pavel Begunkov 6 * Copyright (C) 2026 David Wei 7 * Copyright (C) Meta Platforms, Inc. 8 */ 9 #ifndef LINUX_IO_ZCRX_H 10 #define LINUX_IO_ZCRX_H 11 12 #include <linux/types.h> 13 14 /* Zero copy receive refill queue entry */ 15 struct io_uring_zcrx_rqe { 16 __u64 off; 17 __u32 len; 18 __u32 __pad; 19 }; 20 21 struct io_uring_zcrx_cqe { 22 __u64 off; 23 __u64 __pad; 24 }; 25 26 /* The bit from which area id is encoded into offsets */ 27 #define IORING_ZCRX_AREA_SHIFT 48 28 #define IORING_ZCRX_AREA_MASK (~(((__u64)1 << IORING_ZCRX_AREA_SHIFT) - 1)) 29 30 struct io_uring_zcrx_offsets { 31 __u32 head; 32 __u32 tail; 33 __u32 rqes; 34 __u32 __resv2; 35 __u64 __resv[2]; 36 }; 37 38 enum io_uring_zcrx_area_flags { 39 IORING_ZCRX_AREA_DMABUF = 1, 40 }; 41 42 struct io_uring_zcrx_area_reg { 43 __u64 addr; 44 __u64 len; 45 __u64 rq_area_token; 46 __u32 flags; 47 __u32 dmabuf_fd; 48 __u64 __resv2[2]; 49 }; 50 51 enum zcrx_reg_flags { 52 ZCRX_REG_IMPORT = 1, 53 54 /* 55 * Register a zcrx instance without a net device. All data will be 56 * copied. The refill queue entries might not be automatically 57 * consumed and need to be flushed, see ZCRX_CTRL_FLUSH_RQ. 58 */ 59 ZCRX_REG_NODEV = 2, 60 }; 61 62 enum zcrx_features { 63 /* 64 * The user can ask for the desired rx page size by passing the 65 * value in struct io_uring_zcrx_ifq_reg::rx_buf_len. 66 */ 67 ZCRX_FEATURE_RX_PAGE_SIZE = 1 << 0, 68 ZCRX_FEATURE_EVENT = 1 << 1, 69 }; 70 71 enum zcrx_event_type { 72 ZCRX_EVENT_ALLOC_FAIL, 73 ZCRX_EVENT_COPY, 74 75 __ZCRX_EVENT_TYPE_LAST, 76 }; 77 78 enum zcrx_event_desc_flags { 79 /* If set, stats_offset holds a valid offset to a zcrx_stats struct */ 80 ZCRX_EVENT_DESC_FLAG_STATS = 1 << 0, 81 }; 82 83 struct zcrx_stats { 84 __u64 copy_count; /* cumulative copy-fallback CQEs */ 85 __u64 copy_bytes; /* cumulative bytes copied */ 86 }; 87 88 struct zcrx_event_desc { 89 __u64 user_data; 90 __u32 type_mask; 91 __u32 flags; /* see enum zcrx_event_desc_flags */ 92 __u64 stats_offset; /* offset from the beginning of refill ring region for stats */ 93 __u64 __resv2[9]; 94 }; 95 96 /* 97 * Argument for IORING_REGISTER_ZCRX_IFQ 98 */ 99 struct io_uring_zcrx_ifq_reg { 100 __u32 if_idx; 101 __u32 if_rxq; 102 __u32 rq_entries; 103 __u32 flags; 104 105 __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */ 106 __u64 region_ptr; /* struct io_uring_region_desc * */ 107 108 struct io_uring_zcrx_offsets offsets; 109 __u32 zcrx_id; 110 __u32 rx_buf_len; 111 __u64 event_desc; /* see struct zcrx_event_desc */ 112 __u64 __resv[2]; 113 }; 114 115 enum zcrx_ctrl_op { 116 ZCRX_CTRL_FLUSH_RQ, 117 ZCRX_CTRL_EXPORT, 118 ZCRX_CTRL_ARM_EVENT, 119 ZCRX_CTRL_ADD_AREA, 120 121 __ZCRX_CTRL_LAST, 122 }; 123 124 struct zcrx_ctrl_flush_rq { 125 __u64 __resv[6]; 126 }; 127 128 struct zcrx_ctrl_export { 129 __u32 zcrx_fd; 130 __u32 __resv1[11]; 131 }; 132 133 struct zcrx_ctrl_arm_event { 134 __u32 event_type; /* see enum zcrx_event_type */ 135 __u32 __resv[11]; 136 }; 137 138 struct zcrx_ctrl_add_area { 139 __u64 area_ptr; /* pointer to struct io_uring_zcrx_area_reg */ 140 __u64 __resv[5]; 141 }; 142 143 struct zcrx_ctrl { 144 __u32 zcrx_id; 145 __u32 op; /* see enum zcrx_ctrl_op */ 146 __u64 __resv[2]; 147 148 union { 149 struct zcrx_ctrl_export zc_export; 150 struct zcrx_ctrl_flush_rq zc_flush; 151 struct zcrx_ctrl_arm_event zc_arm_event; 152 struct zcrx_ctrl_add_area zc_area; 153 }; 154 }; 155 156 #endif /* LINUX_IO_ZCRX_H */ 157