1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef TEST_XSK_H_ 3 #define TEST_XSK_H_ 4 5 #include <linux/ethtool.h> 6 #include <linux/if_xdp.h> 7 8 #include "../kselftest.h" 9 #include "xsk.h" 10 11 #ifndef SO_PREFER_BUSY_POLL 12 #define SO_PREFER_BUSY_POLL 69 13 #endif 14 15 #ifndef SO_BUSY_POLL_BUDGET 16 #define SO_BUSY_POLL_BUDGET 70 17 #endif 18 19 #define TEST_PASS 0 20 #define TEST_FAILURE -1 21 #define TEST_CONTINUE 1 22 #define TEST_SKIP 2 23 24 #define DEFAULT_PKT_CNT (4 * 1024) 25 #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) 26 #define HUGEPAGE_SIZE (2 * 1024 * 1024) 27 #define MIN_PKT_SIZE 64 28 #define MAX_ETH_PKT_SIZE 1518 29 #define MAX_INTERFACE_NAME_CHARS 16 30 #define MAX_TEST_NAME_SIZE 48 31 #define SOCK_RECONF_CTR 10 32 #define USLEEP_MAX 10000 33 34 extern bool opt_verbose; 35 #define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) 36 37 38 static inline u32 ceil_u32(u32 a, u32 b) 39 { 40 return (a + b - 1) / b; 41 } 42 43 static inline u64 ceil_u64(u64 a, u64 b) 44 { 45 return (a + b - 1) / b; 46 } 47 48 /* Simple test */ 49 enum test_mode { 50 TEST_MODE_SKB, 51 TEST_MODE_DRV, 52 TEST_MODE_ZC, 53 TEST_MODE_ALL 54 }; 55 56 struct ifobject; 57 struct test_spec; 58 typedef int (*validation_func_t)(struct ifobject *ifobj); 59 typedef void *(*thread_func_t)(void *arg); 60 typedef int (*test_func_t)(struct test_spec *test); 61 62 struct xsk_socket_info { 63 struct xsk_ring_cons rx; 64 struct xsk_ring_prod tx; 65 struct xsk_umem_info *umem; 66 struct xsk_socket *xsk; 67 struct pkt_stream *pkt_stream; 68 u32 outstanding_tx; 69 u32 rxqsize; 70 u32 batch_size; 71 u8 dst_mac[ETH_ALEN]; 72 u8 src_mac[ETH_ALEN]; 73 bool check_consumer; 74 }; 75 76 int kick_rx(struct xsk_socket_info *xsk); 77 int kick_tx(struct xsk_socket_info *xsk); 78 79 struct xsk_umem_info { 80 struct xsk_ring_prod fq; 81 struct xsk_ring_cons cq; 82 struct xsk_umem *umem; 83 u64 next_buffer; 84 u32 num_frames; 85 u32 frame_headroom; 86 void *buffer; 87 u32 frame_size; 88 u32 base_addr; 89 u32 fill_size; 90 u32 comp_size; 91 bool unaligned_mode; 92 }; 93 94 struct set_hw_ring { 95 u32 default_tx; 96 u32 default_rx; 97 }; 98 99 int hw_ring_size_reset(struct ifobject *ifobj); 100 101 struct ifobject { 102 char ifname[MAX_INTERFACE_NAME_CHARS]; 103 struct xsk_socket_info *xsk; 104 struct xsk_socket_info *xsk_arr; 105 struct xsk_umem_info *umem; 106 thread_func_t func_ptr; 107 validation_func_t validation_func; 108 struct xsk_xdp_progs *xdp_progs; 109 struct bpf_map *xskmap; 110 struct bpf_program *xdp_prog; 111 struct ethtool_ringparam ring; 112 struct set_hw_ring set_ring; 113 enum test_mode mode; 114 int ifindex; 115 int mtu; 116 u32 bind_flags; 117 u32 xdp_zc_max_segs; 118 bool tx_on; 119 bool rx_on; 120 bool use_poll; 121 bool busy_poll; 122 bool use_fill_ring; 123 bool release_rx; 124 bool shared_umem; 125 bool use_metadata; 126 bool unaligned_supp; 127 bool multi_buff_supp; 128 bool multi_buff_zc_supp; 129 bool hw_ring_size_supp; 130 }; 131 struct ifobject *ifobject_create(void); 132 void ifobject_delete(struct ifobject *ifobj); 133 int init_iface(struct ifobject *ifobj, thread_func_t func_ptr); 134 135 int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, u64 size); 136 int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 137 struct ifobject *ifobject, bool shared); 138 139 140 struct pkt { 141 int offset; 142 u32 len; 143 u32 pkt_nb; 144 bool valid; 145 u16 options; 146 }; 147 148 struct pkt_stream { 149 u32 nb_pkts; 150 u32 current_pkt_nb; 151 struct pkt *pkts; 152 u32 max_pkt_len; 153 u32 nb_rx_pkts; 154 u32 nb_valid_entries; 155 bool verbatim; 156 }; 157 158 static inline bool pkt_continues(u32 options) 159 { 160 return options & XDP_PKT_CONTD; 161 } 162 163 struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len); 164 void pkt_stream_delete(struct pkt_stream *pkt_stream); 165 void pkt_stream_reset(struct pkt_stream *pkt_stream); 166 void pkt_stream_restore_default(struct test_spec *test); 167 168 struct test_spec { 169 struct ifobject *ifobj_tx; 170 struct ifobject *ifobj_rx; 171 struct pkt_stream *tx_pkt_stream_default; 172 struct pkt_stream *rx_pkt_stream_default; 173 struct bpf_program *xdp_prog_rx; 174 struct bpf_program *xdp_prog_tx; 175 struct bpf_map *xskmap_rx; 176 struct bpf_map *xskmap_tx; 177 test_func_t test_func; 178 int mtu; 179 u16 total_steps; 180 u16 current_step; 181 u16 nb_sockets; 182 bool fail; 183 bool set_ring; 184 bool adjust_tail; 185 bool adjust_tail_support; 186 enum test_mode mode; 187 char name[MAX_TEST_NAME_SIZE]; 188 }; 189 190 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 191 static inline char *mode_string(struct test_spec *test) 192 { 193 switch (test->mode) { 194 case TEST_MODE_SKB: 195 return "SKB"; 196 case TEST_MODE_DRV: 197 return "DRV"; 198 case TEST_MODE_ZC: 199 return "ZC"; 200 default: 201 return "BOGUS"; 202 } 203 } 204 205 void test_init(struct test_spec *test, struct ifobject *ifobj_tx, 206 struct ifobject *ifobj_rx, enum test_mode mode, 207 const struct test_spec *test_to_run); 208 209 int testapp_adjust_tail_grow(struct test_spec *test); 210 int testapp_adjust_tail_grow_mb(struct test_spec *test); 211 int testapp_adjust_tail_shrink(struct test_spec *test); 212 int testapp_adjust_tail_shrink_mb(struct test_spec *test); 213 int testapp_aligned_inv_desc(struct test_spec *test); 214 int testapp_aligned_inv_desc_2k_frame(struct test_spec *test); 215 int testapp_aligned_inv_desc_mb(struct test_spec *test); 216 int testapp_bidirectional(struct test_spec *test); 217 int testapp_headroom(struct test_spec *test); 218 int testapp_hw_sw_max_ring_size(struct test_spec *test); 219 int testapp_hw_sw_min_ring_size(struct test_spec *test); 220 int testapp_poll_rx(struct test_spec *test); 221 int testapp_poll_rxq_tmout(struct test_spec *test); 222 int testapp_poll_tx(struct test_spec *test); 223 int testapp_poll_txq_tmout(struct test_spec *test); 224 int testapp_send_receive(struct test_spec *test); 225 int testapp_send_receive_2k_frame(struct test_spec *test); 226 int testapp_send_receive_mb(struct test_spec *test); 227 int testapp_send_receive_unaligned(struct test_spec *test); 228 int testapp_send_receive_unaligned_mb(struct test_spec *test); 229 int testapp_single_pkt(struct test_spec *test); 230 int testapp_stats_fill_empty(struct test_spec *test); 231 int testapp_stats_rx_dropped(struct test_spec *test); 232 int testapp_stats_tx_invalid_descs(struct test_spec *test); 233 int testapp_stats_rx_full(struct test_spec *test); 234 int testapp_teardown(struct test_spec *test); 235 int testapp_too_many_frags(struct test_spec *test); 236 int testapp_tx_queue_consumer(struct test_spec *test); 237 int testapp_unaligned_inv_desc(struct test_spec *test); 238 int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test); 239 int testapp_unaligned_inv_desc_mb(struct test_spec *test); 240 int testapp_xdp_drop(struct test_spec *test); 241 int testapp_xdp_metadata(struct test_spec *test); 242 int testapp_xdp_metadata_mb(struct test_spec *test); 243 int testapp_xdp_prog_cleanup(struct test_spec *test); 244 int testapp_xdp_shared_umem(struct test_spec *test); 245 246 void *worker_testapp_validate_rx(void *arg); 247 void *worker_testapp_validate_tx(void *arg); 248 249 static const struct test_spec tests[] = { 250 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, 251 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, 252 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, 253 {.name = "POLL_RX", .test_func = testapp_poll_rx}, 254 {.name = "POLL_TX", .test_func = testapp_poll_tx}, 255 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, 256 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, 257 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, 258 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, 259 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, 260 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, 261 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, 262 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, 263 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, 264 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, 265 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, 266 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, 267 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, 268 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, 269 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, 270 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, 271 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, 272 {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, 273 {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, 274 }; 275 276 static const struct test_spec ci_skip_tests[] = { 277 /* Flaky tests */ 278 {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, 279 {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, 280 {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, 281 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, 282 /* Tests with huge page dependency */ 283 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, 284 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, 285 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", 286 .test_func = testapp_unaligned_inv_desc_4001_frame}, 287 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", 288 .test_func = testapp_send_receive_unaligned_mb}, 289 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, 290 /* Test with HW ring size dependency */ 291 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, 292 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, 293 /* Too long test */ 294 {.name = "TEARDOWN", .test_func = testapp_teardown}, 295 }; 296 297 298 #endif /* TEST_XSK_H_ */ 299