1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef TEST_XSK_H_ 3 #define TEST_XSK_H_ 4 5 #include <linux/ethtool.h> 6 #include <linux/if_xdp.h> 7 8 #include "../kselftest.h" 9 #include "xsk.h" 10 11 #ifndef SO_PREFER_BUSY_POLL 12 #define SO_PREFER_BUSY_POLL 69 13 #endif 14 15 #ifndef SO_BUSY_POLL_BUDGET 16 #define SO_BUSY_POLL_BUDGET 70 17 #endif 18 19 #define TEST_PASS 0 20 #define TEST_FAILURE -1 21 #define TEST_CONTINUE 1 22 #define TEST_SKIP 2 23 24 #define DEFAULT_PKT_CNT (4 * 1024) 25 #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) 26 #define HUGEPAGE_SIZE (2 * 1024 * 1024) 27 #define MIN_PKT_SIZE 64 28 #define MAX_ETH_PKT_SIZE 1518 29 #define MAX_INTERFACE_NAME_CHARS 16 30 #define MAX_TEST_NAME_SIZE 48 31 #define SOCK_RECONF_CTR 10 32 #define USLEEP_MAX 10000 33 34 #define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" 35 #define SMP_CACHE_BYTES_PATH "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size" 36 37 extern bool opt_verbose; 38 #define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) 39 40 41 static inline u32 ceil_u32(u32 a, u32 b) 42 { 43 return (a + b - 1) / b; 44 } 45 46 static inline u64 ceil_u64(u64 a, u64 b) 47 { 48 return (a + b - 1) / b; 49 } 50 51 static inline unsigned int read_procfs_val(const char *path) 52 { 53 unsigned int read_val = 0; 54 FILE *file; 55 56 file = fopen(path, "r"); 57 if (!file) { 58 ksft_print_msg("Error opening %s\n", path); 59 return 0; 60 } 61 62 if (fscanf(file, "%u", &read_val) != 1) 63 ksft_print_msg("Error reading %s\n", path); 64 65 fclose(file); 66 return read_val; 67 } 68 69 /* Simple test */ 70 enum test_mode { 71 TEST_MODE_SKB, 72 TEST_MODE_DRV, 73 TEST_MODE_ZC, 74 TEST_MODE_ALL 75 }; 76 77 struct ifobject; 78 struct test_spec; 79 typedef int (*validation_func_t)(struct ifobject *ifobj); 80 typedef void *(*thread_func_t)(void *arg); 81 typedef int (*test_func_t)(struct test_spec *test); 82 83 struct xsk_socket_info { 84 struct xsk_ring_cons rx; 85 struct xsk_ring_prod tx; 86 struct xsk_umem_info *umem; 87 struct xsk_socket *xsk; 88 struct pkt_stream *pkt_stream; 89 u32 outstanding_tx; 90 u32 rxqsize; 91 u32 batch_size; 92 u8 dst_mac[ETH_ALEN]; 93 u8 src_mac[ETH_ALEN]; 94 bool check_consumer; 95 }; 96 97 int kick_rx(struct xsk_socket_info *xsk); 98 int kick_tx(struct xsk_socket_info *xsk); 99 100 struct xsk_umem_info { 101 struct xsk_ring_prod fq; 102 struct xsk_ring_cons cq; 103 struct xsk_umem *umem; 104 u64 next_buffer; 105 u32 num_frames; 106 u32 frame_headroom; 107 void *buffer; 108 u32 frame_size; 109 u32 base_addr; 110 u32 fill_size; 111 u32 comp_size; 112 bool unaligned_mode; 113 }; 114 115 struct set_hw_ring { 116 u32 default_tx; 117 u32 default_rx; 118 }; 119 120 int hw_ring_size_reset(struct ifobject *ifobj); 121 122 struct ifobject { 123 char ifname[MAX_INTERFACE_NAME_CHARS]; 124 struct xsk_socket_info *xsk; 125 struct xsk_socket_info *xsk_arr; 126 struct xsk_umem_info *umem; 127 thread_func_t func_ptr; 128 validation_func_t validation_func; 129 struct xsk_xdp_progs *xdp_progs; 130 struct bpf_map *xskmap; 131 struct bpf_program *xdp_prog; 132 struct ethtool_ringparam ring; 133 struct set_hw_ring set_ring; 134 enum test_mode mode; 135 int ifindex; 136 int mtu; 137 u32 bind_flags; 138 u32 xdp_zc_max_segs; 139 u32 umem_tailroom; 140 u32 max_skb_frags; 141 bool tx_on; 142 bool rx_on; 143 bool use_poll; 144 bool busy_poll; 145 bool use_fill_ring; 146 bool release_rx; 147 bool shared_umem; 148 bool use_metadata; 149 bool unaligned_supp; 150 bool multi_buff_supp; 151 bool multi_buff_zc_supp; 152 bool hw_ring_size_supp; 153 }; 154 struct ifobject *ifobject_create(void); 155 void ifobject_delete(struct ifobject *ifobj); 156 int init_iface(struct ifobject *ifobj, thread_func_t func_ptr); 157 158 int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, u64 size); 159 int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 160 struct ifobject *ifobject, bool shared); 161 162 163 struct pkt { 164 int offset; 165 u32 len; 166 u32 pkt_nb; 167 bool valid; 168 u16 options; 169 }; 170 171 struct pkt_stream { 172 u32 nb_pkts; 173 u32 current_pkt_nb; 174 struct pkt *pkts; 175 u32 max_pkt_len; 176 u32 nb_rx_pkts; 177 u32 nb_valid_entries; 178 bool verbatim; 179 }; 180 181 static inline bool pkt_continues(u32 options) 182 { 183 return options & XDP_PKT_CONTD; 184 } 185 186 struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len); 187 void pkt_stream_delete(struct pkt_stream *pkt_stream); 188 void pkt_stream_reset(struct pkt_stream *pkt_stream); 189 void pkt_stream_restore_default(struct test_spec *test); 190 191 struct test_spec { 192 struct ifobject *ifobj_tx; 193 struct ifobject *ifobj_rx; 194 struct pkt_stream *tx_pkt_stream_default; 195 struct pkt_stream *rx_pkt_stream_default; 196 struct bpf_program *xdp_prog_rx; 197 struct bpf_program *xdp_prog_tx; 198 struct bpf_map *xskmap_rx; 199 struct bpf_map *xskmap_tx; 200 test_func_t test_func; 201 int mtu; 202 u16 total_steps; 203 u16 current_step; 204 u16 nb_sockets; 205 bool fail; 206 bool set_ring; 207 bool adjust_tail; 208 bool adjust_tail_support; 209 enum test_mode mode; 210 char name[MAX_TEST_NAME_SIZE]; 211 }; 212 213 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 214 static inline char *mode_string(struct test_spec *test) 215 { 216 switch (test->mode) { 217 case TEST_MODE_SKB: 218 return "SKB"; 219 case TEST_MODE_DRV: 220 return "DRV"; 221 case TEST_MODE_ZC: 222 return "ZC"; 223 default: 224 return "BOGUS"; 225 } 226 } 227 228 void test_init(struct test_spec *test, struct ifobject *ifobj_tx, 229 struct ifobject *ifobj_rx, enum test_mode mode, 230 const struct test_spec *test_to_run); 231 232 int testapp_adjust_tail_grow(struct test_spec *test); 233 int testapp_adjust_tail_grow_mb(struct test_spec *test); 234 int testapp_adjust_tail_shrink(struct test_spec *test); 235 int testapp_adjust_tail_shrink_mb(struct test_spec *test); 236 int testapp_aligned_inv_desc(struct test_spec *test); 237 int testapp_aligned_inv_desc_2k_frame(struct test_spec *test); 238 int testapp_aligned_inv_desc_mb(struct test_spec *test); 239 int testapp_bidirectional(struct test_spec *test); 240 int testapp_headroom(struct test_spec *test); 241 int testapp_hw_sw_max_ring_size(struct test_spec *test); 242 int testapp_hw_sw_min_ring_size(struct test_spec *test); 243 int testapp_poll_rx(struct test_spec *test); 244 int testapp_poll_rxq_tmout(struct test_spec *test); 245 int testapp_poll_tx(struct test_spec *test); 246 int testapp_poll_txq_tmout(struct test_spec *test); 247 int testapp_send_receive(struct test_spec *test); 248 int testapp_send_receive_2k_frame(struct test_spec *test); 249 int testapp_send_receive_mb(struct test_spec *test); 250 int testapp_send_receive_unaligned(struct test_spec *test); 251 int testapp_send_receive_unaligned_mb(struct test_spec *test); 252 int testapp_single_pkt(struct test_spec *test); 253 int testapp_stats_fill_empty(struct test_spec *test); 254 int testapp_stats_rx_dropped(struct test_spec *test); 255 int testapp_stats_tx_invalid_descs(struct test_spec *test); 256 int testapp_stats_rx_full(struct test_spec *test); 257 int testapp_teardown(struct test_spec *test); 258 int testapp_too_many_frags(struct test_spec *test); 259 int testapp_tx_queue_consumer(struct test_spec *test); 260 int testapp_unaligned_inv_desc(struct test_spec *test); 261 int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test); 262 int testapp_unaligned_inv_desc_mb(struct test_spec *test); 263 int testapp_xdp_drop(struct test_spec *test); 264 int testapp_xdp_metadata(struct test_spec *test); 265 int testapp_xdp_metadata_mb(struct test_spec *test); 266 int testapp_xdp_prog_cleanup(struct test_spec *test); 267 int testapp_xdp_shared_umem(struct test_spec *test); 268 269 void *worker_testapp_validate_rx(void *arg); 270 void *worker_testapp_validate_tx(void *arg); 271 272 static const struct test_spec tests[] = { 273 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, 274 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, 275 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, 276 {.name = "POLL_RX", .test_func = testapp_poll_rx}, 277 {.name = "POLL_TX", .test_func = testapp_poll_tx}, 278 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, 279 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, 280 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, 281 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, 282 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, 283 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, 284 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, 285 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, 286 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, 287 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, 288 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, 289 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, 290 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, 291 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, 292 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, 293 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, 294 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, 295 {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, 296 {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, 297 }; 298 299 static const struct test_spec ci_skip_tests[] = { 300 /* Flaky tests */ 301 {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, 302 {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, 303 {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, 304 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, 305 /* Tests with huge page dependency */ 306 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, 307 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, 308 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", 309 .test_func = testapp_unaligned_inv_desc_4001_frame}, 310 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", 311 .test_func = testapp_send_receive_unaligned_mb}, 312 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, 313 /* Test with HW ring size dependency */ 314 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, 315 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, 316 /* Too long test */ 317 {.name = "TEARDOWN", .test_func = testapp_teardown}, 318 }; 319 320 321 #endif /* TEST_XSK_H_ */ 322