1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef TEST_XSK_H_ 3 #define TEST_XSK_H_ 4 5 #include <linux/ethtool.h> 6 #include <linux/if_xdp.h> 7 8 #include "../kselftest.h" 9 #include "xsk.h" 10 11 #ifndef SO_PREFER_BUSY_POLL 12 #define SO_PREFER_BUSY_POLL 69 13 #endif 14 15 #ifndef SO_BUSY_POLL_BUDGET 16 #define SO_BUSY_POLL_BUDGET 70 17 #endif 18 19 #define TEST_PASS 0 20 #define TEST_FAILURE -1 21 #define TEST_CONTINUE 1 22 #define TEST_SKIP 2 23 24 #define DEFAULT_PKT_CNT (4 * 1024) 25 #define DEFAULT_UMEM_BUFFERS (DEFAULT_PKT_CNT / 4) 26 #define HUGEPAGE_SIZE (2 * 1024 * 1024) 27 #define MIN_PKT_SIZE 64 28 #define MAX_ETH_PKT_SIZE 1518 29 #define MAX_INTERFACE_NAME_CHARS 16 30 #define MAX_TEST_NAME_SIZE 48 31 #define SOCK_RECONF_CTR 10 32 #define USLEEP_MAX 10000 33 34 #define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" 35 #define SMP_CACHE_BYTES_PATH "/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size" 36 37 extern bool opt_verbose; 38 #define print_verbose(x...) do { if (opt_verbose) ksft_print_msg(x); } while (0) 39 40 41 static inline u32 ceil_u32(u32 a, u32 b) 42 { 43 return (a + b - 1) / b; 44 } 45 46 static inline u64 ceil_u64(u64 a, u64 b) 47 { 48 return (a + b - 1) / b; 49 } 50 51 static inline unsigned int read_procfs_val(const char *path) 52 { 53 unsigned int read_val = 0; 54 FILE *file; 55 56 file = fopen(path, "r"); 57 if (!file) { 58 ksft_print_msg("Error opening %s\n", path); 59 return 0; 60 } 61 62 if (fscanf(file, "%u", &read_val) != 1) 63 ksft_print_msg("Error reading %s\n", path); 64 65 fclose(file); 66 return read_val; 67 } 68 69 /* Simple test */ 70 enum test_mode { 71 TEST_MODE_SKB, 72 TEST_MODE_DRV, 73 TEST_MODE_ZC, 74 TEST_MODE_ALL 75 }; 76 77 struct ifobject; 78 struct test_spec; 79 typedef int (*validation_func_t)(struct ifobject *ifobj); 80 typedef void *(*thread_func_t)(void *arg); 81 typedef int (*test_func_t)(struct test_spec *test); 82 83 struct xsk_socket_info { 84 struct xsk_ring_cons rx; 85 struct xsk_ring_prod tx; 86 struct xsk_umem_info *umem_real; 87 struct xsk_umem_info *umem; 88 struct xsk_socket *xsk; 89 struct pkt_stream *pkt_stream; 90 u32 outstanding_tx; 91 u32 rxqsize; 92 u32 batch_size; 93 u8 dst_mac[ETH_ALEN]; 94 u8 src_mac[ETH_ALEN]; 95 bool check_consumer; 96 }; 97 98 int kick_rx(struct xsk_socket_info *xsk); 99 int kick_tx(struct xsk_socket_info *xsk); 100 101 struct xsk_umem_info { 102 struct xsk_ring_prod fq; 103 struct xsk_ring_cons cq; 104 struct xsk_umem *umem; 105 u64 next_buffer; 106 u64 mmap_size; 107 u32 num_frames; 108 u32 frame_headroom; 109 void *buffer; 110 u32 frame_size; 111 u32 base_addr; 112 u32 fill_size; 113 u32 comp_size; 114 bool unaligned_mode; 115 }; 116 117 struct set_hw_ring { 118 u32 default_tx; 119 u32 default_rx; 120 }; 121 122 int hw_ring_size_reset(struct ifobject *ifobj); 123 124 struct ifobject { 125 char ifname[MAX_INTERFACE_NAME_CHARS]; 126 struct xsk_socket_info *xsk; 127 struct xsk_socket_info *xsk_arr; 128 thread_func_t func_ptr; 129 validation_func_t validation_func; 130 struct xsk_xdp_progs *xdp_progs; 131 struct bpf_map *xskmap; 132 struct bpf_program *xdp_prog; 133 struct ethtool_ringparam ring; 134 struct set_hw_ring set_ring; 135 enum test_mode mode; 136 int ifindex; 137 int mtu; 138 u32 bind_flags; 139 u32 xdp_zc_max_segs; 140 u32 umem_tailroom; 141 u32 max_skb_frags; 142 bool tx_on; 143 bool rx_on; 144 bool use_poll; 145 bool busy_poll; 146 bool use_fill_ring; 147 bool release_rx; 148 bool shared_umem; 149 bool use_metadata; 150 bool unaligned_supp; 151 bool multi_buff_supp; 152 bool multi_buff_zc_supp; 153 bool hw_ring_size_supp; 154 }; 155 struct ifobject *ifobject_create(void); 156 void ifobject_delete(struct ifobject *ifobj); 157 int init_iface(struct ifobject *ifobj, thread_func_t func_ptr); 158 159 int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, u64 size); 160 int xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, 161 struct ifobject *ifobject, bool shared); 162 163 164 struct pkt { 165 int offset; 166 u32 len; 167 u32 pkt_nb; 168 bool valid; 169 u16 options; 170 }; 171 172 struct pkt_stream { 173 u32 nb_pkts; 174 u32 current_pkt_nb; 175 struct pkt *pkts; 176 u32 max_pkt_len; 177 u32 nb_rx_pkts; 178 u32 nb_valid_entries; 179 bool verbatim; 180 }; 181 182 static inline bool pkt_continues(u32 options) 183 { 184 return options & XDP_PKT_CONTD; 185 } 186 187 struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len); 188 void pkt_stream_delete(struct pkt_stream *pkt_stream); 189 void pkt_stream_reset(struct pkt_stream *pkt_stream); 190 void pkt_stream_restore_default(struct test_spec *test); 191 192 struct test_spec { 193 struct ifobject *ifobj_tx; 194 struct ifobject *ifobj_rx; 195 struct pkt_stream *tx_pkt_stream_default; 196 struct pkt_stream *rx_pkt_stream_default; 197 struct bpf_program *xdp_prog_rx; 198 struct bpf_program *xdp_prog_tx; 199 struct bpf_map *xskmap_rx; 200 struct bpf_map *xskmap_tx; 201 test_func_t test_func; 202 int mtu; 203 u16 total_steps; 204 u16 current_step; 205 u16 nb_sockets; 206 bool fail; 207 bool set_ring; 208 bool adjust_tail; 209 bool adjust_tail_support; 210 enum test_mode mode; 211 char name[MAX_TEST_NAME_SIZE]; 212 }; 213 214 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" 215 static inline char *mode_string(struct test_spec *test) 216 { 217 switch (test->mode) { 218 case TEST_MODE_SKB: 219 return "SKB"; 220 case TEST_MODE_DRV: 221 return "DRV"; 222 case TEST_MODE_ZC: 223 return "ZC"; 224 default: 225 return "BOGUS"; 226 } 227 } 228 229 void test_init(struct test_spec *test, struct ifobject *ifobj_tx, 230 struct ifobject *ifobj_rx, enum test_mode mode, 231 const struct test_spec *test_to_run); 232 233 int testapp_adjust_tail_grow(struct test_spec *test); 234 int testapp_adjust_tail_grow_mb(struct test_spec *test); 235 int testapp_adjust_tail_shrink(struct test_spec *test); 236 int testapp_adjust_tail_shrink_mb(struct test_spec *test); 237 int testapp_aligned_inv_desc(struct test_spec *test); 238 int testapp_aligned_inv_desc_2k_frame(struct test_spec *test); 239 int testapp_aligned_inv_desc_mb(struct test_spec *test); 240 int testapp_bidirectional(struct test_spec *test); 241 int testapp_headroom(struct test_spec *test); 242 int testapp_hw_sw_max_ring_size(struct test_spec *test); 243 int testapp_hw_sw_min_ring_size(struct test_spec *test); 244 int testapp_poll_rx(struct test_spec *test); 245 int testapp_poll_rxq_tmout(struct test_spec *test); 246 int testapp_poll_tx(struct test_spec *test); 247 int testapp_poll_txq_tmout(struct test_spec *test); 248 int testapp_send_receive(struct test_spec *test); 249 int testapp_send_receive_2k_frame(struct test_spec *test); 250 int testapp_send_receive_mb(struct test_spec *test); 251 int testapp_send_receive_unaligned(struct test_spec *test); 252 int testapp_send_receive_unaligned_mb(struct test_spec *test); 253 int testapp_single_pkt(struct test_spec *test); 254 int testapp_stats_fill_empty(struct test_spec *test); 255 int testapp_stats_rx_dropped(struct test_spec *test); 256 int testapp_stats_tx_invalid_descs(struct test_spec *test); 257 int testapp_stats_rx_full(struct test_spec *test); 258 int testapp_teardown(struct test_spec *test); 259 int testapp_too_many_frags(struct test_spec *test); 260 int testapp_tx_queue_consumer(struct test_spec *test); 261 int testapp_unaligned_inv_desc(struct test_spec *test); 262 int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test); 263 int testapp_unaligned_inv_desc_mb(struct test_spec *test); 264 int testapp_xdp_drop(struct test_spec *test); 265 int testapp_xdp_metadata(struct test_spec *test); 266 int testapp_xdp_metadata_mb(struct test_spec *test); 267 int testapp_xdp_prog_cleanup(struct test_spec *test); 268 int testapp_xdp_shared_umem(struct test_spec *test); 269 270 void *worker_testapp_validate_rx(void *arg); 271 void *worker_testapp_validate_tx(void *arg); 272 273 static const struct test_spec tests[] = { 274 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, 275 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, 276 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, 277 {.name = "POLL_RX", .test_func = testapp_poll_rx}, 278 {.name = "POLL_TX", .test_func = testapp_poll_tx}, 279 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, 280 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, 281 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, 282 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, 283 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, 284 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, 285 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, 286 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, 287 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, 288 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, 289 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, 290 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, 291 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, 292 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, 293 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, 294 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, 295 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, 296 {.name = "XDP_ADJUST_TAIL_SHRINK", .test_func = testapp_adjust_tail_shrink}, 297 {.name = "TX_QUEUE_CONSUMER", .test_func = testapp_tx_queue_consumer}, 298 }; 299 300 static const struct test_spec ci_skip_tests[] = { 301 /* Flaky tests */ 302 {.name = "XDP_ADJUST_TAIL_SHRINK_MULTI_BUFF", .test_func = testapp_adjust_tail_shrink_mb}, 303 {.name = "XDP_ADJUST_TAIL_GROW", .test_func = testapp_adjust_tail_grow}, 304 {.name = "XDP_ADJUST_TAIL_GROW_MULTI_BUFF", .test_func = testapp_adjust_tail_grow_mb}, 305 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, 306 /* Tests with huge page dependency */ 307 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, 308 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, 309 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", 310 .test_func = testapp_unaligned_inv_desc_4001_frame}, 311 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", 312 .test_func = testapp_send_receive_unaligned_mb}, 313 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, 314 /* Test with HW ring size dependency */ 315 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, 316 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, 317 /* Too long test */ 318 {.name = "TEARDOWN", .test_func = testapp_teardown}, 319 }; 320 321 322 #endif /* TEST_XSK_H_ */ 323