127 #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
382 static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,  in parse_command_line()  argument
399 				ifobj = ifobj_tx;  in parse_command_line()
418 			ifobj_tx->busy_poll = true;  in parse_command_line()
474 static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,  in __test_spec_init()  argument
480 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;  in __test_spec_init()
517 	if (ifobj_tx->hw_ring_size_supp)  in __test_spec_init()
518 		hw_ring_size_reset(ifobj_tx);  in __test_spec_init()
520 	test->ifobj_tx = ifobj_tx;  in __test_spec_init()
530 	test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;  in __test_spec_init()
531 	test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;  in __test_spec_init()
534 static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,  in test_spec_init()  argument
549 		struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;  in test_spec_init()
561 	__test_spec_init(test, ifobj_tx, ifobj_rx);  in test_spec_init()
566 	__test_spec_init(test, test->ifobj_tx, test->ifobj_rx);  in test_spec_reset()
589 	if (test->ifobj_tx->mtu != mtu) {  in test_spec_set_mtu()
590 		err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu);  in test_spec_set_mtu()
593 		test->ifobj_tx->mtu = mtu;  in test_spec_set_mtu()
634 	struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream;  in pkt_stream_restore_default()
638 		pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream);  in pkt_stream_restore_default()
639 		test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default;  in pkt_stream_restore_default()
765 	test->ifobj_tx->xsk->pkt_stream = pkt_stream;  in pkt_stream_replace()
785 	__pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);  in pkt_stream_replace_half()
791 	struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream;  in pkt_stream_receive_half()
809 		pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream;  in pkt_stream_even_odd_sequence()
812 		test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream;  in pkt_stream_even_odd_sequence()
908 	pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);  in pkt_stream_generate_custom()
909 	test->ifobj_tx->xsk->pkt_stream = pkt_stream;  in pkt_stream_generate_custom()
1146 			if (!is_umem_valid(test->ifobj_tx))  in __receive_pkts()
1726 	struct ifobject *ifobject = test->ifobj_tx;  in worker_testapp_validate_tx()
1807 	struct ifobject *ifobj = test->ifobj_tx;  in xdp_prog_changed_tx()
1836 				 struct ifobject *ifobj_tx)  in xsk_attach_xdp_progs()  argument
1841 	if (!ifobj_tx || ifobj_tx->shared_umem)  in xsk_attach_xdp_progs()
1845 		xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);  in xsk_attach_xdp_progs()
1923 	struct ifobject *ifobj_tx = test->ifobj_tx;  in testapp_validate_traffic()  local
1926 	    (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) {  in testapp_validate_traffic()
1932 		if (ifobj_tx->hw_ring_size_supp) {  in testapp_validate_traffic()
1933 			if (set_ring_size(ifobj_tx)) {  in testapp_validate_traffic()
1943 	xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx);  in testapp_validate_traffic()
1944 	return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);  in testapp_validate_traffic()
1981 	test->ifobj_tx->rx_on = true;  in testapp_bidirectional()
1988 	swap_directions(&test->ifobj_rx, &test->ifobj_tx);  in testapp_bidirectional()
1989 	res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);  in testapp_bidirectional()
1991 	swap_directions(&test->ifobj_rx, &test->ifobj_tx);  in testapp_bidirectional()
1999 	test->ifobj_tx->xsk_arr[0].pkt_stream = NULL;  in swap_xsk_resources()
2001 	test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default;  in swap_xsk_resources()
2003 	test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1];  in swap_xsk_resources()
2049 	test->ifobj_tx->validation_func = validate_tx_invalid_descs;  in testapp_stats_tx_invalid_descs()
2076 	test->ifobj_tx->umem->unaligned_mode = true;  in testapp_send_receive_unaligned()
2087 	test->ifobj_tx->umem->unaligned_mode = true;  in testapp_send_receive_unaligned_mb()
2111 	struct xsk_umem_info *umem = test->ifobj_tx->umem;  in testapp_invalid_desc_mb()
2151 	struct xsk_umem_info *umem = test->ifobj_tx->umem;  in testapp_invalid_desc()
2184 	if (test->ifobj_tx->shared_umem) {  in testapp_invalid_desc()
2197 	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;  in testapp_xdp_drop()
2209 	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;  in testapp_xdp_metadata_copy()
2236 	struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;  in testapp_xdp_shared_umem()
2252 	test->ifobj_tx->use_poll = true;  in testapp_poll_txq_tmout()
2254 	test->ifobj_tx->umem->frame_size = 2048;  in testapp_poll_txq_tmout()
2256 	return testapp_validate_traffic_single_thread(test, test->ifobj_tx);  in testapp_poll_txq_tmout()
2272 		max_frags = test->ifobj_tx->xdp_zc_max_segs;  in testapp_too_many_frags()
2390 	test->ifobj_tx->umem->frame_size = 2048;  in testapp_send_receive_2k_frame()
2404 	test->ifobj_tx->use_poll = true;  in testapp_poll_tx()
2415 	test->ifobj_tx->umem->frame_size = 2048;  in testapp_aligned_inv_desc_2k_frame()
2422 	test->ifobj_tx->umem->unaligned_mode = true;  in testapp_unaligned_inv_desc()
2432 	test->ifobj_tx->umem->frame_size = 4001;  in testapp_unaligned_inv_desc_4001_frame()
2434 	test->ifobj_tx->umem->unaligned_mode = true;  in testapp_unaligned_inv_desc_4001_frame()
2440 	umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;  in testapp_unaligned_inv_desc_4001_frame()
2454 	test->ifobj_tx->umem->unaligned_mode = true;  in testapp_unaligned_inv_desc_mb()
2476 	test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE;  in testapp_hw_sw_min_ring_size()
2477 	test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2;  in testapp_hw_sw_min_ring_size()
2478 	test->ifobj_tx->xsk->batch_size = 1;  in testapp_hw_sw_min_ring_size()
2485 	test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;  in testapp_hw_sw_min_ring_size()
2497 	test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending;  in testapp_hw_sw_max_ring_size()
2498 	test->ifobj_tx->ring.rx_pending  = test->ifobj_tx->ring.rx_max_pending;  in testapp_hw_sw_max_ring_size()
2502 	test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;  in testapp_hw_sw_max_ring_size()
2512 	test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;  in testapp_hw_sw_max_ring_size()
2513 	test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;  in testapp_hw_sw_max_ring_size()
2639 	struct ifobject *ifobj_tx, *ifobj_rx;  in main()  local
2649 	ifobj_tx = ifobject_create();  in main()
2650 	if (!ifobj_tx)  in main()
2658 	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);  in main()
2669 	shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);  in main()
2670 	ifobj_tx->shared_umem = shared_netdev;  in main()
2673 	if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))  in main()
2676 	if (is_xdp_supported(ifobj_tx->ifindex)) {  in main()
2678 		if (ifobj_zc_avail(ifobj_tx))  in main()
2682 	ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);  in main()
2684 		ifobj_tx->hw_ring_size_supp = true;  in main()
2685 		ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;  in main()
2686 		ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;  in main()
2690 	init_iface(ifobj_tx, worker_testapp_validate_tx);  in main()
2692 	test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);  in main()
2727 			test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);  in main()
2736 	if (ifobj_tx->hw_ring_size_supp)  in main()
2737 		hw_ring_size_reset(ifobj_tx);  in main()
2741 	xsk_unload_xdp_programs(ifobj_tx);  in main()
2743 	ifobject_delete(ifobj_tx);  in main()