1*c5d3705cSYunsheng Lin // SPDX-License-Identifier: GPL-2.0
2*c5d3705cSYunsheng Lin #define _GNU_SOURCE
3*c5d3705cSYunsheng Lin #include <getopt.h>
4*c5d3705cSYunsheng Lin #include <limits.h>
5*c5d3705cSYunsheng Lin #include <string.h>
6*c5d3705cSYunsheng Lin #include <poll.h>
7*c5d3705cSYunsheng Lin #include <sys/eventfd.h>
8*c5d3705cSYunsheng Lin #include <stdlib.h>
9*c5d3705cSYunsheng Lin #include <assert.h>
10*c5d3705cSYunsheng Lin #include <unistd.h>
11*c5d3705cSYunsheng Lin #include <sys/ioctl.h>
12*c5d3705cSYunsheng Lin #include <sys/stat.h>
13*c5d3705cSYunsheng Lin #include <sys/types.h>
14*c5d3705cSYunsheng Lin #include <fcntl.h>
15*c5d3705cSYunsheng Lin #include <stdbool.h>
16*c5d3705cSYunsheng Lin #include <linux/vhost.h>
17*c5d3705cSYunsheng Lin #include <linux/if.h>
18*c5d3705cSYunsheng Lin #include <linux/if_tun.h>
19*c5d3705cSYunsheng Lin #include <linux/in.h>
20*c5d3705cSYunsheng Lin #include <linux/if_packet.h>
21*c5d3705cSYunsheng Lin #include <linux/virtio_net.h>
22*c5d3705cSYunsheng Lin #include <netinet/ether.h>
23*c5d3705cSYunsheng Lin
24*c5d3705cSYunsheng Lin #define HDR_LEN sizeof(struct virtio_net_hdr_mrg_rxbuf)
25*c5d3705cSYunsheng Lin #define TEST_BUF_LEN 256
26*c5d3705cSYunsheng Lin #define TEST_PTYPE ETH_P_LOOPBACK
27*c5d3705cSYunsheng Lin #define DESC_NUM 256
28*c5d3705cSYunsheng Lin
29*c5d3705cSYunsheng Lin /* Used by implementation of kmalloc() in tools/virtio/linux/kernel.h */
30*c5d3705cSYunsheng Lin void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end;
31*c5d3705cSYunsheng Lin
32*c5d3705cSYunsheng Lin struct vq_info {
33*c5d3705cSYunsheng Lin int kick;
34*c5d3705cSYunsheng Lin int call;
35*c5d3705cSYunsheng Lin int idx;
36*c5d3705cSYunsheng Lin long started;
37*c5d3705cSYunsheng Lin long completed;
38*c5d3705cSYunsheng Lin struct pollfd fds;
39*c5d3705cSYunsheng Lin void *ring;
40*c5d3705cSYunsheng Lin /* copy used for control */
41*c5d3705cSYunsheng Lin struct vring vring;
42*c5d3705cSYunsheng Lin struct virtqueue *vq;
43*c5d3705cSYunsheng Lin };
44*c5d3705cSYunsheng Lin
45*c5d3705cSYunsheng Lin struct vdev_info {
46*c5d3705cSYunsheng Lin struct virtio_device vdev;
47*c5d3705cSYunsheng Lin int control;
48*c5d3705cSYunsheng Lin struct vq_info vqs[2];
49*c5d3705cSYunsheng Lin int nvqs;
50*c5d3705cSYunsheng Lin void *buf;
51*c5d3705cSYunsheng Lin size_t buf_size;
52*c5d3705cSYunsheng Lin char *test_buf;
53*c5d3705cSYunsheng Lin char *res_buf;
54*c5d3705cSYunsheng Lin struct vhost_memory *mem;
55*c5d3705cSYunsheng Lin int sock;
56*c5d3705cSYunsheng Lin int ifindex;
57*c5d3705cSYunsheng Lin unsigned char mac[ETHER_ADDR_LEN];
58*c5d3705cSYunsheng Lin };
59*c5d3705cSYunsheng Lin
tun_alloc(struct vdev_info * dev,char * tun_name)60*c5d3705cSYunsheng Lin static int tun_alloc(struct vdev_info *dev, char *tun_name)
61*c5d3705cSYunsheng Lin {
62*c5d3705cSYunsheng Lin struct ifreq ifr;
63*c5d3705cSYunsheng Lin int len = HDR_LEN;
64*c5d3705cSYunsheng Lin int fd, e;
65*c5d3705cSYunsheng Lin
66*c5d3705cSYunsheng Lin fd = open("/dev/net/tun", O_RDWR);
67*c5d3705cSYunsheng Lin if (fd < 0) {
68*c5d3705cSYunsheng Lin perror("Cannot open /dev/net/tun");
69*c5d3705cSYunsheng Lin return fd;
70*c5d3705cSYunsheng Lin }
71*c5d3705cSYunsheng Lin
72*c5d3705cSYunsheng Lin memset(&ifr, 0, sizeof(ifr));
73*c5d3705cSYunsheng Lin
74*c5d3705cSYunsheng Lin ifr.ifr_flags = IFF_TAP | IFF_NO_PI | IFF_VNET_HDR;
75*c5d3705cSYunsheng Lin strncpy(ifr.ifr_name, tun_name, IFNAMSIZ);
76*c5d3705cSYunsheng Lin
77*c5d3705cSYunsheng Lin e = ioctl(fd, TUNSETIFF, &ifr);
78*c5d3705cSYunsheng Lin if (e < 0) {
79*c5d3705cSYunsheng Lin perror("ioctl[TUNSETIFF]");
80*c5d3705cSYunsheng Lin close(fd);
81*c5d3705cSYunsheng Lin return e;
82*c5d3705cSYunsheng Lin }
83*c5d3705cSYunsheng Lin
84*c5d3705cSYunsheng Lin e = ioctl(fd, TUNSETVNETHDRSZ, &len);
85*c5d3705cSYunsheng Lin if (e < 0) {
86*c5d3705cSYunsheng Lin perror("ioctl[TUNSETVNETHDRSZ]");
87*c5d3705cSYunsheng Lin close(fd);
88*c5d3705cSYunsheng Lin return e;
89*c5d3705cSYunsheng Lin }
90*c5d3705cSYunsheng Lin
91*c5d3705cSYunsheng Lin e = ioctl(fd, SIOCGIFHWADDR, &ifr);
92*c5d3705cSYunsheng Lin if (e < 0) {
93*c5d3705cSYunsheng Lin perror("ioctl[SIOCGIFHWADDR]");
94*c5d3705cSYunsheng Lin close(fd);
95*c5d3705cSYunsheng Lin return e;
96*c5d3705cSYunsheng Lin }
97*c5d3705cSYunsheng Lin
98*c5d3705cSYunsheng Lin memcpy(dev->mac, &ifr.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
99*c5d3705cSYunsheng Lin return fd;
100*c5d3705cSYunsheng Lin }
101*c5d3705cSYunsheng Lin
vdev_create_socket(struct vdev_info * dev,char * tun_name)102*c5d3705cSYunsheng Lin static void vdev_create_socket(struct vdev_info *dev, char *tun_name)
103*c5d3705cSYunsheng Lin {
104*c5d3705cSYunsheng Lin struct ifreq ifr;
105*c5d3705cSYunsheng Lin
106*c5d3705cSYunsheng Lin dev->sock = socket(AF_PACKET, SOCK_RAW, htons(TEST_PTYPE));
107*c5d3705cSYunsheng Lin assert(dev->sock != -1);
108*c5d3705cSYunsheng Lin
109*c5d3705cSYunsheng Lin strncpy(ifr.ifr_name, tun_name, IFNAMSIZ);
110*c5d3705cSYunsheng Lin assert(ioctl(dev->sock, SIOCGIFINDEX, &ifr) >= 0);
111*c5d3705cSYunsheng Lin
112*c5d3705cSYunsheng Lin dev->ifindex = ifr.ifr_ifindex;
113*c5d3705cSYunsheng Lin
114*c5d3705cSYunsheng Lin /* Set the flags that bring the device up */
115*c5d3705cSYunsheng Lin assert(ioctl(dev->sock, SIOCGIFFLAGS, &ifr) >= 0);
116*c5d3705cSYunsheng Lin ifr.ifr_flags |= (IFF_UP | IFF_RUNNING);
117*c5d3705cSYunsheng Lin assert(ioctl(dev->sock, SIOCSIFFLAGS, &ifr) >= 0);
118*c5d3705cSYunsheng Lin }
119*c5d3705cSYunsheng Lin
vdev_send_packet(struct vdev_info * dev)120*c5d3705cSYunsheng Lin static void vdev_send_packet(struct vdev_info *dev)
121*c5d3705cSYunsheng Lin {
122*c5d3705cSYunsheng Lin char *sendbuf = dev->test_buf + HDR_LEN;
123*c5d3705cSYunsheng Lin struct sockaddr_ll saddrll = {0};
124*c5d3705cSYunsheng Lin int sockfd = dev->sock;
125*c5d3705cSYunsheng Lin int ret;
126*c5d3705cSYunsheng Lin
127*c5d3705cSYunsheng Lin saddrll.sll_family = PF_PACKET;
128*c5d3705cSYunsheng Lin saddrll.sll_ifindex = dev->ifindex;
129*c5d3705cSYunsheng Lin saddrll.sll_halen = ETH_ALEN;
130*c5d3705cSYunsheng Lin saddrll.sll_protocol = htons(TEST_PTYPE);
131*c5d3705cSYunsheng Lin
132*c5d3705cSYunsheng Lin ret = sendto(sockfd, sendbuf, TEST_BUF_LEN, 0,
133*c5d3705cSYunsheng Lin (struct sockaddr *)&saddrll,
134*c5d3705cSYunsheng Lin sizeof(struct sockaddr_ll));
135*c5d3705cSYunsheng Lin assert(ret >= 0);
136*c5d3705cSYunsheng Lin }
137*c5d3705cSYunsheng Lin
vq_notify(struct virtqueue * vq)138*c5d3705cSYunsheng Lin static bool vq_notify(struct virtqueue *vq)
139*c5d3705cSYunsheng Lin {
140*c5d3705cSYunsheng Lin struct vq_info *info = vq->priv;
141*c5d3705cSYunsheng Lin unsigned long long v = 1;
142*c5d3705cSYunsheng Lin int r;
143*c5d3705cSYunsheng Lin
144*c5d3705cSYunsheng Lin r = write(info->kick, &v, sizeof(v));
145*c5d3705cSYunsheng Lin assert(r == sizeof(v));
146*c5d3705cSYunsheng Lin
147*c5d3705cSYunsheng Lin return true;
148*c5d3705cSYunsheng Lin }
149*c5d3705cSYunsheng Lin
vhost_vq_setup(struct vdev_info * dev,struct vq_info * info)150*c5d3705cSYunsheng Lin static void vhost_vq_setup(struct vdev_info *dev, struct vq_info *info)
151*c5d3705cSYunsheng Lin {
152*c5d3705cSYunsheng Lin struct vhost_vring_addr addr = {
153*c5d3705cSYunsheng Lin .index = info->idx,
154*c5d3705cSYunsheng Lin .desc_user_addr = (uint64_t)(unsigned long)info->vring.desc,
155*c5d3705cSYunsheng Lin .avail_user_addr = (uint64_t)(unsigned long)info->vring.avail,
156*c5d3705cSYunsheng Lin .used_user_addr = (uint64_t)(unsigned long)info->vring.used,
157*c5d3705cSYunsheng Lin };
158*c5d3705cSYunsheng Lin struct vhost_vring_state state = { .index = info->idx };
159*c5d3705cSYunsheng Lin struct vhost_vring_file file = { .index = info->idx };
160*c5d3705cSYunsheng Lin int r;
161*c5d3705cSYunsheng Lin
162*c5d3705cSYunsheng Lin state.num = info->vring.num;
163*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_SET_VRING_NUM, &state);
164*c5d3705cSYunsheng Lin assert(r >= 0);
165*c5d3705cSYunsheng Lin
166*c5d3705cSYunsheng Lin state.num = 0;
167*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_SET_VRING_BASE, &state);
168*c5d3705cSYunsheng Lin assert(r >= 0);
169*c5d3705cSYunsheng Lin
170*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_SET_VRING_ADDR, &addr);
171*c5d3705cSYunsheng Lin assert(r >= 0);
172*c5d3705cSYunsheng Lin
173*c5d3705cSYunsheng Lin file.fd = info->kick;
174*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_SET_VRING_KICK, &file);
175*c5d3705cSYunsheng Lin assert(r >= 0);
176*c5d3705cSYunsheng Lin }
177*c5d3705cSYunsheng Lin
vq_reset(struct vq_info * info,int num,struct virtio_device * vdev)178*c5d3705cSYunsheng Lin static void vq_reset(struct vq_info *info, int num, struct virtio_device *vdev)
179*c5d3705cSYunsheng Lin {
180*c5d3705cSYunsheng Lin if (info->vq)
181*c5d3705cSYunsheng Lin vring_del_virtqueue(info->vq);
182*c5d3705cSYunsheng Lin
183*c5d3705cSYunsheng Lin memset(info->ring, 0, vring_size(num, 4096));
184*c5d3705cSYunsheng Lin vring_init(&info->vring, num, info->ring, 4096);
185*c5d3705cSYunsheng Lin info->vq = vring_new_virtqueue(info->idx, num, 4096, vdev, true, false,
186*c5d3705cSYunsheng Lin info->ring, vq_notify, NULL, "test");
187*c5d3705cSYunsheng Lin assert(info->vq);
188*c5d3705cSYunsheng Lin info->vq->priv = info;
189*c5d3705cSYunsheng Lin }
190*c5d3705cSYunsheng Lin
vq_info_add(struct vdev_info * dev,int idx,int num,int fd)191*c5d3705cSYunsheng Lin static void vq_info_add(struct vdev_info *dev, int idx, int num, int fd)
192*c5d3705cSYunsheng Lin {
193*c5d3705cSYunsheng Lin struct vhost_vring_file backend = { .index = idx, .fd = fd };
194*c5d3705cSYunsheng Lin struct vq_info *info = &dev->vqs[idx];
195*c5d3705cSYunsheng Lin int r;
196*c5d3705cSYunsheng Lin
197*c5d3705cSYunsheng Lin info->idx = idx;
198*c5d3705cSYunsheng Lin info->kick = eventfd(0, EFD_NONBLOCK);
199*c5d3705cSYunsheng Lin r = posix_memalign(&info->ring, 4096, vring_size(num, 4096));
200*c5d3705cSYunsheng Lin assert(r >= 0);
201*c5d3705cSYunsheng Lin vq_reset(info, num, &dev->vdev);
202*c5d3705cSYunsheng Lin vhost_vq_setup(dev, info);
203*c5d3705cSYunsheng Lin
204*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_NET_SET_BACKEND, &backend);
205*c5d3705cSYunsheng Lin assert(!r);
206*c5d3705cSYunsheng Lin }
207*c5d3705cSYunsheng Lin
vdev_info_init(struct vdev_info * dev,unsigned long long features)208*c5d3705cSYunsheng Lin static void vdev_info_init(struct vdev_info *dev, unsigned long long features)
209*c5d3705cSYunsheng Lin {
210*c5d3705cSYunsheng Lin struct ether_header *eh;
211*c5d3705cSYunsheng Lin int i, r;
212*c5d3705cSYunsheng Lin
213*c5d3705cSYunsheng Lin dev->vdev.features = features;
214*c5d3705cSYunsheng Lin INIT_LIST_HEAD(&dev->vdev.vqs);
215*c5d3705cSYunsheng Lin spin_lock_init(&dev->vdev.vqs_list_lock);
216*c5d3705cSYunsheng Lin
217*c5d3705cSYunsheng Lin dev->buf_size = (HDR_LEN + TEST_BUF_LEN) * 2;
218*c5d3705cSYunsheng Lin dev->buf = malloc(dev->buf_size);
219*c5d3705cSYunsheng Lin assert(dev->buf);
220*c5d3705cSYunsheng Lin dev->test_buf = dev->buf;
221*c5d3705cSYunsheng Lin dev->res_buf = dev->test_buf + HDR_LEN + TEST_BUF_LEN;
222*c5d3705cSYunsheng Lin
223*c5d3705cSYunsheng Lin memset(dev->test_buf, 0, HDR_LEN + TEST_BUF_LEN);
224*c5d3705cSYunsheng Lin eh = (struct ether_header *)(dev->test_buf + HDR_LEN);
225*c5d3705cSYunsheng Lin eh->ether_type = htons(TEST_PTYPE);
226*c5d3705cSYunsheng Lin memcpy(eh->ether_dhost, dev->mac, ETHER_ADDR_LEN);
227*c5d3705cSYunsheng Lin memcpy(eh->ether_shost, dev->mac, ETHER_ADDR_LEN);
228*c5d3705cSYunsheng Lin
229*c5d3705cSYunsheng Lin for (i = sizeof(*eh); i < TEST_BUF_LEN; i++)
230*c5d3705cSYunsheng Lin dev->test_buf[i + HDR_LEN] = (char)i;
231*c5d3705cSYunsheng Lin
232*c5d3705cSYunsheng Lin dev->control = open("/dev/vhost-net", O_RDWR);
233*c5d3705cSYunsheng Lin assert(dev->control >= 0);
234*c5d3705cSYunsheng Lin
235*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_SET_OWNER, NULL);
236*c5d3705cSYunsheng Lin assert(r >= 0);
237*c5d3705cSYunsheng Lin
238*c5d3705cSYunsheng Lin dev->mem = malloc(offsetof(struct vhost_memory, regions) +
239*c5d3705cSYunsheng Lin sizeof(dev->mem->regions[0]));
240*c5d3705cSYunsheng Lin assert(dev->mem);
241*c5d3705cSYunsheng Lin memset(dev->mem, 0, offsetof(struct vhost_memory, regions) +
242*c5d3705cSYunsheng Lin sizeof(dev->mem->regions[0]));
243*c5d3705cSYunsheng Lin dev->mem->nregions = 1;
244*c5d3705cSYunsheng Lin dev->mem->regions[0].guest_phys_addr = (long)dev->buf;
245*c5d3705cSYunsheng Lin dev->mem->regions[0].userspace_addr = (long)dev->buf;
246*c5d3705cSYunsheng Lin dev->mem->regions[0].memory_size = dev->buf_size;
247*c5d3705cSYunsheng Lin
248*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_SET_MEM_TABLE, dev->mem);
249*c5d3705cSYunsheng Lin assert(r >= 0);
250*c5d3705cSYunsheng Lin
251*c5d3705cSYunsheng Lin r = ioctl(dev->control, VHOST_SET_FEATURES, &features);
252*c5d3705cSYunsheng Lin assert(r >= 0);
253*c5d3705cSYunsheng Lin
254*c5d3705cSYunsheng Lin dev->nvqs = 2;
255*c5d3705cSYunsheng Lin }
256*c5d3705cSYunsheng Lin
wait_for_interrupt(struct vq_info * vq)257*c5d3705cSYunsheng Lin static void wait_for_interrupt(struct vq_info *vq)
258*c5d3705cSYunsheng Lin {
259*c5d3705cSYunsheng Lin unsigned long long val;
260*c5d3705cSYunsheng Lin
261*c5d3705cSYunsheng Lin poll(&vq->fds, 1, 100);
262*c5d3705cSYunsheng Lin
263*c5d3705cSYunsheng Lin if (vq->fds.revents & POLLIN)
264*c5d3705cSYunsheng Lin read(vq->fds.fd, &val, sizeof(val));
265*c5d3705cSYunsheng Lin }
266*c5d3705cSYunsheng Lin
verify_res_buf(char * res_buf)267*c5d3705cSYunsheng Lin static void verify_res_buf(char *res_buf)
268*c5d3705cSYunsheng Lin {
269*c5d3705cSYunsheng Lin int i;
270*c5d3705cSYunsheng Lin
271*c5d3705cSYunsheng Lin for (i = ETHER_HDR_LEN; i < TEST_BUF_LEN; i++)
272*c5d3705cSYunsheng Lin assert(res_buf[i] == (char)i);
273*c5d3705cSYunsheng Lin }
274*c5d3705cSYunsheng Lin
run_tx_test(struct vdev_info * dev,struct vq_info * vq,bool delayed,int bufs)275*c5d3705cSYunsheng Lin static void run_tx_test(struct vdev_info *dev, struct vq_info *vq,
276*c5d3705cSYunsheng Lin bool delayed, int bufs)
277*c5d3705cSYunsheng Lin {
278*c5d3705cSYunsheng Lin long long spurious = 0;
279*c5d3705cSYunsheng Lin struct scatterlist sl;
280*c5d3705cSYunsheng Lin unsigned int len;
281*c5d3705cSYunsheng Lin int r;
282*c5d3705cSYunsheng Lin
283*c5d3705cSYunsheng Lin for (;;) {
284*c5d3705cSYunsheng Lin long started_before = vq->started;
285*c5d3705cSYunsheng Lin long completed_before = vq->completed;
286*c5d3705cSYunsheng Lin
287*c5d3705cSYunsheng Lin virtqueue_disable_cb(vq->vq);
288*c5d3705cSYunsheng Lin do {
289*c5d3705cSYunsheng Lin while (vq->started < bufs &&
290*c5d3705cSYunsheng Lin (vq->started - vq->completed) < 1) {
291*c5d3705cSYunsheng Lin sg_init_one(&sl, dev->test_buf, HDR_LEN + TEST_BUF_LEN);
292*c5d3705cSYunsheng Lin r = virtqueue_add_outbuf(vq->vq, &sl, 1,
293*c5d3705cSYunsheng Lin dev->test_buf + vq->started,
294*c5d3705cSYunsheng Lin GFP_ATOMIC);
295*c5d3705cSYunsheng Lin if (unlikely(r != 0))
296*c5d3705cSYunsheng Lin break;
297*c5d3705cSYunsheng Lin
298*c5d3705cSYunsheng Lin ++vq->started;
299*c5d3705cSYunsheng Lin
300*c5d3705cSYunsheng Lin if (unlikely(!virtqueue_kick(vq->vq))) {
301*c5d3705cSYunsheng Lin r = -1;
302*c5d3705cSYunsheng Lin break;
303*c5d3705cSYunsheng Lin }
304*c5d3705cSYunsheng Lin }
305*c5d3705cSYunsheng Lin
306*c5d3705cSYunsheng Lin if (vq->started >= bufs)
307*c5d3705cSYunsheng Lin r = -1;
308*c5d3705cSYunsheng Lin
309*c5d3705cSYunsheng Lin /* Flush out completed bufs if any */
310*c5d3705cSYunsheng Lin while (virtqueue_get_buf(vq->vq, &len)) {
311*c5d3705cSYunsheng Lin int n;
312*c5d3705cSYunsheng Lin
313*c5d3705cSYunsheng Lin n = recvfrom(dev->sock, dev->res_buf, TEST_BUF_LEN, 0, NULL, NULL);
314*c5d3705cSYunsheng Lin assert(n == TEST_BUF_LEN);
315*c5d3705cSYunsheng Lin verify_res_buf(dev->res_buf);
316*c5d3705cSYunsheng Lin
317*c5d3705cSYunsheng Lin ++vq->completed;
318*c5d3705cSYunsheng Lin r = 0;
319*c5d3705cSYunsheng Lin }
320*c5d3705cSYunsheng Lin } while (r == 0);
321*c5d3705cSYunsheng Lin
322*c5d3705cSYunsheng Lin if (vq->completed == completed_before && vq->started == started_before)
323*c5d3705cSYunsheng Lin ++spurious;
324*c5d3705cSYunsheng Lin
325*c5d3705cSYunsheng Lin assert(vq->completed <= bufs);
326*c5d3705cSYunsheng Lin assert(vq->started <= bufs);
327*c5d3705cSYunsheng Lin if (vq->completed == bufs)
328*c5d3705cSYunsheng Lin break;
329*c5d3705cSYunsheng Lin
330*c5d3705cSYunsheng Lin if (delayed) {
331*c5d3705cSYunsheng Lin if (virtqueue_enable_cb_delayed(vq->vq))
332*c5d3705cSYunsheng Lin wait_for_interrupt(vq);
333*c5d3705cSYunsheng Lin } else {
334*c5d3705cSYunsheng Lin if (virtqueue_enable_cb(vq->vq))
335*c5d3705cSYunsheng Lin wait_for_interrupt(vq);
336*c5d3705cSYunsheng Lin }
337*c5d3705cSYunsheng Lin }
338*c5d3705cSYunsheng Lin printf("TX spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n",
339*c5d3705cSYunsheng Lin spurious, vq->started, vq->completed);
340*c5d3705cSYunsheng Lin }
341*c5d3705cSYunsheng Lin
run_rx_test(struct vdev_info * dev,struct vq_info * vq,bool delayed,int bufs)342*c5d3705cSYunsheng Lin static void run_rx_test(struct vdev_info *dev, struct vq_info *vq,
343*c5d3705cSYunsheng Lin bool delayed, int bufs)
344*c5d3705cSYunsheng Lin {
345*c5d3705cSYunsheng Lin long long spurious = 0;
346*c5d3705cSYunsheng Lin struct scatterlist sl;
347*c5d3705cSYunsheng Lin unsigned int len;
348*c5d3705cSYunsheng Lin int r;
349*c5d3705cSYunsheng Lin
350*c5d3705cSYunsheng Lin for (;;) {
351*c5d3705cSYunsheng Lin long started_before = vq->started;
352*c5d3705cSYunsheng Lin long completed_before = vq->completed;
353*c5d3705cSYunsheng Lin
354*c5d3705cSYunsheng Lin do {
355*c5d3705cSYunsheng Lin while (vq->started < bufs &&
356*c5d3705cSYunsheng Lin (vq->started - vq->completed) < 1) {
357*c5d3705cSYunsheng Lin sg_init_one(&sl, dev->res_buf, HDR_LEN + TEST_BUF_LEN);
358*c5d3705cSYunsheng Lin
359*c5d3705cSYunsheng Lin r = virtqueue_add_inbuf(vq->vq, &sl, 1,
360*c5d3705cSYunsheng Lin dev->res_buf + vq->started,
361*c5d3705cSYunsheng Lin GFP_ATOMIC);
362*c5d3705cSYunsheng Lin if (unlikely(r != 0))
363*c5d3705cSYunsheng Lin break;
364*c5d3705cSYunsheng Lin
365*c5d3705cSYunsheng Lin ++vq->started;
366*c5d3705cSYunsheng Lin
367*c5d3705cSYunsheng Lin vdev_send_packet(dev);
368*c5d3705cSYunsheng Lin
369*c5d3705cSYunsheng Lin if (unlikely(!virtqueue_kick(vq->vq))) {
370*c5d3705cSYunsheng Lin r = -1;
371*c5d3705cSYunsheng Lin break;
372*c5d3705cSYunsheng Lin }
373*c5d3705cSYunsheng Lin }
374*c5d3705cSYunsheng Lin
375*c5d3705cSYunsheng Lin if (vq->started >= bufs)
376*c5d3705cSYunsheng Lin r = -1;
377*c5d3705cSYunsheng Lin
378*c5d3705cSYunsheng Lin /* Flush out completed bufs if any */
379*c5d3705cSYunsheng Lin while (virtqueue_get_buf(vq->vq, &len)) {
380*c5d3705cSYunsheng Lin struct ether_header *eh;
381*c5d3705cSYunsheng Lin
382*c5d3705cSYunsheng Lin eh = (struct ether_header *)(dev->res_buf + HDR_LEN);
383*c5d3705cSYunsheng Lin
384*c5d3705cSYunsheng Lin /* tun netdev is up and running, only handle the
385*c5d3705cSYunsheng Lin * TEST_PTYPE packet.
386*c5d3705cSYunsheng Lin */
387*c5d3705cSYunsheng Lin if (eh->ether_type == htons(TEST_PTYPE)) {
388*c5d3705cSYunsheng Lin assert(len == TEST_BUF_LEN + HDR_LEN);
389*c5d3705cSYunsheng Lin verify_res_buf(dev->res_buf + HDR_LEN);
390*c5d3705cSYunsheng Lin }
391*c5d3705cSYunsheng Lin
392*c5d3705cSYunsheng Lin ++vq->completed;
393*c5d3705cSYunsheng Lin r = 0;
394*c5d3705cSYunsheng Lin }
395*c5d3705cSYunsheng Lin } while (r == 0);
396*c5d3705cSYunsheng Lin
397*c5d3705cSYunsheng Lin if (vq->completed == completed_before && vq->started == started_before)
398*c5d3705cSYunsheng Lin ++spurious;
399*c5d3705cSYunsheng Lin
400*c5d3705cSYunsheng Lin assert(vq->completed <= bufs);
401*c5d3705cSYunsheng Lin assert(vq->started <= bufs);
402*c5d3705cSYunsheng Lin if (vq->completed == bufs)
403*c5d3705cSYunsheng Lin break;
404*c5d3705cSYunsheng Lin }
405*c5d3705cSYunsheng Lin
406*c5d3705cSYunsheng Lin printf("RX spurious wakeups: 0x%llx started=0x%lx completed=0x%lx\n",
407*c5d3705cSYunsheng Lin spurious, vq->started, vq->completed);
408*c5d3705cSYunsheng Lin }
409*c5d3705cSYunsheng Lin
410*c5d3705cSYunsheng Lin static const char optstring[] = "h";
411*c5d3705cSYunsheng Lin static const struct option longopts[] = {
412*c5d3705cSYunsheng Lin {
413*c5d3705cSYunsheng Lin .name = "help",
414*c5d3705cSYunsheng Lin .val = 'h',
415*c5d3705cSYunsheng Lin },
416*c5d3705cSYunsheng Lin {
417*c5d3705cSYunsheng Lin .name = "event-idx",
418*c5d3705cSYunsheng Lin .val = 'E',
419*c5d3705cSYunsheng Lin },
420*c5d3705cSYunsheng Lin {
421*c5d3705cSYunsheng Lin .name = "no-event-idx",
422*c5d3705cSYunsheng Lin .val = 'e',
423*c5d3705cSYunsheng Lin },
424*c5d3705cSYunsheng Lin {
425*c5d3705cSYunsheng Lin .name = "indirect",
426*c5d3705cSYunsheng Lin .val = 'I',
427*c5d3705cSYunsheng Lin },
428*c5d3705cSYunsheng Lin {
429*c5d3705cSYunsheng Lin .name = "no-indirect",
430*c5d3705cSYunsheng Lin .val = 'i',
431*c5d3705cSYunsheng Lin },
432*c5d3705cSYunsheng Lin {
433*c5d3705cSYunsheng Lin .name = "virtio-1",
434*c5d3705cSYunsheng Lin .val = '1',
435*c5d3705cSYunsheng Lin },
436*c5d3705cSYunsheng Lin {
437*c5d3705cSYunsheng Lin .name = "no-virtio-1",
438*c5d3705cSYunsheng Lin .val = '0',
439*c5d3705cSYunsheng Lin },
440*c5d3705cSYunsheng Lin {
441*c5d3705cSYunsheng Lin .name = "delayed-interrupt",
442*c5d3705cSYunsheng Lin .val = 'D',
443*c5d3705cSYunsheng Lin },
444*c5d3705cSYunsheng Lin {
445*c5d3705cSYunsheng Lin .name = "no-delayed-interrupt",
446*c5d3705cSYunsheng Lin .val = 'd',
447*c5d3705cSYunsheng Lin },
448*c5d3705cSYunsheng Lin {
449*c5d3705cSYunsheng Lin .name = "buf-num",
450*c5d3705cSYunsheng Lin .val = 'n',
451*c5d3705cSYunsheng Lin .has_arg = required_argument,
452*c5d3705cSYunsheng Lin },
453*c5d3705cSYunsheng Lin {
454*c5d3705cSYunsheng Lin .name = "batch",
455*c5d3705cSYunsheng Lin .val = 'b',
456*c5d3705cSYunsheng Lin .has_arg = required_argument,
457*c5d3705cSYunsheng Lin },
458*c5d3705cSYunsheng Lin {
459*c5d3705cSYunsheng Lin }
460*c5d3705cSYunsheng Lin };
461*c5d3705cSYunsheng Lin
help(int status)462*c5d3705cSYunsheng Lin static void help(int status)
463*c5d3705cSYunsheng Lin {
464*c5d3705cSYunsheng Lin fprintf(stderr, "Usage: vhost_net_test [--help]"
465*c5d3705cSYunsheng Lin " [--no-indirect]"
466*c5d3705cSYunsheng Lin " [--no-event-idx]"
467*c5d3705cSYunsheng Lin " [--no-virtio-1]"
468*c5d3705cSYunsheng Lin " [--delayed-interrupt]"
469*c5d3705cSYunsheng Lin " [--buf-num]"
470*c5d3705cSYunsheng Lin "\n");
471*c5d3705cSYunsheng Lin
472*c5d3705cSYunsheng Lin exit(status);
473*c5d3705cSYunsheng Lin }
474*c5d3705cSYunsheng Lin
main(int argc,char ** argv)475*c5d3705cSYunsheng Lin int main(int argc, char **argv)
476*c5d3705cSYunsheng Lin {
477*c5d3705cSYunsheng Lin unsigned long long features = (1ULL << VIRTIO_RING_F_INDIRECT_DESC) |
478*c5d3705cSYunsheng Lin (1ULL << VIRTIO_RING_F_EVENT_IDX) | (1ULL << VIRTIO_F_VERSION_1);
479*c5d3705cSYunsheng Lin char tun_name[IFNAMSIZ];
480*c5d3705cSYunsheng Lin long nbufs = 0x100000;
481*c5d3705cSYunsheng Lin struct vdev_info dev;
482*c5d3705cSYunsheng Lin bool delayed = false;
483*c5d3705cSYunsheng Lin int o, fd;
484*c5d3705cSYunsheng Lin
485*c5d3705cSYunsheng Lin for (;;) {
486*c5d3705cSYunsheng Lin o = getopt_long(argc, argv, optstring, longopts, NULL);
487*c5d3705cSYunsheng Lin switch (o) {
488*c5d3705cSYunsheng Lin case -1:
489*c5d3705cSYunsheng Lin goto done;
490*c5d3705cSYunsheng Lin case '?':
491*c5d3705cSYunsheng Lin help(2);
492*c5d3705cSYunsheng Lin case 'e':
493*c5d3705cSYunsheng Lin features &= ~(1ULL << VIRTIO_RING_F_EVENT_IDX);
494*c5d3705cSYunsheng Lin break;
495*c5d3705cSYunsheng Lin case 'h':
496*c5d3705cSYunsheng Lin help(0);
497*c5d3705cSYunsheng Lin case 'i':
498*c5d3705cSYunsheng Lin features &= ~(1ULL << VIRTIO_RING_F_INDIRECT_DESC);
499*c5d3705cSYunsheng Lin break;
500*c5d3705cSYunsheng Lin case '0':
501*c5d3705cSYunsheng Lin features &= ~(1ULL << VIRTIO_F_VERSION_1);
502*c5d3705cSYunsheng Lin break;
503*c5d3705cSYunsheng Lin case 'D':
504*c5d3705cSYunsheng Lin delayed = true;
505*c5d3705cSYunsheng Lin break;
506*c5d3705cSYunsheng Lin case 'n':
507*c5d3705cSYunsheng Lin nbufs = strtol(optarg, NULL, 10);
508*c5d3705cSYunsheng Lin assert(nbufs > 0);
509*c5d3705cSYunsheng Lin break;
510*c5d3705cSYunsheng Lin default:
511*c5d3705cSYunsheng Lin assert(0);
512*c5d3705cSYunsheng Lin break;
513*c5d3705cSYunsheng Lin }
514*c5d3705cSYunsheng Lin }
515*c5d3705cSYunsheng Lin
516*c5d3705cSYunsheng Lin done:
517*c5d3705cSYunsheng Lin memset(&dev, 0, sizeof(dev));
518*c5d3705cSYunsheng Lin snprintf(tun_name, IFNAMSIZ, "tun_%d", getpid());
519*c5d3705cSYunsheng Lin
520*c5d3705cSYunsheng Lin fd = tun_alloc(&dev, tun_name);
521*c5d3705cSYunsheng Lin assert(fd >= 0);
522*c5d3705cSYunsheng Lin
523*c5d3705cSYunsheng Lin vdev_info_init(&dev, features);
524*c5d3705cSYunsheng Lin vq_info_add(&dev, 0, DESC_NUM, fd);
525*c5d3705cSYunsheng Lin vq_info_add(&dev, 1, DESC_NUM, fd);
526*c5d3705cSYunsheng Lin vdev_create_socket(&dev, tun_name);
527*c5d3705cSYunsheng Lin
528*c5d3705cSYunsheng Lin run_rx_test(&dev, &dev.vqs[0], delayed, nbufs);
529*c5d3705cSYunsheng Lin run_tx_test(&dev, &dev.vqs[1], delayed, nbufs);
530*c5d3705cSYunsheng Lin
531*c5d3705cSYunsheng Lin return 0;
532*c5d3705cSYunsheng Lin }
533