1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * VDPA simulator for networking device.
4 *
5 * Copyright (c) 2020, Red Hat Inc. All rights reserved.
6 * Author: Jason Wang <jasowang@redhat.com>
7 *
8 */
9
10 #include <linux/init.h>
11 #include <linux/module.h>
12 #include <linux/device.h>
13 #include <linux/kernel.h>
14 #include <linux/etherdevice.h>
15 #include <linux/vringh.h>
16 #include <linux/vdpa.h>
17 #include <net/netlink.h>
18 #include <uapi/linux/virtio_net.h>
19 #include <uapi/linux/vdpa.h>
20
21 #include "vdpa_sim.h"
22
23 #define DRV_VERSION "0.1"
24 #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>"
25 #define DRV_DESC "vDPA Device Simulator for networking device"
26 #define DRV_LICENSE "GPL v2"
27
28 #define VDPASIM_NET_FEATURES (VDPASIM_FEATURES | \
29 (1ULL << VIRTIO_NET_F_MAC) | \
30 (1ULL << VIRTIO_NET_F_STATUS) | \
31 (1ULL << VIRTIO_NET_F_MTU) | \
32 (1ULL << VIRTIO_NET_F_CTRL_VQ) | \
33 (1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR))
34
35 /* 3 virtqueues, 2 address spaces, 2 virtqueue groups */
36 #define VDPASIM_NET_VQ_NUM 3
37 #define VDPASIM_NET_AS_NUM 2
38 #define VDPASIM_NET_GROUP_NUM 2
39
40 struct vdpasim_dataq_stats {
41 struct u64_stats_sync syncp;
42 u64 pkts;
43 u64 bytes;
44 u64 drops;
45 u64 errors;
46 u64 overruns;
47 };
48
49 struct vdpasim_cq_stats {
50 struct u64_stats_sync syncp;
51 u64 requests;
52 u64 successes;
53 u64 errors;
54 };
55
56 struct vdpasim_net{
57 struct vdpasim vdpasim;
58 struct vdpasim_dataq_stats tx_stats;
59 struct vdpasim_dataq_stats rx_stats;
60 struct vdpasim_cq_stats cq_stats;
61 void *buffer;
62 };
63
sim_to_net(struct vdpasim * vdpasim)64 static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim)
65 {
66 return container_of(vdpasim, struct vdpasim_net, vdpasim);
67 }
68
vdpasim_net_complete(struct vdpasim_virtqueue * vq,size_t len)69 static void vdpasim_net_complete(struct vdpasim_virtqueue *vq, size_t len)
70 {
71 /* Make sure data is wrote before advancing index */
72 smp_wmb();
73
74 vringh_complete_iotlb(&vq->vring, vq->head, len);
75
76 /* Make sure used is visible before rasing the interrupt. */
77 smp_wmb();
78
79 local_bh_disable();
80 if (vringh_need_notify_iotlb(&vq->vring) > 0)
81 vringh_notify(&vq->vring);
82 local_bh_enable();
83 }
84
receive_filter(struct vdpasim * vdpasim,size_t len)85 static bool receive_filter(struct vdpasim *vdpasim, size_t len)
86 {
87 bool modern = vdpasim->features & (1ULL << VIRTIO_F_VERSION_1);
88 size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) :
89 sizeof(struct virtio_net_hdr);
90 struct virtio_net_config *vio_config = vdpasim->config;
91 struct vdpasim_net *net = sim_to_net(vdpasim);
92
93 if (len < ETH_ALEN + hdr_len)
94 return false;
95
96 if (is_broadcast_ether_addr(net->buffer + hdr_len) ||
97 is_multicast_ether_addr(net->buffer + hdr_len))
98 return true;
99 if (!strncmp(net->buffer + hdr_len, vio_config->mac, ETH_ALEN))
100 return true;
101
102 return false;
103 }
104
vdpasim_handle_ctrl_mac(struct vdpasim * vdpasim,u8 cmd)105 static virtio_net_ctrl_ack vdpasim_handle_ctrl_mac(struct vdpasim *vdpasim,
106 u8 cmd)
107 {
108 struct virtio_net_config *vio_config = vdpasim->config;
109 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2];
110 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
111 size_t read;
112
113 switch (cmd) {
114 case VIRTIO_NET_CTRL_MAC_ADDR_SET:
115 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov,
116 vio_config->mac, ETH_ALEN);
117 if (read == ETH_ALEN)
118 status = VIRTIO_NET_OK;
119 break;
120 default:
121 break;
122 }
123
124 return status;
125 }
126
vdpasim_handle_cvq(struct vdpasim * vdpasim)127 static void vdpasim_handle_cvq(struct vdpasim *vdpasim)
128 {
129 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2];
130 struct vdpasim_net *net = sim_to_net(vdpasim);
131 virtio_net_ctrl_ack status = VIRTIO_NET_ERR;
132 struct virtio_net_ctrl_hdr ctrl;
133 size_t read, write;
134 u64 requests = 0, errors = 0, successes = 0;
135 int err;
136
137 if (!(vdpasim->features & (1ULL << VIRTIO_NET_F_CTRL_VQ)))
138 return;
139
140 if (!cvq->ready)
141 return;
142
143 while (true) {
144 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->in_iov,
145 &cvq->out_iov,
146 &cvq->head, GFP_ATOMIC);
147 if (err <= 0)
148 break;
149
150 ++requests;
151 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov, &ctrl,
152 sizeof(ctrl));
153 if (read != sizeof(ctrl)) {
154 ++errors;
155 break;
156 }
157
158 switch (ctrl.class) {
159 case VIRTIO_NET_CTRL_MAC:
160 status = vdpasim_handle_ctrl_mac(vdpasim, ctrl.cmd);
161 break;
162 default:
163 break;
164 }
165
166 if (status == VIRTIO_NET_OK)
167 ++successes;
168 else
169 ++errors;
170
171 /* Make sure data is wrote before advancing index */
172 smp_wmb();
173
174 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->out_iov,
175 &status, sizeof(status));
176 vringh_complete_iotlb(&cvq->vring, cvq->head, write);
177 vringh_kiov_cleanup(&cvq->in_iov);
178 vringh_kiov_cleanup(&cvq->out_iov);
179
180 /* Make sure used is visible before rasing the interrupt. */
181 smp_wmb();
182
183 local_bh_disable();
184 if (cvq->cb)
185 cvq->cb(cvq->private);
186 local_bh_enable();
187 }
188
189 u64_stats_update_begin(&net->cq_stats.syncp);
190 net->cq_stats.requests += requests;
191 net->cq_stats.errors += errors;
192 net->cq_stats.successes += successes;
193 u64_stats_update_end(&net->cq_stats.syncp);
194 }
195
vdpasim_net_work(struct vdpasim * vdpasim)196 static void vdpasim_net_work(struct vdpasim *vdpasim)
197 {
198 struct vdpasim_virtqueue *txq = &vdpasim->vqs[1];
199 struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0];
200 struct vdpasim_net *net = sim_to_net(vdpasim);
201 ssize_t read, write;
202 u64 tx_pkts = 0, rx_pkts = 0, tx_bytes = 0, rx_bytes = 0;
203 u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0;
204 int err;
205
206 mutex_lock(&vdpasim->mutex);
207
208 if (!vdpasim->running)
209 goto out;
210
211 if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK))
212 goto out;
213
214 vdpasim_handle_cvq(vdpasim);
215
216 if (!txq->ready || !rxq->ready)
217 goto out;
218
219 while (true) {
220 err = vringh_getdesc_iotlb(&txq->vring, &txq->out_iov, NULL,
221 &txq->head, GFP_ATOMIC);
222 if (err <= 0) {
223 if (err)
224 ++tx_errors;
225 break;
226 }
227
228 read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov,
229 net->buffer, PAGE_SIZE);
230 if (read <= 0) {
231 ++tx_errors;
232 vdpasim_net_complete(txq, 0);
233 continue;
234 }
235
236 ++tx_pkts;
237 tx_bytes += read;
238
239 if (!receive_filter(vdpasim, read)) {
240 ++rx_drops;
241 vdpasim_net_complete(txq, 0);
242 continue;
243 }
244
245 err = vringh_getdesc_iotlb(&rxq->vring, NULL, &rxq->in_iov,
246 &rxq->head, GFP_ATOMIC);
247 if (err <= 0) {
248 ++rx_overruns;
249 vdpasim_net_complete(txq, 0);
250 break;
251 }
252
253 write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov,
254 net->buffer, read);
255 if (write <= 0) {
256 ++rx_errors;
257 break;
258 }
259
260 ++rx_pkts;
261 rx_bytes += write;
262
263 vdpasim_net_complete(txq, 0);
264 vdpasim_net_complete(rxq, write);
265
266 if (tx_pkts > 4) {
267 vdpasim_schedule_work(vdpasim);
268 goto out;
269 }
270 }
271
272 out:
273 mutex_unlock(&vdpasim->mutex);
274
275 u64_stats_update_begin(&net->tx_stats.syncp);
276 net->tx_stats.pkts += tx_pkts;
277 net->tx_stats.bytes += tx_bytes;
278 net->tx_stats.errors += tx_errors;
279 u64_stats_update_end(&net->tx_stats.syncp);
280
281 u64_stats_update_begin(&net->rx_stats.syncp);
282 net->rx_stats.pkts += rx_pkts;
283 net->rx_stats.bytes += rx_bytes;
284 net->rx_stats.drops += rx_drops;
285 net->rx_stats.errors += rx_errors;
286 net->rx_stats.overruns += rx_overruns;
287 u64_stats_update_end(&net->rx_stats.syncp);
288 }
289
vdpasim_net_get_stats(struct vdpasim * vdpasim,u16 idx,struct sk_buff * msg,struct netlink_ext_ack * extack)290 static int vdpasim_net_get_stats(struct vdpasim *vdpasim, u16 idx,
291 struct sk_buff *msg,
292 struct netlink_ext_ack *extack)
293 {
294 struct vdpasim_net *net = sim_to_net(vdpasim);
295 u64 rx_pkts, rx_bytes, rx_errors, rx_overruns, rx_drops;
296 u64 tx_pkts, tx_bytes, tx_errors, tx_drops;
297 u64 cq_requests, cq_successes, cq_errors;
298 unsigned int start;
299 int err = -EMSGSIZE;
300
301 switch(idx) {
302 case 0:
303 do {
304 start = u64_stats_fetch_begin(&net->rx_stats.syncp);
305 rx_pkts = net->rx_stats.pkts;
306 rx_bytes = net->rx_stats.bytes;
307 rx_errors = net->rx_stats.errors;
308 rx_overruns = net->rx_stats.overruns;
309 rx_drops = net->rx_stats.drops;
310 } while (u64_stats_fetch_retry(&net->rx_stats.syncp, start));
311
312 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
313 "rx packets"))
314 break;
315 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
316 rx_pkts, VDPA_ATTR_PAD))
317 break;
318 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
319 "rx bytes"))
320 break;
321 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
322 rx_bytes, VDPA_ATTR_PAD))
323 break;
324 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
325 "rx errors"))
326 break;
327 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
328 rx_errors, VDPA_ATTR_PAD))
329 break;
330 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
331 "rx overruns"))
332 break;
333 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
334 rx_overruns, VDPA_ATTR_PAD))
335 break;
336 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
337 "rx drops"))
338 break;
339 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
340 rx_drops, VDPA_ATTR_PAD))
341 break;
342 err = 0;
343 break;
344 case 1:
345 do {
346 start = u64_stats_fetch_begin(&net->tx_stats.syncp);
347 tx_pkts = net->tx_stats.pkts;
348 tx_bytes = net->tx_stats.bytes;
349 tx_errors = net->tx_stats.errors;
350 tx_drops = net->tx_stats.drops;
351 } while (u64_stats_fetch_retry(&net->tx_stats.syncp, start));
352
353 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
354 "tx packets"))
355 break;
356 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
357 tx_pkts, VDPA_ATTR_PAD))
358 break;
359 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
360 "tx bytes"))
361 break;
362 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
363 tx_bytes, VDPA_ATTR_PAD))
364 break;
365 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
366 "tx errors"))
367 break;
368 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
369 tx_errors, VDPA_ATTR_PAD))
370 break;
371 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
372 "tx drops"))
373 break;
374 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
375 tx_drops, VDPA_ATTR_PAD))
376 break;
377 err = 0;
378 break;
379 case 2:
380 do {
381 start = u64_stats_fetch_begin(&net->cq_stats.syncp);
382 cq_requests = net->cq_stats.requests;
383 cq_successes = net->cq_stats.successes;
384 cq_errors = net->cq_stats.errors;
385 } while (u64_stats_fetch_retry(&net->cq_stats.syncp, start));
386
387 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
388 "cvq requests"))
389 break;
390 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
391 cq_requests, VDPA_ATTR_PAD))
392 break;
393 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
394 "cvq successes"))
395 break;
396 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
397 cq_successes, VDPA_ATTR_PAD))
398 break;
399 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME,
400 "cvq errors"))
401 break;
402 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE,
403 cq_errors, VDPA_ATTR_PAD))
404 break;
405 err = 0;
406 break;
407 default:
408 err = -EINVAL;
409 break;
410 }
411
412 return err;
413 }
414
vdpasim_net_get_config(struct vdpasim * vdpasim,void * config)415 static void vdpasim_net_get_config(struct vdpasim *vdpasim, void *config)
416 {
417 struct virtio_net_config *net_config = config;
418
419 net_config->status = cpu_to_vdpasim16(vdpasim, VIRTIO_NET_S_LINK_UP);
420 }
421
vdpasim_net_set_attr(struct vdpa_mgmt_dev * mdev,struct vdpa_device * dev,const struct vdpa_dev_set_config * config)422 static int vdpasim_net_set_attr(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev,
423 const struct vdpa_dev_set_config *config)
424 {
425 struct vdpasim *vdpasim = container_of(dev, struct vdpasim, vdpa);
426 struct virtio_net_config *vio_config = vdpasim->config;
427
428 mutex_lock(&vdpasim->mutex);
429
430 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) {
431 ether_addr_copy(vio_config->mac, config->net.mac);
432 mutex_unlock(&vdpasim->mutex);
433 return 0;
434 }
435
436 mutex_unlock(&vdpasim->mutex);
437 return -EOPNOTSUPP;
438 }
439
vdpasim_net_setup_config(struct vdpasim * vdpasim,const struct vdpa_dev_set_config * config)440 static void vdpasim_net_setup_config(struct vdpasim *vdpasim,
441 const struct vdpa_dev_set_config *config)
442 {
443 struct virtio_net_config *vio_config = vdpasim->config;
444
445 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR))
446 memcpy(vio_config->mac, config->net.mac, ETH_ALEN);
447 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MTU))
448 vio_config->mtu = cpu_to_vdpasim16(vdpasim, config->net.mtu);
449 else
450 /* Setup default MTU to be 1500 */
451 vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500);
452 }
453
vdpasim_net_free(struct vdpasim * vdpasim)454 static void vdpasim_net_free(struct vdpasim *vdpasim)
455 {
456 struct vdpasim_net *net = sim_to_net(vdpasim);
457
458 kvfree(net->buffer);
459 }
460
461 static struct device *vdpasim_net_mgmtdev;
462
vdpasim_net_dev_add(struct vdpa_mgmt_dev * mdev,const char * name,const struct vdpa_dev_set_config * config)463 static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
464 const struct vdpa_dev_set_config *config)
465 {
466 struct vdpasim_dev_attr dev_attr = {};
467 struct vdpasim_net *net;
468 struct vdpasim *simdev;
469 int ret;
470
471 dev_attr.mgmt_dev = mdev;
472 dev_attr.name = name;
473 dev_attr.id = VIRTIO_ID_NET;
474 dev_attr.supported_features = VDPASIM_NET_FEATURES;
475 dev_attr.nvqs = VDPASIM_NET_VQ_NUM;
476 dev_attr.ngroups = VDPASIM_NET_GROUP_NUM;
477 dev_attr.nas = VDPASIM_NET_AS_NUM;
478 dev_attr.alloc_size = sizeof(struct vdpasim_net);
479 dev_attr.config_size = sizeof(struct virtio_net_config);
480 dev_attr.get_config = vdpasim_net_get_config;
481 dev_attr.work_fn = vdpasim_net_work;
482 dev_attr.get_stats = vdpasim_net_get_stats;
483 dev_attr.free = vdpasim_net_free;
484
485 simdev = vdpasim_create(&dev_attr, config);
486 if (IS_ERR(simdev))
487 return PTR_ERR(simdev);
488
489 vdpasim_net_setup_config(simdev, config);
490
491 net = sim_to_net(simdev);
492
493 u64_stats_init(&net->tx_stats.syncp);
494 u64_stats_init(&net->rx_stats.syncp);
495 u64_stats_init(&net->cq_stats.syncp);
496
497 net->buffer = kvmalloc(PAGE_SIZE, GFP_KERNEL);
498 if (!net->buffer) {
499 ret = -ENOMEM;
500 goto reg_err;
501 }
502
503 /*
504 * Initialization must be completed before this call, since it can
505 * connect the device to the vDPA bus, so requests can arrive after
506 * this call.
507 */
508 ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_NET_VQ_NUM);
509 if (ret)
510 goto reg_err;
511
512 return 0;
513
514 reg_err:
515 put_device(&simdev->vdpa.dev);
516 return ret;
517 }
518
vdpasim_net_dev_del(struct vdpa_mgmt_dev * mdev,struct vdpa_device * dev)519 static void vdpasim_net_dev_del(struct vdpa_mgmt_dev *mdev,
520 struct vdpa_device *dev)
521 {
522 struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa);
523
524 _vdpa_unregister_device(&simdev->vdpa);
525 }
526
527 static const struct vdpa_mgmtdev_ops vdpasim_net_mgmtdev_ops = {
528 .dev_add = vdpasim_net_dev_add,
529 .dev_del = vdpasim_net_dev_del,
530 .dev_set_attr = vdpasim_net_set_attr
531 };
532
533 static struct virtio_device_id id_table[] = {
534 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID },
535 { 0 },
536 };
537
538 static struct vdpa_mgmt_dev mgmt_dev = {
539 .id_table = id_table,
540 .ops = &vdpasim_net_mgmtdev_ops,
541 .config_attr_mask = (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR |
542 1 << VDPA_ATTR_DEV_NET_CFG_MTU |
543 1 << VDPA_ATTR_DEV_FEATURES),
544 .max_supported_vqs = VDPASIM_NET_VQ_NUM,
545 .supported_features = VDPASIM_NET_FEATURES,
546 };
547
vdpasim_net_init(void)548 static int __init vdpasim_net_init(void)
549 {
550 int ret;
551
552 vdpasim_net_mgmtdev = root_device_register("vdpasim_net");
553 if (IS_ERR(vdpasim_net_mgmtdev))
554 return PTR_ERR(vdpasim_net_mgmtdev);
555
556 mgmt_dev.device = vdpasim_net_mgmtdev;
557 ret = vdpa_mgmtdev_register(&mgmt_dev);
558 if (ret)
559 goto parent_err;
560 return 0;
561
562 parent_err:
563 root_device_unregister(vdpasim_net_mgmtdev);
564 return ret;
565 }
566
vdpasim_net_exit(void)567 static void __exit vdpasim_net_exit(void)
568 {
569 vdpa_mgmtdev_unregister(&mgmt_dev);
570 root_device_unregister(vdpasim_net_mgmtdev);
571 }
572
573 module_init(vdpasim_net_init);
574 module_exit(vdpasim_net_exit);
575
576 MODULE_VERSION(DRV_VERSION);
577 MODULE_LICENSE(DRV_LICENSE);
578 MODULE_AUTHOR(DRV_AUTHOR);
579 MODULE_DESCRIPTION(DRV_DESC);
580