1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDPA simulator for networking device. 4 * 5 * Copyright (c) 2020, Red Hat Inc. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/uaccess.h> 15 #include <linux/etherdevice.h> 16 #include <linux/vringh.h> 17 #include <linux/vdpa.h> 18 #include <net/netlink.h> 19 #include <uapi/linux/virtio_net.h> 20 #include <uapi/linux/vdpa.h> 21 22 #include "vdpa_sim.h" 23 24 #define DRV_VERSION "0.1" 25 #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>" 26 #define DRV_DESC "vDPA Device Simulator for networking device" 27 #define DRV_LICENSE "GPL v2" 28 29 #define VDPASIM_NET_FEATURES (VDPASIM_FEATURES | \ 30 (1ULL << VIRTIO_NET_F_MAC) | \ 31 (1ULL << VIRTIO_NET_F_STATUS) | \ 32 (1ULL << VIRTIO_NET_F_MTU) | \ 33 (1ULL << VIRTIO_NET_F_CTRL_VQ) | \ 34 (1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR)) 35 36 /* 3 virtqueues, 2 address spaces, 2 virtqueue groups */ 37 #define VDPASIM_NET_VQ_NUM 3 38 #define VDPASIM_NET_AS_NUM 2 39 #define VDPASIM_NET_GROUP_NUM 2 40 41 struct vdpasim_dataq_stats { 42 struct u64_stats_sync syncp; 43 u64 pkts; 44 u64 bytes; 45 u64 drops; 46 u64 errors; 47 u64 overruns; 48 }; 49 50 struct vdpasim_cq_stats { 51 struct u64_stats_sync syncp; 52 u64 requests; 53 u64 successes; 54 u64 errors; 55 }; 56 57 struct vdpasim_net{ 58 struct vdpasim vdpasim; 59 struct vdpasim_dataq_stats tx_stats; 60 struct vdpasim_dataq_stats rx_stats; 61 struct vdpasim_cq_stats cq_stats; 62 void *buffer; 63 }; 64 65 static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim) 66 { 67 return container_of(vdpasim, struct vdpasim_net, vdpasim); 68 } 69 70 static void vdpasim_net_complete(struct vdpasim_virtqueue *vq, size_t len) 71 { 72 bool notify; 73 74 /* Make sure data is wrote before advancing index */ 75 smp_wmb(); 76 77 vringh_complete_iotlb(&vq->vring, vq->head, len); 78 79 /* Make sure used is visible before rasing the interrupt. */ 80 smp_wmb(); 81 82 local_bh_disable(); 83 /* The ring may be in user memory, and we can't take faults here */ 84 pagefault_disable(); 85 notify = vringh_need_notify_iotlb(&vq->vring) > 0; 86 pagefault_enable(); 87 if (notify) 88 vringh_notify(&vq->vring); 89 local_bh_enable(); 90 } 91 92 static bool receive_filter(struct vdpasim *vdpasim, size_t len) 93 { 94 bool modern = vdpasim->features & (1ULL << VIRTIO_F_VERSION_1); 95 size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) : 96 sizeof(struct virtio_net_hdr); 97 struct virtio_net_config *vio_config = vdpasim->config; 98 struct vdpasim_net *net = sim_to_net(vdpasim); 99 100 if (len < ETH_ALEN + hdr_len) 101 return false; 102 103 if (is_broadcast_ether_addr(net->buffer + hdr_len) || 104 is_multicast_ether_addr(net->buffer + hdr_len)) 105 return true; 106 if (!strncmp(net->buffer + hdr_len, vio_config->mac, ETH_ALEN)) 107 return true; 108 109 return false; 110 } 111 112 static virtio_net_ctrl_ack vdpasim_handle_ctrl_mac(struct vdpasim *vdpasim, 113 u8 cmd) 114 { 115 struct virtio_net_config *vio_config = vdpasim->config; 116 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2]; 117 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 118 size_t read; 119 120 switch (cmd) { 121 case VIRTIO_NET_CTRL_MAC_ADDR_SET: 122 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov, 123 vio_config->mac, ETH_ALEN); 124 if (read == ETH_ALEN) 125 status = VIRTIO_NET_OK; 126 break; 127 default: 128 break; 129 } 130 131 return status; 132 } 133 134 static void vdpasim_handle_cvq(struct vdpasim *vdpasim) 135 { 136 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2]; 137 struct vdpasim_net *net = sim_to_net(vdpasim); 138 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 139 struct virtio_net_ctrl_hdr ctrl; 140 size_t read, write; 141 u64 requests = 0, errors = 0, successes = 0; 142 int err; 143 144 if (!(vdpasim->features & (1ULL << VIRTIO_NET_F_CTRL_VQ))) 145 return; 146 147 if (!cvq->ready) 148 return; 149 150 while (true) { 151 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->in_iov, 152 &cvq->out_iov, 153 &cvq->head, GFP_ATOMIC); 154 if (err <= 0) 155 break; 156 157 ++requests; 158 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov, &ctrl, 159 sizeof(ctrl)); 160 if (read != sizeof(ctrl)) { 161 ++errors; 162 break; 163 } 164 165 switch (ctrl.class) { 166 case VIRTIO_NET_CTRL_MAC: 167 status = vdpasim_handle_ctrl_mac(vdpasim, ctrl.cmd); 168 break; 169 default: 170 break; 171 } 172 173 if (status == VIRTIO_NET_OK) 174 ++successes; 175 else 176 ++errors; 177 178 /* Make sure data is wrote before advancing index */ 179 smp_wmb(); 180 181 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->out_iov, 182 &status, sizeof(status)); 183 vringh_complete_iotlb(&cvq->vring, cvq->head, write); 184 vringh_kiov_cleanup(&cvq->in_iov); 185 vringh_kiov_cleanup(&cvq->out_iov); 186 187 /* Make sure used is visible before rasing the interrupt. */ 188 smp_wmb(); 189 190 local_bh_disable(); 191 if (cvq->cb) 192 cvq->cb(cvq->private); 193 local_bh_enable(); 194 } 195 196 u64_stats_update_begin(&net->cq_stats.syncp); 197 net->cq_stats.requests += requests; 198 net->cq_stats.errors += errors; 199 net->cq_stats.successes += successes; 200 u64_stats_update_end(&net->cq_stats.syncp); 201 } 202 203 static void vdpasim_net_work(struct vdpasim *vdpasim) 204 { 205 struct vdpasim_virtqueue *txq = &vdpasim->vqs[1]; 206 struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0]; 207 struct vdpasim_net *net = sim_to_net(vdpasim); 208 ssize_t read, write; 209 u64 tx_pkts = 0, rx_pkts = 0, tx_bytes = 0, rx_bytes = 0; 210 u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0; 211 int err; 212 213 mutex_lock(&vdpasim->mutex); 214 215 if (!vdpasim->running) 216 goto out; 217 218 if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) 219 goto out; 220 221 vdpasim_handle_cvq(vdpasim); 222 223 if (!txq->ready || !rxq->ready) 224 goto out; 225 226 while (true) { 227 err = vringh_getdesc_iotlb(&txq->vring, &txq->out_iov, NULL, 228 &txq->head, GFP_ATOMIC); 229 if (err <= 0) { 230 if (err) 231 ++tx_errors; 232 break; 233 } 234 235 read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov, 236 net->buffer, PAGE_SIZE); 237 if (read <= 0) { 238 ++tx_errors; 239 vdpasim_net_complete(txq, 0); 240 continue; 241 } 242 243 ++tx_pkts; 244 tx_bytes += read; 245 246 if (!receive_filter(vdpasim, read)) { 247 ++rx_drops; 248 vdpasim_net_complete(txq, 0); 249 continue; 250 } 251 252 err = vringh_getdesc_iotlb(&rxq->vring, NULL, &rxq->in_iov, 253 &rxq->head, GFP_ATOMIC); 254 if (err <= 0) { 255 ++rx_overruns; 256 vdpasim_net_complete(txq, 0); 257 break; 258 } 259 260 write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov, 261 net->buffer, read); 262 if (write <= 0) { 263 ++rx_errors; 264 break; 265 } 266 267 ++rx_pkts; 268 rx_bytes += write; 269 270 vdpasim_net_complete(txq, 0); 271 vdpasim_net_complete(rxq, write); 272 273 if (tx_pkts > 4) { 274 vdpasim_schedule_work(vdpasim); 275 goto out; 276 } 277 } 278 279 out: 280 mutex_unlock(&vdpasim->mutex); 281 282 u64_stats_update_begin(&net->tx_stats.syncp); 283 net->tx_stats.pkts += tx_pkts; 284 net->tx_stats.bytes += tx_bytes; 285 net->tx_stats.errors += tx_errors; 286 u64_stats_update_end(&net->tx_stats.syncp); 287 288 u64_stats_update_begin(&net->rx_stats.syncp); 289 net->rx_stats.pkts += rx_pkts; 290 net->rx_stats.bytes += rx_bytes; 291 net->rx_stats.drops += rx_drops; 292 net->rx_stats.errors += rx_errors; 293 net->rx_stats.overruns += rx_overruns; 294 u64_stats_update_end(&net->rx_stats.syncp); 295 } 296 297 static int vdpasim_net_get_stats(struct vdpasim *vdpasim, u16 idx, 298 struct sk_buff *msg, 299 struct netlink_ext_ack *extack) 300 { 301 struct vdpasim_net *net = sim_to_net(vdpasim); 302 u64 rx_pkts, rx_bytes, rx_errors, rx_overruns, rx_drops; 303 u64 tx_pkts, tx_bytes, tx_errors, tx_drops; 304 u64 cq_requests, cq_successes, cq_errors; 305 unsigned int start; 306 int err = -EMSGSIZE; 307 308 switch(idx) { 309 case 0: 310 do { 311 start = u64_stats_fetch_begin(&net->rx_stats.syncp); 312 rx_pkts = net->rx_stats.pkts; 313 rx_bytes = net->rx_stats.bytes; 314 rx_errors = net->rx_stats.errors; 315 rx_overruns = net->rx_stats.overruns; 316 rx_drops = net->rx_stats.drops; 317 } while (u64_stats_fetch_retry(&net->rx_stats.syncp, start)); 318 319 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 320 "rx packets")) 321 break; 322 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 323 rx_pkts, VDPA_ATTR_PAD)) 324 break; 325 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 326 "rx bytes")) 327 break; 328 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 329 rx_bytes, VDPA_ATTR_PAD)) 330 break; 331 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 332 "rx errors")) 333 break; 334 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 335 rx_errors, VDPA_ATTR_PAD)) 336 break; 337 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 338 "rx overruns")) 339 break; 340 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 341 rx_overruns, VDPA_ATTR_PAD)) 342 break; 343 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 344 "rx drops")) 345 break; 346 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 347 rx_drops, VDPA_ATTR_PAD)) 348 break; 349 err = 0; 350 break; 351 case 1: 352 do { 353 start = u64_stats_fetch_begin(&net->tx_stats.syncp); 354 tx_pkts = net->tx_stats.pkts; 355 tx_bytes = net->tx_stats.bytes; 356 tx_errors = net->tx_stats.errors; 357 tx_drops = net->tx_stats.drops; 358 } while (u64_stats_fetch_retry(&net->tx_stats.syncp, start)); 359 360 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 361 "tx packets")) 362 break; 363 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 364 tx_pkts, VDPA_ATTR_PAD)) 365 break; 366 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 367 "tx bytes")) 368 break; 369 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 370 tx_bytes, VDPA_ATTR_PAD)) 371 break; 372 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 373 "tx errors")) 374 break; 375 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 376 tx_errors, VDPA_ATTR_PAD)) 377 break; 378 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 379 "tx drops")) 380 break; 381 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 382 tx_drops, VDPA_ATTR_PAD)) 383 break; 384 err = 0; 385 break; 386 case 2: 387 do { 388 start = u64_stats_fetch_begin(&net->cq_stats.syncp); 389 cq_requests = net->cq_stats.requests; 390 cq_successes = net->cq_stats.successes; 391 cq_errors = net->cq_stats.errors; 392 } while (u64_stats_fetch_retry(&net->cq_stats.syncp, start)); 393 394 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 395 "cvq requests")) 396 break; 397 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 398 cq_requests, VDPA_ATTR_PAD)) 399 break; 400 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 401 "cvq successes")) 402 break; 403 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 404 cq_successes, VDPA_ATTR_PAD)) 405 break; 406 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 407 "cvq errors")) 408 break; 409 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 410 cq_errors, VDPA_ATTR_PAD)) 411 break; 412 err = 0; 413 break; 414 default: 415 err = -EINVAL; 416 break; 417 } 418 419 return err; 420 } 421 422 static void vdpasim_net_get_config(struct vdpasim *vdpasim, void *config) 423 { 424 struct virtio_net_config *net_config = config; 425 426 net_config->status = cpu_to_vdpasim16(vdpasim, VIRTIO_NET_S_LINK_UP); 427 } 428 429 static int vdpasim_net_set_attr(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev, 430 const struct vdpa_dev_set_config *config) 431 { 432 struct vdpasim *vdpasim = container_of(dev, struct vdpasim, vdpa); 433 struct virtio_net_config *vio_config = vdpasim->config; 434 435 mutex_lock(&vdpasim->mutex); 436 437 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) { 438 ether_addr_copy(vio_config->mac, config->net.mac); 439 mutex_unlock(&vdpasim->mutex); 440 return 0; 441 } 442 443 mutex_unlock(&vdpasim->mutex); 444 return -EOPNOTSUPP; 445 } 446 447 static void vdpasim_net_setup_config(struct vdpasim *vdpasim, 448 const struct vdpa_dev_set_config *config) 449 { 450 struct virtio_net_config *vio_config = vdpasim->config; 451 452 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) 453 memcpy(vio_config->mac, config->net.mac, ETH_ALEN); 454 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MTU)) 455 vio_config->mtu = cpu_to_vdpasim16(vdpasim, config->net.mtu); 456 else 457 /* Setup default MTU to be 1500 */ 458 vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500); 459 } 460 461 static void vdpasim_net_free(struct vdpasim *vdpasim) 462 { 463 struct vdpasim_net *net = sim_to_net(vdpasim); 464 465 kvfree(net->buffer); 466 } 467 468 static struct device *vdpasim_net_mgmtdev; 469 470 static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 471 const struct vdpa_dev_set_config *config) 472 { 473 struct vdpasim_dev_attr dev_attr = {}; 474 struct vdpasim_net *net; 475 struct vdpasim *simdev; 476 int ret; 477 478 dev_attr.mgmt_dev = mdev; 479 dev_attr.name = name; 480 dev_attr.id = VIRTIO_ID_NET; 481 dev_attr.supported_features = VDPASIM_NET_FEATURES; 482 dev_attr.nvqs = VDPASIM_NET_VQ_NUM; 483 dev_attr.ngroups = VDPASIM_NET_GROUP_NUM; 484 dev_attr.nas = VDPASIM_NET_AS_NUM; 485 dev_attr.alloc_size = sizeof(struct vdpasim_net); 486 dev_attr.config_size = sizeof(struct virtio_net_config); 487 dev_attr.get_config = vdpasim_net_get_config; 488 dev_attr.work_fn = vdpasim_net_work; 489 dev_attr.get_stats = vdpasim_net_get_stats; 490 dev_attr.free = vdpasim_net_free; 491 492 simdev = vdpasim_create(&dev_attr, config); 493 if (IS_ERR(simdev)) 494 return PTR_ERR(simdev); 495 496 vdpasim_net_setup_config(simdev, config); 497 498 net = sim_to_net(simdev); 499 500 u64_stats_init(&net->tx_stats.syncp); 501 u64_stats_init(&net->rx_stats.syncp); 502 u64_stats_init(&net->cq_stats.syncp); 503 504 net->buffer = kvmalloc(PAGE_SIZE, GFP_KERNEL); 505 if (!net->buffer) { 506 ret = -ENOMEM; 507 goto reg_err; 508 } 509 510 /* 511 * Initialization must be completed before this call, since it can 512 * connect the device to the vDPA bus, so requests can arrive after 513 * this call. 514 */ 515 ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_NET_VQ_NUM); 516 if (ret) 517 goto reg_err; 518 519 return 0; 520 521 reg_err: 522 put_device(&simdev->vdpa.dev); 523 return ret; 524 } 525 526 static void vdpasim_net_dev_del(struct vdpa_mgmt_dev *mdev, 527 struct vdpa_device *dev) 528 { 529 struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa); 530 531 _vdpa_unregister_device(&simdev->vdpa); 532 } 533 534 static const struct vdpa_mgmtdev_ops vdpasim_net_mgmtdev_ops = { 535 .dev_add = vdpasim_net_dev_add, 536 .dev_del = vdpasim_net_dev_del, 537 .dev_set_attr = vdpasim_net_set_attr 538 }; 539 540 static struct virtio_device_id id_table[] = { 541 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 542 { 0 }, 543 }; 544 545 static struct vdpa_mgmt_dev mgmt_dev = { 546 .id_table = id_table, 547 .ops = &vdpasim_net_mgmtdev_ops, 548 .config_attr_mask = (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR | 549 1 << VDPA_ATTR_DEV_NET_CFG_MTU | 550 1 << VDPA_ATTR_DEV_FEATURES), 551 .max_supported_vqs = VDPASIM_NET_VQ_NUM, 552 .supported_features = VDPASIM_NET_FEATURES, 553 }; 554 555 static int __init vdpasim_net_init(void) 556 { 557 int ret; 558 559 vdpasim_net_mgmtdev = root_device_register("vdpasim_net"); 560 if (IS_ERR(vdpasim_net_mgmtdev)) 561 return PTR_ERR(vdpasim_net_mgmtdev); 562 563 mgmt_dev.device = vdpasim_net_mgmtdev; 564 ret = vdpa_mgmtdev_register(&mgmt_dev); 565 if (ret) 566 goto parent_err; 567 return 0; 568 569 parent_err: 570 root_device_unregister(vdpasim_net_mgmtdev); 571 return ret; 572 } 573 574 static void __exit vdpasim_net_exit(void) 575 { 576 vdpa_mgmtdev_unregister(&mgmt_dev); 577 root_device_unregister(vdpasim_net_mgmtdev); 578 } 579 580 module_init(vdpasim_net_init); 581 module_exit(vdpasim_net_exit); 582 583 MODULE_VERSION(DRV_VERSION); 584 MODULE_LICENSE(DRV_LICENSE); 585 MODULE_AUTHOR(DRV_AUTHOR); 586 MODULE_DESCRIPTION(DRV_DESC); 587