1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDPA simulator for networking device. 4 * 5 * Copyright (c) 2020, Red Hat Inc. All rights reserved. 6 * Author: Jason Wang <jasowang@redhat.com> 7 * 8 */ 9 10 #include <linux/init.h> 11 #include <linux/module.h> 12 #include <linux/device.h> 13 #include <linux/kernel.h> 14 #include <linux/etherdevice.h> 15 #include <linux/vringh.h> 16 #include <linux/vdpa.h> 17 #include <net/netlink.h> 18 #include <uapi/linux/virtio_net.h> 19 #include <uapi/linux/vdpa.h> 20 21 #include "vdpa_sim.h" 22 23 #define DRV_VERSION "0.1" 24 #define DRV_AUTHOR "Jason Wang <jasowang@redhat.com>" 25 #define DRV_DESC "vDPA Device Simulator for networking device" 26 #define DRV_LICENSE "GPL v2" 27 28 #define VDPASIM_NET_FEATURES (VDPASIM_FEATURES | \ 29 (1ULL << VIRTIO_NET_F_MAC) | \ 30 (1ULL << VIRTIO_NET_F_STATUS) | \ 31 (1ULL << VIRTIO_NET_F_MTU) | \ 32 (1ULL << VIRTIO_NET_F_CTRL_VQ) | \ 33 (1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR)) 34 35 /* 3 virtqueues, 2 address spaces, 2 virtqueue groups */ 36 #define VDPASIM_NET_VQ_NUM 3 37 #define VDPASIM_NET_AS_NUM 2 38 #define VDPASIM_NET_GROUP_NUM 2 39 40 struct vdpasim_dataq_stats { 41 struct u64_stats_sync syncp; 42 u64 pkts; 43 u64 bytes; 44 u64 drops; 45 u64 errors; 46 u64 overruns; 47 }; 48 49 struct vdpasim_cq_stats { 50 struct u64_stats_sync syncp; 51 u64 requests; 52 u64 successes; 53 u64 errors; 54 }; 55 56 struct vdpasim_net{ 57 struct vdpasim vdpasim; 58 struct vdpasim_dataq_stats tx_stats; 59 struct vdpasim_dataq_stats rx_stats; 60 struct vdpasim_cq_stats cq_stats; 61 }; 62 63 static struct vdpasim_net *sim_to_net(struct vdpasim *vdpasim) 64 { 65 return container_of(vdpasim, struct vdpasim_net, vdpasim); 66 } 67 68 static void vdpasim_net_complete(struct vdpasim_virtqueue *vq, size_t len) 69 { 70 /* Make sure data is wrote before advancing index */ 71 smp_wmb(); 72 73 vringh_complete_iotlb(&vq->vring, vq->head, len); 74 75 /* Make sure used is visible before rasing the interrupt. */ 76 smp_wmb(); 77 78 local_bh_disable(); 79 if (vringh_need_notify_iotlb(&vq->vring) > 0) 80 vringh_notify(&vq->vring); 81 local_bh_enable(); 82 } 83 84 static bool receive_filter(struct vdpasim *vdpasim, size_t len) 85 { 86 bool modern = vdpasim->features & (1ULL << VIRTIO_F_VERSION_1); 87 size_t hdr_len = modern ? sizeof(struct virtio_net_hdr_v1) : 88 sizeof(struct virtio_net_hdr); 89 struct virtio_net_config *vio_config = vdpasim->config; 90 91 if (len < ETH_ALEN + hdr_len) 92 return false; 93 94 if (is_broadcast_ether_addr(vdpasim->buffer + hdr_len) || 95 is_multicast_ether_addr(vdpasim->buffer + hdr_len)) 96 return true; 97 if (!strncmp(vdpasim->buffer + hdr_len, vio_config->mac, ETH_ALEN)) 98 return true; 99 100 return false; 101 } 102 103 static virtio_net_ctrl_ack vdpasim_handle_ctrl_mac(struct vdpasim *vdpasim, 104 u8 cmd) 105 { 106 struct virtio_net_config *vio_config = vdpasim->config; 107 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2]; 108 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 109 size_t read; 110 111 switch (cmd) { 112 case VIRTIO_NET_CTRL_MAC_ADDR_SET: 113 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov, 114 vio_config->mac, ETH_ALEN); 115 if (read == ETH_ALEN) 116 status = VIRTIO_NET_OK; 117 break; 118 default: 119 break; 120 } 121 122 return status; 123 } 124 125 static void vdpasim_handle_cvq(struct vdpasim *vdpasim) 126 { 127 struct vdpasim_virtqueue *cvq = &vdpasim->vqs[2]; 128 struct vdpasim_net *net = sim_to_net(vdpasim); 129 virtio_net_ctrl_ack status = VIRTIO_NET_ERR; 130 struct virtio_net_ctrl_hdr ctrl; 131 size_t read, write; 132 u64 requests = 0, errors = 0, successes = 0; 133 int err; 134 135 if (!(vdpasim->features & (1ULL << VIRTIO_NET_F_CTRL_VQ))) 136 return; 137 138 if (!cvq->ready) 139 return; 140 141 while (true) { 142 err = vringh_getdesc_iotlb(&cvq->vring, &cvq->in_iov, 143 &cvq->out_iov, 144 &cvq->head, GFP_ATOMIC); 145 if (err <= 0) 146 break; 147 148 ++requests; 149 read = vringh_iov_pull_iotlb(&cvq->vring, &cvq->in_iov, &ctrl, 150 sizeof(ctrl)); 151 if (read != sizeof(ctrl)) { 152 ++errors; 153 break; 154 } 155 156 switch (ctrl.class) { 157 case VIRTIO_NET_CTRL_MAC: 158 status = vdpasim_handle_ctrl_mac(vdpasim, ctrl.cmd); 159 break; 160 default: 161 break; 162 } 163 164 if (status == VIRTIO_NET_OK) 165 ++successes; 166 else 167 ++errors; 168 169 /* Make sure data is wrote before advancing index */ 170 smp_wmb(); 171 172 write = vringh_iov_push_iotlb(&cvq->vring, &cvq->out_iov, 173 &status, sizeof(status)); 174 vringh_complete_iotlb(&cvq->vring, cvq->head, write); 175 vringh_kiov_cleanup(&cvq->in_iov); 176 vringh_kiov_cleanup(&cvq->out_iov); 177 178 /* Make sure used is visible before rasing the interrupt. */ 179 smp_wmb(); 180 181 local_bh_disable(); 182 if (cvq->cb) 183 cvq->cb(cvq->private); 184 local_bh_enable(); 185 } 186 187 u64_stats_update_begin(&net->cq_stats.syncp); 188 net->cq_stats.requests += requests; 189 net->cq_stats.errors += errors; 190 net->cq_stats.successes += successes; 191 u64_stats_update_end(&net->cq_stats.syncp); 192 } 193 194 static void vdpasim_net_work(struct vdpasim *vdpasim) 195 { 196 struct vdpasim_virtqueue *txq = &vdpasim->vqs[1]; 197 struct vdpasim_virtqueue *rxq = &vdpasim->vqs[0]; 198 struct vdpasim_net *net = sim_to_net(vdpasim); 199 ssize_t read, write; 200 u64 tx_pkts = 0, rx_pkts = 0, tx_bytes = 0, rx_bytes = 0; 201 u64 rx_drops = 0, rx_overruns = 0, rx_errors = 0, tx_errors = 0; 202 int err; 203 204 mutex_lock(&vdpasim->mutex); 205 206 if (!vdpasim->running) 207 goto out; 208 209 if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) 210 goto out; 211 212 vdpasim_handle_cvq(vdpasim); 213 214 if (!txq->ready || !rxq->ready) 215 goto out; 216 217 while (true) { 218 err = vringh_getdesc_iotlb(&txq->vring, &txq->out_iov, NULL, 219 &txq->head, GFP_ATOMIC); 220 if (err <= 0) { 221 if (err) 222 ++tx_errors; 223 break; 224 } 225 226 ++tx_pkts; 227 read = vringh_iov_pull_iotlb(&txq->vring, &txq->out_iov, 228 vdpasim->buffer, 229 PAGE_SIZE); 230 231 tx_bytes += read; 232 233 if (!receive_filter(vdpasim, read)) { 234 ++rx_drops; 235 vdpasim_net_complete(txq, 0); 236 continue; 237 } 238 239 err = vringh_getdesc_iotlb(&rxq->vring, NULL, &rxq->in_iov, 240 &rxq->head, GFP_ATOMIC); 241 if (err <= 0) { 242 ++rx_overruns; 243 vdpasim_net_complete(txq, 0); 244 break; 245 } 246 247 write = vringh_iov_push_iotlb(&rxq->vring, &rxq->in_iov, 248 vdpasim->buffer, read); 249 if (write <= 0) { 250 ++rx_errors; 251 break; 252 } 253 254 ++rx_pkts; 255 rx_bytes += write; 256 257 vdpasim_net_complete(txq, 0); 258 vdpasim_net_complete(rxq, write); 259 260 if (tx_pkts > 4) { 261 vdpasim_schedule_work(vdpasim); 262 goto out; 263 } 264 } 265 266 out: 267 mutex_unlock(&vdpasim->mutex); 268 269 u64_stats_update_begin(&net->tx_stats.syncp); 270 net->tx_stats.pkts += tx_pkts; 271 net->tx_stats.bytes += tx_bytes; 272 net->tx_stats.errors += tx_errors; 273 u64_stats_update_end(&net->tx_stats.syncp); 274 275 u64_stats_update_begin(&net->rx_stats.syncp); 276 net->rx_stats.pkts += rx_pkts; 277 net->rx_stats.bytes += rx_bytes; 278 net->rx_stats.drops += rx_drops; 279 net->rx_stats.errors += rx_errors; 280 net->rx_stats.overruns += rx_overruns; 281 u64_stats_update_end(&net->rx_stats.syncp); 282 } 283 284 static int vdpasim_net_get_stats(struct vdpasim *vdpasim, u16 idx, 285 struct sk_buff *msg, 286 struct netlink_ext_ack *extack) 287 { 288 struct vdpasim_net *net = sim_to_net(vdpasim); 289 u64 rx_pkts, rx_bytes, rx_errors, rx_overruns, rx_drops; 290 u64 tx_pkts, tx_bytes, tx_errors, tx_drops; 291 u64 cq_requests, cq_successes, cq_errors; 292 unsigned int start; 293 int err = -EMSGSIZE; 294 295 switch(idx) { 296 case 0: 297 do { 298 start = u64_stats_fetch_begin(&net->rx_stats.syncp); 299 rx_pkts = net->rx_stats.pkts; 300 rx_bytes = net->rx_stats.bytes; 301 rx_errors = net->rx_stats.errors; 302 rx_overruns = net->rx_stats.overruns; 303 rx_drops = net->rx_stats.drops; 304 } while (u64_stats_fetch_retry(&net->rx_stats.syncp, start)); 305 306 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 307 "rx packets")) 308 break; 309 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 310 rx_pkts, VDPA_ATTR_PAD)) 311 break; 312 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 313 "rx bytes")) 314 break; 315 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 316 rx_bytes, VDPA_ATTR_PAD)) 317 break; 318 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 319 "rx errors")) 320 break; 321 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 322 rx_errors, VDPA_ATTR_PAD)) 323 break; 324 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 325 "rx overruns")) 326 break; 327 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 328 rx_overruns, VDPA_ATTR_PAD)) 329 break; 330 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 331 "rx drops")) 332 break; 333 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 334 rx_drops, VDPA_ATTR_PAD)) 335 break; 336 err = 0; 337 break; 338 case 1: 339 do { 340 start = u64_stats_fetch_begin(&net->tx_stats.syncp); 341 tx_pkts = net->tx_stats.pkts; 342 tx_bytes = net->tx_stats.bytes; 343 tx_errors = net->tx_stats.errors; 344 tx_drops = net->tx_stats.drops; 345 } while (u64_stats_fetch_retry(&net->tx_stats.syncp, start)); 346 347 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 348 "tx packets")) 349 break; 350 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 351 tx_pkts, VDPA_ATTR_PAD)) 352 break; 353 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 354 "tx bytes")) 355 break; 356 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 357 tx_bytes, VDPA_ATTR_PAD)) 358 break; 359 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 360 "tx errors")) 361 break; 362 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 363 tx_errors, VDPA_ATTR_PAD)) 364 break; 365 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 366 "tx drops")) 367 break; 368 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 369 tx_drops, VDPA_ATTR_PAD)) 370 break; 371 err = 0; 372 break; 373 case 2: 374 do { 375 start = u64_stats_fetch_begin(&net->cq_stats.syncp); 376 cq_requests = net->cq_stats.requests; 377 cq_successes = net->cq_stats.successes; 378 cq_errors = net->cq_stats.errors; 379 } while (u64_stats_fetch_retry(&net->cq_stats.syncp, start)); 380 381 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 382 "cvq requests")) 383 break; 384 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 385 cq_requests, VDPA_ATTR_PAD)) 386 break; 387 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 388 "cvq successes")) 389 break; 390 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 391 cq_successes, VDPA_ATTR_PAD)) 392 break; 393 if (nla_put_string(msg, VDPA_ATTR_DEV_VENDOR_ATTR_NAME, 394 "cvq errors")) 395 break; 396 if (nla_put_u64_64bit(msg, VDPA_ATTR_DEV_VENDOR_ATTR_VALUE, 397 cq_errors, VDPA_ATTR_PAD)) 398 break; 399 err = 0; 400 break; 401 default: 402 err = -EINVAL; 403 break; 404 } 405 406 return err; 407 } 408 409 static void vdpasim_net_get_config(struct vdpasim *vdpasim, void *config) 410 { 411 struct virtio_net_config *net_config = config; 412 413 net_config->status = cpu_to_vdpasim16(vdpasim, VIRTIO_NET_S_LINK_UP); 414 } 415 416 static void vdpasim_net_setup_config(struct vdpasim *vdpasim, 417 const struct vdpa_dev_set_config *config) 418 { 419 struct virtio_net_config *vio_config = vdpasim->config; 420 421 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR)) 422 memcpy(vio_config->mac, config->net.mac, ETH_ALEN); 423 if (config->mask & (1 << VDPA_ATTR_DEV_NET_CFG_MTU)) 424 vio_config->mtu = cpu_to_vdpasim16(vdpasim, config->net.mtu); 425 else 426 /* Setup default MTU to be 1500 */ 427 vio_config->mtu = cpu_to_vdpasim16(vdpasim, 1500); 428 } 429 430 static void vdpasim_net_mgmtdev_release(struct device *dev) 431 { 432 } 433 434 static struct device vdpasim_net_mgmtdev = { 435 .init_name = "vdpasim_net", 436 .release = vdpasim_net_mgmtdev_release, 437 }; 438 439 static int vdpasim_net_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, 440 const struct vdpa_dev_set_config *config) 441 { 442 struct vdpasim_dev_attr dev_attr = {}; 443 struct vdpasim_net *net; 444 struct vdpasim *simdev; 445 int ret; 446 447 dev_attr.mgmt_dev = mdev; 448 dev_attr.name = name; 449 dev_attr.id = VIRTIO_ID_NET; 450 dev_attr.supported_features = VDPASIM_NET_FEATURES; 451 dev_attr.nvqs = VDPASIM_NET_VQ_NUM; 452 dev_attr.ngroups = VDPASIM_NET_GROUP_NUM; 453 dev_attr.nas = VDPASIM_NET_AS_NUM; 454 dev_attr.alloc_size = sizeof(struct vdpasim_net); 455 dev_attr.config_size = sizeof(struct virtio_net_config); 456 dev_attr.get_config = vdpasim_net_get_config; 457 dev_attr.work_fn = vdpasim_net_work; 458 dev_attr.get_stats = vdpasim_net_get_stats; 459 dev_attr.buffer_size = PAGE_SIZE; 460 461 simdev = vdpasim_create(&dev_attr, config); 462 if (IS_ERR(simdev)) 463 return PTR_ERR(simdev); 464 465 vdpasim_net_setup_config(simdev, config); 466 467 net = sim_to_net(simdev); 468 469 u64_stats_init(&net->tx_stats.syncp); 470 u64_stats_init(&net->rx_stats.syncp); 471 u64_stats_init(&net->cq_stats.syncp); 472 473 /* 474 * Initialization must be completed before this call, since it can 475 * connect the device to the vDPA bus, so requests can arrive after 476 * this call. 477 */ 478 ret = _vdpa_register_device(&simdev->vdpa, VDPASIM_NET_VQ_NUM); 479 if (ret) 480 goto reg_err; 481 482 return 0; 483 484 reg_err: 485 put_device(&simdev->vdpa.dev); 486 return ret; 487 } 488 489 static void vdpasim_net_dev_del(struct vdpa_mgmt_dev *mdev, 490 struct vdpa_device *dev) 491 { 492 struct vdpasim *simdev = container_of(dev, struct vdpasim, vdpa); 493 494 _vdpa_unregister_device(&simdev->vdpa); 495 } 496 497 static const struct vdpa_mgmtdev_ops vdpasim_net_mgmtdev_ops = { 498 .dev_add = vdpasim_net_dev_add, 499 .dev_del = vdpasim_net_dev_del 500 }; 501 502 static struct virtio_device_id id_table[] = { 503 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 504 { 0 }, 505 }; 506 507 static struct vdpa_mgmt_dev mgmt_dev = { 508 .device = &vdpasim_net_mgmtdev, 509 .id_table = id_table, 510 .ops = &vdpasim_net_mgmtdev_ops, 511 .config_attr_mask = (1 << VDPA_ATTR_DEV_NET_CFG_MACADDR | 512 1 << VDPA_ATTR_DEV_NET_CFG_MTU | 513 1 << VDPA_ATTR_DEV_FEATURES), 514 .max_supported_vqs = VDPASIM_NET_VQ_NUM, 515 .supported_features = VDPASIM_NET_FEATURES, 516 }; 517 518 static int __init vdpasim_net_init(void) 519 { 520 int ret; 521 522 ret = device_register(&vdpasim_net_mgmtdev); 523 if (ret) { 524 put_device(&vdpasim_net_mgmtdev); 525 return ret; 526 } 527 528 ret = vdpa_mgmtdev_register(&mgmt_dev); 529 if (ret) 530 goto parent_err; 531 return 0; 532 533 parent_err: 534 device_unregister(&vdpasim_net_mgmtdev); 535 return ret; 536 } 537 538 static void __exit vdpasim_net_exit(void) 539 { 540 vdpa_mgmtdev_unregister(&mgmt_dev); 541 device_unregister(&vdpasim_net_mgmtdev); 542 } 543 544 module_init(vdpasim_net_init); 545 module_exit(vdpasim_net_exit); 546 547 MODULE_VERSION(DRV_VERSION); 548 MODULE_LICENSE(DRV_LICENSE); 549 MODULE_AUTHOR(DRV_AUTHOR); 550 MODULE_DESCRIPTION(DRV_DESC); 551