1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* A network driver using virtio. 3 * 4 * Copyright 2007 Rusty Russell <rusty@rustcorp.com.au> IBM Corporation 5 */ 6 //#define DEBUG 7 #include <linux/netdevice.h> 8 #include <linux/etherdevice.h> 9 #include <linux/ethtool.h> 10 #include <linux/module.h> 11 #include <linux/virtio.h> 12 #include <linux/virtio_net.h> 13 #include <linux/bpf.h> 14 #include <linux/bpf_trace.h> 15 #include <linux/scatterlist.h> 16 #include <linux/if_vlan.h> 17 #include <linux/slab.h> 18 #include <linux/cpu.h> 19 #include <linux/average.h> 20 #include <linux/filter.h> 21 #include <linux/kernel.h> 22 #include <linux/dim.h> 23 #include <net/route.h> 24 #include <net/xdp.h> 25 #include <net/net_failover.h> 26 #include <net/netdev_rx_queue.h> 27 #include <net/netdev_queues.h> 28 #include <net/xdp_sock_drv.h> 29 30 static int napi_weight = NAPI_POLL_WEIGHT; 31 module_param(napi_weight, int, 0444); 32 33 static bool csum = true, gso = true, napi_tx = true; 34 module_param(csum, bool, 0444); 35 module_param(gso, bool, 0444); 36 module_param(napi_tx, bool, 0644); 37 38 /* FIXME: MTU in config. */ 39 #define GOOD_PACKET_LEN (ETH_HLEN + VLAN_HLEN + ETH_DATA_LEN) 40 #define GOOD_COPY_LEN 128 41 42 #define VIRTNET_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 43 44 /* Separating two types of XDP xmit */ 45 #define VIRTIO_XDP_TX BIT(0) 46 #define VIRTIO_XDP_REDIR BIT(1) 47 48 /* RX packet size EWMA. The average packet size is used to determine the packet 49 * buffer size when refilling RX rings. As the entire RX ring may be refilled 50 * at once, the weight is chosen so that the EWMA will be insensitive to short- 51 * term, transient changes in packet size. 52 */ 53 DECLARE_EWMA(pkt_len, 0, 64) 54 55 #define VIRTNET_DRIVER_VERSION "1.0.0" 56 57 static const unsigned long guest_offloads[] = { 58 VIRTIO_NET_F_GUEST_TSO4, 59 VIRTIO_NET_F_GUEST_TSO6, 60 VIRTIO_NET_F_GUEST_ECN, 61 VIRTIO_NET_F_GUEST_UFO, 62 VIRTIO_NET_F_GUEST_CSUM, 63 VIRTIO_NET_F_GUEST_USO4, 64 VIRTIO_NET_F_GUEST_USO6, 65 VIRTIO_NET_F_GUEST_HDRLEN 66 }; 67 68 #define GUEST_OFFLOAD_GRO_HW_MASK ((1ULL << VIRTIO_NET_F_GUEST_TSO4) | \ 69 (1ULL << VIRTIO_NET_F_GUEST_TSO6) | \ 70 (1ULL << VIRTIO_NET_F_GUEST_ECN) | \ 71 (1ULL << VIRTIO_NET_F_GUEST_UFO) | \ 72 (1ULL << VIRTIO_NET_F_GUEST_USO4) | \ 73 (1ULL << VIRTIO_NET_F_GUEST_USO6)) 74 75 struct virtnet_stat_desc { 76 char desc[ETH_GSTRING_LEN]; 77 size_t offset; 78 size_t qstat_offset; 79 }; 80 81 struct virtnet_sq_free_stats { 82 u64 packets; 83 u64 bytes; 84 u64 napi_packets; 85 u64 napi_bytes; 86 u64 xsk; 87 }; 88 89 struct virtnet_sq_stats { 90 struct u64_stats_sync syncp; 91 u64_stats_t packets; 92 u64_stats_t bytes; 93 u64_stats_t xdp_tx; 94 u64_stats_t xdp_tx_drops; 95 u64_stats_t kicks; 96 u64_stats_t tx_timeouts; 97 u64_stats_t stop; 98 u64_stats_t wake; 99 }; 100 101 struct virtnet_rq_stats { 102 struct u64_stats_sync syncp; 103 u64_stats_t packets; 104 u64_stats_t bytes; 105 u64_stats_t drops; 106 u64_stats_t xdp_packets; 107 u64_stats_t xdp_tx; 108 u64_stats_t xdp_redirects; 109 u64_stats_t xdp_drops; 110 u64_stats_t kicks; 111 }; 112 113 #define VIRTNET_SQ_STAT(name, m) {name, offsetof(struct virtnet_sq_stats, m), -1} 114 #define VIRTNET_RQ_STAT(name, m) {name, offsetof(struct virtnet_rq_stats, m), -1} 115 116 #define VIRTNET_SQ_STAT_QSTAT(name, m) \ 117 { \ 118 name, \ 119 offsetof(struct virtnet_sq_stats, m), \ 120 offsetof(struct netdev_queue_stats_tx, m), \ 121 } 122 123 #define VIRTNET_RQ_STAT_QSTAT(name, m) \ 124 { \ 125 name, \ 126 offsetof(struct virtnet_rq_stats, m), \ 127 offsetof(struct netdev_queue_stats_rx, m), \ 128 } 129 130 static const struct virtnet_stat_desc virtnet_sq_stats_desc[] = { 131 VIRTNET_SQ_STAT("xdp_tx", xdp_tx), 132 VIRTNET_SQ_STAT("xdp_tx_drops", xdp_tx_drops), 133 VIRTNET_SQ_STAT("kicks", kicks), 134 VIRTNET_SQ_STAT("tx_timeouts", tx_timeouts), 135 }; 136 137 static const struct virtnet_stat_desc virtnet_rq_stats_desc[] = { 138 VIRTNET_RQ_STAT("drops", drops), 139 VIRTNET_RQ_STAT("xdp_packets", xdp_packets), 140 VIRTNET_RQ_STAT("xdp_tx", xdp_tx), 141 VIRTNET_RQ_STAT("xdp_redirects", xdp_redirects), 142 VIRTNET_RQ_STAT("xdp_drops", xdp_drops), 143 VIRTNET_RQ_STAT("kicks", kicks), 144 }; 145 146 static const struct virtnet_stat_desc virtnet_sq_stats_desc_qstat[] = { 147 VIRTNET_SQ_STAT_QSTAT("packets", packets), 148 VIRTNET_SQ_STAT_QSTAT("bytes", bytes), 149 VIRTNET_SQ_STAT_QSTAT("stop", stop), 150 VIRTNET_SQ_STAT_QSTAT("wake", wake), 151 }; 152 153 static const struct virtnet_stat_desc virtnet_rq_stats_desc_qstat[] = { 154 VIRTNET_RQ_STAT_QSTAT("packets", packets), 155 VIRTNET_RQ_STAT_QSTAT("bytes", bytes), 156 }; 157 158 #define VIRTNET_STATS_DESC_CQ(name) \ 159 {#name, offsetof(struct virtio_net_stats_cvq, name), -1} 160 161 #define VIRTNET_STATS_DESC_RX(class, name) \ 162 {#name, offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), -1} 163 164 #define VIRTNET_STATS_DESC_TX(class, name) \ 165 {#name, offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), -1} 166 167 168 static const struct virtnet_stat_desc virtnet_stats_cvq_desc[] = { 169 VIRTNET_STATS_DESC_CQ(command_num), 170 VIRTNET_STATS_DESC_CQ(ok_num), 171 }; 172 173 static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc[] = { 174 VIRTNET_STATS_DESC_RX(basic, packets), 175 VIRTNET_STATS_DESC_RX(basic, bytes), 176 177 VIRTNET_STATS_DESC_RX(basic, notifications), 178 VIRTNET_STATS_DESC_RX(basic, interrupts), 179 }; 180 181 static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc[] = { 182 VIRTNET_STATS_DESC_TX(basic, packets), 183 VIRTNET_STATS_DESC_TX(basic, bytes), 184 185 VIRTNET_STATS_DESC_TX(basic, notifications), 186 VIRTNET_STATS_DESC_TX(basic, interrupts), 187 }; 188 189 static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc[] = { 190 VIRTNET_STATS_DESC_RX(csum, needs_csum), 191 }; 192 193 static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc[] = { 194 VIRTNET_STATS_DESC_TX(gso, gso_packets_noseg), 195 VIRTNET_STATS_DESC_TX(gso, gso_bytes_noseg), 196 }; 197 198 static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc[] = { 199 VIRTNET_STATS_DESC_RX(speed, ratelimit_bytes), 200 }; 201 202 static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc[] = { 203 VIRTNET_STATS_DESC_TX(speed, ratelimit_bytes), 204 }; 205 206 #define VIRTNET_STATS_DESC_RX_QSTAT(class, name, qstat_field) \ 207 { \ 208 #name, \ 209 offsetof(struct virtio_net_stats_rx_ ## class, rx_ ## name), \ 210 offsetof(struct netdev_queue_stats_rx, qstat_field), \ 211 } 212 213 #define VIRTNET_STATS_DESC_TX_QSTAT(class, name, qstat_field) \ 214 { \ 215 #name, \ 216 offsetof(struct virtio_net_stats_tx_ ## class, tx_ ## name), \ 217 offsetof(struct netdev_queue_stats_tx, qstat_field), \ 218 } 219 220 static const struct virtnet_stat_desc virtnet_stats_rx_basic_desc_qstat[] = { 221 VIRTNET_STATS_DESC_RX_QSTAT(basic, drops, hw_drops), 222 VIRTNET_STATS_DESC_RX_QSTAT(basic, drop_overruns, hw_drop_overruns), 223 }; 224 225 static const struct virtnet_stat_desc virtnet_stats_tx_basic_desc_qstat[] = { 226 VIRTNET_STATS_DESC_TX_QSTAT(basic, drops, hw_drops), 227 VIRTNET_STATS_DESC_TX_QSTAT(basic, drop_malformed, hw_drop_errors), 228 }; 229 230 static const struct virtnet_stat_desc virtnet_stats_rx_csum_desc_qstat[] = { 231 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_valid, csum_unnecessary), 232 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_none, csum_none), 233 VIRTNET_STATS_DESC_RX_QSTAT(csum, csum_bad, csum_bad), 234 }; 235 236 static const struct virtnet_stat_desc virtnet_stats_tx_csum_desc_qstat[] = { 237 VIRTNET_STATS_DESC_TX_QSTAT(csum, csum_none, csum_none), 238 VIRTNET_STATS_DESC_TX_QSTAT(csum, needs_csum, needs_csum), 239 }; 240 241 static const struct virtnet_stat_desc virtnet_stats_rx_gso_desc_qstat[] = { 242 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets, hw_gro_packets), 243 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes, hw_gro_bytes), 244 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_packets_coalesced, hw_gro_wire_packets), 245 VIRTNET_STATS_DESC_RX_QSTAT(gso, gso_bytes_coalesced, hw_gro_wire_bytes), 246 }; 247 248 static const struct virtnet_stat_desc virtnet_stats_tx_gso_desc_qstat[] = { 249 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_packets, hw_gso_packets), 250 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_bytes, hw_gso_bytes), 251 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments, hw_gso_wire_packets), 252 VIRTNET_STATS_DESC_TX_QSTAT(gso, gso_segments_bytes, hw_gso_wire_bytes), 253 }; 254 255 static const struct virtnet_stat_desc virtnet_stats_rx_speed_desc_qstat[] = { 256 VIRTNET_STATS_DESC_RX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits), 257 }; 258 259 static const struct virtnet_stat_desc virtnet_stats_tx_speed_desc_qstat[] = { 260 VIRTNET_STATS_DESC_TX_QSTAT(speed, ratelimit_packets, hw_drop_ratelimits), 261 }; 262 263 #define VIRTNET_Q_TYPE_RX 0 264 #define VIRTNET_Q_TYPE_TX 1 265 #define VIRTNET_Q_TYPE_CQ 2 266 267 struct virtnet_interrupt_coalesce { 268 u32 max_packets; 269 u32 max_usecs; 270 }; 271 272 /* The dma information of pages allocated at a time. */ 273 struct virtnet_rq_dma { 274 dma_addr_t addr; 275 u32 ref; 276 u16 len; 277 u16 need_sync; 278 }; 279 280 /* Internal representation of a send virtqueue */ 281 struct send_queue { 282 /* Virtqueue associated with this send _queue */ 283 struct virtqueue *vq; 284 285 /* TX: fragments + linear part + virtio header */ 286 struct scatterlist sg[MAX_SKB_FRAGS + 2]; 287 288 /* Name of the send queue: output.$index */ 289 char name[16]; 290 291 struct virtnet_sq_stats stats; 292 293 struct virtnet_interrupt_coalesce intr_coal; 294 295 struct napi_struct napi; 296 297 /* Record whether sq is in reset state. */ 298 bool reset; 299 300 struct xsk_buff_pool *xsk_pool; 301 302 dma_addr_t xsk_hdr_dma_addr; 303 }; 304 305 /* Internal representation of a receive virtqueue */ 306 struct receive_queue { 307 /* Virtqueue associated with this receive_queue */ 308 struct virtqueue *vq; 309 310 struct napi_struct napi; 311 312 struct bpf_prog __rcu *xdp_prog; 313 314 struct virtnet_rq_stats stats; 315 316 /* The number of rx notifications */ 317 u16 calls; 318 319 /* Is dynamic interrupt moderation enabled? */ 320 bool dim_enabled; 321 322 /* Used to protect dim_enabled and inter_coal */ 323 struct mutex dim_lock; 324 325 /* Dynamic Interrupt Moderation */ 326 struct dim dim; 327 328 u32 packets_in_napi; 329 330 struct virtnet_interrupt_coalesce intr_coal; 331 332 /* Chain pages by the private ptr. */ 333 struct page *pages; 334 335 /* Average packet length for mergeable receive buffers. */ 336 struct ewma_pkt_len mrg_avg_pkt_len; 337 338 /* Page frag for packet buffer allocation. */ 339 struct page_frag alloc_frag; 340 341 /* RX: fragments + linear part + virtio header */ 342 struct scatterlist sg[MAX_SKB_FRAGS + 2]; 343 344 /* Min single buffer size for mergeable buffers case. */ 345 unsigned int min_buf_len; 346 347 /* Name of this receive queue: input.$index */ 348 char name[16]; 349 350 struct xdp_rxq_info xdp_rxq; 351 352 /* Record the last dma info to free after new pages is allocated. */ 353 struct virtnet_rq_dma *last_dma; 354 355 struct xsk_buff_pool *xsk_pool; 356 357 /* xdp rxq used by xsk */ 358 struct xdp_rxq_info xsk_rxq_info; 359 360 struct xdp_buff **xsk_buffs; 361 }; 362 363 #define VIRTIO_NET_RSS_MAX_KEY_SIZE 40 364 365 /* Control VQ buffers: protected by the rtnl lock */ 366 struct control_buf { 367 struct virtio_net_ctrl_hdr hdr; 368 virtio_net_ctrl_ack status; 369 }; 370 371 struct virtnet_info { 372 struct virtio_device *vdev; 373 struct virtqueue *cvq; 374 struct net_device *dev; 375 struct send_queue *sq; 376 struct receive_queue *rq; 377 unsigned int status; 378 379 /* Max # of queue pairs supported by the device */ 380 u16 max_queue_pairs; 381 382 /* # of queue pairs currently used by the driver */ 383 u16 curr_queue_pairs; 384 385 /* # of XDP queue pairs currently used by the driver */ 386 u16 xdp_queue_pairs; 387 388 /* xdp_queue_pairs may be 0, when xdp is already loaded. So add this. */ 389 bool xdp_enabled; 390 391 /* I like... big packets and I cannot lie! */ 392 bool big_packets; 393 394 /* number of sg entries allocated for big packets */ 395 unsigned int big_packets_num_skbfrags; 396 397 /* Host will merge rx buffers for big packets (shake it! shake it!) */ 398 bool mergeable_rx_bufs; 399 400 /* Host supports rss and/or hash report */ 401 bool has_rss; 402 bool has_rss_hash_report; 403 u8 rss_key_size; 404 u16 rss_indir_table_size; 405 u32 rss_hash_types_supported; 406 u32 rss_hash_types_saved; 407 struct virtio_net_rss_config_hdr *rss_hdr; 408 struct virtio_net_rss_config_trailer rss_trailer; 409 u8 rss_hash_key_data[VIRTIO_NET_RSS_MAX_KEY_SIZE]; 410 411 /* Has control virtqueue */ 412 bool has_cvq; 413 414 /* Lock to protect the control VQ */ 415 struct mutex cvq_lock; 416 417 /* Host can handle any s/g split between our header and packet data */ 418 bool any_header_sg; 419 420 /* Packet virtio header size */ 421 u8 hdr_len; 422 423 /* Work struct for delayed refilling if we run low on memory. */ 424 struct delayed_work refill; 425 426 /* Is delayed refill enabled? */ 427 bool refill_enabled; 428 429 /* The lock to synchronize the access to refill_enabled */ 430 spinlock_t refill_lock; 431 432 /* Work struct for config space updates */ 433 struct work_struct config_work; 434 435 /* Work struct for setting rx mode */ 436 struct work_struct rx_mode_work; 437 438 /* OK to queue work setting RX mode? */ 439 bool rx_mode_work_enabled; 440 441 /* Does the affinity hint is set for virtqueues? */ 442 bool affinity_hint_set; 443 444 /* CPU hotplug instances for online & dead */ 445 struct hlist_node node; 446 struct hlist_node node_dead; 447 448 struct control_buf *ctrl; 449 450 /* Ethtool settings */ 451 u8 duplex; 452 u32 speed; 453 454 /* Is rx dynamic interrupt moderation enabled? */ 455 bool rx_dim_enabled; 456 457 /* Interrupt coalescing settings */ 458 struct virtnet_interrupt_coalesce intr_coal_tx; 459 struct virtnet_interrupt_coalesce intr_coal_rx; 460 461 unsigned long guest_offloads; 462 unsigned long guest_offloads_capable; 463 464 /* failover when STANDBY feature enabled */ 465 struct failover *failover; 466 467 u64 device_stats_cap; 468 }; 469 470 struct padded_vnet_hdr { 471 struct virtio_net_hdr_v1_hash hdr; 472 /* 473 * hdr is in a separate sg buffer, and data sg buffer shares same page 474 * with this header sg. This padding makes next sg 16 byte aligned 475 * after the header. 476 */ 477 char padding[12]; 478 }; 479 480 struct virtio_net_common_hdr { 481 union { 482 struct virtio_net_hdr hdr; 483 struct virtio_net_hdr_mrg_rxbuf mrg_hdr; 484 struct virtio_net_hdr_v1_hash hash_v1_hdr; 485 }; 486 }; 487 488 static struct virtio_net_common_hdr xsk_hdr; 489 490 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf); 491 static void virtnet_sq_free_unused_buf_done(struct virtqueue *vq); 492 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp, 493 struct net_device *dev, 494 unsigned int *xdp_xmit, 495 struct virtnet_rq_stats *stats); 496 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq, 497 struct sk_buff *skb, u8 flags); 498 static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb, 499 struct sk_buff *curr_skb, 500 struct page *page, void *buf, 501 int len, int truesize); 502 static void virtnet_xsk_completed(struct send_queue *sq, int num); 503 504 enum virtnet_xmit_type { 505 VIRTNET_XMIT_TYPE_SKB, 506 VIRTNET_XMIT_TYPE_SKB_ORPHAN, 507 VIRTNET_XMIT_TYPE_XDP, 508 VIRTNET_XMIT_TYPE_XSK, 509 }; 510 511 static size_t virtnet_rss_hdr_size(const struct virtnet_info *vi) 512 { 513 u16 indir_table_size = vi->has_rss ? vi->rss_indir_table_size : 1; 514 515 return struct_size(vi->rss_hdr, indirection_table, indir_table_size); 516 } 517 518 static size_t virtnet_rss_trailer_size(const struct virtnet_info *vi) 519 { 520 return struct_size(&vi->rss_trailer, hash_key_data, vi->rss_key_size); 521 } 522 523 /* We use the last two bits of the pointer to distinguish the xmit type. */ 524 #define VIRTNET_XMIT_TYPE_MASK (BIT(0) | BIT(1)) 525 526 #define VIRTIO_XSK_FLAG_OFFSET 2 527 528 static enum virtnet_xmit_type virtnet_xmit_ptr_unpack(void **ptr) 529 { 530 unsigned long p = (unsigned long)*ptr; 531 532 *ptr = (void *)(p & ~VIRTNET_XMIT_TYPE_MASK); 533 534 return p & VIRTNET_XMIT_TYPE_MASK; 535 } 536 537 static void *virtnet_xmit_ptr_pack(void *ptr, enum virtnet_xmit_type type) 538 { 539 return (void *)((unsigned long)ptr | type); 540 } 541 542 static int virtnet_add_outbuf(struct send_queue *sq, int num, void *data, 543 enum virtnet_xmit_type type) 544 { 545 return virtqueue_add_outbuf(sq->vq, sq->sg, num, 546 virtnet_xmit_ptr_pack(data, type), 547 GFP_ATOMIC); 548 } 549 550 static u32 virtnet_ptr_to_xsk_buff_len(void *ptr) 551 { 552 return ((unsigned long)ptr) >> VIRTIO_XSK_FLAG_OFFSET; 553 } 554 555 static void sg_fill_dma(struct scatterlist *sg, dma_addr_t addr, u32 len) 556 { 557 sg_dma_address(sg) = addr; 558 sg_dma_len(sg) = len; 559 } 560 561 static void __free_old_xmit(struct send_queue *sq, struct netdev_queue *txq, 562 bool in_napi, struct virtnet_sq_free_stats *stats) 563 { 564 struct xdp_frame *frame; 565 struct sk_buff *skb; 566 unsigned int len; 567 void *ptr; 568 569 while ((ptr = virtqueue_get_buf(sq->vq, &len)) != NULL) { 570 switch (virtnet_xmit_ptr_unpack(&ptr)) { 571 case VIRTNET_XMIT_TYPE_SKB: 572 skb = ptr; 573 574 pr_debug("Sent skb %p\n", skb); 575 stats->napi_packets++; 576 stats->napi_bytes += skb->len; 577 napi_consume_skb(skb, in_napi); 578 break; 579 580 case VIRTNET_XMIT_TYPE_SKB_ORPHAN: 581 skb = ptr; 582 583 stats->packets++; 584 stats->bytes += skb->len; 585 napi_consume_skb(skb, in_napi); 586 break; 587 588 case VIRTNET_XMIT_TYPE_XDP: 589 frame = ptr; 590 591 stats->packets++; 592 stats->bytes += xdp_get_frame_len(frame); 593 xdp_return_frame(frame); 594 break; 595 596 case VIRTNET_XMIT_TYPE_XSK: 597 stats->bytes += virtnet_ptr_to_xsk_buff_len(ptr); 598 stats->xsk++; 599 break; 600 } 601 } 602 netdev_tx_completed_queue(txq, stats->napi_packets, stats->napi_bytes); 603 } 604 605 static void virtnet_free_old_xmit(struct send_queue *sq, 606 struct netdev_queue *txq, 607 bool in_napi, 608 struct virtnet_sq_free_stats *stats) 609 { 610 __free_old_xmit(sq, txq, in_napi, stats); 611 612 if (stats->xsk) 613 virtnet_xsk_completed(sq, stats->xsk); 614 } 615 616 /* Converting between virtqueue no. and kernel tx/rx queue no. 617 * 0:rx0 1:tx0 2:rx1 3:tx1 ... 2N:rxN 2N+1:txN 2N+2:cvq 618 */ 619 static int vq2txq(struct virtqueue *vq) 620 { 621 return (vq->index - 1) / 2; 622 } 623 624 static int txq2vq(int txq) 625 { 626 return txq * 2 + 1; 627 } 628 629 static int vq2rxq(struct virtqueue *vq) 630 { 631 return vq->index / 2; 632 } 633 634 static int rxq2vq(int rxq) 635 { 636 return rxq * 2; 637 } 638 639 static int vq_type(struct virtnet_info *vi, int qid) 640 { 641 if (qid == vi->max_queue_pairs * 2) 642 return VIRTNET_Q_TYPE_CQ; 643 644 if (qid % 2) 645 return VIRTNET_Q_TYPE_TX; 646 647 return VIRTNET_Q_TYPE_RX; 648 } 649 650 static inline struct virtio_net_common_hdr * 651 skb_vnet_common_hdr(struct sk_buff *skb) 652 { 653 return (struct virtio_net_common_hdr *)skb->cb; 654 } 655 656 /* 657 * private is used to chain pages for big packets, put the whole 658 * most recent used list in the beginning for reuse 659 */ 660 static void give_pages(struct receive_queue *rq, struct page *page) 661 { 662 struct page *end; 663 664 /* Find end of list, sew whole thing into vi->rq.pages. */ 665 for (end = page; end->private; end = (struct page *)end->private); 666 end->private = (unsigned long)rq->pages; 667 rq->pages = page; 668 } 669 670 static struct page *get_a_page(struct receive_queue *rq, gfp_t gfp_mask) 671 { 672 struct page *p = rq->pages; 673 674 if (p) { 675 rq->pages = (struct page *)p->private; 676 /* clear private here, it is used to chain pages */ 677 p->private = 0; 678 } else 679 p = alloc_page(gfp_mask); 680 return p; 681 } 682 683 static void virtnet_rq_free_buf(struct virtnet_info *vi, 684 struct receive_queue *rq, void *buf) 685 { 686 if (vi->mergeable_rx_bufs) 687 put_page(virt_to_head_page(buf)); 688 else if (vi->big_packets) 689 give_pages(rq, buf); 690 else 691 put_page(virt_to_head_page(buf)); 692 } 693 694 static void enable_delayed_refill(struct virtnet_info *vi) 695 { 696 spin_lock_bh(&vi->refill_lock); 697 vi->refill_enabled = true; 698 spin_unlock_bh(&vi->refill_lock); 699 } 700 701 static void disable_delayed_refill(struct virtnet_info *vi) 702 { 703 spin_lock_bh(&vi->refill_lock); 704 vi->refill_enabled = false; 705 spin_unlock_bh(&vi->refill_lock); 706 } 707 708 static void enable_rx_mode_work(struct virtnet_info *vi) 709 { 710 rtnl_lock(); 711 vi->rx_mode_work_enabled = true; 712 rtnl_unlock(); 713 } 714 715 static void disable_rx_mode_work(struct virtnet_info *vi) 716 { 717 rtnl_lock(); 718 vi->rx_mode_work_enabled = false; 719 rtnl_unlock(); 720 } 721 722 static void virtqueue_napi_schedule(struct napi_struct *napi, 723 struct virtqueue *vq) 724 { 725 if (napi_schedule_prep(napi)) { 726 virtqueue_disable_cb(vq); 727 __napi_schedule(napi); 728 } 729 } 730 731 static bool virtqueue_napi_complete(struct napi_struct *napi, 732 struct virtqueue *vq, int processed) 733 { 734 int opaque; 735 736 opaque = virtqueue_enable_cb_prepare(vq); 737 if (napi_complete_done(napi, processed)) { 738 if (unlikely(virtqueue_poll(vq, opaque))) 739 virtqueue_napi_schedule(napi, vq); 740 else 741 return true; 742 } else { 743 virtqueue_disable_cb(vq); 744 } 745 746 return false; 747 } 748 749 static void skb_xmit_done(struct virtqueue *vq) 750 { 751 struct virtnet_info *vi = vq->vdev->priv; 752 struct napi_struct *napi = &vi->sq[vq2txq(vq)].napi; 753 754 /* Suppress further interrupts. */ 755 virtqueue_disable_cb(vq); 756 757 if (napi->weight) 758 virtqueue_napi_schedule(napi, vq); 759 else 760 /* We were probably waiting for more output buffers. */ 761 netif_wake_subqueue(vi->dev, vq2txq(vq)); 762 } 763 764 #define MRG_CTX_HEADER_SHIFT 22 765 static void *mergeable_len_to_ctx(unsigned int truesize, 766 unsigned int headroom) 767 { 768 return (void *)(unsigned long)((headroom << MRG_CTX_HEADER_SHIFT) | truesize); 769 } 770 771 static unsigned int mergeable_ctx_to_headroom(void *mrg_ctx) 772 { 773 return (unsigned long)mrg_ctx >> MRG_CTX_HEADER_SHIFT; 774 } 775 776 static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx) 777 { 778 return (unsigned long)mrg_ctx & ((1 << MRG_CTX_HEADER_SHIFT) - 1); 779 } 780 781 static struct sk_buff *virtnet_build_skb(void *buf, unsigned int buflen, 782 unsigned int headroom, 783 unsigned int len) 784 { 785 struct sk_buff *skb; 786 787 skb = build_skb(buf, buflen); 788 if (unlikely(!skb)) 789 return NULL; 790 791 skb_reserve(skb, headroom); 792 skb_put(skb, len); 793 794 return skb; 795 } 796 797 /* Called from bottom half context */ 798 static struct sk_buff *page_to_skb(struct virtnet_info *vi, 799 struct receive_queue *rq, 800 struct page *page, unsigned int offset, 801 unsigned int len, unsigned int truesize, 802 unsigned int headroom) 803 { 804 struct sk_buff *skb; 805 struct virtio_net_common_hdr *hdr; 806 unsigned int copy, hdr_len, hdr_padded_len; 807 struct page *page_to_free = NULL; 808 int tailroom, shinfo_size; 809 char *p, *hdr_p, *buf; 810 811 p = page_address(page) + offset; 812 hdr_p = p; 813 814 hdr_len = vi->hdr_len; 815 if (vi->mergeable_rx_bufs) 816 hdr_padded_len = hdr_len; 817 else 818 hdr_padded_len = sizeof(struct padded_vnet_hdr); 819 820 buf = p - headroom; 821 len -= hdr_len; 822 offset += hdr_padded_len; 823 p += hdr_padded_len; 824 tailroom = truesize - headroom - hdr_padded_len - len; 825 826 shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 827 828 if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) { 829 skb = virtnet_build_skb(buf, truesize, p - buf, len); 830 if (unlikely(!skb)) 831 return NULL; 832 833 page = (struct page *)page->private; 834 if (page) 835 give_pages(rq, page); 836 goto ok; 837 } 838 839 /* copy small packet so we can reuse these pages for small data */ 840 skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN); 841 if (unlikely(!skb)) 842 return NULL; 843 844 /* Copy all frame if it fits skb->head, otherwise 845 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed. 846 */ 847 if (len <= skb_tailroom(skb)) 848 copy = len; 849 else 850 copy = ETH_HLEN; 851 skb_put_data(skb, p, copy); 852 853 len -= copy; 854 offset += copy; 855 856 if (vi->mergeable_rx_bufs) { 857 if (len) 858 skb_add_rx_frag(skb, 0, page, offset, len, truesize); 859 else 860 page_to_free = page; 861 goto ok; 862 } 863 864 /* 865 * Verify that we can indeed put this data into a skb. 866 * This is here to handle cases when the device erroneously 867 * tries to receive more than is possible. This is usually 868 * the case of a broken device. 869 */ 870 if (unlikely(len > MAX_SKB_FRAGS * PAGE_SIZE)) { 871 net_dbg_ratelimited("%s: too much data\n", skb->dev->name); 872 dev_kfree_skb(skb); 873 return NULL; 874 } 875 BUG_ON(offset >= PAGE_SIZE); 876 while (len) { 877 unsigned int frag_size = min((unsigned)PAGE_SIZE - offset, len); 878 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page, offset, 879 frag_size, truesize); 880 len -= frag_size; 881 page = (struct page *)page->private; 882 offset = 0; 883 } 884 885 if (page) 886 give_pages(rq, page); 887 888 ok: 889 hdr = skb_vnet_common_hdr(skb); 890 memcpy(hdr, hdr_p, hdr_len); 891 if (page_to_free) 892 put_page(page_to_free); 893 894 return skb; 895 } 896 897 static void virtnet_rq_unmap(struct receive_queue *rq, void *buf, u32 len) 898 { 899 struct virtnet_info *vi = rq->vq->vdev->priv; 900 struct page *page = virt_to_head_page(buf); 901 struct virtnet_rq_dma *dma; 902 void *head; 903 int offset; 904 905 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 906 907 head = page_address(page); 908 909 dma = head; 910 911 --dma->ref; 912 913 if (dma->need_sync && len) { 914 offset = buf - (head + sizeof(*dma)); 915 916 virtqueue_dma_sync_single_range_for_cpu(rq->vq, dma->addr, 917 offset, len, 918 DMA_FROM_DEVICE); 919 } 920 921 if (dma->ref) 922 return; 923 924 virtqueue_dma_unmap_single_attrs(rq->vq, dma->addr, dma->len, 925 DMA_FROM_DEVICE, DMA_ATTR_SKIP_CPU_SYNC); 926 put_page(page); 927 } 928 929 static void *virtnet_rq_get_buf(struct receive_queue *rq, u32 *len, void **ctx) 930 { 931 struct virtnet_info *vi = rq->vq->vdev->priv; 932 void *buf; 933 934 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 935 936 buf = virtqueue_get_buf_ctx(rq->vq, len, ctx); 937 if (buf) 938 virtnet_rq_unmap(rq, buf, *len); 939 940 return buf; 941 } 942 943 static void virtnet_rq_init_one_sg(struct receive_queue *rq, void *buf, u32 len) 944 { 945 struct virtnet_info *vi = rq->vq->vdev->priv; 946 struct virtnet_rq_dma *dma; 947 dma_addr_t addr; 948 u32 offset; 949 void *head; 950 951 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 952 953 head = page_address(rq->alloc_frag.page); 954 955 offset = buf - head; 956 957 dma = head; 958 959 addr = dma->addr - sizeof(*dma) + offset; 960 961 sg_init_table(rq->sg, 1); 962 sg_fill_dma(rq->sg, addr, len); 963 } 964 965 static void *virtnet_rq_alloc(struct receive_queue *rq, u32 size, gfp_t gfp) 966 { 967 struct page_frag *alloc_frag = &rq->alloc_frag; 968 struct virtnet_info *vi = rq->vq->vdev->priv; 969 struct virtnet_rq_dma *dma; 970 void *buf, *head; 971 dma_addr_t addr; 972 973 BUG_ON(vi->big_packets && !vi->mergeable_rx_bufs); 974 975 head = page_address(alloc_frag->page); 976 977 dma = head; 978 979 /* new pages */ 980 if (!alloc_frag->offset) { 981 if (rq->last_dma) { 982 /* Now, the new page is allocated, the last dma 983 * will not be used. So the dma can be unmapped 984 * if the ref is 0. 985 */ 986 virtnet_rq_unmap(rq, rq->last_dma, 0); 987 rq->last_dma = NULL; 988 } 989 990 dma->len = alloc_frag->size - sizeof(*dma); 991 992 addr = virtqueue_dma_map_single_attrs(rq->vq, dma + 1, 993 dma->len, DMA_FROM_DEVICE, 0); 994 if (virtqueue_dma_mapping_error(rq->vq, addr)) 995 return NULL; 996 997 dma->addr = addr; 998 dma->need_sync = virtqueue_dma_need_sync(rq->vq, addr); 999 1000 /* Add a reference to dma to prevent the entire dma from 1001 * being released during error handling. This reference 1002 * will be freed after the pages are no longer used. 1003 */ 1004 get_page(alloc_frag->page); 1005 dma->ref = 1; 1006 alloc_frag->offset = sizeof(*dma); 1007 1008 rq->last_dma = dma; 1009 } 1010 1011 ++dma->ref; 1012 1013 buf = head + alloc_frag->offset; 1014 1015 get_page(alloc_frag->page); 1016 alloc_frag->offset += size; 1017 1018 return buf; 1019 } 1020 1021 static void virtnet_rq_unmap_free_buf(struct virtqueue *vq, void *buf) 1022 { 1023 struct virtnet_info *vi = vq->vdev->priv; 1024 struct receive_queue *rq; 1025 int i = vq2rxq(vq); 1026 1027 rq = &vi->rq[i]; 1028 1029 if (rq->xsk_pool) { 1030 xsk_buff_free((struct xdp_buff *)buf); 1031 return; 1032 } 1033 1034 if (!vi->big_packets || vi->mergeable_rx_bufs) 1035 virtnet_rq_unmap(rq, buf, 0); 1036 1037 virtnet_rq_free_buf(vi, rq, buf); 1038 } 1039 1040 static void free_old_xmit(struct send_queue *sq, struct netdev_queue *txq, 1041 bool in_napi) 1042 { 1043 struct virtnet_sq_free_stats stats = {0}; 1044 1045 virtnet_free_old_xmit(sq, txq, in_napi, &stats); 1046 1047 /* Avoid overhead when no packets have been processed 1048 * happens when called speculatively from start_xmit. 1049 */ 1050 if (!stats.packets && !stats.napi_packets) 1051 return; 1052 1053 u64_stats_update_begin(&sq->stats.syncp); 1054 u64_stats_add(&sq->stats.bytes, stats.bytes + stats.napi_bytes); 1055 u64_stats_add(&sq->stats.packets, stats.packets + stats.napi_packets); 1056 u64_stats_update_end(&sq->stats.syncp); 1057 } 1058 1059 static bool is_xdp_raw_buffer_queue(struct virtnet_info *vi, int q) 1060 { 1061 if (q < (vi->curr_queue_pairs - vi->xdp_queue_pairs)) 1062 return false; 1063 else if (q < vi->curr_queue_pairs) 1064 return true; 1065 else 1066 return false; 1067 } 1068 1069 static bool tx_may_stop(struct virtnet_info *vi, 1070 struct net_device *dev, 1071 struct send_queue *sq) 1072 { 1073 int qnum; 1074 1075 qnum = sq - vi->sq; 1076 1077 /* If running out of space, stop queue to avoid getting packets that we 1078 * are then unable to transmit. 1079 * An alternative would be to force queuing layer to requeue the skb by 1080 * returning NETDEV_TX_BUSY. However, NETDEV_TX_BUSY should not be 1081 * returned in a normal path of operation: it means that driver is not 1082 * maintaining the TX queue stop/start state properly, and causes 1083 * the stack to do a non-trivial amount of useless work. 1084 * Since most packets only take 1 or 2 ring slots, stopping the queue 1085 * early means 16 slots are typically wasted. 1086 */ 1087 if (sq->vq->num_free < 2+MAX_SKB_FRAGS) { 1088 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 1089 1090 netif_tx_stop_queue(txq); 1091 u64_stats_update_begin(&sq->stats.syncp); 1092 u64_stats_inc(&sq->stats.stop); 1093 u64_stats_update_end(&sq->stats.syncp); 1094 1095 return true; 1096 } 1097 1098 return false; 1099 } 1100 1101 static void check_sq_full_and_disable(struct virtnet_info *vi, 1102 struct net_device *dev, 1103 struct send_queue *sq) 1104 { 1105 bool use_napi = sq->napi.weight; 1106 int qnum; 1107 1108 qnum = sq - vi->sq; 1109 1110 if (tx_may_stop(vi, dev, sq)) { 1111 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 1112 1113 if (use_napi) { 1114 if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) 1115 virtqueue_napi_schedule(&sq->napi, sq->vq); 1116 } else if (unlikely(!virtqueue_enable_cb_delayed(sq->vq))) { 1117 /* More just got used, free them then recheck. */ 1118 free_old_xmit(sq, txq, false); 1119 if (sq->vq->num_free >= 2+MAX_SKB_FRAGS) { 1120 netif_start_subqueue(dev, qnum); 1121 u64_stats_update_begin(&sq->stats.syncp); 1122 u64_stats_inc(&sq->stats.wake); 1123 u64_stats_update_end(&sq->stats.syncp); 1124 virtqueue_disable_cb(sq->vq); 1125 } 1126 } 1127 } 1128 } 1129 1130 static struct xdp_buff *buf_to_xdp(struct virtnet_info *vi, 1131 struct receive_queue *rq, void *buf, u32 len) 1132 { 1133 struct xdp_buff *xdp; 1134 u32 bufsize; 1135 1136 xdp = (struct xdp_buff *)buf; 1137 1138 bufsize = xsk_pool_get_rx_frame_size(rq->xsk_pool) + vi->hdr_len; 1139 1140 if (unlikely(len > bufsize)) { 1141 pr_debug("%s: rx error: len %u exceeds truesize %u\n", 1142 vi->dev->name, len, bufsize); 1143 DEV_STATS_INC(vi->dev, rx_length_errors); 1144 xsk_buff_free(xdp); 1145 return NULL; 1146 } 1147 1148 xsk_buff_set_size(xdp, len); 1149 xsk_buff_dma_sync_for_cpu(xdp); 1150 1151 return xdp; 1152 } 1153 1154 static struct sk_buff *xsk_construct_skb(struct receive_queue *rq, 1155 struct xdp_buff *xdp) 1156 { 1157 unsigned int metasize = xdp->data - xdp->data_meta; 1158 struct sk_buff *skb; 1159 unsigned int size; 1160 1161 size = xdp->data_end - xdp->data_hard_start; 1162 skb = napi_alloc_skb(&rq->napi, size); 1163 if (unlikely(!skb)) { 1164 xsk_buff_free(xdp); 1165 return NULL; 1166 } 1167 1168 skb_reserve(skb, xdp->data_meta - xdp->data_hard_start); 1169 1170 size = xdp->data_end - xdp->data_meta; 1171 memcpy(__skb_put(skb, size), xdp->data_meta, size); 1172 1173 if (metasize) { 1174 __skb_pull(skb, metasize); 1175 skb_metadata_set(skb, metasize); 1176 } 1177 1178 xsk_buff_free(xdp); 1179 1180 return skb; 1181 } 1182 1183 static struct sk_buff *virtnet_receive_xsk_small(struct net_device *dev, struct virtnet_info *vi, 1184 struct receive_queue *rq, struct xdp_buff *xdp, 1185 unsigned int *xdp_xmit, 1186 struct virtnet_rq_stats *stats) 1187 { 1188 struct bpf_prog *prog; 1189 u32 ret; 1190 1191 ret = XDP_PASS; 1192 rcu_read_lock(); 1193 prog = rcu_dereference(rq->xdp_prog); 1194 if (prog) 1195 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats); 1196 rcu_read_unlock(); 1197 1198 switch (ret) { 1199 case XDP_PASS: 1200 return xsk_construct_skb(rq, xdp); 1201 1202 case XDP_TX: 1203 case XDP_REDIRECT: 1204 return NULL; 1205 1206 default: 1207 /* drop packet */ 1208 xsk_buff_free(xdp); 1209 u64_stats_inc(&stats->drops); 1210 return NULL; 1211 } 1212 } 1213 1214 static void xsk_drop_follow_bufs(struct net_device *dev, 1215 struct receive_queue *rq, 1216 u32 num_buf, 1217 struct virtnet_rq_stats *stats) 1218 { 1219 struct xdp_buff *xdp; 1220 u32 len; 1221 1222 while (num_buf-- > 1) { 1223 xdp = virtqueue_get_buf(rq->vq, &len); 1224 if (unlikely(!xdp)) { 1225 pr_debug("%s: rx error: %d buffers missing\n", 1226 dev->name, num_buf); 1227 DEV_STATS_INC(dev, rx_length_errors); 1228 break; 1229 } 1230 u64_stats_add(&stats->bytes, len); 1231 xsk_buff_free(xdp); 1232 } 1233 } 1234 1235 static int xsk_append_merge_buffer(struct virtnet_info *vi, 1236 struct receive_queue *rq, 1237 struct sk_buff *head_skb, 1238 u32 num_buf, 1239 struct virtio_net_hdr_mrg_rxbuf *hdr, 1240 struct virtnet_rq_stats *stats) 1241 { 1242 struct sk_buff *curr_skb; 1243 struct xdp_buff *xdp; 1244 u32 len, truesize; 1245 struct page *page; 1246 void *buf; 1247 1248 curr_skb = head_skb; 1249 1250 while (--num_buf) { 1251 buf = virtqueue_get_buf(rq->vq, &len); 1252 if (unlikely(!buf)) { 1253 pr_debug("%s: rx error: %d buffers out of %d missing\n", 1254 vi->dev->name, num_buf, 1255 virtio16_to_cpu(vi->vdev, 1256 hdr->num_buffers)); 1257 DEV_STATS_INC(vi->dev, rx_length_errors); 1258 return -EINVAL; 1259 } 1260 1261 u64_stats_add(&stats->bytes, len); 1262 1263 xdp = buf_to_xdp(vi, rq, buf, len); 1264 if (!xdp) 1265 goto err; 1266 1267 buf = napi_alloc_frag(len); 1268 if (!buf) { 1269 xsk_buff_free(xdp); 1270 goto err; 1271 } 1272 1273 memcpy(buf, xdp->data - vi->hdr_len, len); 1274 1275 xsk_buff_free(xdp); 1276 1277 page = virt_to_page(buf); 1278 1279 truesize = len; 1280 1281 curr_skb = virtnet_skb_append_frag(head_skb, curr_skb, page, 1282 buf, len, truesize); 1283 if (!curr_skb) { 1284 put_page(page); 1285 goto err; 1286 } 1287 } 1288 1289 return 0; 1290 1291 err: 1292 xsk_drop_follow_bufs(vi->dev, rq, num_buf, stats); 1293 return -EINVAL; 1294 } 1295 1296 static struct sk_buff *virtnet_receive_xsk_merge(struct net_device *dev, struct virtnet_info *vi, 1297 struct receive_queue *rq, struct xdp_buff *xdp, 1298 unsigned int *xdp_xmit, 1299 struct virtnet_rq_stats *stats) 1300 { 1301 struct virtio_net_hdr_mrg_rxbuf *hdr; 1302 struct bpf_prog *prog; 1303 struct sk_buff *skb; 1304 u32 ret, num_buf; 1305 1306 hdr = xdp->data - vi->hdr_len; 1307 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 1308 1309 ret = XDP_PASS; 1310 rcu_read_lock(); 1311 prog = rcu_dereference(rq->xdp_prog); 1312 /* TODO: support multi buffer. */ 1313 if (prog && num_buf == 1) 1314 ret = virtnet_xdp_handler(prog, xdp, dev, xdp_xmit, stats); 1315 rcu_read_unlock(); 1316 1317 switch (ret) { 1318 case XDP_PASS: 1319 skb = xsk_construct_skb(rq, xdp); 1320 if (!skb) 1321 goto drop_bufs; 1322 1323 if (xsk_append_merge_buffer(vi, rq, skb, num_buf, hdr, stats)) { 1324 dev_kfree_skb(skb); 1325 goto drop; 1326 } 1327 1328 return skb; 1329 1330 case XDP_TX: 1331 case XDP_REDIRECT: 1332 return NULL; 1333 1334 default: 1335 /* drop packet */ 1336 xsk_buff_free(xdp); 1337 } 1338 1339 drop_bufs: 1340 xsk_drop_follow_bufs(dev, rq, num_buf, stats); 1341 1342 drop: 1343 u64_stats_inc(&stats->drops); 1344 return NULL; 1345 } 1346 1347 static void virtnet_receive_xsk_buf(struct virtnet_info *vi, struct receive_queue *rq, 1348 void *buf, u32 len, 1349 unsigned int *xdp_xmit, 1350 struct virtnet_rq_stats *stats) 1351 { 1352 struct net_device *dev = vi->dev; 1353 struct sk_buff *skb = NULL; 1354 struct xdp_buff *xdp; 1355 u8 flags; 1356 1357 len -= vi->hdr_len; 1358 1359 u64_stats_add(&stats->bytes, len); 1360 1361 xdp = buf_to_xdp(vi, rq, buf, len); 1362 if (!xdp) 1363 return; 1364 1365 if (unlikely(len < ETH_HLEN)) { 1366 pr_debug("%s: short packet %i\n", dev->name, len); 1367 DEV_STATS_INC(dev, rx_length_errors); 1368 xsk_buff_free(xdp); 1369 return; 1370 } 1371 1372 flags = ((struct virtio_net_common_hdr *)(xdp->data - vi->hdr_len))->hdr.flags; 1373 1374 if (!vi->mergeable_rx_bufs) 1375 skb = virtnet_receive_xsk_small(dev, vi, rq, xdp, xdp_xmit, stats); 1376 else 1377 skb = virtnet_receive_xsk_merge(dev, vi, rq, xdp, xdp_xmit, stats); 1378 1379 if (skb) 1380 virtnet_receive_done(vi, rq, skb, flags); 1381 } 1382 1383 static int virtnet_add_recvbuf_xsk(struct virtnet_info *vi, struct receive_queue *rq, 1384 struct xsk_buff_pool *pool, gfp_t gfp) 1385 { 1386 struct xdp_buff **xsk_buffs; 1387 dma_addr_t addr; 1388 int err = 0; 1389 u32 len, i; 1390 int num; 1391 1392 xsk_buffs = rq->xsk_buffs; 1393 1394 num = xsk_buff_alloc_batch(pool, xsk_buffs, rq->vq->num_free); 1395 if (!num) 1396 return -ENOMEM; 1397 1398 len = xsk_pool_get_rx_frame_size(pool) + vi->hdr_len; 1399 1400 for (i = 0; i < num; ++i) { 1401 /* Use the part of XDP_PACKET_HEADROOM as the virtnet hdr space. 1402 * We assume XDP_PACKET_HEADROOM is larger than hdr->len. 1403 * (see function virtnet_xsk_pool_enable) 1404 */ 1405 addr = xsk_buff_xdp_get_dma(xsk_buffs[i]) - vi->hdr_len; 1406 1407 sg_init_table(rq->sg, 1); 1408 sg_fill_dma(rq->sg, addr, len); 1409 1410 err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, 1411 xsk_buffs[i], NULL, gfp); 1412 if (err) 1413 goto err; 1414 } 1415 1416 return num; 1417 1418 err: 1419 for (; i < num; ++i) 1420 xsk_buff_free(xsk_buffs[i]); 1421 1422 return err; 1423 } 1424 1425 static void *virtnet_xsk_to_ptr(u32 len) 1426 { 1427 unsigned long p; 1428 1429 p = len << VIRTIO_XSK_FLAG_OFFSET; 1430 1431 return virtnet_xmit_ptr_pack((void *)p, VIRTNET_XMIT_TYPE_XSK); 1432 } 1433 1434 static int virtnet_xsk_xmit_one(struct send_queue *sq, 1435 struct xsk_buff_pool *pool, 1436 struct xdp_desc *desc) 1437 { 1438 struct virtnet_info *vi; 1439 dma_addr_t addr; 1440 1441 vi = sq->vq->vdev->priv; 1442 1443 addr = xsk_buff_raw_get_dma(pool, desc->addr); 1444 xsk_buff_raw_dma_sync_for_device(pool, addr, desc->len); 1445 1446 sg_init_table(sq->sg, 2); 1447 sg_fill_dma(sq->sg, sq->xsk_hdr_dma_addr, vi->hdr_len); 1448 sg_fill_dma(sq->sg + 1, addr, desc->len); 1449 1450 return virtqueue_add_outbuf_premapped(sq->vq, sq->sg, 2, 1451 virtnet_xsk_to_ptr(desc->len), 1452 GFP_ATOMIC); 1453 } 1454 1455 static int virtnet_xsk_xmit_batch(struct send_queue *sq, 1456 struct xsk_buff_pool *pool, 1457 unsigned int budget, 1458 u64 *kicks) 1459 { 1460 struct xdp_desc *descs = pool->tx_descs; 1461 bool kick = false; 1462 u32 nb_pkts, i; 1463 int err; 1464 1465 budget = min_t(u32, budget, sq->vq->num_free); 1466 1467 nb_pkts = xsk_tx_peek_release_desc_batch(pool, budget); 1468 if (!nb_pkts) 1469 return 0; 1470 1471 for (i = 0; i < nb_pkts; i++) { 1472 err = virtnet_xsk_xmit_one(sq, pool, &descs[i]); 1473 if (unlikely(err)) { 1474 xsk_tx_completed(sq->xsk_pool, nb_pkts - i); 1475 break; 1476 } 1477 1478 kick = true; 1479 } 1480 1481 if (kick && virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 1482 (*kicks)++; 1483 1484 return i; 1485 } 1486 1487 static bool virtnet_xsk_xmit(struct send_queue *sq, struct xsk_buff_pool *pool, 1488 int budget) 1489 { 1490 struct virtnet_info *vi = sq->vq->vdev->priv; 1491 struct virtnet_sq_free_stats stats = {}; 1492 struct net_device *dev = vi->dev; 1493 u64 kicks = 0; 1494 int sent; 1495 1496 /* Avoid to wakeup napi meanless, so call __free_old_xmit instead of 1497 * free_old_xmit(). 1498 */ 1499 __free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq), true, &stats); 1500 1501 if (stats.xsk) 1502 xsk_tx_completed(sq->xsk_pool, stats.xsk); 1503 1504 sent = virtnet_xsk_xmit_batch(sq, pool, budget, &kicks); 1505 1506 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) 1507 check_sq_full_and_disable(vi, vi->dev, sq); 1508 1509 if (sent) { 1510 struct netdev_queue *txq; 1511 1512 txq = netdev_get_tx_queue(vi->dev, sq - vi->sq); 1513 txq_trans_cond_update(txq); 1514 } 1515 1516 u64_stats_update_begin(&sq->stats.syncp); 1517 u64_stats_add(&sq->stats.packets, stats.packets); 1518 u64_stats_add(&sq->stats.bytes, stats.bytes); 1519 u64_stats_add(&sq->stats.kicks, kicks); 1520 u64_stats_add(&sq->stats.xdp_tx, sent); 1521 u64_stats_update_end(&sq->stats.syncp); 1522 1523 if (xsk_uses_need_wakeup(pool)) 1524 xsk_set_tx_need_wakeup(pool); 1525 1526 return sent; 1527 } 1528 1529 static void xsk_wakeup(struct send_queue *sq) 1530 { 1531 if (napi_if_scheduled_mark_missed(&sq->napi)) 1532 return; 1533 1534 local_bh_disable(); 1535 virtqueue_napi_schedule(&sq->napi, sq->vq); 1536 local_bh_enable(); 1537 } 1538 1539 static int virtnet_xsk_wakeup(struct net_device *dev, u32 qid, u32 flag) 1540 { 1541 struct virtnet_info *vi = netdev_priv(dev); 1542 struct send_queue *sq; 1543 1544 if (!netif_running(dev)) 1545 return -ENETDOWN; 1546 1547 if (qid >= vi->curr_queue_pairs) 1548 return -EINVAL; 1549 1550 sq = &vi->sq[qid]; 1551 1552 xsk_wakeup(sq); 1553 return 0; 1554 } 1555 1556 static void virtnet_xsk_completed(struct send_queue *sq, int num) 1557 { 1558 xsk_tx_completed(sq->xsk_pool, num); 1559 1560 /* If this is called by rx poll, start_xmit and xdp xmit we should 1561 * wakeup the tx napi to consume the xsk tx queue, because the tx 1562 * interrupt may not be triggered. 1563 */ 1564 xsk_wakeup(sq); 1565 } 1566 1567 static int __virtnet_xdp_xmit_one(struct virtnet_info *vi, 1568 struct send_queue *sq, 1569 struct xdp_frame *xdpf) 1570 { 1571 struct virtio_net_hdr_mrg_rxbuf *hdr; 1572 struct skb_shared_info *shinfo; 1573 u8 nr_frags = 0; 1574 int err, i; 1575 1576 if (unlikely(xdpf->headroom < vi->hdr_len)) 1577 return -EOVERFLOW; 1578 1579 if (unlikely(xdp_frame_has_frags(xdpf))) { 1580 shinfo = xdp_get_shared_info_from_frame(xdpf); 1581 nr_frags = shinfo->nr_frags; 1582 } 1583 1584 /* In wrapping function virtnet_xdp_xmit(), we need to free 1585 * up the pending old buffers, where we need to calculate the 1586 * position of skb_shared_info in xdp_get_frame_len() and 1587 * xdp_return_frame(), which will involve to xdpf->data and 1588 * xdpf->headroom. Therefore, we need to update the value of 1589 * headroom synchronously here. 1590 */ 1591 xdpf->headroom -= vi->hdr_len; 1592 xdpf->data -= vi->hdr_len; 1593 /* Zero header and leave csum up to XDP layers */ 1594 hdr = xdpf->data; 1595 memset(hdr, 0, vi->hdr_len); 1596 xdpf->len += vi->hdr_len; 1597 1598 sg_init_table(sq->sg, nr_frags + 1); 1599 sg_set_buf(sq->sg, xdpf->data, xdpf->len); 1600 for (i = 0; i < nr_frags; i++) { 1601 skb_frag_t *frag = &shinfo->frags[i]; 1602 1603 sg_set_page(&sq->sg[i + 1], skb_frag_page(frag), 1604 skb_frag_size(frag), skb_frag_off(frag)); 1605 } 1606 1607 err = virtnet_add_outbuf(sq, nr_frags + 1, xdpf, VIRTNET_XMIT_TYPE_XDP); 1608 if (unlikely(err)) 1609 return -ENOSPC; /* Caller handle free/refcnt */ 1610 1611 return 0; 1612 } 1613 1614 /* when vi->curr_queue_pairs > nr_cpu_ids, the txq/sq is only used for xdp tx on 1615 * the current cpu, so it does not need to be locked. 1616 * 1617 * Here we use marco instead of inline functions because we have to deal with 1618 * three issues at the same time: 1. the choice of sq. 2. judge and execute the 1619 * lock/unlock of txq 3. make sparse happy. It is difficult for two inline 1620 * functions to perfectly solve these three problems at the same time. 1621 */ 1622 #define virtnet_xdp_get_sq(vi) ({ \ 1623 int cpu = smp_processor_id(); \ 1624 struct netdev_queue *txq; \ 1625 typeof(vi) v = (vi); \ 1626 unsigned int qp; \ 1627 \ 1628 if (v->curr_queue_pairs > nr_cpu_ids) { \ 1629 qp = v->curr_queue_pairs - v->xdp_queue_pairs; \ 1630 qp += cpu; \ 1631 txq = netdev_get_tx_queue(v->dev, qp); \ 1632 __netif_tx_acquire(txq); \ 1633 } else { \ 1634 qp = cpu % v->curr_queue_pairs; \ 1635 txq = netdev_get_tx_queue(v->dev, qp); \ 1636 __netif_tx_lock(txq, cpu); \ 1637 } \ 1638 v->sq + qp; \ 1639 }) 1640 1641 #define virtnet_xdp_put_sq(vi, q) { \ 1642 struct netdev_queue *txq; \ 1643 typeof(vi) v = (vi); \ 1644 \ 1645 txq = netdev_get_tx_queue(v->dev, (q) - v->sq); \ 1646 if (v->curr_queue_pairs > nr_cpu_ids) \ 1647 __netif_tx_release(txq); \ 1648 else \ 1649 __netif_tx_unlock(txq); \ 1650 } 1651 1652 static int virtnet_xdp_xmit(struct net_device *dev, 1653 int n, struct xdp_frame **frames, u32 flags) 1654 { 1655 struct virtnet_info *vi = netdev_priv(dev); 1656 struct virtnet_sq_free_stats stats = {0}; 1657 struct receive_queue *rq = vi->rq; 1658 struct bpf_prog *xdp_prog; 1659 struct send_queue *sq; 1660 int nxmit = 0; 1661 int kicks = 0; 1662 int ret; 1663 int i; 1664 1665 /* Only allow ndo_xdp_xmit if XDP is loaded on dev, as this 1666 * indicate XDP resources have been successfully allocated. 1667 */ 1668 xdp_prog = rcu_access_pointer(rq->xdp_prog); 1669 if (!xdp_prog) 1670 return -ENXIO; 1671 1672 sq = virtnet_xdp_get_sq(vi); 1673 1674 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) { 1675 ret = -EINVAL; 1676 goto out; 1677 } 1678 1679 /* Free up any pending old buffers before queueing new ones. */ 1680 virtnet_free_old_xmit(sq, netdev_get_tx_queue(dev, sq - vi->sq), 1681 false, &stats); 1682 1683 for (i = 0; i < n; i++) { 1684 struct xdp_frame *xdpf = frames[i]; 1685 1686 if (__virtnet_xdp_xmit_one(vi, sq, xdpf)) 1687 break; 1688 nxmit++; 1689 } 1690 ret = nxmit; 1691 1692 if (!is_xdp_raw_buffer_queue(vi, sq - vi->sq)) 1693 check_sq_full_and_disable(vi, dev, sq); 1694 1695 if (flags & XDP_XMIT_FLUSH) { 1696 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) 1697 kicks = 1; 1698 } 1699 out: 1700 u64_stats_update_begin(&sq->stats.syncp); 1701 u64_stats_add(&sq->stats.bytes, stats.bytes); 1702 u64_stats_add(&sq->stats.packets, stats.packets); 1703 u64_stats_add(&sq->stats.xdp_tx, n); 1704 u64_stats_add(&sq->stats.xdp_tx_drops, n - nxmit); 1705 u64_stats_add(&sq->stats.kicks, kicks); 1706 u64_stats_update_end(&sq->stats.syncp); 1707 1708 virtnet_xdp_put_sq(vi, sq); 1709 return ret; 1710 } 1711 1712 static void put_xdp_frags(struct xdp_buff *xdp) 1713 { 1714 struct skb_shared_info *shinfo; 1715 struct page *xdp_page; 1716 int i; 1717 1718 if (xdp_buff_has_frags(xdp)) { 1719 shinfo = xdp_get_shared_info_from_buff(xdp); 1720 for (i = 0; i < shinfo->nr_frags; i++) { 1721 xdp_page = skb_frag_page(&shinfo->frags[i]); 1722 put_page(xdp_page); 1723 } 1724 } 1725 } 1726 1727 static int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp, 1728 struct net_device *dev, 1729 unsigned int *xdp_xmit, 1730 struct virtnet_rq_stats *stats) 1731 { 1732 struct xdp_frame *xdpf; 1733 int err; 1734 u32 act; 1735 1736 act = bpf_prog_run_xdp(xdp_prog, xdp); 1737 u64_stats_inc(&stats->xdp_packets); 1738 1739 switch (act) { 1740 case XDP_PASS: 1741 return act; 1742 1743 case XDP_TX: 1744 u64_stats_inc(&stats->xdp_tx); 1745 xdpf = xdp_convert_buff_to_frame(xdp); 1746 if (unlikely(!xdpf)) { 1747 netdev_dbg(dev, "convert buff to frame failed for xdp\n"); 1748 return XDP_DROP; 1749 } 1750 1751 err = virtnet_xdp_xmit(dev, 1, &xdpf, 0); 1752 if (unlikely(!err)) { 1753 xdp_return_frame_rx_napi(xdpf); 1754 } else if (unlikely(err < 0)) { 1755 trace_xdp_exception(dev, xdp_prog, act); 1756 return XDP_DROP; 1757 } 1758 *xdp_xmit |= VIRTIO_XDP_TX; 1759 return act; 1760 1761 case XDP_REDIRECT: 1762 u64_stats_inc(&stats->xdp_redirects); 1763 err = xdp_do_redirect(dev, xdp, xdp_prog); 1764 if (err) 1765 return XDP_DROP; 1766 1767 *xdp_xmit |= VIRTIO_XDP_REDIR; 1768 return act; 1769 1770 default: 1771 bpf_warn_invalid_xdp_action(dev, xdp_prog, act); 1772 fallthrough; 1773 case XDP_ABORTED: 1774 trace_xdp_exception(dev, xdp_prog, act); 1775 fallthrough; 1776 case XDP_DROP: 1777 return XDP_DROP; 1778 } 1779 } 1780 1781 static unsigned int virtnet_get_headroom(struct virtnet_info *vi) 1782 { 1783 return vi->xdp_enabled ? XDP_PACKET_HEADROOM : 0; 1784 } 1785 1786 /* We copy the packet for XDP in the following cases: 1787 * 1788 * 1) Packet is scattered across multiple rx buffers. 1789 * 2) Headroom space is insufficient. 1790 * 1791 * This is inefficient but it's a temporary condition that 1792 * we hit right after XDP is enabled and until queue is refilled 1793 * with large buffers with sufficient headroom - so it should affect 1794 * at most queue size packets. 1795 * Afterwards, the conditions to enable 1796 * XDP should preclude the underlying device from sending packets 1797 * across multiple buffers (num_buf > 1), and we make sure buffers 1798 * have enough headroom. 1799 */ 1800 static struct page *xdp_linearize_page(struct receive_queue *rq, 1801 int *num_buf, 1802 struct page *p, 1803 int offset, 1804 int page_off, 1805 unsigned int *len) 1806 { 1807 int tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1808 struct page *page; 1809 1810 if (page_off + *len + tailroom > PAGE_SIZE) 1811 return NULL; 1812 1813 page = alloc_page(GFP_ATOMIC); 1814 if (!page) 1815 return NULL; 1816 1817 memcpy(page_address(page) + page_off, page_address(p) + offset, *len); 1818 page_off += *len; 1819 1820 while (--*num_buf) { 1821 unsigned int buflen; 1822 void *buf; 1823 int off; 1824 1825 buf = virtnet_rq_get_buf(rq, &buflen, NULL); 1826 if (unlikely(!buf)) 1827 goto err_buf; 1828 1829 p = virt_to_head_page(buf); 1830 off = buf - page_address(p); 1831 1832 /* guard against a misconfigured or uncooperative backend that 1833 * is sending packet larger than the MTU. 1834 */ 1835 if ((page_off + buflen + tailroom) > PAGE_SIZE) { 1836 put_page(p); 1837 goto err_buf; 1838 } 1839 1840 memcpy(page_address(page) + page_off, 1841 page_address(p) + off, buflen); 1842 page_off += buflen; 1843 put_page(p); 1844 } 1845 1846 /* Headroom does not contribute to packet length */ 1847 *len = page_off - XDP_PACKET_HEADROOM; 1848 return page; 1849 err_buf: 1850 __free_pages(page, 0); 1851 return NULL; 1852 } 1853 1854 static struct sk_buff *receive_small_build_skb(struct virtnet_info *vi, 1855 unsigned int xdp_headroom, 1856 void *buf, 1857 unsigned int len) 1858 { 1859 unsigned int header_offset; 1860 unsigned int headroom; 1861 unsigned int buflen; 1862 struct sk_buff *skb; 1863 1864 header_offset = VIRTNET_RX_PAD + xdp_headroom; 1865 headroom = vi->hdr_len + header_offset; 1866 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 1867 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1868 1869 skb = virtnet_build_skb(buf, buflen, headroom, len); 1870 if (unlikely(!skb)) 1871 return NULL; 1872 1873 buf += header_offset; 1874 memcpy(skb_vnet_common_hdr(skb), buf, vi->hdr_len); 1875 1876 return skb; 1877 } 1878 1879 static struct sk_buff *receive_small_xdp(struct net_device *dev, 1880 struct virtnet_info *vi, 1881 struct receive_queue *rq, 1882 struct bpf_prog *xdp_prog, 1883 void *buf, 1884 unsigned int xdp_headroom, 1885 unsigned int len, 1886 unsigned int *xdp_xmit, 1887 struct virtnet_rq_stats *stats) 1888 { 1889 unsigned int header_offset = VIRTNET_RX_PAD + xdp_headroom; 1890 unsigned int headroom = vi->hdr_len + header_offset; 1891 struct virtio_net_hdr_mrg_rxbuf *hdr = buf + header_offset; 1892 struct page *page = virt_to_head_page(buf); 1893 struct page *xdp_page; 1894 unsigned int buflen; 1895 struct xdp_buff xdp; 1896 struct sk_buff *skb; 1897 unsigned int metasize = 0; 1898 u32 act; 1899 1900 if (unlikely(hdr->hdr.gso_type)) 1901 goto err_xdp; 1902 1903 /* Partially checksummed packets must be dropped. */ 1904 if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) 1905 goto err_xdp; 1906 1907 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 1908 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1909 1910 if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) { 1911 int offset = buf - page_address(page) + header_offset; 1912 unsigned int tlen = len + vi->hdr_len; 1913 int num_buf = 1; 1914 1915 xdp_headroom = virtnet_get_headroom(vi); 1916 header_offset = VIRTNET_RX_PAD + xdp_headroom; 1917 headroom = vi->hdr_len + header_offset; 1918 buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) + 1919 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1920 xdp_page = xdp_linearize_page(rq, &num_buf, page, 1921 offset, header_offset, 1922 &tlen); 1923 if (!xdp_page) 1924 goto err_xdp; 1925 1926 buf = page_address(xdp_page); 1927 put_page(page); 1928 page = xdp_page; 1929 } 1930 1931 xdp_init_buff(&xdp, buflen, &rq->xdp_rxq); 1932 xdp_prepare_buff(&xdp, buf + VIRTNET_RX_PAD + vi->hdr_len, 1933 xdp_headroom, len, true); 1934 1935 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats); 1936 1937 switch (act) { 1938 case XDP_PASS: 1939 /* Recalculate length in case bpf program changed it */ 1940 len = xdp.data_end - xdp.data; 1941 metasize = xdp.data - xdp.data_meta; 1942 break; 1943 1944 case XDP_TX: 1945 case XDP_REDIRECT: 1946 goto xdp_xmit; 1947 1948 default: 1949 goto err_xdp; 1950 } 1951 1952 skb = virtnet_build_skb(buf, buflen, xdp.data - buf, len); 1953 if (unlikely(!skb)) 1954 goto err; 1955 1956 if (metasize) 1957 skb_metadata_set(skb, metasize); 1958 1959 return skb; 1960 1961 err_xdp: 1962 u64_stats_inc(&stats->xdp_drops); 1963 err: 1964 u64_stats_inc(&stats->drops); 1965 put_page(page); 1966 xdp_xmit: 1967 return NULL; 1968 } 1969 1970 static struct sk_buff *receive_small(struct net_device *dev, 1971 struct virtnet_info *vi, 1972 struct receive_queue *rq, 1973 void *buf, void *ctx, 1974 unsigned int len, 1975 unsigned int *xdp_xmit, 1976 struct virtnet_rq_stats *stats) 1977 { 1978 unsigned int xdp_headroom = (unsigned long)ctx; 1979 struct page *page = virt_to_head_page(buf); 1980 struct sk_buff *skb; 1981 1982 /* We passed the address of virtnet header to virtio-core, 1983 * so truncate the padding. 1984 */ 1985 buf -= VIRTNET_RX_PAD + xdp_headroom; 1986 1987 len -= vi->hdr_len; 1988 u64_stats_add(&stats->bytes, len); 1989 1990 if (unlikely(len > GOOD_PACKET_LEN)) { 1991 pr_debug("%s: rx error: len %u exceeds max size %d\n", 1992 dev->name, len, GOOD_PACKET_LEN); 1993 DEV_STATS_INC(dev, rx_length_errors); 1994 goto err; 1995 } 1996 1997 if (unlikely(vi->xdp_enabled)) { 1998 struct bpf_prog *xdp_prog; 1999 2000 rcu_read_lock(); 2001 xdp_prog = rcu_dereference(rq->xdp_prog); 2002 if (xdp_prog) { 2003 skb = receive_small_xdp(dev, vi, rq, xdp_prog, buf, 2004 xdp_headroom, len, xdp_xmit, 2005 stats); 2006 rcu_read_unlock(); 2007 return skb; 2008 } 2009 rcu_read_unlock(); 2010 } 2011 2012 skb = receive_small_build_skb(vi, xdp_headroom, buf, len); 2013 if (likely(skb)) 2014 return skb; 2015 2016 err: 2017 u64_stats_inc(&stats->drops); 2018 put_page(page); 2019 return NULL; 2020 } 2021 2022 static struct sk_buff *receive_big(struct net_device *dev, 2023 struct virtnet_info *vi, 2024 struct receive_queue *rq, 2025 void *buf, 2026 unsigned int len, 2027 struct virtnet_rq_stats *stats) 2028 { 2029 struct page *page = buf; 2030 struct sk_buff *skb = 2031 page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, 0); 2032 2033 u64_stats_add(&stats->bytes, len - vi->hdr_len); 2034 if (unlikely(!skb)) 2035 goto err; 2036 2037 return skb; 2038 2039 err: 2040 u64_stats_inc(&stats->drops); 2041 give_pages(rq, page); 2042 return NULL; 2043 } 2044 2045 static void mergeable_buf_free(struct receive_queue *rq, int num_buf, 2046 struct net_device *dev, 2047 struct virtnet_rq_stats *stats) 2048 { 2049 struct page *page; 2050 void *buf; 2051 int len; 2052 2053 while (num_buf-- > 1) { 2054 buf = virtnet_rq_get_buf(rq, &len, NULL); 2055 if (unlikely(!buf)) { 2056 pr_debug("%s: rx error: %d buffers missing\n", 2057 dev->name, num_buf); 2058 DEV_STATS_INC(dev, rx_length_errors); 2059 break; 2060 } 2061 u64_stats_add(&stats->bytes, len); 2062 page = virt_to_head_page(buf); 2063 put_page(page); 2064 } 2065 } 2066 2067 /* Why not use xdp_build_skb_from_frame() ? 2068 * XDP core assumes that xdp frags are PAGE_SIZE in length, while in 2069 * virtio-net there are 2 points that do not match its requirements: 2070 * 1. The size of the prefilled buffer is not fixed before xdp is set. 2071 * 2. xdp_build_skb_from_frame() does more checks that we don't need, 2072 * like eth_type_trans() (which virtio-net does in receive_buf()). 2073 */ 2074 static struct sk_buff *build_skb_from_xdp_buff(struct net_device *dev, 2075 struct virtnet_info *vi, 2076 struct xdp_buff *xdp, 2077 unsigned int xdp_frags_truesz) 2078 { 2079 struct skb_shared_info *sinfo = xdp_get_shared_info_from_buff(xdp); 2080 unsigned int headroom, data_len; 2081 struct sk_buff *skb; 2082 int metasize; 2083 u8 nr_frags; 2084 2085 if (unlikely(xdp->data_end > xdp_data_hard_end(xdp))) { 2086 pr_debug("Error building skb as missing reserved tailroom for xdp"); 2087 return NULL; 2088 } 2089 2090 if (unlikely(xdp_buff_has_frags(xdp))) 2091 nr_frags = sinfo->nr_frags; 2092 2093 skb = build_skb(xdp->data_hard_start, xdp->frame_sz); 2094 if (unlikely(!skb)) 2095 return NULL; 2096 2097 headroom = xdp->data - xdp->data_hard_start; 2098 data_len = xdp->data_end - xdp->data; 2099 skb_reserve(skb, headroom); 2100 __skb_put(skb, data_len); 2101 2102 metasize = xdp->data - xdp->data_meta; 2103 metasize = metasize > 0 ? metasize : 0; 2104 if (metasize) 2105 skb_metadata_set(skb, metasize); 2106 2107 if (unlikely(xdp_buff_has_frags(xdp))) 2108 xdp_update_skb_shared_info(skb, nr_frags, 2109 sinfo->xdp_frags_size, 2110 xdp_frags_truesz, 2111 xdp_buff_is_frag_pfmemalloc(xdp)); 2112 2113 return skb; 2114 } 2115 2116 /* TODO: build xdp in big mode */ 2117 static int virtnet_build_xdp_buff_mrg(struct net_device *dev, 2118 struct virtnet_info *vi, 2119 struct receive_queue *rq, 2120 struct xdp_buff *xdp, 2121 void *buf, 2122 unsigned int len, 2123 unsigned int frame_sz, 2124 int *num_buf, 2125 unsigned int *xdp_frags_truesize, 2126 struct virtnet_rq_stats *stats) 2127 { 2128 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 2129 unsigned int headroom, tailroom, room; 2130 unsigned int truesize, cur_frag_size; 2131 struct skb_shared_info *shinfo; 2132 unsigned int xdp_frags_truesz = 0; 2133 struct page *page; 2134 skb_frag_t *frag; 2135 int offset; 2136 void *ctx; 2137 2138 xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq); 2139 xdp_prepare_buff(xdp, buf - XDP_PACKET_HEADROOM, 2140 XDP_PACKET_HEADROOM + vi->hdr_len, len - vi->hdr_len, true); 2141 2142 if (!*num_buf) 2143 return 0; 2144 2145 if (*num_buf > 1) { 2146 /* If we want to build multi-buffer xdp, we need 2147 * to specify that the flags of xdp_buff have the 2148 * XDP_FLAGS_HAS_FRAG bit. 2149 */ 2150 if (!xdp_buff_has_frags(xdp)) 2151 xdp_buff_set_frags_flag(xdp); 2152 2153 shinfo = xdp_get_shared_info_from_buff(xdp); 2154 shinfo->nr_frags = 0; 2155 shinfo->xdp_frags_size = 0; 2156 } 2157 2158 if (*num_buf > MAX_SKB_FRAGS + 1) 2159 return -EINVAL; 2160 2161 while (--*num_buf > 0) { 2162 buf = virtnet_rq_get_buf(rq, &len, &ctx); 2163 if (unlikely(!buf)) { 2164 pr_debug("%s: rx error: %d buffers out of %d missing\n", 2165 dev->name, *num_buf, 2166 virtio16_to_cpu(vi->vdev, hdr->num_buffers)); 2167 DEV_STATS_INC(dev, rx_length_errors); 2168 goto err; 2169 } 2170 2171 u64_stats_add(&stats->bytes, len); 2172 page = virt_to_head_page(buf); 2173 offset = buf - page_address(page); 2174 2175 truesize = mergeable_ctx_to_truesize(ctx); 2176 headroom = mergeable_ctx_to_headroom(ctx); 2177 tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 2178 room = SKB_DATA_ALIGN(headroom + tailroom); 2179 2180 cur_frag_size = truesize; 2181 xdp_frags_truesz += cur_frag_size; 2182 if (unlikely(len > truesize - room || cur_frag_size > PAGE_SIZE)) { 2183 put_page(page); 2184 pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 2185 dev->name, len, (unsigned long)(truesize - room)); 2186 DEV_STATS_INC(dev, rx_length_errors); 2187 goto err; 2188 } 2189 2190 frag = &shinfo->frags[shinfo->nr_frags++]; 2191 skb_frag_fill_page_desc(frag, page, offset, len); 2192 if (page_is_pfmemalloc(page)) 2193 xdp_buff_set_frag_pfmemalloc(xdp); 2194 2195 shinfo->xdp_frags_size += len; 2196 } 2197 2198 *xdp_frags_truesize = xdp_frags_truesz; 2199 return 0; 2200 2201 err: 2202 put_xdp_frags(xdp); 2203 return -EINVAL; 2204 } 2205 2206 static void *mergeable_xdp_get_buf(struct virtnet_info *vi, 2207 struct receive_queue *rq, 2208 struct bpf_prog *xdp_prog, 2209 void *ctx, 2210 unsigned int *frame_sz, 2211 int *num_buf, 2212 struct page **page, 2213 int offset, 2214 unsigned int *len, 2215 struct virtio_net_hdr_mrg_rxbuf *hdr) 2216 { 2217 unsigned int truesize = mergeable_ctx_to_truesize(ctx); 2218 unsigned int headroom = mergeable_ctx_to_headroom(ctx); 2219 struct page *xdp_page; 2220 unsigned int xdp_room; 2221 2222 /* Transient failure which in theory could occur if 2223 * in-flight packets from before XDP was enabled reach 2224 * the receive path after XDP is loaded. 2225 */ 2226 if (unlikely(hdr->hdr.gso_type)) 2227 return NULL; 2228 2229 /* Partially checksummed packets must be dropped. */ 2230 if (unlikely(hdr->hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)) 2231 return NULL; 2232 2233 /* Now XDP core assumes frag size is PAGE_SIZE, but buffers 2234 * with headroom may add hole in truesize, which 2235 * make their length exceed PAGE_SIZE. So we disabled the 2236 * hole mechanism for xdp. See add_recvbuf_mergeable(). 2237 */ 2238 *frame_sz = truesize; 2239 2240 if (likely(headroom >= virtnet_get_headroom(vi) && 2241 (*num_buf == 1 || xdp_prog->aux->xdp_has_frags))) { 2242 return page_address(*page) + offset; 2243 } 2244 2245 /* This happens when headroom is not enough because 2246 * of the buffer was prefilled before XDP is set. 2247 * This should only happen for the first several packets. 2248 * In fact, vq reset can be used here to help us clean up 2249 * the prefilled buffers, but many existing devices do not 2250 * support it, and we don't want to bother users who are 2251 * using xdp normally. 2252 */ 2253 if (!xdp_prog->aux->xdp_has_frags) { 2254 /* linearize data for XDP */ 2255 xdp_page = xdp_linearize_page(rq, num_buf, 2256 *page, offset, 2257 XDP_PACKET_HEADROOM, 2258 len); 2259 if (!xdp_page) 2260 return NULL; 2261 } else { 2262 xdp_room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM + 2263 sizeof(struct skb_shared_info)); 2264 if (*len + xdp_room > PAGE_SIZE) 2265 return NULL; 2266 2267 xdp_page = alloc_page(GFP_ATOMIC); 2268 if (!xdp_page) 2269 return NULL; 2270 2271 memcpy(page_address(xdp_page) + XDP_PACKET_HEADROOM, 2272 page_address(*page) + offset, *len); 2273 } 2274 2275 *frame_sz = PAGE_SIZE; 2276 2277 put_page(*page); 2278 2279 *page = xdp_page; 2280 2281 return page_address(*page) + XDP_PACKET_HEADROOM; 2282 } 2283 2284 static struct sk_buff *receive_mergeable_xdp(struct net_device *dev, 2285 struct virtnet_info *vi, 2286 struct receive_queue *rq, 2287 struct bpf_prog *xdp_prog, 2288 void *buf, 2289 void *ctx, 2290 unsigned int len, 2291 unsigned int *xdp_xmit, 2292 struct virtnet_rq_stats *stats) 2293 { 2294 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 2295 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 2296 struct page *page = virt_to_head_page(buf); 2297 int offset = buf - page_address(page); 2298 unsigned int xdp_frags_truesz = 0; 2299 struct sk_buff *head_skb; 2300 unsigned int frame_sz; 2301 struct xdp_buff xdp; 2302 void *data; 2303 u32 act; 2304 int err; 2305 2306 data = mergeable_xdp_get_buf(vi, rq, xdp_prog, ctx, &frame_sz, &num_buf, &page, 2307 offset, &len, hdr); 2308 if (unlikely(!data)) 2309 goto err_xdp; 2310 2311 err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz, 2312 &num_buf, &xdp_frags_truesz, stats); 2313 if (unlikely(err)) 2314 goto err_xdp; 2315 2316 act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats); 2317 2318 switch (act) { 2319 case XDP_PASS: 2320 head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz); 2321 if (unlikely(!head_skb)) 2322 break; 2323 return head_skb; 2324 2325 case XDP_TX: 2326 case XDP_REDIRECT: 2327 return NULL; 2328 2329 default: 2330 break; 2331 } 2332 2333 put_xdp_frags(&xdp); 2334 2335 err_xdp: 2336 put_page(page); 2337 mergeable_buf_free(rq, num_buf, dev, stats); 2338 2339 u64_stats_inc(&stats->xdp_drops); 2340 u64_stats_inc(&stats->drops); 2341 return NULL; 2342 } 2343 2344 static struct sk_buff *virtnet_skb_append_frag(struct sk_buff *head_skb, 2345 struct sk_buff *curr_skb, 2346 struct page *page, void *buf, 2347 int len, int truesize) 2348 { 2349 int num_skb_frags; 2350 int offset; 2351 2352 num_skb_frags = skb_shinfo(curr_skb)->nr_frags; 2353 if (unlikely(num_skb_frags == MAX_SKB_FRAGS)) { 2354 struct sk_buff *nskb = alloc_skb(0, GFP_ATOMIC); 2355 2356 if (unlikely(!nskb)) 2357 return NULL; 2358 2359 if (curr_skb == head_skb) 2360 skb_shinfo(curr_skb)->frag_list = nskb; 2361 else 2362 curr_skb->next = nskb; 2363 curr_skb = nskb; 2364 head_skb->truesize += nskb->truesize; 2365 num_skb_frags = 0; 2366 } 2367 2368 if (curr_skb != head_skb) { 2369 head_skb->data_len += len; 2370 head_skb->len += len; 2371 head_skb->truesize += truesize; 2372 } 2373 2374 offset = buf - page_address(page); 2375 if (skb_can_coalesce(curr_skb, num_skb_frags, page, offset)) { 2376 put_page(page); 2377 skb_coalesce_rx_frag(curr_skb, num_skb_frags - 1, 2378 len, truesize); 2379 } else { 2380 skb_add_rx_frag(curr_skb, num_skb_frags, page, 2381 offset, len, truesize); 2382 } 2383 2384 return curr_skb; 2385 } 2386 2387 static struct sk_buff *receive_mergeable(struct net_device *dev, 2388 struct virtnet_info *vi, 2389 struct receive_queue *rq, 2390 void *buf, 2391 void *ctx, 2392 unsigned int len, 2393 unsigned int *xdp_xmit, 2394 struct virtnet_rq_stats *stats) 2395 { 2396 struct virtio_net_hdr_mrg_rxbuf *hdr = buf; 2397 int num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers); 2398 struct page *page = virt_to_head_page(buf); 2399 int offset = buf - page_address(page); 2400 struct sk_buff *head_skb, *curr_skb; 2401 unsigned int truesize = mergeable_ctx_to_truesize(ctx); 2402 unsigned int headroom = mergeable_ctx_to_headroom(ctx); 2403 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 2404 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 2405 2406 head_skb = NULL; 2407 u64_stats_add(&stats->bytes, len - vi->hdr_len); 2408 2409 if (unlikely(len > truesize - room)) { 2410 pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 2411 dev->name, len, (unsigned long)(truesize - room)); 2412 DEV_STATS_INC(dev, rx_length_errors); 2413 goto err_skb; 2414 } 2415 2416 if (unlikely(vi->xdp_enabled)) { 2417 struct bpf_prog *xdp_prog; 2418 2419 rcu_read_lock(); 2420 xdp_prog = rcu_dereference(rq->xdp_prog); 2421 if (xdp_prog) { 2422 head_skb = receive_mergeable_xdp(dev, vi, rq, xdp_prog, buf, ctx, 2423 len, xdp_xmit, stats); 2424 rcu_read_unlock(); 2425 return head_skb; 2426 } 2427 rcu_read_unlock(); 2428 } 2429 2430 head_skb = page_to_skb(vi, rq, page, offset, len, truesize, headroom); 2431 curr_skb = head_skb; 2432 2433 if (unlikely(!curr_skb)) 2434 goto err_skb; 2435 while (--num_buf) { 2436 buf = virtnet_rq_get_buf(rq, &len, &ctx); 2437 if (unlikely(!buf)) { 2438 pr_debug("%s: rx error: %d buffers out of %d missing\n", 2439 dev->name, num_buf, 2440 virtio16_to_cpu(vi->vdev, 2441 hdr->num_buffers)); 2442 DEV_STATS_INC(dev, rx_length_errors); 2443 goto err_buf; 2444 } 2445 2446 u64_stats_add(&stats->bytes, len); 2447 page = virt_to_head_page(buf); 2448 2449 truesize = mergeable_ctx_to_truesize(ctx); 2450 headroom = mergeable_ctx_to_headroom(ctx); 2451 tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 2452 room = SKB_DATA_ALIGN(headroom + tailroom); 2453 if (unlikely(len > truesize - room)) { 2454 pr_debug("%s: rx error: len %u exceeds truesize %lu\n", 2455 dev->name, len, (unsigned long)(truesize - room)); 2456 DEV_STATS_INC(dev, rx_length_errors); 2457 goto err_skb; 2458 } 2459 2460 curr_skb = virtnet_skb_append_frag(head_skb, curr_skb, page, 2461 buf, len, truesize); 2462 if (!curr_skb) 2463 goto err_skb; 2464 } 2465 2466 ewma_pkt_len_add(&rq->mrg_avg_pkt_len, head_skb->len); 2467 return head_skb; 2468 2469 err_skb: 2470 put_page(page); 2471 mergeable_buf_free(rq, num_buf, dev, stats); 2472 2473 err_buf: 2474 u64_stats_inc(&stats->drops); 2475 dev_kfree_skb(head_skb); 2476 return NULL; 2477 } 2478 2479 static void virtio_skb_set_hash(const struct virtio_net_hdr_v1_hash *hdr_hash, 2480 struct sk_buff *skb) 2481 { 2482 enum pkt_hash_types rss_hash_type; 2483 2484 if (!hdr_hash || !skb) 2485 return; 2486 2487 switch (__le16_to_cpu(hdr_hash->hash_report)) { 2488 case VIRTIO_NET_HASH_REPORT_TCPv4: 2489 case VIRTIO_NET_HASH_REPORT_UDPv4: 2490 case VIRTIO_NET_HASH_REPORT_TCPv6: 2491 case VIRTIO_NET_HASH_REPORT_UDPv6: 2492 case VIRTIO_NET_HASH_REPORT_TCPv6_EX: 2493 case VIRTIO_NET_HASH_REPORT_UDPv6_EX: 2494 rss_hash_type = PKT_HASH_TYPE_L4; 2495 break; 2496 case VIRTIO_NET_HASH_REPORT_IPv4: 2497 case VIRTIO_NET_HASH_REPORT_IPv6: 2498 case VIRTIO_NET_HASH_REPORT_IPv6_EX: 2499 rss_hash_type = PKT_HASH_TYPE_L3; 2500 break; 2501 case VIRTIO_NET_HASH_REPORT_NONE: 2502 default: 2503 rss_hash_type = PKT_HASH_TYPE_NONE; 2504 } 2505 skb_set_hash(skb, __le32_to_cpu(hdr_hash->hash_value), rss_hash_type); 2506 } 2507 2508 static void virtnet_receive_done(struct virtnet_info *vi, struct receive_queue *rq, 2509 struct sk_buff *skb, u8 flags) 2510 { 2511 struct virtio_net_common_hdr *hdr; 2512 struct net_device *dev = vi->dev; 2513 2514 hdr = skb_vnet_common_hdr(skb); 2515 if (dev->features & NETIF_F_RXHASH && vi->has_rss_hash_report) 2516 virtio_skb_set_hash(&hdr->hash_v1_hdr, skb); 2517 2518 if (flags & VIRTIO_NET_HDR_F_DATA_VALID) 2519 skb->ip_summed = CHECKSUM_UNNECESSARY; 2520 2521 if (virtio_net_hdr_to_skb(skb, &hdr->hdr, 2522 virtio_is_little_endian(vi->vdev))) { 2523 net_warn_ratelimited("%s: bad gso: type: %u, size: %u\n", 2524 dev->name, hdr->hdr.gso_type, 2525 hdr->hdr.gso_size); 2526 goto frame_err; 2527 } 2528 2529 skb_record_rx_queue(skb, vq2rxq(rq->vq)); 2530 skb->protocol = eth_type_trans(skb, dev); 2531 pr_debug("Receiving skb proto 0x%04x len %i type %i\n", 2532 ntohs(skb->protocol), skb->len, skb->pkt_type); 2533 2534 napi_gro_receive(&rq->napi, skb); 2535 return; 2536 2537 frame_err: 2538 DEV_STATS_INC(dev, rx_frame_errors); 2539 dev_kfree_skb(skb); 2540 } 2541 2542 static void receive_buf(struct virtnet_info *vi, struct receive_queue *rq, 2543 void *buf, unsigned int len, void **ctx, 2544 unsigned int *xdp_xmit, 2545 struct virtnet_rq_stats *stats) 2546 { 2547 struct net_device *dev = vi->dev; 2548 struct sk_buff *skb; 2549 u8 flags; 2550 2551 if (unlikely(len < vi->hdr_len + ETH_HLEN)) { 2552 pr_debug("%s: short packet %i\n", dev->name, len); 2553 DEV_STATS_INC(dev, rx_length_errors); 2554 virtnet_rq_free_buf(vi, rq, buf); 2555 return; 2556 } 2557 2558 /* 1. Save the flags early, as the XDP program might overwrite them. 2559 * These flags ensure packets marked as VIRTIO_NET_HDR_F_DATA_VALID 2560 * stay valid after XDP processing. 2561 * 2. XDP doesn't work with partially checksummed packets (refer to 2562 * virtnet_xdp_set()), so packets marked as 2563 * VIRTIO_NET_HDR_F_NEEDS_CSUM get dropped during XDP processing. 2564 */ 2565 flags = ((struct virtio_net_common_hdr *)buf)->hdr.flags; 2566 2567 if (vi->mergeable_rx_bufs) 2568 skb = receive_mergeable(dev, vi, rq, buf, ctx, len, xdp_xmit, 2569 stats); 2570 else if (vi->big_packets) 2571 skb = receive_big(dev, vi, rq, buf, len, stats); 2572 else 2573 skb = receive_small(dev, vi, rq, buf, ctx, len, xdp_xmit, stats); 2574 2575 if (unlikely(!skb)) 2576 return; 2577 2578 virtnet_receive_done(vi, rq, skb, flags); 2579 } 2580 2581 /* Unlike mergeable buffers, all buffers are allocated to the 2582 * same size, except for the headroom. For this reason we do 2583 * not need to use mergeable_len_to_ctx here - it is enough 2584 * to store the headroom as the context ignoring the truesize. 2585 */ 2586 static int add_recvbuf_small(struct virtnet_info *vi, struct receive_queue *rq, 2587 gfp_t gfp) 2588 { 2589 char *buf; 2590 unsigned int xdp_headroom = virtnet_get_headroom(vi); 2591 void *ctx = (void *)(unsigned long)xdp_headroom; 2592 int len = vi->hdr_len + VIRTNET_RX_PAD + GOOD_PACKET_LEN + xdp_headroom; 2593 int err; 2594 2595 len = SKB_DATA_ALIGN(len) + 2596 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 2597 2598 if (unlikely(!skb_page_frag_refill(len, &rq->alloc_frag, gfp))) 2599 return -ENOMEM; 2600 2601 buf = virtnet_rq_alloc(rq, len, gfp); 2602 if (unlikely(!buf)) 2603 return -ENOMEM; 2604 2605 buf += VIRTNET_RX_PAD + xdp_headroom; 2606 2607 virtnet_rq_init_one_sg(rq, buf, vi->hdr_len + GOOD_PACKET_LEN); 2608 2609 err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, buf, ctx, gfp); 2610 if (err < 0) { 2611 virtnet_rq_unmap(rq, buf, 0); 2612 put_page(virt_to_head_page(buf)); 2613 } 2614 2615 return err; 2616 } 2617 2618 static int add_recvbuf_big(struct virtnet_info *vi, struct receive_queue *rq, 2619 gfp_t gfp) 2620 { 2621 struct page *first, *list = NULL; 2622 char *p; 2623 int i, err, offset; 2624 2625 sg_init_table(rq->sg, vi->big_packets_num_skbfrags + 2); 2626 2627 /* page in rq->sg[vi->big_packets_num_skbfrags + 1] is list tail */ 2628 for (i = vi->big_packets_num_skbfrags + 1; i > 1; --i) { 2629 first = get_a_page(rq, gfp); 2630 if (!first) { 2631 if (list) 2632 give_pages(rq, list); 2633 return -ENOMEM; 2634 } 2635 sg_set_buf(&rq->sg[i], page_address(first), PAGE_SIZE); 2636 2637 /* chain new page in list head to match sg */ 2638 first->private = (unsigned long)list; 2639 list = first; 2640 } 2641 2642 first = get_a_page(rq, gfp); 2643 if (!first) { 2644 give_pages(rq, list); 2645 return -ENOMEM; 2646 } 2647 p = page_address(first); 2648 2649 /* rq->sg[0], rq->sg[1] share the same page */ 2650 /* a separated rq->sg[0] for header - required in case !any_header_sg */ 2651 sg_set_buf(&rq->sg[0], p, vi->hdr_len); 2652 2653 /* rq->sg[1] for data packet, from offset */ 2654 offset = sizeof(struct padded_vnet_hdr); 2655 sg_set_buf(&rq->sg[1], p + offset, PAGE_SIZE - offset); 2656 2657 /* chain first in list head */ 2658 first->private = (unsigned long)list; 2659 err = virtqueue_add_inbuf(rq->vq, rq->sg, vi->big_packets_num_skbfrags + 2, 2660 first, gfp); 2661 if (err < 0) 2662 give_pages(rq, first); 2663 2664 return err; 2665 } 2666 2667 static unsigned int get_mergeable_buf_len(struct receive_queue *rq, 2668 struct ewma_pkt_len *avg_pkt_len, 2669 unsigned int room) 2670 { 2671 struct virtnet_info *vi = rq->vq->vdev->priv; 2672 const size_t hdr_len = vi->hdr_len; 2673 unsigned int len; 2674 2675 if (room) 2676 return PAGE_SIZE - room; 2677 2678 len = hdr_len + clamp_t(unsigned int, ewma_pkt_len_read(avg_pkt_len), 2679 rq->min_buf_len, PAGE_SIZE - hdr_len); 2680 2681 return ALIGN(len, L1_CACHE_BYTES); 2682 } 2683 2684 static int add_recvbuf_mergeable(struct virtnet_info *vi, 2685 struct receive_queue *rq, gfp_t gfp) 2686 { 2687 struct page_frag *alloc_frag = &rq->alloc_frag; 2688 unsigned int headroom = virtnet_get_headroom(vi); 2689 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 2690 unsigned int room = SKB_DATA_ALIGN(headroom + tailroom); 2691 unsigned int len, hole; 2692 void *ctx; 2693 char *buf; 2694 int err; 2695 2696 /* Extra tailroom is needed to satisfy XDP's assumption. This 2697 * means rx frags coalescing won't work, but consider we've 2698 * disabled GSO for XDP, it won't be a big issue. 2699 */ 2700 len = get_mergeable_buf_len(rq, &rq->mrg_avg_pkt_len, room); 2701 2702 if (unlikely(!skb_page_frag_refill(len + room, alloc_frag, gfp))) 2703 return -ENOMEM; 2704 2705 if (!alloc_frag->offset && len + room + sizeof(struct virtnet_rq_dma) > alloc_frag->size) 2706 len -= sizeof(struct virtnet_rq_dma); 2707 2708 buf = virtnet_rq_alloc(rq, len + room, gfp); 2709 if (unlikely(!buf)) 2710 return -ENOMEM; 2711 2712 buf += headroom; /* advance address leaving hole at front of pkt */ 2713 hole = alloc_frag->size - alloc_frag->offset; 2714 if (hole < len + room) { 2715 /* To avoid internal fragmentation, if there is very likely not 2716 * enough space for another buffer, add the remaining space to 2717 * the current buffer. 2718 * XDP core assumes that frame_size of xdp_buff and the length 2719 * of the frag are PAGE_SIZE, so we disable the hole mechanism. 2720 */ 2721 if (!headroom) 2722 len += hole; 2723 alloc_frag->offset += hole; 2724 } 2725 2726 virtnet_rq_init_one_sg(rq, buf, len); 2727 2728 ctx = mergeable_len_to_ctx(len + room, headroom); 2729 err = virtqueue_add_inbuf_premapped(rq->vq, rq->sg, 1, buf, ctx, gfp); 2730 if (err < 0) { 2731 virtnet_rq_unmap(rq, buf, 0); 2732 put_page(virt_to_head_page(buf)); 2733 } 2734 2735 return err; 2736 } 2737 2738 /* 2739 * Returns false if we couldn't fill entirely (OOM). 2740 * 2741 * Normally run in the receive path, but can also be run from ndo_open 2742 * before we're receiving packets, or from refill_work which is 2743 * careful to disable receiving (using napi_disable). 2744 */ 2745 static bool try_fill_recv(struct virtnet_info *vi, struct receive_queue *rq, 2746 gfp_t gfp) 2747 { 2748 int err; 2749 2750 if (rq->xsk_pool) { 2751 err = virtnet_add_recvbuf_xsk(vi, rq, rq->xsk_pool, gfp); 2752 goto kick; 2753 } 2754 2755 do { 2756 if (vi->mergeable_rx_bufs) 2757 err = add_recvbuf_mergeable(vi, rq, gfp); 2758 else if (vi->big_packets) 2759 err = add_recvbuf_big(vi, rq, gfp); 2760 else 2761 err = add_recvbuf_small(vi, rq, gfp); 2762 2763 if (err) 2764 break; 2765 } while (rq->vq->num_free); 2766 2767 kick: 2768 if (virtqueue_kick_prepare(rq->vq) && virtqueue_notify(rq->vq)) { 2769 unsigned long flags; 2770 2771 flags = u64_stats_update_begin_irqsave(&rq->stats.syncp); 2772 u64_stats_inc(&rq->stats.kicks); 2773 u64_stats_update_end_irqrestore(&rq->stats.syncp, flags); 2774 } 2775 2776 return err != -ENOMEM; 2777 } 2778 2779 static void skb_recv_done(struct virtqueue *rvq) 2780 { 2781 struct virtnet_info *vi = rvq->vdev->priv; 2782 struct receive_queue *rq = &vi->rq[vq2rxq(rvq)]; 2783 2784 rq->calls++; 2785 virtqueue_napi_schedule(&rq->napi, rvq); 2786 } 2787 2788 static void virtnet_napi_do_enable(struct virtqueue *vq, 2789 struct napi_struct *napi) 2790 { 2791 napi_enable(napi); 2792 2793 /* If all buffers were filled by other side before we napi_enabled, we 2794 * won't get another interrupt, so process any outstanding packets now. 2795 * Call local_bh_enable after to trigger softIRQ processing. 2796 */ 2797 local_bh_disable(); 2798 virtqueue_napi_schedule(napi, vq); 2799 local_bh_enable(); 2800 } 2801 2802 static void virtnet_napi_enable(struct receive_queue *rq) 2803 { 2804 struct virtnet_info *vi = rq->vq->vdev->priv; 2805 int qidx = vq2rxq(rq->vq); 2806 2807 virtnet_napi_do_enable(rq->vq, &rq->napi); 2808 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_RX, &rq->napi); 2809 } 2810 2811 static void virtnet_napi_tx_enable(struct send_queue *sq) 2812 { 2813 struct virtnet_info *vi = sq->vq->vdev->priv; 2814 struct napi_struct *napi = &sq->napi; 2815 int qidx = vq2txq(sq->vq); 2816 2817 if (!napi->weight) 2818 return; 2819 2820 /* Tx napi touches cachelines on the cpu handling tx interrupts. Only 2821 * enable the feature if this is likely affine with the transmit path. 2822 */ 2823 if (!vi->affinity_hint_set) { 2824 napi->weight = 0; 2825 return; 2826 } 2827 2828 virtnet_napi_do_enable(sq->vq, napi); 2829 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_TX, napi); 2830 } 2831 2832 static void virtnet_napi_tx_disable(struct send_queue *sq) 2833 { 2834 struct virtnet_info *vi = sq->vq->vdev->priv; 2835 struct napi_struct *napi = &sq->napi; 2836 int qidx = vq2txq(sq->vq); 2837 2838 if (napi->weight) { 2839 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_TX, NULL); 2840 napi_disable(napi); 2841 } 2842 } 2843 2844 static void virtnet_napi_disable(struct receive_queue *rq) 2845 { 2846 struct virtnet_info *vi = rq->vq->vdev->priv; 2847 struct napi_struct *napi = &rq->napi; 2848 int qidx = vq2rxq(rq->vq); 2849 2850 netif_queue_set_napi(vi->dev, qidx, NETDEV_QUEUE_TYPE_RX, NULL); 2851 napi_disable(napi); 2852 } 2853 2854 static void refill_work(struct work_struct *work) 2855 { 2856 struct virtnet_info *vi = 2857 container_of(work, struct virtnet_info, refill.work); 2858 bool still_empty; 2859 int i; 2860 2861 for (i = 0; i < vi->curr_queue_pairs; i++) { 2862 struct receive_queue *rq = &vi->rq[i]; 2863 2864 /* 2865 * When queue API support is added in the future and the call 2866 * below becomes napi_disable_locked, this driver will need to 2867 * be refactored. 2868 * 2869 * One possible solution would be to: 2870 * - cancel refill_work with cancel_delayed_work (note: 2871 * non-sync) 2872 * - cancel refill_work with cancel_delayed_work_sync in 2873 * virtnet_remove after the netdev is unregistered 2874 * - wrap all of the work in a lock (perhaps the netdev 2875 * instance lock) 2876 * - check netif_running() and return early to avoid a race 2877 */ 2878 napi_disable(&rq->napi); 2879 still_empty = !try_fill_recv(vi, rq, GFP_KERNEL); 2880 virtnet_napi_do_enable(rq->vq, &rq->napi); 2881 2882 /* In theory, this can happen: if we don't get any buffers in 2883 * we will *never* try to fill again. 2884 */ 2885 if (still_empty) 2886 schedule_delayed_work(&vi->refill, HZ/2); 2887 } 2888 } 2889 2890 static int virtnet_receive_xsk_bufs(struct virtnet_info *vi, 2891 struct receive_queue *rq, 2892 int budget, 2893 unsigned int *xdp_xmit, 2894 struct virtnet_rq_stats *stats) 2895 { 2896 unsigned int len; 2897 int packets = 0; 2898 void *buf; 2899 2900 while (packets < budget) { 2901 buf = virtqueue_get_buf(rq->vq, &len); 2902 if (!buf) 2903 break; 2904 2905 virtnet_receive_xsk_buf(vi, rq, buf, len, xdp_xmit, stats); 2906 packets++; 2907 } 2908 2909 return packets; 2910 } 2911 2912 static int virtnet_receive_packets(struct virtnet_info *vi, 2913 struct receive_queue *rq, 2914 int budget, 2915 unsigned int *xdp_xmit, 2916 struct virtnet_rq_stats *stats) 2917 { 2918 unsigned int len; 2919 int packets = 0; 2920 void *buf; 2921 2922 if (!vi->big_packets || vi->mergeable_rx_bufs) { 2923 void *ctx; 2924 while (packets < budget && 2925 (buf = virtnet_rq_get_buf(rq, &len, &ctx))) { 2926 receive_buf(vi, rq, buf, len, ctx, xdp_xmit, stats); 2927 packets++; 2928 } 2929 } else { 2930 while (packets < budget && 2931 (buf = virtqueue_get_buf(rq->vq, &len)) != NULL) { 2932 receive_buf(vi, rq, buf, len, NULL, xdp_xmit, stats); 2933 packets++; 2934 } 2935 } 2936 2937 return packets; 2938 } 2939 2940 static int virtnet_receive(struct receive_queue *rq, int budget, 2941 unsigned int *xdp_xmit) 2942 { 2943 struct virtnet_info *vi = rq->vq->vdev->priv; 2944 struct virtnet_rq_stats stats = {}; 2945 int i, packets; 2946 2947 if (rq->xsk_pool) 2948 packets = virtnet_receive_xsk_bufs(vi, rq, budget, xdp_xmit, &stats); 2949 else 2950 packets = virtnet_receive_packets(vi, rq, budget, xdp_xmit, &stats); 2951 2952 if (rq->vq->num_free > min((unsigned int)budget, virtqueue_get_vring_size(rq->vq)) / 2) { 2953 if (!try_fill_recv(vi, rq, GFP_ATOMIC)) { 2954 spin_lock(&vi->refill_lock); 2955 if (vi->refill_enabled) 2956 schedule_delayed_work(&vi->refill, 0); 2957 spin_unlock(&vi->refill_lock); 2958 } 2959 } 2960 2961 u64_stats_set(&stats.packets, packets); 2962 u64_stats_update_begin(&rq->stats.syncp); 2963 for (i = 0; i < ARRAY_SIZE(virtnet_rq_stats_desc); i++) { 2964 size_t offset = virtnet_rq_stats_desc[i].offset; 2965 u64_stats_t *item, *src; 2966 2967 item = (u64_stats_t *)((u8 *)&rq->stats + offset); 2968 src = (u64_stats_t *)((u8 *)&stats + offset); 2969 u64_stats_add(item, u64_stats_read(src)); 2970 } 2971 2972 u64_stats_add(&rq->stats.packets, u64_stats_read(&stats.packets)); 2973 u64_stats_add(&rq->stats.bytes, u64_stats_read(&stats.bytes)); 2974 2975 u64_stats_update_end(&rq->stats.syncp); 2976 2977 return packets; 2978 } 2979 2980 static void virtnet_poll_cleantx(struct receive_queue *rq, int budget) 2981 { 2982 struct virtnet_info *vi = rq->vq->vdev->priv; 2983 unsigned int index = vq2rxq(rq->vq); 2984 struct send_queue *sq = &vi->sq[index]; 2985 struct netdev_queue *txq = netdev_get_tx_queue(vi->dev, index); 2986 2987 if (!sq->napi.weight || is_xdp_raw_buffer_queue(vi, index)) 2988 return; 2989 2990 if (__netif_tx_trylock(txq)) { 2991 if (sq->reset) { 2992 __netif_tx_unlock(txq); 2993 return; 2994 } 2995 2996 do { 2997 virtqueue_disable_cb(sq->vq); 2998 free_old_xmit(sq, txq, !!budget); 2999 } while (unlikely(!virtqueue_enable_cb_delayed(sq->vq))); 3000 3001 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) { 3002 if (netif_tx_queue_stopped(txq)) { 3003 u64_stats_update_begin(&sq->stats.syncp); 3004 u64_stats_inc(&sq->stats.wake); 3005 u64_stats_update_end(&sq->stats.syncp); 3006 } 3007 netif_tx_wake_queue(txq); 3008 } 3009 3010 __netif_tx_unlock(txq); 3011 } 3012 } 3013 3014 static void virtnet_rx_dim_update(struct virtnet_info *vi, struct receive_queue *rq) 3015 { 3016 struct dim_sample cur_sample = {}; 3017 3018 if (!rq->packets_in_napi) 3019 return; 3020 3021 /* Don't need protection when fetching stats, since fetcher and 3022 * updater of the stats are in same context 3023 */ 3024 dim_update_sample(rq->calls, 3025 u64_stats_read(&rq->stats.packets), 3026 u64_stats_read(&rq->stats.bytes), 3027 &cur_sample); 3028 3029 net_dim(&rq->dim, &cur_sample); 3030 rq->packets_in_napi = 0; 3031 } 3032 3033 static int virtnet_poll(struct napi_struct *napi, int budget) 3034 { 3035 struct receive_queue *rq = 3036 container_of(napi, struct receive_queue, napi); 3037 struct virtnet_info *vi = rq->vq->vdev->priv; 3038 struct send_queue *sq; 3039 unsigned int received; 3040 unsigned int xdp_xmit = 0; 3041 bool napi_complete; 3042 3043 virtnet_poll_cleantx(rq, budget); 3044 3045 received = virtnet_receive(rq, budget, &xdp_xmit); 3046 rq->packets_in_napi += received; 3047 3048 if (xdp_xmit & VIRTIO_XDP_REDIR) 3049 xdp_do_flush(); 3050 3051 /* Out of packets? */ 3052 if (received < budget) { 3053 napi_complete = virtqueue_napi_complete(napi, rq->vq, received); 3054 /* Intentionally not taking dim_lock here. This may result in a 3055 * spurious net_dim call. But if that happens virtnet_rx_dim_work 3056 * will not act on the scheduled work. 3057 */ 3058 if (napi_complete && rq->dim_enabled) 3059 virtnet_rx_dim_update(vi, rq); 3060 } 3061 3062 if (xdp_xmit & VIRTIO_XDP_TX) { 3063 sq = virtnet_xdp_get_sq(vi); 3064 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 3065 u64_stats_update_begin(&sq->stats.syncp); 3066 u64_stats_inc(&sq->stats.kicks); 3067 u64_stats_update_end(&sq->stats.syncp); 3068 } 3069 virtnet_xdp_put_sq(vi, sq); 3070 } 3071 3072 return received; 3073 } 3074 3075 static void virtnet_disable_queue_pair(struct virtnet_info *vi, int qp_index) 3076 { 3077 virtnet_napi_tx_disable(&vi->sq[qp_index]); 3078 virtnet_napi_disable(&vi->rq[qp_index]); 3079 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); 3080 } 3081 3082 static int virtnet_enable_queue_pair(struct virtnet_info *vi, int qp_index) 3083 { 3084 struct net_device *dev = vi->dev; 3085 int err; 3086 3087 err = xdp_rxq_info_reg(&vi->rq[qp_index].xdp_rxq, dev, qp_index, 3088 vi->rq[qp_index].napi.napi_id); 3089 if (err < 0) 3090 return err; 3091 3092 err = xdp_rxq_info_reg_mem_model(&vi->rq[qp_index].xdp_rxq, 3093 MEM_TYPE_PAGE_SHARED, NULL); 3094 if (err < 0) 3095 goto err_xdp_reg_mem_model; 3096 3097 virtnet_napi_enable(&vi->rq[qp_index]); 3098 virtnet_napi_tx_enable(&vi->sq[qp_index]); 3099 3100 return 0; 3101 3102 err_xdp_reg_mem_model: 3103 xdp_rxq_info_unreg(&vi->rq[qp_index].xdp_rxq); 3104 return err; 3105 } 3106 3107 static void virtnet_cancel_dim(struct virtnet_info *vi, struct dim *dim) 3108 { 3109 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 3110 return; 3111 net_dim_work_cancel(dim); 3112 } 3113 3114 static void virtnet_update_settings(struct virtnet_info *vi) 3115 { 3116 u32 speed; 3117 u8 duplex; 3118 3119 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_SPEED_DUPLEX)) 3120 return; 3121 3122 virtio_cread_le(vi->vdev, struct virtio_net_config, speed, &speed); 3123 3124 if (ethtool_validate_speed(speed)) 3125 vi->speed = speed; 3126 3127 virtio_cread_le(vi->vdev, struct virtio_net_config, duplex, &duplex); 3128 3129 if (ethtool_validate_duplex(duplex)) 3130 vi->duplex = duplex; 3131 } 3132 3133 static int virtnet_open(struct net_device *dev) 3134 { 3135 struct virtnet_info *vi = netdev_priv(dev); 3136 int i, err; 3137 3138 enable_delayed_refill(vi); 3139 3140 for (i = 0; i < vi->max_queue_pairs; i++) { 3141 if (i < vi->curr_queue_pairs) 3142 /* Make sure we have some buffers: if oom use wq. */ 3143 if (!try_fill_recv(vi, &vi->rq[i], GFP_KERNEL)) 3144 schedule_delayed_work(&vi->refill, 0); 3145 3146 err = virtnet_enable_queue_pair(vi, i); 3147 if (err < 0) 3148 goto err_enable_qp; 3149 } 3150 3151 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 3152 if (vi->status & VIRTIO_NET_S_LINK_UP) 3153 netif_carrier_on(vi->dev); 3154 virtio_config_driver_enable(vi->vdev); 3155 } else { 3156 vi->status = VIRTIO_NET_S_LINK_UP; 3157 netif_carrier_on(dev); 3158 } 3159 3160 return 0; 3161 3162 err_enable_qp: 3163 disable_delayed_refill(vi); 3164 cancel_delayed_work_sync(&vi->refill); 3165 3166 for (i--; i >= 0; i--) { 3167 virtnet_disable_queue_pair(vi, i); 3168 virtnet_cancel_dim(vi, &vi->rq[i].dim); 3169 } 3170 3171 return err; 3172 } 3173 3174 static int virtnet_poll_tx(struct napi_struct *napi, int budget) 3175 { 3176 struct send_queue *sq = container_of(napi, struct send_queue, napi); 3177 struct virtnet_info *vi = sq->vq->vdev->priv; 3178 unsigned int index = vq2txq(sq->vq); 3179 struct netdev_queue *txq; 3180 int opaque, xsk_done = 0; 3181 bool done; 3182 3183 if (unlikely(is_xdp_raw_buffer_queue(vi, index))) { 3184 /* We don't need to enable cb for XDP */ 3185 napi_complete_done(napi, 0); 3186 return 0; 3187 } 3188 3189 txq = netdev_get_tx_queue(vi->dev, index); 3190 __netif_tx_lock(txq, raw_smp_processor_id()); 3191 virtqueue_disable_cb(sq->vq); 3192 3193 if (sq->xsk_pool) 3194 xsk_done = virtnet_xsk_xmit(sq, sq->xsk_pool, budget); 3195 else 3196 free_old_xmit(sq, txq, !!budget); 3197 3198 if (sq->vq->num_free >= 2 + MAX_SKB_FRAGS) { 3199 if (netif_tx_queue_stopped(txq)) { 3200 u64_stats_update_begin(&sq->stats.syncp); 3201 u64_stats_inc(&sq->stats.wake); 3202 u64_stats_update_end(&sq->stats.syncp); 3203 } 3204 netif_tx_wake_queue(txq); 3205 } 3206 3207 if (xsk_done >= budget) { 3208 __netif_tx_unlock(txq); 3209 return budget; 3210 } 3211 3212 opaque = virtqueue_enable_cb_prepare(sq->vq); 3213 3214 done = napi_complete_done(napi, 0); 3215 3216 if (!done) 3217 virtqueue_disable_cb(sq->vq); 3218 3219 __netif_tx_unlock(txq); 3220 3221 if (done) { 3222 if (unlikely(virtqueue_poll(sq->vq, opaque))) { 3223 if (napi_schedule_prep(napi)) { 3224 __netif_tx_lock(txq, raw_smp_processor_id()); 3225 virtqueue_disable_cb(sq->vq); 3226 __netif_tx_unlock(txq); 3227 __napi_schedule(napi); 3228 } 3229 } 3230 } 3231 3232 return 0; 3233 } 3234 3235 static int xmit_skb(struct send_queue *sq, struct sk_buff *skb, bool orphan) 3236 { 3237 struct virtio_net_hdr_mrg_rxbuf *hdr; 3238 const unsigned char *dest = ((struct ethhdr *)skb->data)->h_dest; 3239 struct virtnet_info *vi = sq->vq->vdev->priv; 3240 int num_sg; 3241 unsigned hdr_len = vi->hdr_len; 3242 bool can_push; 3243 3244 pr_debug("%s: xmit %p %pM\n", vi->dev->name, skb, dest); 3245 3246 can_push = vi->any_header_sg && 3247 !((unsigned long)skb->data & (__alignof__(*hdr) - 1)) && 3248 !skb_header_cloned(skb) && skb_headroom(skb) >= hdr_len; 3249 /* Even if we can, don't push here yet as this would skew 3250 * csum_start offset below. */ 3251 if (can_push) 3252 hdr = (struct virtio_net_hdr_mrg_rxbuf *)(skb->data - hdr_len); 3253 else 3254 hdr = &skb_vnet_common_hdr(skb)->mrg_hdr; 3255 3256 if (virtio_net_hdr_from_skb(skb, &hdr->hdr, 3257 virtio_is_little_endian(vi->vdev), false, 3258 0)) 3259 return -EPROTO; 3260 3261 if (vi->mergeable_rx_bufs) 3262 hdr->num_buffers = 0; 3263 3264 sg_init_table(sq->sg, skb_shinfo(skb)->nr_frags + (can_push ? 1 : 2)); 3265 if (can_push) { 3266 __skb_push(skb, hdr_len); 3267 num_sg = skb_to_sgvec(skb, sq->sg, 0, skb->len); 3268 if (unlikely(num_sg < 0)) 3269 return num_sg; 3270 /* Pull header back to avoid skew in tx bytes calculations. */ 3271 __skb_pull(skb, hdr_len); 3272 } else { 3273 sg_set_buf(sq->sg, hdr, hdr_len); 3274 num_sg = skb_to_sgvec(skb, sq->sg + 1, 0, skb->len); 3275 if (unlikely(num_sg < 0)) 3276 return num_sg; 3277 num_sg++; 3278 } 3279 3280 return virtnet_add_outbuf(sq, num_sg, skb, 3281 orphan ? VIRTNET_XMIT_TYPE_SKB_ORPHAN : VIRTNET_XMIT_TYPE_SKB); 3282 } 3283 3284 static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) 3285 { 3286 struct virtnet_info *vi = netdev_priv(dev); 3287 int qnum = skb_get_queue_mapping(skb); 3288 struct send_queue *sq = &vi->sq[qnum]; 3289 int err; 3290 struct netdev_queue *txq = netdev_get_tx_queue(dev, qnum); 3291 bool xmit_more = netdev_xmit_more(); 3292 bool use_napi = sq->napi.weight; 3293 bool kick; 3294 3295 if (!use_napi) 3296 free_old_xmit(sq, txq, false); 3297 else 3298 virtqueue_disable_cb(sq->vq); 3299 3300 /* timestamp packet in software */ 3301 skb_tx_timestamp(skb); 3302 3303 /* Try to transmit */ 3304 err = xmit_skb(sq, skb, !use_napi); 3305 3306 /* This should not happen! */ 3307 if (unlikely(err)) { 3308 DEV_STATS_INC(dev, tx_fifo_errors); 3309 if (net_ratelimit()) 3310 dev_warn(&dev->dev, 3311 "Unexpected TXQ (%d) queue failure: %d\n", 3312 qnum, err); 3313 DEV_STATS_INC(dev, tx_dropped); 3314 dev_kfree_skb_any(skb); 3315 return NETDEV_TX_OK; 3316 } 3317 3318 /* Don't wait up for transmitted skbs to be freed. */ 3319 if (!use_napi) { 3320 skb_orphan(skb); 3321 nf_reset_ct(skb); 3322 } 3323 3324 if (use_napi) 3325 tx_may_stop(vi, dev, sq); 3326 else 3327 check_sq_full_and_disable(vi, dev,sq); 3328 3329 kick = use_napi ? __netdev_tx_sent_queue(txq, skb->len, xmit_more) : 3330 !xmit_more || netif_xmit_stopped(txq); 3331 if (kick) { 3332 if (virtqueue_kick_prepare(sq->vq) && virtqueue_notify(sq->vq)) { 3333 u64_stats_update_begin(&sq->stats.syncp); 3334 u64_stats_inc(&sq->stats.kicks); 3335 u64_stats_update_end(&sq->stats.syncp); 3336 } 3337 } 3338 3339 if (use_napi && kick && unlikely(!virtqueue_enable_cb_delayed(sq->vq))) 3340 virtqueue_napi_schedule(&sq->napi, sq->vq); 3341 3342 return NETDEV_TX_OK; 3343 } 3344 3345 static void __virtnet_rx_pause(struct virtnet_info *vi, 3346 struct receive_queue *rq) 3347 { 3348 bool running = netif_running(vi->dev); 3349 3350 if (running) { 3351 virtnet_napi_disable(rq); 3352 virtnet_cancel_dim(vi, &rq->dim); 3353 } 3354 } 3355 3356 static void virtnet_rx_pause_all(struct virtnet_info *vi) 3357 { 3358 int i; 3359 3360 /* 3361 * Make sure refill_work does not run concurrently to 3362 * avoid napi_disable race which leads to deadlock. 3363 */ 3364 disable_delayed_refill(vi); 3365 cancel_delayed_work_sync(&vi->refill); 3366 for (i = 0; i < vi->max_queue_pairs; i++) 3367 __virtnet_rx_pause(vi, &vi->rq[i]); 3368 } 3369 3370 static void virtnet_rx_pause(struct virtnet_info *vi, struct receive_queue *rq) 3371 { 3372 /* 3373 * Make sure refill_work does not run concurrently to 3374 * avoid napi_disable race which leads to deadlock. 3375 */ 3376 disable_delayed_refill(vi); 3377 cancel_delayed_work_sync(&vi->refill); 3378 __virtnet_rx_pause(vi, rq); 3379 } 3380 3381 static void __virtnet_rx_resume(struct virtnet_info *vi, 3382 struct receive_queue *rq, 3383 bool refill) 3384 { 3385 bool running = netif_running(vi->dev); 3386 3387 if (refill && !try_fill_recv(vi, rq, GFP_KERNEL)) 3388 schedule_delayed_work(&vi->refill, 0); 3389 3390 if (running) 3391 virtnet_napi_enable(rq); 3392 } 3393 3394 static void virtnet_rx_resume_all(struct virtnet_info *vi) 3395 { 3396 int i; 3397 3398 enable_delayed_refill(vi); 3399 for (i = 0; i < vi->max_queue_pairs; i++) { 3400 if (i < vi->curr_queue_pairs) 3401 __virtnet_rx_resume(vi, &vi->rq[i], true); 3402 else 3403 __virtnet_rx_resume(vi, &vi->rq[i], false); 3404 } 3405 } 3406 3407 static void virtnet_rx_resume(struct virtnet_info *vi, struct receive_queue *rq) 3408 { 3409 enable_delayed_refill(vi); 3410 __virtnet_rx_resume(vi, rq, true); 3411 } 3412 3413 static int virtnet_rx_resize(struct virtnet_info *vi, 3414 struct receive_queue *rq, u32 ring_num) 3415 { 3416 int err, qindex; 3417 3418 qindex = rq - vi->rq; 3419 3420 virtnet_rx_pause(vi, rq); 3421 3422 err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_unmap_free_buf, NULL); 3423 if (err) 3424 netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); 3425 3426 virtnet_rx_resume(vi, rq); 3427 return err; 3428 } 3429 3430 static void virtnet_tx_pause(struct virtnet_info *vi, struct send_queue *sq) 3431 { 3432 bool running = netif_running(vi->dev); 3433 struct netdev_queue *txq; 3434 int qindex; 3435 3436 qindex = sq - vi->sq; 3437 3438 if (running) 3439 virtnet_napi_tx_disable(sq); 3440 3441 txq = netdev_get_tx_queue(vi->dev, qindex); 3442 3443 /* 1. wait all ximt complete 3444 * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue() 3445 */ 3446 __netif_tx_lock_bh(txq); 3447 3448 /* Prevent rx poll from accessing sq. */ 3449 sq->reset = true; 3450 3451 /* Prevent the upper layer from trying to send packets. */ 3452 netif_stop_subqueue(vi->dev, qindex); 3453 3454 __netif_tx_unlock_bh(txq); 3455 } 3456 3457 static void virtnet_tx_resume(struct virtnet_info *vi, struct send_queue *sq) 3458 { 3459 bool running = netif_running(vi->dev); 3460 struct netdev_queue *txq; 3461 int qindex; 3462 3463 qindex = sq - vi->sq; 3464 3465 txq = netdev_get_tx_queue(vi->dev, qindex); 3466 3467 __netif_tx_lock_bh(txq); 3468 sq->reset = false; 3469 netif_tx_wake_queue(txq); 3470 __netif_tx_unlock_bh(txq); 3471 3472 if (running) 3473 virtnet_napi_tx_enable(sq); 3474 } 3475 3476 static int virtnet_tx_resize(struct virtnet_info *vi, struct send_queue *sq, 3477 u32 ring_num) 3478 { 3479 int qindex, err; 3480 3481 qindex = sq - vi->sq; 3482 3483 virtnet_tx_pause(vi, sq); 3484 3485 err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf, 3486 virtnet_sq_free_unused_buf_done); 3487 if (err) 3488 netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err); 3489 3490 virtnet_tx_resume(vi, sq); 3491 3492 return err; 3493 } 3494 3495 /* 3496 * Send command via the control virtqueue and check status. Commands 3497 * supported by the hypervisor, as indicated by feature bits, should 3498 * never fail unless improperly formatted. 3499 */ 3500 static bool virtnet_send_command_reply(struct virtnet_info *vi, u8 class, u8 cmd, 3501 struct scatterlist *out, 3502 struct scatterlist *in) 3503 { 3504 struct scatterlist *sgs[5], hdr, stat; 3505 u32 out_num = 0, tmp, in_num = 0; 3506 bool ok; 3507 int ret; 3508 3509 /* Caller should know better */ 3510 BUG_ON(!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ)); 3511 3512 mutex_lock(&vi->cvq_lock); 3513 vi->ctrl->status = ~0; 3514 vi->ctrl->hdr.class = class; 3515 vi->ctrl->hdr.cmd = cmd; 3516 /* Add header */ 3517 sg_init_one(&hdr, &vi->ctrl->hdr, sizeof(vi->ctrl->hdr)); 3518 sgs[out_num++] = &hdr; 3519 3520 if (out) 3521 sgs[out_num++] = out; 3522 3523 /* Add return status. */ 3524 sg_init_one(&stat, &vi->ctrl->status, sizeof(vi->ctrl->status)); 3525 sgs[out_num + in_num++] = &stat; 3526 3527 if (in) 3528 sgs[out_num + in_num++] = in; 3529 3530 BUG_ON(out_num + in_num > ARRAY_SIZE(sgs)); 3531 ret = virtqueue_add_sgs(vi->cvq, sgs, out_num, in_num, vi, GFP_ATOMIC); 3532 if (ret < 0) { 3533 dev_warn(&vi->vdev->dev, 3534 "Failed to add sgs for command vq: %d\n.", ret); 3535 mutex_unlock(&vi->cvq_lock); 3536 return false; 3537 } 3538 3539 if (unlikely(!virtqueue_kick(vi->cvq))) 3540 goto unlock; 3541 3542 /* Spin for a response, the kick causes an ioport write, trapping 3543 * into the hypervisor, so the request should be handled immediately. 3544 */ 3545 while (!virtqueue_get_buf(vi->cvq, &tmp) && 3546 !virtqueue_is_broken(vi->cvq)) { 3547 cond_resched(); 3548 cpu_relax(); 3549 } 3550 3551 unlock: 3552 ok = vi->ctrl->status == VIRTIO_NET_OK; 3553 mutex_unlock(&vi->cvq_lock); 3554 return ok; 3555 } 3556 3557 static bool virtnet_send_command(struct virtnet_info *vi, u8 class, u8 cmd, 3558 struct scatterlist *out) 3559 { 3560 return virtnet_send_command_reply(vi, class, cmd, out, NULL); 3561 } 3562 3563 static int virtnet_set_mac_address(struct net_device *dev, void *p) 3564 { 3565 struct virtnet_info *vi = netdev_priv(dev); 3566 struct virtio_device *vdev = vi->vdev; 3567 int ret; 3568 struct sockaddr *addr; 3569 struct scatterlist sg; 3570 3571 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 3572 return -EOPNOTSUPP; 3573 3574 addr = kmemdup(p, sizeof(*addr), GFP_KERNEL); 3575 if (!addr) 3576 return -ENOMEM; 3577 3578 ret = eth_prepare_mac_addr_change(dev, addr); 3579 if (ret) 3580 goto out; 3581 3582 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 3583 sg_init_one(&sg, addr->sa_data, dev->addr_len); 3584 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 3585 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 3586 dev_warn(&vdev->dev, 3587 "Failed to set mac address by vq command.\n"); 3588 ret = -EINVAL; 3589 goto out; 3590 } 3591 } else if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 3592 !virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) { 3593 unsigned int i; 3594 3595 /* Naturally, this has an atomicity problem. */ 3596 for (i = 0; i < dev->addr_len; i++) 3597 virtio_cwrite8(vdev, 3598 offsetof(struct virtio_net_config, mac) + 3599 i, addr->sa_data[i]); 3600 } 3601 3602 eth_commit_mac_addr_change(dev, p); 3603 ret = 0; 3604 3605 out: 3606 kfree(addr); 3607 return ret; 3608 } 3609 3610 static void virtnet_stats(struct net_device *dev, 3611 struct rtnl_link_stats64 *tot) 3612 { 3613 struct virtnet_info *vi = netdev_priv(dev); 3614 unsigned int start; 3615 int i; 3616 3617 for (i = 0; i < vi->max_queue_pairs; i++) { 3618 u64 tpackets, tbytes, terrors, rpackets, rbytes, rdrops; 3619 struct receive_queue *rq = &vi->rq[i]; 3620 struct send_queue *sq = &vi->sq[i]; 3621 3622 do { 3623 start = u64_stats_fetch_begin(&sq->stats.syncp); 3624 tpackets = u64_stats_read(&sq->stats.packets); 3625 tbytes = u64_stats_read(&sq->stats.bytes); 3626 terrors = u64_stats_read(&sq->stats.tx_timeouts); 3627 } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 3628 3629 do { 3630 start = u64_stats_fetch_begin(&rq->stats.syncp); 3631 rpackets = u64_stats_read(&rq->stats.packets); 3632 rbytes = u64_stats_read(&rq->stats.bytes); 3633 rdrops = u64_stats_read(&rq->stats.drops); 3634 } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 3635 3636 tot->rx_packets += rpackets; 3637 tot->tx_packets += tpackets; 3638 tot->rx_bytes += rbytes; 3639 tot->tx_bytes += tbytes; 3640 tot->rx_dropped += rdrops; 3641 tot->tx_errors += terrors; 3642 } 3643 3644 tot->tx_dropped = DEV_STATS_READ(dev, tx_dropped); 3645 tot->tx_fifo_errors = DEV_STATS_READ(dev, tx_fifo_errors); 3646 tot->rx_length_errors = DEV_STATS_READ(dev, rx_length_errors); 3647 tot->rx_frame_errors = DEV_STATS_READ(dev, rx_frame_errors); 3648 } 3649 3650 static void virtnet_ack_link_announce(struct virtnet_info *vi) 3651 { 3652 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_ANNOUNCE, 3653 VIRTIO_NET_CTRL_ANNOUNCE_ACK, NULL)) 3654 dev_warn(&vi->dev->dev, "Failed to ack link announce.\n"); 3655 } 3656 3657 static bool virtnet_commit_rss_command(struct virtnet_info *vi); 3658 3659 static void virtnet_rss_update_by_qpairs(struct virtnet_info *vi, u16 queue_pairs) 3660 { 3661 u32 indir_val = 0; 3662 int i = 0; 3663 3664 for (; i < vi->rss_indir_table_size; ++i) { 3665 indir_val = ethtool_rxfh_indir_default(i, queue_pairs); 3666 vi->rss_hdr->indirection_table[i] = cpu_to_le16(indir_val); 3667 } 3668 vi->rss_trailer.max_tx_vq = cpu_to_le16(queue_pairs); 3669 } 3670 3671 static int virtnet_set_queues(struct virtnet_info *vi, u16 queue_pairs) 3672 { 3673 struct virtio_net_ctrl_mq *mq __free(kfree) = NULL; 3674 struct virtio_net_rss_config_hdr *old_rss_hdr; 3675 struct virtio_net_rss_config_trailer old_rss_trailer; 3676 struct net_device *dev = vi->dev; 3677 struct scatterlist sg; 3678 3679 if (!vi->has_cvq || !virtio_has_feature(vi->vdev, VIRTIO_NET_F_MQ)) 3680 return 0; 3681 3682 /* Firstly check if we need update rss. Do updating if both (1) rss enabled and 3683 * (2) no user configuration. 3684 * 3685 * During rss command processing, device updates queue_pairs using rss.max_tx_vq. That is, 3686 * the device updates queue_pairs together with rss, so we can skip the sperate queue_pairs 3687 * update (VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET below) and return directly. 3688 */ 3689 if (vi->has_rss && !netif_is_rxfh_configured(dev)) { 3690 old_rss_hdr = vi->rss_hdr; 3691 old_rss_trailer = vi->rss_trailer; 3692 vi->rss_hdr = devm_kzalloc(&dev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); 3693 if (!vi->rss_hdr) { 3694 vi->rss_hdr = old_rss_hdr; 3695 return -ENOMEM; 3696 } 3697 3698 *vi->rss_hdr = *old_rss_hdr; 3699 virtnet_rss_update_by_qpairs(vi, queue_pairs); 3700 3701 if (!virtnet_commit_rss_command(vi)) { 3702 /* restore ctrl_rss if commit_rss_command failed */ 3703 devm_kfree(&dev->dev, vi->rss_hdr); 3704 vi->rss_hdr = old_rss_hdr; 3705 vi->rss_trailer = old_rss_trailer; 3706 3707 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d, because committing RSS failed\n", 3708 queue_pairs); 3709 return -EINVAL; 3710 } 3711 devm_kfree(&dev->dev, old_rss_hdr); 3712 goto succ; 3713 } 3714 3715 mq = kzalloc(sizeof(*mq), GFP_KERNEL); 3716 if (!mq) 3717 return -ENOMEM; 3718 3719 mq->virtqueue_pairs = cpu_to_virtio16(vi->vdev, queue_pairs); 3720 sg_init_one(&sg, mq, sizeof(*mq)); 3721 3722 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 3723 VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET, &sg)) { 3724 dev_warn(&dev->dev, "Fail to set num of queue pairs to %d\n", 3725 queue_pairs); 3726 return -EINVAL; 3727 } 3728 succ: 3729 vi->curr_queue_pairs = queue_pairs; 3730 /* virtnet_open() will refill when device is going to up. */ 3731 if (dev->flags & IFF_UP) 3732 schedule_delayed_work(&vi->refill, 0); 3733 3734 return 0; 3735 } 3736 3737 static int virtnet_close(struct net_device *dev) 3738 { 3739 struct virtnet_info *vi = netdev_priv(dev); 3740 int i; 3741 3742 /* Make sure NAPI doesn't schedule refill work */ 3743 disable_delayed_refill(vi); 3744 /* Make sure refill_work doesn't re-enable napi! */ 3745 cancel_delayed_work_sync(&vi->refill); 3746 /* Prevent the config change callback from changing carrier 3747 * after close 3748 */ 3749 virtio_config_driver_disable(vi->vdev); 3750 /* Stop getting status/speed updates: we don't care until next 3751 * open 3752 */ 3753 cancel_work_sync(&vi->config_work); 3754 3755 for (i = 0; i < vi->max_queue_pairs; i++) { 3756 virtnet_disable_queue_pair(vi, i); 3757 virtnet_cancel_dim(vi, &vi->rq[i].dim); 3758 } 3759 3760 netif_carrier_off(dev); 3761 3762 return 0; 3763 } 3764 3765 static void virtnet_rx_mode_work(struct work_struct *work) 3766 { 3767 struct virtnet_info *vi = 3768 container_of(work, struct virtnet_info, rx_mode_work); 3769 u8 *promisc_allmulti __free(kfree) = NULL; 3770 struct net_device *dev = vi->dev; 3771 struct scatterlist sg[2]; 3772 struct virtio_net_ctrl_mac *mac_data; 3773 struct netdev_hw_addr *ha; 3774 int uc_count; 3775 int mc_count; 3776 void *buf; 3777 int i; 3778 3779 /* We can't dynamically set ndo_set_rx_mode, so return gracefully */ 3780 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_RX)) 3781 return; 3782 3783 promisc_allmulti = kzalloc(sizeof(*promisc_allmulti), GFP_KERNEL); 3784 if (!promisc_allmulti) { 3785 dev_warn(&dev->dev, "Failed to set RX mode, no memory.\n"); 3786 return; 3787 } 3788 3789 rtnl_lock(); 3790 3791 *promisc_allmulti = !!(dev->flags & IFF_PROMISC); 3792 sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti)); 3793 3794 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 3795 VIRTIO_NET_CTRL_RX_PROMISC, sg)) 3796 dev_warn(&dev->dev, "Failed to %sable promisc mode.\n", 3797 *promisc_allmulti ? "en" : "dis"); 3798 3799 *promisc_allmulti = !!(dev->flags & IFF_ALLMULTI); 3800 sg_init_one(sg, promisc_allmulti, sizeof(*promisc_allmulti)); 3801 3802 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_RX, 3803 VIRTIO_NET_CTRL_RX_ALLMULTI, sg)) 3804 dev_warn(&dev->dev, "Failed to %sable allmulti mode.\n", 3805 *promisc_allmulti ? "en" : "dis"); 3806 3807 netif_addr_lock_bh(dev); 3808 3809 uc_count = netdev_uc_count(dev); 3810 mc_count = netdev_mc_count(dev); 3811 /* MAC filter - use one buffer for both lists */ 3812 buf = kzalloc(((uc_count + mc_count) * ETH_ALEN) + 3813 (2 * sizeof(mac_data->entries)), GFP_ATOMIC); 3814 mac_data = buf; 3815 if (!buf) { 3816 netif_addr_unlock_bh(dev); 3817 rtnl_unlock(); 3818 return; 3819 } 3820 3821 sg_init_table(sg, 2); 3822 3823 /* Store the unicast list and count in the front of the buffer */ 3824 mac_data->entries = cpu_to_virtio32(vi->vdev, uc_count); 3825 i = 0; 3826 netdev_for_each_uc_addr(ha, dev) 3827 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 3828 3829 sg_set_buf(&sg[0], mac_data, 3830 sizeof(mac_data->entries) + (uc_count * ETH_ALEN)); 3831 3832 /* multicast list and count fill the end */ 3833 mac_data = (void *)&mac_data->macs[uc_count][0]; 3834 3835 mac_data->entries = cpu_to_virtio32(vi->vdev, mc_count); 3836 i = 0; 3837 netdev_for_each_mc_addr(ha, dev) 3838 memcpy(&mac_data->macs[i++][0], ha->addr, ETH_ALEN); 3839 3840 netif_addr_unlock_bh(dev); 3841 3842 sg_set_buf(&sg[1], mac_data, 3843 sizeof(mac_data->entries) + (mc_count * ETH_ALEN)); 3844 3845 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 3846 VIRTIO_NET_CTRL_MAC_TABLE_SET, sg)) 3847 dev_warn(&dev->dev, "Failed to set MAC filter table.\n"); 3848 3849 rtnl_unlock(); 3850 3851 kfree(buf); 3852 } 3853 3854 static void virtnet_set_rx_mode(struct net_device *dev) 3855 { 3856 struct virtnet_info *vi = netdev_priv(dev); 3857 3858 if (vi->rx_mode_work_enabled) 3859 schedule_work(&vi->rx_mode_work); 3860 } 3861 3862 static int virtnet_vlan_rx_add_vid(struct net_device *dev, 3863 __be16 proto, u16 vid) 3864 { 3865 struct virtnet_info *vi = netdev_priv(dev); 3866 __virtio16 *_vid __free(kfree) = NULL; 3867 struct scatterlist sg; 3868 3869 _vid = kzalloc(sizeof(*_vid), GFP_KERNEL); 3870 if (!_vid) 3871 return -ENOMEM; 3872 3873 *_vid = cpu_to_virtio16(vi->vdev, vid); 3874 sg_init_one(&sg, _vid, sizeof(*_vid)); 3875 3876 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 3877 VIRTIO_NET_CTRL_VLAN_ADD, &sg)) 3878 dev_warn(&dev->dev, "Failed to add VLAN ID %d.\n", vid); 3879 return 0; 3880 } 3881 3882 static int virtnet_vlan_rx_kill_vid(struct net_device *dev, 3883 __be16 proto, u16 vid) 3884 { 3885 struct virtnet_info *vi = netdev_priv(dev); 3886 __virtio16 *_vid __free(kfree) = NULL; 3887 struct scatterlist sg; 3888 3889 _vid = kzalloc(sizeof(*_vid), GFP_KERNEL); 3890 if (!_vid) 3891 return -ENOMEM; 3892 3893 *_vid = cpu_to_virtio16(vi->vdev, vid); 3894 sg_init_one(&sg, _vid, sizeof(*_vid)); 3895 3896 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_VLAN, 3897 VIRTIO_NET_CTRL_VLAN_DEL, &sg)) 3898 dev_warn(&dev->dev, "Failed to kill VLAN ID %d.\n", vid); 3899 return 0; 3900 } 3901 3902 static void virtnet_clean_affinity(struct virtnet_info *vi) 3903 { 3904 int i; 3905 3906 if (vi->affinity_hint_set) { 3907 for (i = 0; i < vi->max_queue_pairs; i++) { 3908 virtqueue_set_affinity(vi->rq[i].vq, NULL); 3909 virtqueue_set_affinity(vi->sq[i].vq, NULL); 3910 } 3911 3912 vi->affinity_hint_set = false; 3913 } 3914 } 3915 3916 static void virtnet_set_affinity(struct virtnet_info *vi) 3917 { 3918 cpumask_var_t mask; 3919 int stragglers; 3920 int group_size; 3921 int i, start = 0, cpu; 3922 int num_cpu; 3923 int stride; 3924 3925 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) { 3926 virtnet_clean_affinity(vi); 3927 return; 3928 } 3929 3930 num_cpu = num_online_cpus(); 3931 stride = max_t(int, num_cpu / vi->curr_queue_pairs, 1); 3932 stragglers = num_cpu >= vi->curr_queue_pairs ? 3933 num_cpu % vi->curr_queue_pairs : 3934 0; 3935 3936 for (i = 0; i < vi->curr_queue_pairs; i++) { 3937 group_size = stride + (i < stragglers ? 1 : 0); 3938 3939 for_each_online_cpu_wrap(cpu, start) { 3940 if (!group_size--) { 3941 start = cpu; 3942 break; 3943 } 3944 cpumask_set_cpu(cpu, mask); 3945 } 3946 3947 virtqueue_set_affinity(vi->rq[i].vq, mask); 3948 virtqueue_set_affinity(vi->sq[i].vq, mask); 3949 __netif_set_xps_queue(vi->dev, cpumask_bits(mask), i, XPS_CPUS); 3950 cpumask_clear(mask); 3951 } 3952 3953 vi->affinity_hint_set = true; 3954 free_cpumask_var(mask); 3955 } 3956 3957 static int virtnet_cpu_online(unsigned int cpu, struct hlist_node *node) 3958 { 3959 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 3960 node); 3961 virtnet_set_affinity(vi); 3962 return 0; 3963 } 3964 3965 static int virtnet_cpu_dead(unsigned int cpu, struct hlist_node *node) 3966 { 3967 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 3968 node_dead); 3969 virtnet_set_affinity(vi); 3970 return 0; 3971 } 3972 3973 static int virtnet_cpu_down_prep(unsigned int cpu, struct hlist_node *node) 3974 { 3975 struct virtnet_info *vi = hlist_entry_safe(node, struct virtnet_info, 3976 node); 3977 3978 virtnet_clean_affinity(vi); 3979 return 0; 3980 } 3981 3982 static enum cpuhp_state virtionet_online; 3983 3984 static int virtnet_cpu_notif_add(struct virtnet_info *vi) 3985 { 3986 int ret; 3987 3988 ret = cpuhp_state_add_instance_nocalls(virtionet_online, &vi->node); 3989 if (ret) 3990 return ret; 3991 ret = cpuhp_state_add_instance_nocalls(CPUHP_VIRT_NET_DEAD, 3992 &vi->node_dead); 3993 if (!ret) 3994 return ret; 3995 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 3996 return ret; 3997 } 3998 3999 static void virtnet_cpu_notif_remove(struct virtnet_info *vi) 4000 { 4001 cpuhp_state_remove_instance_nocalls(virtionet_online, &vi->node); 4002 cpuhp_state_remove_instance_nocalls(CPUHP_VIRT_NET_DEAD, 4003 &vi->node_dead); 4004 } 4005 4006 static int virtnet_send_ctrl_coal_vq_cmd(struct virtnet_info *vi, 4007 u16 vqn, u32 max_usecs, u32 max_packets) 4008 { 4009 struct virtio_net_ctrl_coal_vq *coal_vq __free(kfree) = NULL; 4010 struct scatterlist sgs; 4011 4012 coal_vq = kzalloc(sizeof(*coal_vq), GFP_KERNEL); 4013 if (!coal_vq) 4014 return -ENOMEM; 4015 4016 coal_vq->vqn = cpu_to_le16(vqn); 4017 coal_vq->coal.max_usecs = cpu_to_le32(max_usecs); 4018 coal_vq->coal.max_packets = cpu_to_le32(max_packets); 4019 sg_init_one(&sgs, coal_vq, sizeof(*coal_vq)); 4020 4021 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 4022 VIRTIO_NET_CTRL_NOTF_COAL_VQ_SET, 4023 &sgs)) 4024 return -EINVAL; 4025 4026 return 0; 4027 } 4028 4029 static int virtnet_send_rx_ctrl_coal_vq_cmd(struct virtnet_info *vi, 4030 u16 queue, u32 max_usecs, 4031 u32 max_packets) 4032 { 4033 int err; 4034 4035 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 4036 return -EOPNOTSUPP; 4037 4038 err = virtnet_send_ctrl_coal_vq_cmd(vi, rxq2vq(queue), 4039 max_usecs, max_packets); 4040 if (err) 4041 return err; 4042 4043 vi->rq[queue].intr_coal.max_usecs = max_usecs; 4044 vi->rq[queue].intr_coal.max_packets = max_packets; 4045 4046 return 0; 4047 } 4048 4049 static int virtnet_send_tx_ctrl_coal_vq_cmd(struct virtnet_info *vi, 4050 u16 queue, u32 max_usecs, 4051 u32 max_packets) 4052 { 4053 int err; 4054 4055 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 4056 return -EOPNOTSUPP; 4057 4058 err = virtnet_send_ctrl_coal_vq_cmd(vi, txq2vq(queue), 4059 max_usecs, max_packets); 4060 if (err) 4061 return err; 4062 4063 vi->sq[queue].intr_coal.max_usecs = max_usecs; 4064 vi->sq[queue].intr_coal.max_packets = max_packets; 4065 4066 return 0; 4067 } 4068 4069 static void virtnet_get_ringparam(struct net_device *dev, 4070 struct ethtool_ringparam *ring, 4071 struct kernel_ethtool_ringparam *kernel_ring, 4072 struct netlink_ext_ack *extack) 4073 { 4074 struct virtnet_info *vi = netdev_priv(dev); 4075 4076 ring->rx_max_pending = vi->rq[0].vq->num_max; 4077 ring->tx_max_pending = vi->sq[0].vq->num_max; 4078 ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 4079 ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 4080 } 4081 4082 static int virtnet_set_ringparam(struct net_device *dev, 4083 struct ethtool_ringparam *ring, 4084 struct kernel_ethtool_ringparam *kernel_ring, 4085 struct netlink_ext_ack *extack) 4086 { 4087 struct virtnet_info *vi = netdev_priv(dev); 4088 u32 rx_pending, tx_pending; 4089 struct receive_queue *rq; 4090 struct send_queue *sq; 4091 int i, err; 4092 4093 if (ring->rx_mini_pending || ring->rx_jumbo_pending) 4094 return -EINVAL; 4095 4096 rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); 4097 tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); 4098 4099 if (ring->rx_pending == rx_pending && 4100 ring->tx_pending == tx_pending) 4101 return 0; 4102 4103 if (ring->rx_pending > vi->rq[0].vq->num_max) 4104 return -EINVAL; 4105 4106 if (ring->tx_pending > vi->sq[0].vq->num_max) 4107 return -EINVAL; 4108 4109 for (i = 0; i < vi->max_queue_pairs; i++) { 4110 rq = vi->rq + i; 4111 sq = vi->sq + i; 4112 4113 if (ring->tx_pending != tx_pending) { 4114 err = virtnet_tx_resize(vi, sq, ring->tx_pending); 4115 if (err) 4116 return err; 4117 4118 /* Upon disabling and re-enabling a transmit virtqueue, the device must 4119 * set the coalescing parameters of the virtqueue to those configured 4120 * through the VIRTIO_NET_CTRL_NOTF_COAL_TX_SET command, or, if the driver 4121 * did not set any TX coalescing parameters, to 0. 4122 */ 4123 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, i, 4124 vi->intr_coal_tx.max_usecs, 4125 vi->intr_coal_tx.max_packets); 4126 4127 /* Don't break the tx resize action if the vq coalescing is not 4128 * supported. The same is true for rx resize below. 4129 */ 4130 if (err && err != -EOPNOTSUPP) 4131 return err; 4132 } 4133 4134 if (ring->rx_pending != rx_pending) { 4135 err = virtnet_rx_resize(vi, rq, ring->rx_pending); 4136 if (err) 4137 return err; 4138 4139 /* The reason is same as the transmit virtqueue reset */ 4140 mutex_lock(&vi->rq[i].dim_lock); 4141 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, i, 4142 vi->intr_coal_rx.max_usecs, 4143 vi->intr_coal_rx.max_packets); 4144 mutex_unlock(&vi->rq[i].dim_lock); 4145 if (err && err != -EOPNOTSUPP) 4146 return err; 4147 } 4148 } 4149 4150 return 0; 4151 } 4152 4153 static bool virtnet_commit_rss_command(struct virtnet_info *vi) 4154 { 4155 struct net_device *dev = vi->dev; 4156 struct scatterlist sgs[2]; 4157 4158 /* prepare sgs */ 4159 sg_init_table(sgs, 2); 4160 sg_set_buf(&sgs[0], vi->rss_hdr, virtnet_rss_hdr_size(vi)); 4161 sg_set_buf(&sgs[1], &vi->rss_trailer, virtnet_rss_trailer_size(vi)); 4162 4163 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MQ, 4164 vi->has_rss ? VIRTIO_NET_CTRL_MQ_RSS_CONFIG 4165 : VIRTIO_NET_CTRL_MQ_HASH_CONFIG, sgs)) 4166 goto err; 4167 4168 return true; 4169 4170 err: 4171 dev_warn(&dev->dev, "VIRTIONET issue with committing RSS sgs\n"); 4172 return false; 4173 4174 } 4175 4176 static void virtnet_init_default_rss(struct virtnet_info *vi) 4177 { 4178 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_supported); 4179 vi->rss_hash_types_saved = vi->rss_hash_types_supported; 4180 vi->rss_hdr->indirection_table_mask = vi->rss_indir_table_size 4181 ? cpu_to_le16(vi->rss_indir_table_size - 1) : 0; 4182 vi->rss_hdr->unclassified_queue = 0; 4183 4184 virtnet_rss_update_by_qpairs(vi, vi->curr_queue_pairs); 4185 4186 vi->rss_trailer.hash_key_length = vi->rss_key_size; 4187 4188 netdev_rss_key_fill(vi->rss_hash_key_data, vi->rss_key_size); 4189 } 4190 4191 static void virtnet_get_hashflow(const struct virtnet_info *vi, struct ethtool_rxnfc *info) 4192 { 4193 info->data = 0; 4194 switch (info->flow_type) { 4195 case TCP_V4_FLOW: 4196 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv4) { 4197 info->data = RXH_IP_SRC | RXH_IP_DST | 4198 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4199 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 4200 info->data = RXH_IP_SRC | RXH_IP_DST; 4201 } 4202 break; 4203 case TCP_V6_FLOW: 4204 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_TCPv6) { 4205 info->data = RXH_IP_SRC | RXH_IP_DST | 4206 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4207 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 4208 info->data = RXH_IP_SRC | RXH_IP_DST; 4209 } 4210 break; 4211 case UDP_V4_FLOW: 4212 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv4) { 4213 info->data = RXH_IP_SRC | RXH_IP_DST | 4214 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4215 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) { 4216 info->data = RXH_IP_SRC | RXH_IP_DST; 4217 } 4218 break; 4219 case UDP_V6_FLOW: 4220 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_UDPv6) { 4221 info->data = RXH_IP_SRC | RXH_IP_DST | 4222 RXH_L4_B_0_1 | RXH_L4_B_2_3; 4223 } else if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) { 4224 info->data = RXH_IP_SRC | RXH_IP_DST; 4225 } 4226 break; 4227 case IPV4_FLOW: 4228 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv4) 4229 info->data = RXH_IP_SRC | RXH_IP_DST; 4230 4231 break; 4232 case IPV6_FLOW: 4233 if (vi->rss_hash_types_saved & VIRTIO_NET_RSS_HASH_TYPE_IPv6) 4234 info->data = RXH_IP_SRC | RXH_IP_DST; 4235 4236 break; 4237 default: 4238 info->data = 0; 4239 break; 4240 } 4241 } 4242 4243 static bool virtnet_set_hashflow(struct virtnet_info *vi, struct ethtool_rxnfc *info) 4244 { 4245 u32 new_hashtypes = vi->rss_hash_types_saved; 4246 bool is_disable = info->data & RXH_DISCARD; 4247 bool is_l4 = info->data == (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3); 4248 4249 /* supports only 'sd', 'sdfn' and 'r' */ 4250 if (!((info->data == (RXH_IP_SRC | RXH_IP_DST)) | is_l4 | is_disable)) 4251 return false; 4252 4253 switch (info->flow_type) { 4254 case TCP_V4_FLOW: 4255 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_TCPv4); 4256 if (!is_disable) 4257 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 4258 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv4 : 0); 4259 break; 4260 case UDP_V4_FLOW: 4261 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv4 | VIRTIO_NET_RSS_HASH_TYPE_UDPv4); 4262 if (!is_disable) 4263 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv4 4264 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv4 : 0); 4265 break; 4266 case IPV4_FLOW: 4267 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv4; 4268 if (!is_disable) 4269 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv4; 4270 break; 4271 case TCP_V6_FLOW: 4272 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_TCPv6); 4273 if (!is_disable) 4274 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 4275 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_TCPv6 : 0); 4276 break; 4277 case UDP_V6_FLOW: 4278 new_hashtypes &= ~(VIRTIO_NET_RSS_HASH_TYPE_IPv6 | VIRTIO_NET_RSS_HASH_TYPE_UDPv6); 4279 if (!is_disable) 4280 new_hashtypes |= VIRTIO_NET_RSS_HASH_TYPE_IPv6 4281 | (is_l4 ? VIRTIO_NET_RSS_HASH_TYPE_UDPv6 : 0); 4282 break; 4283 case IPV6_FLOW: 4284 new_hashtypes &= ~VIRTIO_NET_RSS_HASH_TYPE_IPv6; 4285 if (!is_disable) 4286 new_hashtypes = VIRTIO_NET_RSS_HASH_TYPE_IPv6; 4287 break; 4288 default: 4289 /* unsupported flow */ 4290 return false; 4291 } 4292 4293 /* if unsupported hashtype was set */ 4294 if (new_hashtypes != (new_hashtypes & vi->rss_hash_types_supported)) 4295 return false; 4296 4297 if (new_hashtypes != vi->rss_hash_types_saved) { 4298 vi->rss_hash_types_saved = new_hashtypes; 4299 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_saved); 4300 if (vi->dev->features & NETIF_F_RXHASH) 4301 return virtnet_commit_rss_command(vi); 4302 } 4303 4304 return true; 4305 } 4306 4307 static void virtnet_get_drvinfo(struct net_device *dev, 4308 struct ethtool_drvinfo *info) 4309 { 4310 struct virtnet_info *vi = netdev_priv(dev); 4311 struct virtio_device *vdev = vi->vdev; 4312 4313 strscpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 4314 strscpy(info->version, VIRTNET_DRIVER_VERSION, sizeof(info->version)); 4315 strscpy(info->bus_info, virtio_bus_name(vdev), sizeof(info->bus_info)); 4316 4317 } 4318 4319 /* TODO: Eliminate OOO packets during switching */ 4320 static int virtnet_set_channels(struct net_device *dev, 4321 struct ethtool_channels *channels) 4322 { 4323 struct virtnet_info *vi = netdev_priv(dev); 4324 u16 queue_pairs = channels->combined_count; 4325 int err; 4326 4327 /* We don't support separate rx/tx channels. 4328 * We don't allow setting 'other' channels. 4329 */ 4330 if (channels->rx_count || channels->tx_count || channels->other_count) 4331 return -EINVAL; 4332 4333 if (queue_pairs > vi->max_queue_pairs || queue_pairs == 0) 4334 return -EINVAL; 4335 4336 /* For now we don't support modifying channels while XDP is loaded 4337 * also when XDP is loaded all RX queues have XDP programs so we only 4338 * need to check a single RX queue. 4339 */ 4340 if (vi->rq[0].xdp_prog) 4341 return -EINVAL; 4342 4343 cpus_read_lock(); 4344 err = virtnet_set_queues(vi, queue_pairs); 4345 if (err) { 4346 cpus_read_unlock(); 4347 goto err; 4348 } 4349 virtnet_set_affinity(vi); 4350 cpus_read_unlock(); 4351 4352 netif_set_real_num_tx_queues(dev, queue_pairs); 4353 netif_set_real_num_rx_queues(dev, queue_pairs); 4354 err: 4355 return err; 4356 } 4357 4358 static void virtnet_stats_sprintf(u8 **p, const char *fmt, const char *noq_fmt, 4359 int num, int qid, const struct virtnet_stat_desc *desc) 4360 { 4361 int i; 4362 4363 if (qid < 0) { 4364 for (i = 0; i < num; ++i) 4365 ethtool_sprintf(p, noq_fmt, desc[i].desc); 4366 } else { 4367 for (i = 0; i < num; ++i) 4368 ethtool_sprintf(p, fmt, qid, desc[i].desc); 4369 } 4370 } 4371 4372 /* qid == -1: for rx/tx queue total field */ 4373 static void virtnet_get_stats_string(struct virtnet_info *vi, int type, int qid, u8 **data) 4374 { 4375 const struct virtnet_stat_desc *desc; 4376 const char *fmt, *noq_fmt; 4377 u8 *p = *data; 4378 u32 num; 4379 4380 if (type == VIRTNET_Q_TYPE_CQ && qid >= 0) { 4381 noq_fmt = "cq_hw_%s"; 4382 4383 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) { 4384 desc = &virtnet_stats_cvq_desc[0]; 4385 num = ARRAY_SIZE(virtnet_stats_cvq_desc); 4386 4387 virtnet_stats_sprintf(&p, NULL, noq_fmt, num, -1, desc); 4388 } 4389 } 4390 4391 if (type == VIRTNET_Q_TYPE_RX) { 4392 fmt = "rx%u_%s"; 4393 noq_fmt = "rx_%s"; 4394 4395 desc = &virtnet_rq_stats_desc[0]; 4396 num = ARRAY_SIZE(virtnet_rq_stats_desc); 4397 4398 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4399 4400 fmt = "rx%u_hw_%s"; 4401 noq_fmt = "rx_hw_%s"; 4402 4403 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4404 desc = &virtnet_stats_rx_basic_desc[0]; 4405 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc); 4406 4407 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4408 } 4409 4410 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4411 desc = &virtnet_stats_rx_csum_desc[0]; 4412 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc); 4413 4414 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4415 } 4416 4417 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4418 desc = &virtnet_stats_rx_speed_desc[0]; 4419 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc); 4420 4421 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4422 } 4423 } 4424 4425 if (type == VIRTNET_Q_TYPE_TX) { 4426 fmt = "tx%u_%s"; 4427 noq_fmt = "tx_%s"; 4428 4429 desc = &virtnet_sq_stats_desc[0]; 4430 num = ARRAY_SIZE(virtnet_sq_stats_desc); 4431 4432 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4433 4434 fmt = "tx%u_hw_%s"; 4435 noq_fmt = "tx_hw_%s"; 4436 4437 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4438 desc = &virtnet_stats_tx_basic_desc[0]; 4439 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc); 4440 4441 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4442 } 4443 4444 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4445 desc = &virtnet_stats_tx_gso_desc[0]; 4446 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc); 4447 4448 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4449 } 4450 4451 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4452 desc = &virtnet_stats_tx_speed_desc[0]; 4453 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc); 4454 4455 virtnet_stats_sprintf(&p, fmt, noq_fmt, num, qid, desc); 4456 } 4457 } 4458 4459 *data = p; 4460 } 4461 4462 struct virtnet_stats_ctx { 4463 /* The stats are write to qstats or ethtool -S */ 4464 bool to_qstat; 4465 4466 /* Used to calculate the offset inside the output buffer. */ 4467 u32 desc_num[3]; 4468 4469 /* The actual supported stat types. */ 4470 u64 bitmap[3]; 4471 4472 /* Used to calculate the reply buffer size. */ 4473 u32 size[3]; 4474 4475 /* Record the output buffer. */ 4476 u64 *data; 4477 }; 4478 4479 static void virtnet_stats_ctx_init(struct virtnet_info *vi, 4480 struct virtnet_stats_ctx *ctx, 4481 u64 *data, bool to_qstat) 4482 { 4483 u32 queue_type; 4484 4485 ctx->data = data; 4486 ctx->to_qstat = to_qstat; 4487 4488 if (to_qstat) { 4489 ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc_qstat); 4490 ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc_qstat); 4491 4492 queue_type = VIRTNET_Q_TYPE_RX; 4493 4494 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4495 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_BASIC; 4496 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat); 4497 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_basic); 4498 } 4499 4500 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4501 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_CSUM; 4502 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat); 4503 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_csum); 4504 } 4505 4506 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) { 4507 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_GSO; 4508 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat); 4509 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_gso); 4510 } 4511 4512 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4513 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_SPEED; 4514 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat); 4515 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_speed); 4516 } 4517 4518 queue_type = VIRTNET_Q_TYPE_TX; 4519 4520 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4521 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_BASIC; 4522 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat); 4523 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_basic); 4524 } 4525 4526 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { 4527 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_CSUM; 4528 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat); 4529 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_csum); 4530 } 4531 4532 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4533 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_GSO; 4534 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat); 4535 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_gso); 4536 } 4537 4538 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4539 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_SPEED; 4540 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat); 4541 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_speed); 4542 } 4543 4544 return; 4545 } 4546 4547 ctx->desc_num[VIRTNET_Q_TYPE_RX] = ARRAY_SIZE(virtnet_rq_stats_desc); 4548 ctx->desc_num[VIRTNET_Q_TYPE_TX] = ARRAY_SIZE(virtnet_sq_stats_desc); 4549 4550 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_CVQ) { 4551 queue_type = VIRTNET_Q_TYPE_CQ; 4552 4553 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_CVQ; 4554 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_cvq_desc); 4555 ctx->size[queue_type] += sizeof(struct virtio_net_stats_cvq); 4556 } 4557 4558 queue_type = VIRTNET_Q_TYPE_RX; 4559 4560 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4561 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_BASIC; 4562 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_basic_desc); 4563 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_basic); 4564 } 4565 4566 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4567 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_CSUM; 4568 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_csum_desc); 4569 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_csum); 4570 } 4571 4572 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4573 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_RX_SPEED; 4574 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_rx_speed_desc); 4575 ctx->size[queue_type] += sizeof(struct virtio_net_stats_rx_speed); 4576 } 4577 4578 queue_type = VIRTNET_Q_TYPE_TX; 4579 4580 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4581 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_BASIC; 4582 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_basic_desc); 4583 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_basic); 4584 } 4585 4586 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4587 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_GSO; 4588 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_gso_desc); 4589 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_gso); 4590 } 4591 4592 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4593 ctx->bitmap[queue_type] |= VIRTIO_NET_STATS_TYPE_TX_SPEED; 4594 ctx->desc_num[queue_type] += ARRAY_SIZE(virtnet_stats_tx_speed_desc); 4595 ctx->size[queue_type] += sizeof(struct virtio_net_stats_tx_speed); 4596 } 4597 } 4598 4599 /* stats_sum_queue - Calculate the sum of the same fields in sq or rq. 4600 * @sum: the position to store the sum values 4601 * @num: field num 4602 * @q_value: the first queue fields 4603 * @q_num: number of the queues 4604 */ 4605 static void stats_sum_queue(u64 *sum, u32 num, u64 *q_value, u32 q_num) 4606 { 4607 u32 step = num; 4608 int i, j; 4609 u64 *p; 4610 4611 for (i = 0; i < num; ++i) { 4612 p = sum + i; 4613 *p = 0; 4614 4615 for (j = 0; j < q_num; ++j) 4616 *p += *(q_value + i + j * step); 4617 } 4618 } 4619 4620 static void virtnet_fill_total_fields(struct virtnet_info *vi, 4621 struct virtnet_stats_ctx *ctx) 4622 { 4623 u64 *data, *first_rx_q, *first_tx_q; 4624 u32 num_cq, num_rx, num_tx; 4625 4626 num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ]; 4627 num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX]; 4628 num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX]; 4629 4630 first_rx_q = ctx->data + num_rx + num_tx + num_cq; 4631 first_tx_q = first_rx_q + vi->curr_queue_pairs * num_rx; 4632 4633 data = ctx->data; 4634 4635 stats_sum_queue(data, num_rx, first_rx_q, vi->curr_queue_pairs); 4636 4637 data = ctx->data + num_rx; 4638 4639 stats_sum_queue(data, num_tx, first_tx_q, vi->curr_queue_pairs); 4640 } 4641 4642 static void virtnet_fill_stats_qstat(struct virtnet_info *vi, u32 qid, 4643 struct virtnet_stats_ctx *ctx, 4644 const u8 *base, bool drv_stats, u8 reply_type) 4645 { 4646 const struct virtnet_stat_desc *desc; 4647 const u64_stats_t *v_stat; 4648 u64 offset, bitmap; 4649 const __le64 *v; 4650 u32 queue_type; 4651 int i, num; 4652 4653 queue_type = vq_type(vi, qid); 4654 bitmap = ctx->bitmap[queue_type]; 4655 4656 if (drv_stats) { 4657 if (queue_type == VIRTNET_Q_TYPE_RX) { 4658 desc = &virtnet_rq_stats_desc_qstat[0]; 4659 num = ARRAY_SIZE(virtnet_rq_stats_desc_qstat); 4660 } else { 4661 desc = &virtnet_sq_stats_desc_qstat[0]; 4662 num = ARRAY_SIZE(virtnet_sq_stats_desc_qstat); 4663 } 4664 4665 for (i = 0; i < num; ++i) { 4666 offset = desc[i].qstat_offset / sizeof(*ctx->data); 4667 v_stat = (const u64_stats_t *)(base + desc[i].offset); 4668 ctx->data[offset] = u64_stats_read(v_stat); 4669 } 4670 return; 4671 } 4672 4673 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4674 desc = &virtnet_stats_rx_basic_desc_qstat[0]; 4675 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc_qstat); 4676 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC) 4677 goto found; 4678 } 4679 4680 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4681 desc = &virtnet_stats_rx_csum_desc_qstat[0]; 4682 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc_qstat); 4683 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM) 4684 goto found; 4685 } 4686 4687 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_GSO) { 4688 desc = &virtnet_stats_rx_gso_desc_qstat[0]; 4689 num = ARRAY_SIZE(virtnet_stats_rx_gso_desc_qstat); 4690 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_GSO) 4691 goto found; 4692 } 4693 4694 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4695 desc = &virtnet_stats_rx_speed_desc_qstat[0]; 4696 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc_qstat); 4697 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED) 4698 goto found; 4699 } 4700 4701 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4702 desc = &virtnet_stats_tx_basic_desc_qstat[0]; 4703 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc_qstat); 4704 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC) 4705 goto found; 4706 } 4707 4708 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { 4709 desc = &virtnet_stats_tx_csum_desc_qstat[0]; 4710 num = ARRAY_SIZE(virtnet_stats_tx_csum_desc_qstat); 4711 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_CSUM) 4712 goto found; 4713 } 4714 4715 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4716 desc = &virtnet_stats_tx_gso_desc_qstat[0]; 4717 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc_qstat); 4718 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO) 4719 goto found; 4720 } 4721 4722 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4723 desc = &virtnet_stats_tx_speed_desc_qstat[0]; 4724 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc_qstat); 4725 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED) 4726 goto found; 4727 } 4728 4729 return; 4730 4731 found: 4732 for (i = 0; i < num; ++i) { 4733 offset = desc[i].qstat_offset / sizeof(*ctx->data); 4734 v = (const __le64 *)(base + desc[i].offset); 4735 ctx->data[offset] = le64_to_cpu(*v); 4736 } 4737 } 4738 4739 /* virtnet_fill_stats - copy the stats to qstats or ethtool -S 4740 * The stats source is the device or the driver. 4741 * 4742 * @vi: virtio net info 4743 * @qid: the vq id 4744 * @ctx: stats ctx (initiated by virtnet_stats_ctx_init()) 4745 * @base: pointer to the device reply or the driver stats structure. 4746 * @drv_stats: designate the base type (device reply, driver stats) 4747 * @type: the type of the device reply (if drv_stats is true, this must be zero) 4748 */ 4749 static void virtnet_fill_stats(struct virtnet_info *vi, u32 qid, 4750 struct virtnet_stats_ctx *ctx, 4751 const u8 *base, bool drv_stats, u8 reply_type) 4752 { 4753 u32 queue_type, num_rx, num_tx, num_cq; 4754 const struct virtnet_stat_desc *desc; 4755 const u64_stats_t *v_stat; 4756 u64 offset, bitmap; 4757 const __le64 *v; 4758 int i, num; 4759 4760 if (ctx->to_qstat) 4761 return virtnet_fill_stats_qstat(vi, qid, ctx, base, drv_stats, reply_type); 4762 4763 num_cq = ctx->desc_num[VIRTNET_Q_TYPE_CQ]; 4764 num_rx = ctx->desc_num[VIRTNET_Q_TYPE_RX]; 4765 num_tx = ctx->desc_num[VIRTNET_Q_TYPE_TX]; 4766 4767 queue_type = vq_type(vi, qid); 4768 bitmap = ctx->bitmap[queue_type]; 4769 4770 /* skip the total fields of pairs */ 4771 offset = num_rx + num_tx; 4772 4773 if (queue_type == VIRTNET_Q_TYPE_TX) { 4774 offset += num_cq + num_rx * vi->curr_queue_pairs + num_tx * (qid / 2); 4775 4776 num = ARRAY_SIZE(virtnet_sq_stats_desc); 4777 if (drv_stats) { 4778 desc = &virtnet_sq_stats_desc[0]; 4779 goto drv_stats; 4780 } 4781 4782 offset += num; 4783 4784 } else if (queue_type == VIRTNET_Q_TYPE_RX) { 4785 offset += num_cq + num_rx * (qid / 2); 4786 4787 num = ARRAY_SIZE(virtnet_rq_stats_desc); 4788 if (drv_stats) { 4789 desc = &virtnet_rq_stats_desc[0]; 4790 goto drv_stats; 4791 } 4792 4793 offset += num; 4794 } 4795 4796 if (bitmap & VIRTIO_NET_STATS_TYPE_CVQ) { 4797 desc = &virtnet_stats_cvq_desc[0]; 4798 num = ARRAY_SIZE(virtnet_stats_cvq_desc); 4799 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_CVQ) 4800 goto found; 4801 4802 offset += num; 4803 } 4804 4805 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 4806 desc = &virtnet_stats_rx_basic_desc[0]; 4807 num = ARRAY_SIZE(virtnet_stats_rx_basic_desc); 4808 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_BASIC) 4809 goto found; 4810 4811 offset += num; 4812 } 4813 4814 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 4815 desc = &virtnet_stats_rx_csum_desc[0]; 4816 num = ARRAY_SIZE(virtnet_stats_rx_csum_desc); 4817 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_CSUM) 4818 goto found; 4819 4820 offset += num; 4821 } 4822 4823 if (bitmap & VIRTIO_NET_STATS_TYPE_RX_SPEED) { 4824 desc = &virtnet_stats_rx_speed_desc[0]; 4825 num = ARRAY_SIZE(virtnet_stats_rx_speed_desc); 4826 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_RX_SPEED) 4827 goto found; 4828 4829 offset += num; 4830 } 4831 4832 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 4833 desc = &virtnet_stats_tx_basic_desc[0]; 4834 num = ARRAY_SIZE(virtnet_stats_tx_basic_desc); 4835 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_BASIC) 4836 goto found; 4837 4838 offset += num; 4839 } 4840 4841 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 4842 desc = &virtnet_stats_tx_gso_desc[0]; 4843 num = ARRAY_SIZE(virtnet_stats_tx_gso_desc); 4844 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_GSO) 4845 goto found; 4846 4847 offset += num; 4848 } 4849 4850 if (bitmap & VIRTIO_NET_STATS_TYPE_TX_SPEED) { 4851 desc = &virtnet_stats_tx_speed_desc[0]; 4852 num = ARRAY_SIZE(virtnet_stats_tx_speed_desc); 4853 if (reply_type == VIRTIO_NET_STATS_TYPE_REPLY_TX_SPEED) 4854 goto found; 4855 4856 offset += num; 4857 } 4858 4859 return; 4860 4861 found: 4862 for (i = 0; i < num; ++i) { 4863 v = (const __le64 *)(base + desc[i].offset); 4864 ctx->data[offset + i] = le64_to_cpu(*v); 4865 } 4866 4867 return; 4868 4869 drv_stats: 4870 for (i = 0; i < num; ++i) { 4871 v_stat = (const u64_stats_t *)(base + desc[i].offset); 4872 ctx->data[offset + i] = u64_stats_read(v_stat); 4873 } 4874 } 4875 4876 static int __virtnet_get_hw_stats(struct virtnet_info *vi, 4877 struct virtnet_stats_ctx *ctx, 4878 struct virtio_net_ctrl_queue_stats *req, 4879 int req_size, void *reply, int res_size) 4880 { 4881 struct virtio_net_stats_reply_hdr *hdr; 4882 struct scatterlist sgs_in, sgs_out; 4883 void *p; 4884 u32 qid; 4885 int ok; 4886 4887 sg_init_one(&sgs_out, req, req_size); 4888 sg_init_one(&sgs_in, reply, res_size); 4889 4890 ok = virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS, 4891 VIRTIO_NET_CTRL_STATS_GET, 4892 &sgs_out, &sgs_in); 4893 4894 if (!ok) 4895 return ok; 4896 4897 for (p = reply; p - reply < res_size; p += le16_to_cpu(hdr->size)) { 4898 hdr = p; 4899 qid = le16_to_cpu(hdr->vq_index); 4900 virtnet_fill_stats(vi, qid, ctx, p, false, hdr->type); 4901 } 4902 4903 return 0; 4904 } 4905 4906 static void virtnet_make_stat_req(struct virtnet_info *vi, 4907 struct virtnet_stats_ctx *ctx, 4908 struct virtio_net_ctrl_queue_stats *req, 4909 int qid, int *idx) 4910 { 4911 int qtype = vq_type(vi, qid); 4912 u64 bitmap = ctx->bitmap[qtype]; 4913 4914 if (!bitmap) 4915 return; 4916 4917 req->stats[*idx].vq_index = cpu_to_le16(qid); 4918 req->stats[*idx].types_bitmap[0] = cpu_to_le64(bitmap); 4919 *idx += 1; 4920 } 4921 4922 /* qid: -1: get stats of all vq. 4923 * > 0: get the stats for the special vq. This must not be cvq. 4924 */ 4925 static int virtnet_get_hw_stats(struct virtnet_info *vi, 4926 struct virtnet_stats_ctx *ctx, int qid) 4927 { 4928 int qnum, i, j, res_size, qtype, last_vq, first_vq; 4929 struct virtio_net_ctrl_queue_stats *req; 4930 bool enable_cvq; 4931 void *reply; 4932 int ok; 4933 4934 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) 4935 return 0; 4936 4937 if (qid == -1) { 4938 last_vq = vi->curr_queue_pairs * 2 - 1; 4939 first_vq = 0; 4940 enable_cvq = true; 4941 } else { 4942 last_vq = qid; 4943 first_vq = qid; 4944 enable_cvq = false; 4945 } 4946 4947 qnum = 0; 4948 res_size = 0; 4949 for (i = first_vq; i <= last_vq ; ++i) { 4950 qtype = vq_type(vi, i); 4951 if (ctx->bitmap[qtype]) { 4952 ++qnum; 4953 res_size += ctx->size[qtype]; 4954 } 4955 } 4956 4957 if (enable_cvq && ctx->bitmap[VIRTNET_Q_TYPE_CQ]) { 4958 res_size += ctx->size[VIRTNET_Q_TYPE_CQ]; 4959 qnum += 1; 4960 } 4961 4962 req = kcalloc(qnum, sizeof(*req), GFP_KERNEL); 4963 if (!req) 4964 return -ENOMEM; 4965 4966 reply = kmalloc(res_size, GFP_KERNEL); 4967 if (!reply) { 4968 kfree(req); 4969 return -ENOMEM; 4970 } 4971 4972 j = 0; 4973 for (i = first_vq; i <= last_vq ; ++i) 4974 virtnet_make_stat_req(vi, ctx, req, i, &j); 4975 4976 if (enable_cvq) 4977 virtnet_make_stat_req(vi, ctx, req, vi->max_queue_pairs * 2, &j); 4978 4979 ok = __virtnet_get_hw_stats(vi, ctx, req, sizeof(*req) * j, reply, res_size); 4980 4981 kfree(req); 4982 kfree(reply); 4983 4984 return ok; 4985 } 4986 4987 static void virtnet_get_strings(struct net_device *dev, u32 stringset, u8 *data) 4988 { 4989 struct virtnet_info *vi = netdev_priv(dev); 4990 unsigned int i; 4991 u8 *p = data; 4992 4993 switch (stringset) { 4994 case ETH_SS_STATS: 4995 /* Generate the total field names. */ 4996 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, -1, &p); 4997 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, -1, &p); 4998 4999 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_CQ, 0, &p); 5000 5001 for (i = 0; i < vi->curr_queue_pairs; ++i) 5002 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_RX, i, &p); 5003 5004 for (i = 0; i < vi->curr_queue_pairs; ++i) 5005 virtnet_get_stats_string(vi, VIRTNET_Q_TYPE_TX, i, &p); 5006 break; 5007 } 5008 } 5009 5010 static int virtnet_get_sset_count(struct net_device *dev, int sset) 5011 { 5012 struct virtnet_info *vi = netdev_priv(dev); 5013 struct virtnet_stats_ctx ctx = {0}; 5014 u32 pair_count; 5015 5016 switch (sset) { 5017 case ETH_SS_STATS: 5018 virtnet_stats_ctx_init(vi, &ctx, NULL, false); 5019 5020 pair_count = ctx.desc_num[VIRTNET_Q_TYPE_RX] + ctx.desc_num[VIRTNET_Q_TYPE_TX]; 5021 5022 return pair_count + ctx.desc_num[VIRTNET_Q_TYPE_CQ] + 5023 vi->curr_queue_pairs * pair_count; 5024 default: 5025 return -EOPNOTSUPP; 5026 } 5027 } 5028 5029 static void virtnet_get_ethtool_stats(struct net_device *dev, 5030 struct ethtool_stats *stats, u64 *data) 5031 { 5032 struct virtnet_info *vi = netdev_priv(dev); 5033 struct virtnet_stats_ctx ctx = {0}; 5034 unsigned int start, i; 5035 const u8 *stats_base; 5036 5037 virtnet_stats_ctx_init(vi, &ctx, data, false); 5038 if (virtnet_get_hw_stats(vi, &ctx, -1)) 5039 dev_warn(&vi->dev->dev, "Failed to get hw stats.\n"); 5040 5041 for (i = 0; i < vi->curr_queue_pairs; i++) { 5042 struct receive_queue *rq = &vi->rq[i]; 5043 struct send_queue *sq = &vi->sq[i]; 5044 5045 stats_base = (const u8 *)&rq->stats; 5046 do { 5047 start = u64_stats_fetch_begin(&rq->stats.syncp); 5048 virtnet_fill_stats(vi, i * 2, &ctx, stats_base, true, 0); 5049 } while (u64_stats_fetch_retry(&rq->stats.syncp, start)); 5050 5051 stats_base = (const u8 *)&sq->stats; 5052 do { 5053 start = u64_stats_fetch_begin(&sq->stats.syncp); 5054 virtnet_fill_stats(vi, i * 2 + 1, &ctx, stats_base, true, 0); 5055 } while (u64_stats_fetch_retry(&sq->stats.syncp, start)); 5056 } 5057 5058 virtnet_fill_total_fields(vi, &ctx); 5059 } 5060 5061 static void virtnet_get_channels(struct net_device *dev, 5062 struct ethtool_channels *channels) 5063 { 5064 struct virtnet_info *vi = netdev_priv(dev); 5065 5066 channels->combined_count = vi->curr_queue_pairs; 5067 channels->max_combined = vi->max_queue_pairs; 5068 channels->max_other = 0; 5069 channels->rx_count = 0; 5070 channels->tx_count = 0; 5071 channels->other_count = 0; 5072 } 5073 5074 static int virtnet_set_link_ksettings(struct net_device *dev, 5075 const struct ethtool_link_ksettings *cmd) 5076 { 5077 struct virtnet_info *vi = netdev_priv(dev); 5078 5079 return ethtool_virtdev_set_link_ksettings(dev, cmd, 5080 &vi->speed, &vi->duplex); 5081 } 5082 5083 static int virtnet_get_link_ksettings(struct net_device *dev, 5084 struct ethtool_link_ksettings *cmd) 5085 { 5086 struct virtnet_info *vi = netdev_priv(dev); 5087 5088 cmd->base.speed = vi->speed; 5089 cmd->base.duplex = vi->duplex; 5090 cmd->base.port = PORT_OTHER; 5091 5092 return 0; 5093 } 5094 5095 static int virtnet_send_tx_notf_coal_cmds(struct virtnet_info *vi, 5096 struct ethtool_coalesce *ec) 5097 { 5098 struct virtio_net_ctrl_coal_tx *coal_tx __free(kfree) = NULL; 5099 struct scatterlist sgs_tx; 5100 int i; 5101 5102 coal_tx = kzalloc(sizeof(*coal_tx), GFP_KERNEL); 5103 if (!coal_tx) 5104 return -ENOMEM; 5105 5106 coal_tx->tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs); 5107 coal_tx->tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames); 5108 sg_init_one(&sgs_tx, coal_tx, sizeof(*coal_tx)); 5109 5110 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 5111 VIRTIO_NET_CTRL_NOTF_COAL_TX_SET, 5112 &sgs_tx)) 5113 return -EINVAL; 5114 5115 vi->intr_coal_tx.max_usecs = ec->tx_coalesce_usecs; 5116 vi->intr_coal_tx.max_packets = ec->tx_max_coalesced_frames; 5117 for (i = 0; i < vi->max_queue_pairs; i++) { 5118 vi->sq[i].intr_coal.max_usecs = ec->tx_coalesce_usecs; 5119 vi->sq[i].intr_coal.max_packets = ec->tx_max_coalesced_frames; 5120 } 5121 5122 return 0; 5123 } 5124 5125 static int virtnet_send_rx_notf_coal_cmds(struct virtnet_info *vi, 5126 struct ethtool_coalesce *ec) 5127 { 5128 struct virtio_net_ctrl_coal_rx *coal_rx __free(kfree) = NULL; 5129 bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce; 5130 struct scatterlist sgs_rx; 5131 int i; 5132 5133 if (rx_ctrl_dim_on && !virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 5134 return -EOPNOTSUPP; 5135 5136 if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != vi->intr_coal_rx.max_usecs || 5137 ec->rx_max_coalesced_frames != vi->intr_coal_rx.max_packets)) 5138 return -EINVAL; 5139 5140 if (rx_ctrl_dim_on && !vi->rx_dim_enabled) { 5141 vi->rx_dim_enabled = true; 5142 for (i = 0; i < vi->max_queue_pairs; i++) { 5143 mutex_lock(&vi->rq[i].dim_lock); 5144 vi->rq[i].dim_enabled = true; 5145 mutex_unlock(&vi->rq[i].dim_lock); 5146 } 5147 return 0; 5148 } 5149 5150 coal_rx = kzalloc(sizeof(*coal_rx), GFP_KERNEL); 5151 if (!coal_rx) 5152 return -ENOMEM; 5153 5154 if (!rx_ctrl_dim_on && vi->rx_dim_enabled) { 5155 vi->rx_dim_enabled = false; 5156 for (i = 0; i < vi->max_queue_pairs; i++) { 5157 mutex_lock(&vi->rq[i].dim_lock); 5158 vi->rq[i].dim_enabled = false; 5159 mutex_unlock(&vi->rq[i].dim_lock); 5160 } 5161 } 5162 5163 /* Since the per-queue coalescing params can be set, 5164 * we need apply the global new params even if they 5165 * are not updated. 5166 */ 5167 coal_rx->rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs); 5168 coal_rx->rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames); 5169 sg_init_one(&sgs_rx, coal_rx, sizeof(*coal_rx)); 5170 5171 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, 5172 VIRTIO_NET_CTRL_NOTF_COAL_RX_SET, 5173 &sgs_rx)) 5174 return -EINVAL; 5175 5176 vi->intr_coal_rx.max_usecs = ec->rx_coalesce_usecs; 5177 vi->intr_coal_rx.max_packets = ec->rx_max_coalesced_frames; 5178 for (i = 0; i < vi->max_queue_pairs; i++) { 5179 mutex_lock(&vi->rq[i].dim_lock); 5180 vi->rq[i].intr_coal.max_usecs = ec->rx_coalesce_usecs; 5181 vi->rq[i].intr_coal.max_packets = ec->rx_max_coalesced_frames; 5182 mutex_unlock(&vi->rq[i].dim_lock); 5183 } 5184 5185 return 0; 5186 } 5187 5188 static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, 5189 struct ethtool_coalesce *ec) 5190 { 5191 int err; 5192 5193 err = virtnet_send_tx_notf_coal_cmds(vi, ec); 5194 if (err) 5195 return err; 5196 5197 err = virtnet_send_rx_notf_coal_cmds(vi, ec); 5198 if (err) 5199 return err; 5200 5201 return 0; 5202 } 5203 5204 static int virtnet_send_rx_notf_coal_vq_cmds(struct virtnet_info *vi, 5205 struct ethtool_coalesce *ec, 5206 u16 queue) 5207 { 5208 bool rx_ctrl_dim_on = !!ec->use_adaptive_rx_coalesce; 5209 u32 max_usecs, max_packets; 5210 bool cur_rx_dim; 5211 int err; 5212 5213 mutex_lock(&vi->rq[queue].dim_lock); 5214 cur_rx_dim = vi->rq[queue].dim_enabled; 5215 max_usecs = vi->rq[queue].intr_coal.max_usecs; 5216 max_packets = vi->rq[queue].intr_coal.max_packets; 5217 5218 if (rx_ctrl_dim_on && (ec->rx_coalesce_usecs != max_usecs || 5219 ec->rx_max_coalesced_frames != max_packets)) { 5220 mutex_unlock(&vi->rq[queue].dim_lock); 5221 return -EINVAL; 5222 } 5223 5224 if (rx_ctrl_dim_on && !cur_rx_dim) { 5225 vi->rq[queue].dim_enabled = true; 5226 mutex_unlock(&vi->rq[queue].dim_lock); 5227 return 0; 5228 } 5229 5230 if (!rx_ctrl_dim_on && cur_rx_dim) 5231 vi->rq[queue].dim_enabled = false; 5232 5233 /* If no params are updated, userspace ethtool will 5234 * reject the modification. 5235 */ 5236 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, queue, 5237 ec->rx_coalesce_usecs, 5238 ec->rx_max_coalesced_frames); 5239 mutex_unlock(&vi->rq[queue].dim_lock); 5240 return err; 5241 } 5242 5243 static int virtnet_send_notf_coal_vq_cmds(struct virtnet_info *vi, 5244 struct ethtool_coalesce *ec, 5245 u16 queue) 5246 { 5247 int err; 5248 5249 err = virtnet_send_rx_notf_coal_vq_cmds(vi, ec, queue); 5250 if (err) 5251 return err; 5252 5253 err = virtnet_send_tx_ctrl_coal_vq_cmd(vi, queue, 5254 ec->tx_coalesce_usecs, 5255 ec->tx_max_coalesced_frames); 5256 if (err) 5257 return err; 5258 5259 return 0; 5260 } 5261 5262 static void virtnet_rx_dim_work(struct work_struct *work) 5263 { 5264 struct dim *dim = container_of(work, struct dim, work); 5265 struct receive_queue *rq = container_of(dim, 5266 struct receive_queue, dim); 5267 struct virtnet_info *vi = rq->vq->vdev->priv; 5268 struct net_device *dev = vi->dev; 5269 struct dim_cq_moder update_moder; 5270 int qnum, err; 5271 5272 qnum = rq - vi->rq; 5273 5274 mutex_lock(&rq->dim_lock); 5275 if (!rq->dim_enabled) 5276 goto out; 5277 5278 update_moder = net_dim_get_rx_irq_moder(dev, dim); 5279 if (update_moder.usec != rq->intr_coal.max_usecs || 5280 update_moder.pkts != rq->intr_coal.max_packets) { 5281 err = virtnet_send_rx_ctrl_coal_vq_cmd(vi, qnum, 5282 update_moder.usec, 5283 update_moder.pkts); 5284 if (err) 5285 pr_debug("%s: Failed to send dim parameters on rxq%d\n", 5286 dev->name, qnum); 5287 } 5288 out: 5289 dim->state = DIM_START_MEASURE; 5290 mutex_unlock(&rq->dim_lock); 5291 } 5292 5293 static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) 5294 { 5295 /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL 5296 * or VIRTIO_NET_F_VQ_NOTF_COAL feature is negotiated. 5297 */ 5298 if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs) 5299 return -EOPNOTSUPP; 5300 5301 if (ec->tx_max_coalesced_frames > 1 || 5302 ec->rx_max_coalesced_frames != 1) 5303 return -EINVAL; 5304 5305 return 0; 5306 } 5307 5308 static int virtnet_should_update_vq_weight(int dev_flags, int weight, 5309 int vq_weight, bool *should_update) 5310 { 5311 if (weight ^ vq_weight) { 5312 if (dev_flags & IFF_UP) 5313 return -EBUSY; 5314 *should_update = true; 5315 } 5316 5317 return 0; 5318 } 5319 5320 static int virtnet_set_coalesce(struct net_device *dev, 5321 struct ethtool_coalesce *ec, 5322 struct kernel_ethtool_coalesce *kernel_coal, 5323 struct netlink_ext_ack *extack) 5324 { 5325 struct virtnet_info *vi = netdev_priv(dev); 5326 int ret, queue_number, napi_weight, i; 5327 bool update_napi = false; 5328 5329 /* Can't change NAPI weight if the link is up */ 5330 napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 5331 for (queue_number = 0; queue_number < vi->max_queue_pairs; queue_number++) { 5332 ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, 5333 vi->sq[queue_number].napi.weight, 5334 &update_napi); 5335 if (ret) 5336 return ret; 5337 5338 if (update_napi) { 5339 /* All queues that belong to [queue_number, vi->max_queue_pairs] will be 5340 * updated for the sake of simplicity, which might not be necessary 5341 */ 5342 break; 5343 } 5344 } 5345 5346 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) 5347 ret = virtnet_send_notf_coal_cmds(vi, ec); 5348 else 5349 ret = virtnet_coal_params_supported(ec); 5350 5351 if (ret) 5352 return ret; 5353 5354 if (update_napi) { 5355 /* xsk xmit depends on the tx napi. So if xsk is active, 5356 * prevent modifications to tx napi. 5357 */ 5358 for (i = queue_number; i < vi->max_queue_pairs; i++) { 5359 if (vi->sq[i].xsk_pool) 5360 return -EBUSY; 5361 } 5362 5363 for (; queue_number < vi->max_queue_pairs; queue_number++) 5364 vi->sq[queue_number].napi.weight = napi_weight; 5365 } 5366 5367 return ret; 5368 } 5369 5370 static int virtnet_get_coalesce(struct net_device *dev, 5371 struct ethtool_coalesce *ec, 5372 struct kernel_ethtool_coalesce *kernel_coal, 5373 struct netlink_ext_ack *extack) 5374 { 5375 struct virtnet_info *vi = netdev_priv(dev); 5376 5377 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 5378 ec->rx_coalesce_usecs = vi->intr_coal_rx.max_usecs; 5379 ec->tx_coalesce_usecs = vi->intr_coal_tx.max_usecs; 5380 ec->tx_max_coalesced_frames = vi->intr_coal_tx.max_packets; 5381 ec->rx_max_coalesced_frames = vi->intr_coal_rx.max_packets; 5382 ec->use_adaptive_rx_coalesce = vi->rx_dim_enabled; 5383 } else { 5384 ec->rx_max_coalesced_frames = 1; 5385 5386 if (vi->sq[0].napi.weight) 5387 ec->tx_max_coalesced_frames = 1; 5388 } 5389 5390 return 0; 5391 } 5392 5393 static int virtnet_set_per_queue_coalesce(struct net_device *dev, 5394 u32 queue, 5395 struct ethtool_coalesce *ec) 5396 { 5397 struct virtnet_info *vi = netdev_priv(dev); 5398 int ret, napi_weight; 5399 bool update_napi = false; 5400 5401 if (queue >= vi->max_queue_pairs) 5402 return -EINVAL; 5403 5404 /* Can't change NAPI weight if the link is up */ 5405 napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; 5406 ret = virtnet_should_update_vq_weight(dev->flags, napi_weight, 5407 vi->sq[queue].napi.weight, 5408 &update_napi); 5409 if (ret) 5410 return ret; 5411 5412 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 5413 ret = virtnet_send_notf_coal_vq_cmds(vi, ec, queue); 5414 else 5415 ret = virtnet_coal_params_supported(ec); 5416 5417 if (ret) 5418 return ret; 5419 5420 if (update_napi) 5421 vi->sq[queue].napi.weight = napi_weight; 5422 5423 return 0; 5424 } 5425 5426 static int virtnet_get_per_queue_coalesce(struct net_device *dev, 5427 u32 queue, 5428 struct ethtool_coalesce *ec) 5429 { 5430 struct virtnet_info *vi = netdev_priv(dev); 5431 5432 if (queue >= vi->max_queue_pairs) 5433 return -EINVAL; 5434 5435 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { 5436 mutex_lock(&vi->rq[queue].dim_lock); 5437 ec->rx_coalesce_usecs = vi->rq[queue].intr_coal.max_usecs; 5438 ec->tx_coalesce_usecs = vi->sq[queue].intr_coal.max_usecs; 5439 ec->tx_max_coalesced_frames = vi->sq[queue].intr_coal.max_packets; 5440 ec->rx_max_coalesced_frames = vi->rq[queue].intr_coal.max_packets; 5441 ec->use_adaptive_rx_coalesce = vi->rq[queue].dim_enabled; 5442 mutex_unlock(&vi->rq[queue].dim_lock); 5443 } else { 5444 ec->rx_max_coalesced_frames = 1; 5445 5446 if (vi->sq[queue].napi.weight) 5447 ec->tx_max_coalesced_frames = 1; 5448 } 5449 5450 return 0; 5451 } 5452 5453 static void virtnet_init_settings(struct net_device *dev) 5454 { 5455 struct virtnet_info *vi = netdev_priv(dev); 5456 5457 vi->speed = SPEED_UNKNOWN; 5458 vi->duplex = DUPLEX_UNKNOWN; 5459 } 5460 5461 static u32 virtnet_get_rxfh_key_size(struct net_device *dev) 5462 { 5463 return ((struct virtnet_info *)netdev_priv(dev))->rss_key_size; 5464 } 5465 5466 static u32 virtnet_get_rxfh_indir_size(struct net_device *dev) 5467 { 5468 return ((struct virtnet_info *)netdev_priv(dev))->rss_indir_table_size; 5469 } 5470 5471 static int virtnet_get_rxfh(struct net_device *dev, 5472 struct ethtool_rxfh_param *rxfh) 5473 { 5474 struct virtnet_info *vi = netdev_priv(dev); 5475 int i; 5476 5477 if (rxfh->indir) { 5478 for (i = 0; i < vi->rss_indir_table_size; ++i) 5479 rxfh->indir[i] = le16_to_cpu(vi->rss_hdr->indirection_table[i]); 5480 } 5481 5482 if (rxfh->key) 5483 memcpy(rxfh->key, vi->rss_hash_key_data, vi->rss_key_size); 5484 5485 rxfh->hfunc = ETH_RSS_HASH_TOP; 5486 5487 return 0; 5488 } 5489 5490 static int virtnet_set_rxfh(struct net_device *dev, 5491 struct ethtool_rxfh_param *rxfh, 5492 struct netlink_ext_ack *extack) 5493 { 5494 struct virtnet_info *vi = netdev_priv(dev); 5495 bool update = false; 5496 int i; 5497 5498 if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && 5499 rxfh->hfunc != ETH_RSS_HASH_TOP) 5500 return -EOPNOTSUPP; 5501 5502 if (rxfh->indir) { 5503 if (!vi->has_rss) 5504 return -EOPNOTSUPP; 5505 5506 for (i = 0; i < vi->rss_indir_table_size; ++i) 5507 vi->rss_hdr->indirection_table[i] = cpu_to_le16(rxfh->indir[i]); 5508 update = true; 5509 } 5510 5511 if (rxfh->key) { 5512 /* If either _F_HASH_REPORT or _F_RSS are negotiated, the 5513 * device provides hash calculation capabilities, that is, 5514 * hash_key is configured. 5515 */ 5516 if (!vi->has_rss && !vi->has_rss_hash_report) 5517 return -EOPNOTSUPP; 5518 5519 memcpy(vi->rss_hash_key_data, rxfh->key, vi->rss_key_size); 5520 update = true; 5521 } 5522 5523 if (update) 5524 virtnet_commit_rss_command(vi); 5525 5526 return 0; 5527 } 5528 5529 static int virtnet_get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, u32 *rule_locs) 5530 { 5531 struct virtnet_info *vi = netdev_priv(dev); 5532 int rc = 0; 5533 5534 switch (info->cmd) { 5535 case ETHTOOL_GRXRINGS: 5536 info->data = vi->curr_queue_pairs; 5537 break; 5538 case ETHTOOL_GRXFH: 5539 virtnet_get_hashflow(vi, info); 5540 break; 5541 default: 5542 rc = -EOPNOTSUPP; 5543 } 5544 5545 return rc; 5546 } 5547 5548 static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info) 5549 { 5550 struct virtnet_info *vi = netdev_priv(dev); 5551 int rc = 0; 5552 5553 switch (info->cmd) { 5554 case ETHTOOL_SRXFH: 5555 if (!virtnet_set_hashflow(vi, info)) 5556 rc = -EINVAL; 5557 5558 break; 5559 default: 5560 rc = -EOPNOTSUPP; 5561 } 5562 5563 return rc; 5564 } 5565 5566 static const struct ethtool_ops virtnet_ethtool_ops = { 5567 .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | 5568 ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_USE_ADAPTIVE_RX, 5569 .get_drvinfo = virtnet_get_drvinfo, 5570 .get_link = ethtool_op_get_link, 5571 .get_ringparam = virtnet_get_ringparam, 5572 .set_ringparam = virtnet_set_ringparam, 5573 .get_strings = virtnet_get_strings, 5574 .get_sset_count = virtnet_get_sset_count, 5575 .get_ethtool_stats = virtnet_get_ethtool_stats, 5576 .set_channels = virtnet_set_channels, 5577 .get_channels = virtnet_get_channels, 5578 .get_ts_info = ethtool_op_get_ts_info, 5579 .get_link_ksettings = virtnet_get_link_ksettings, 5580 .set_link_ksettings = virtnet_set_link_ksettings, 5581 .set_coalesce = virtnet_set_coalesce, 5582 .get_coalesce = virtnet_get_coalesce, 5583 .set_per_queue_coalesce = virtnet_set_per_queue_coalesce, 5584 .get_per_queue_coalesce = virtnet_get_per_queue_coalesce, 5585 .get_rxfh_key_size = virtnet_get_rxfh_key_size, 5586 .get_rxfh_indir_size = virtnet_get_rxfh_indir_size, 5587 .get_rxfh = virtnet_get_rxfh, 5588 .set_rxfh = virtnet_set_rxfh, 5589 .get_rxnfc = virtnet_get_rxnfc, 5590 .set_rxnfc = virtnet_set_rxnfc, 5591 }; 5592 5593 static void virtnet_get_queue_stats_rx(struct net_device *dev, int i, 5594 struct netdev_queue_stats_rx *stats) 5595 { 5596 struct virtnet_info *vi = netdev_priv(dev); 5597 struct receive_queue *rq = &vi->rq[i]; 5598 struct virtnet_stats_ctx ctx = {0}; 5599 5600 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true); 5601 5602 virtnet_get_hw_stats(vi, &ctx, i * 2); 5603 virtnet_fill_stats(vi, i * 2, &ctx, (void *)&rq->stats, true, 0); 5604 } 5605 5606 static void virtnet_get_queue_stats_tx(struct net_device *dev, int i, 5607 struct netdev_queue_stats_tx *stats) 5608 { 5609 struct virtnet_info *vi = netdev_priv(dev); 5610 struct send_queue *sq = &vi->sq[i]; 5611 struct virtnet_stats_ctx ctx = {0}; 5612 5613 virtnet_stats_ctx_init(vi, &ctx, (void *)stats, true); 5614 5615 virtnet_get_hw_stats(vi, &ctx, i * 2 + 1); 5616 virtnet_fill_stats(vi, i * 2 + 1, &ctx, (void *)&sq->stats, true, 0); 5617 } 5618 5619 static void virtnet_get_base_stats(struct net_device *dev, 5620 struct netdev_queue_stats_rx *rx, 5621 struct netdev_queue_stats_tx *tx) 5622 { 5623 struct virtnet_info *vi = netdev_priv(dev); 5624 5625 /* The queue stats of the virtio-net will not be reset. So here we 5626 * return 0. 5627 */ 5628 rx->bytes = 0; 5629 rx->packets = 0; 5630 5631 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_BASIC) { 5632 rx->hw_drops = 0; 5633 rx->hw_drop_overruns = 0; 5634 } 5635 5636 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_CSUM) { 5637 rx->csum_unnecessary = 0; 5638 rx->csum_none = 0; 5639 rx->csum_bad = 0; 5640 } 5641 5642 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_GSO) { 5643 rx->hw_gro_packets = 0; 5644 rx->hw_gro_bytes = 0; 5645 rx->hw_gro_wire_packets = 0; 5646 rx->hw_gro_wire_bytes = 0; 5647 } 5648 5649 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_RX_SPEED) 5650 rx->hw_drop_ratelimits = 0; 5651 5652 tx->bytes = 0; 5653 tx->packets = 0; 5654 tx->stop = 0; 5655 tx->wake = 0; 5656 5657 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_BASIC) { 5658 tx->hw_drops = 0; 5659 tx->hw_drop_errors = 0; 5660 } 5661 5662 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_CSUM) { 5663 tx->csum_none = 0; 5664 tx->needs_csum = 0; 5665 } 5666 5667 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_GSO) { 5668 tx->hw_gso_packets = 0; 5669 tx->hw_gso_bytes = 0; 5670 tx->hw_gso_wire_packets = 0; 5671 tx->hw_gso_wire_bytes = 0; 5672 } 5673 5674 if (vi->device_stats_cap & VIRTIO_NET_STATS_TYPE_TX_SPEED) 5675 tx->hw_drop_ratelimits = 0; 5676 } 5677 5678 static const struct netdev_stat_ops virtnet_stat_ops = { 5679 .get_queue_stats_rx = virtnet_get_queue_stats_rx, 5680 .get_queue_stats_tx = virtnet_get_queue_stats_tx, 5681 .get_base_stats = virtnet_get_base_stats, 5682 }; 5683 5684 static void virtnet_freeze_down(struct virtio_device *vdev) 5685 { 5686 struct virtnet_info *vi = vdev->priv; 5687 5688 /* Make sure no work handler is accessing the device */ 5689 flush_work(&vi->config_work); 5690 disable_rx_mode_work(vi); 5691 flush_work(&vi->rx_mode_work); 5692 5693 netif_tx_lock_bh(vi->dev); 5694 netif_device_detach(vi->dev); 5695 netif_tx_unlock_bh(vi->dev); 5696 if (netif_running(vi->dev)) { 5697 rtnl_lock(); 5698 virtnet_close(vi->dev); 5699 rtnl_unlock(); 5700 } 5701 } 5702 5703 static int init_vqs(struct virtnet_info *vi); 5704 5705 static int virtnet_restore_up(struct virtio_device *vdev) 5706 { 5707 struct virtnet_info *vi = vdev->priv; 5708 int err; 5709 5710 err = init_vqs(vi); 5711 if (err) 5712 return err; 5713 5714 virtio_device_ready(vdev); 5715 5716 enable_delayed_refill(vi); 5717 enable_rx_mode_work(vi); 5718 5719 if (netif_running(vi->dev)) { 5720 rtnl_lock(); 5721 err = virtnet_open(vi->dev); 5722 rtnl_unlock(); 5723 if (err) 5724 return err; 5725 } 5726 5727 netif_tx_lock_bh(vi->dev); 5728 netif_device_attach(vi->dev); 5729 netif_tx_unlock_bh(vi->dev); 5730 return err; 5731 } 5732 5733 static int virtnet_set_guest_offloads(struct virtnet_info *vi, u64 offloads) 5734 { 5735 __virtio64 *_offloads __free(kfree) = NULL; 5736 struct scatterlist sg; 5737 5738 _offloads = kzalloc(sizeof(*_offloads), GFP_KERNEL); 5739 if (!_offloads) 5740 return -ENOMEM; 5741 5742 *_offloads = cpu_to_virtio64(vi->vdev, offloads); 5743 5744 sg_init_one(&sg, _offloads, sizeof(*_offloads)); 5745 5746 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_GUEST_OFFLOADS, 5747 VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET, &sg)) { 5748 dev_warn(&vi->dev->dev, "Fail to set guest offload.\n"); 5749 return -EINVAL; 5750 } 5751 5752 return 0; 5753 } 5754 5755 static int virtnet_clear_guest_offloads(struct virtnet_info *vi) 5756 { 5757 u64 offloads = 0; 5758 5759 if (!vi->guest_offloads) 5760 return 0; 5761 5762 return virtnet_set_guest_offloads(vi, offloads); 5763 } 5764 5765 static int virtnet_restore_guest_offloads(struct virtnet_info *vi) 5766 { 5767 u64 offloads = vi->guest_offloads; 5768 5769 if (!vi->guest_offloads) 5770 return 0; 5771 5772 return virtnet_set_guest_offloads(vi, offloads); 5773 } 5774 5775 static int virtnet_rq_bind_xsk_pool(struct virtnet_info *vi, struct receive_queue *rq, 5776 struct xsk_buff_pool *pool) 5777 { 5778 int err, qindex; 5779 5780 qindex = rq - vi->rq; 5781 5782 if (pool) { 5783 err = xdp_rxq_info_reg(&rq->xsk_rxq_info, vi->dev, qindex, rq->napi.napi_id); 5784 if (err < 0) 5785 return err; 5786 5787 err = xdp_rxq_info_reg_mem_model(&rq->xsk_rxq_info, 5788 MEM_TYPE_XSK_BUFF_POOL, NULL); 5789 if (err < 0) 5790 goto unreg; 5791 5792 xsk_pool_set_rxq_info(pool, &rq->xsk_rxq_info); 5793 } 5794 5795 virtnet_rx_pause(vi, rq); 5796 5797 err = virtqueue_reset(rq->vq, virtnet_rq_unmap_free_buf, NULL); 5798 if (err) { 5799 netdev_err(vi->dev, "reset rx fail: rx queue index: %d err: %d\n", qindex, err); 5800 5801 pool = NULL; 5802 } 5803 5804 rq->xsk_pool = pool; 5805 5806 virtnet_rx_resume(vi, rq); 5807 5808 if (pool) 5809 return 0; 5810 5811 unreg: 5812 xdp_rxq_info_unreg(&rq->xsk_rxq_info); 5813 return err; 5814 } 5815 5816 static int virtnet_sq_bind_xsk_pool(struct virtnet_info *vi, 5817 struct send_queue *sq, 5818 struct xsk_buff_pool *pool) 5819 { 5820 int err, qindex; 5821 5822 qindex = sq - vi->sq; 5823 5824 virtnet_tx_pause(vi, sq); 5825 5826 err = virtqueue_reset(sq->vq, virtnet_sq_free_unused_buf, 5827 virtnet_sq_free_unused_buf_done); 5828 if (err) { 5829 netdev_err(vi->dev, "reset tx fail: tx queue index: %d err: %d\n", qindex, err); 5830 pool = NULL; 5831 } 5832 5833 sq->xsk_pool = pool; 5834 5835 virtnet_tx_resume(vi, sq); 5836 5837 return err; 5838 } 5839 5840 static int virtnet_xsk_pool_enable(struct net_device *dev, 5841 struct xsk_buff_pool *pool, 5842 u16 qid) 5843 { 5844 struct virtnet_info *vi = netdev_priv(dev); 5845 struct receive_queue *rq; 5846 struct device *dma_dev; 5847 struct send_queue *sq; 5848 dma_addr_t hdr_dma; 5849 int err, size; 5850 5851 if (vi->hdr_len > xsk_pool_get_headroom(pool)) 5852 return -EINVAL; 5853 5854 /* In big_packets mode, xdp cannot work, so there is no need to 5855 * initialize xsk of rq. 5856 */ 5857 if (vi->big_packets && !vi->mergeable_rx_bufs) 5858 return -ENOENT; 5859 5860 if (qid >= vi->curr_queue_pairs) 5861 return -EINVAL; 5862 5863 sq = &vi->sq[qid]; 5864 rq = &vi->rq[qid]; 5865 5866 /* xsk assumes that tx and rx must have the same dma device. The af-xdp 5867 * may use one buffer to receive from the rx and reuse this buffer to 5868 * send by the tx. So the dma dev of sq and rq must be the same one. 5869 * 5870 * But vq->dma_dev allows every vq has the respective dma dev. So I 5871 * check the dma dev of vq and sq is the same dev. 5872 */ 5873 if (virtqueue_dma_dev(rq->vq) != virtqueue_dma_dev(sq->vq)) 5874 return -EINVAL; 5875 5876 dma_dev = virtqueue_dma_dev(rq->vq); 5877 if (!dma_dev) 5878 return -EINVAL; 5879 5880 size = virtqueue_get_vring_size(rq->vq); 5881 5882 rq->xsk_buffs = kvcalloc(size, sizeof(*rq->xsk_buffs), GFP_KERNEL); 5883 if (!rq->xsk_buffs) 5884 return -ENOMEM; 5885 5886 hdr_dma = virtqueue_dma_map_single_attrs(sq->vq, &xsk_hdr, vi->hdr_len, 5887 DMA_TO_DEVICE, 0); 5888 if (virtqueue_dma_mapping_error(sq->vq, hdr_dma)) 5889 return -ENOMEM; 5890 5891 err = xsk_pool_dma_map(pool, dma_dev, 0); 5892 if (err) 5893 goto err_xsk_map; 5894 5895 err = virtnet_rq_bind_xsk_pool(vi, rq, pool); 5896 if (err) 5897 goto err_rq; 5898 5899 err = virtnet_sq_bind_xsk_pool(vi, sq, pool); 5900 if (err) 5901 goto err_sq; 5902 5903 /* Now, we do not support tx offload(such as tx csum), so all the tx 5904 * virtnet hdr is zero. So all the tx packets can share a single hdr. 5905 */ 5906 sq->xsk_hdr_dma_addr = hdr_dma; 5907 5908 return 0; 5909 5910 err_sq: 5911 virtnet_rq_bind_xsk_pool(vi, rq, NULL); 5912 err_rq: 5913 xsk_pool_dma_unmap(pool, 0); 5914 err_xsk_map: 5915 virtqueue_dma_unmap_single_attrs(rq->vq, hdr_dma, vi->hdr_len, 5916 DMA_TO_DEVICE, 0); 5917 return err; 5918 } 5919 5920 static int virtnet_xsk_pool_disable(struct net_device *dev, u16 qid) 5921 { 5922 struct virtnet_info *vi = netdev_priv(dev); 5923 struct xsk_buff_pool *pool; 5924 struct receive_queue *rq; 5925 struct send_queue *sq; 5926 int err; 5927 5928 if (qid >= vi->curr_queue_pairs) 5929 return -EINVAL; 5930 5931 sq = &vi->sq[qid]; 5932 rq = &vi->rq[qid]; 5933 5934 pool = rq->xsk_pool; 5935 5936 err = virtnet_rq_bind_xsk_pool(vi, rq, NULL); 5937 err |= virtnet_sq_bind_xsk_pool(vi, sq, NULL); 5938 5939 xsk_pool_dma_unmap(pool, 0); 5940 5941 virtqueue_dma_unmap_single_attrs(sq->vq, sq->xsk_hdr_dma_addr, 5942 vi->hdr_len, DMA_TO_DEVICE, 0); 5943 kvfree(rq->xsk_buffs); 5944 5945 return err; 5946 } 5947 5948 static int virtnet_xsk_pool_setup(struct net_device *dev, struct netdev_bpf *xdp) 5949 { 5950 if (xdp->xsk.pool) 5951 return virtnet_xsk_pool_enable(dev, xdp->xsk.pool, 5952 xdp->xsk.queue_id); 5953 else 5954 return virtnet_xsk_pool_disable(dev, xdp->xsk.queue_id); 5955 } 5956 5957 static int virtnet_xdp_set(struct net_device *dev, struct bpf_prog *prog, 5958 struct netlink_ext_ack *extack) 5959 { 5960 unsigned int room = SKB_DATA_ALIGN(XDP_PACKET_HEADROOM + 5961 sizeof(struct skb_shared_info)); 5962 unsigned int max_sz = PAGE_SIZE - room - ETH_HLEN; 5963 struct virtnet_info *vi = netdev_priv(dev); 5964 struct bpf_prog *old_prog; 5965 u16 xdp_qp = 0, curr_qp; 5966 int i, err; 5967 5968 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) 5969 && (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 5970 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 5971 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 5972 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 5973 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_CSUM) || 5974 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) || 5975 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6))) { 5976 NL_SET_ERR_MSG_MOD(extack, "Can't set XDP while host is implementing GRO_HW/CSUM, disable GRO_HW/CSUM first"); 5977 return -EOPNOTSUPP; 5978 } 5979 5980 if (vi->mergeable_rx_bufs && !vi->any_header_sg) { 5981 NL_SET_ERR_MSG_MOD(extack, "XDP expects header/data in single page, any_header_sg required"); 5982 return -EINVAL; 5983 } 5984 5985 if (prog && !prog->aux->xdp_has_frags && dev->mtu > max_sz) { 5986 NL_SET_ERR_MSG_MOD(extack, "MTU too large to enable XDP without frags"); 5987 netdev_warn(dev, "single-buffer XDP requires MTU less than %u\n", max_sz); 5988 return -EINVAL; 5989 } 5990 5991 curr_qp = vi->curr_queue_pairs - vi->xdp_queue_pairs; 5992 if (prog) 5993 xdp_qp = nr_cpu_ids; 5994 5995 /* XDP requires extra queues for XDP_TX */ 5996 if (curr_qp + xdp_qp > vi->max_queue_pairs) { 5997 netdev_warn_once(dev, "XDP request %i queues but max is %i. XDP_TX and XDP_REDIRECT will operate in a slower locked tx mode.\n", 5998 curr_qp + xdp_qp, vi->max_queue_pairs); 5999 xdp_qp = 0; 6000 } 6001 6002 old_prog = rtnl_dereference(vi->rq[0].xdp_prog); 6003 if (!prog && !old_prog) 6004 return 0; 6005 6006 if (prog) 6007 bpf_prog_add(prog, vi->max_queue_pairs - 1); 6008 6009 virtnet_rx_pause_all(vi); 6010 6011 /* Make sure NAPI is not using any XDP TX queues for RX. */ 6012 if (netif_running(dev)) { 6013 for (i = 0; i < vi->max_queue_pairs; i++) 6014 virtnet_napi_tx_disable(&vi->sq[i]); 6015 } 6016 6017 if (!prog) { 6018 for (i = 0; i < vi->max_queue_pairs; i++) { 6019 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 6020 if (i == 0) 6021 virtnet_restore_guest_offloads(vi); 6022 } 6023 synchronize_net(); 6024 } 6025 6026 err = virtnet_set_queues(vi, curr_qp + xdp_qp); 6027 if (err) 6028 goto err; 6029 netif_set_real_num_rx_queues(dev, curr_qp + xdp_qp); 6030 vi->xdp_queue_pairs = xdp_qp; 6031 6032 if (prog) { 6033 vi->xdp_enabled = true; 6034 for (i = 0; i < vi->max_queue_pairs; i++) { 6035 rcu_assign_pointer(vi->rq[i].xdp_prog, prog); 6036 if (i == 0 && !old_prog) 6037 virtnet_clear_guest_offloads(vi); 6038 } 6039 if (!old_prog) 6040 xdp_features_set_redirect_target(dev, true); 6041 } else { 6042 xdp_features_clear_redirect_target(dev); 6043 vi->xdp_enabled = false; 6044 } 6045 6046 virtnet_rx_resume_all(vi); 6047 for (i = 0; i < vi->max_queue_pairs; i++) { 6048 if (old_prog) 6049 bpf_prog_put(old_prog); 6050 if (netif_running(dev)) 6051 virtnet_napi_tx_enable(&vi->sq[i]); 6052 } 6053 6054 return 0; 6055 6056 err: 6057 if (!prog) { 6058 virtnet_clear_guest_offloads(vi); 6059 for (i = 0; i < vi->max_queue_pairs; i++) 6060 rcu_assign_pointer(vi->rq[i].xdp_prog, old_prog); 6061 } 6062 6063 virtnet_rx_resume_all(vi); 6064 if (netif_running(dev)) { 6065 for (i = 0; i < vi->max_queue_pairs; i++) 6066 virtnet_napi_tx_enable(&vi->sq[i]); 6067 } 6068 if (prog) 6069 bpf_prog_sub(prog, vi->max_queue_pairs - 1); 6070 return err; 6071 } 6072 6073 static int virtnet_xdp(struct net_device *dev, struct netdev_bpf *xdp) 6074 { 6075 switch (xdp->command) { 6076 case XDP_SETUP_PROG: 6077 return virtnet_xdp_set(dev, xdp->prog, xdp->extack); 6078 case XDP_SETUP_XSK_POOL: 6079 return virtnet_xsk_pool_setup(dev, xdp); 6080 default: 6081 return -EINVAL; 6082 } 6083 } 6084 6085 static int virtnet_get_phys_port_name(struct net_device *dev, char *buf, 6086 size_t len) 6087 { 6088 struct virtnet_info *vi = netdev_priv(dev); 6089 int ret; 6090 6091 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_STANDBY)) 6092 return -EOPNOTSUPP; 6093 6094 ret = snprintf(buf, len, "sby"); 6095 if (ret >= len) 6096 return -EOPNOTSUPP; 6097 6098 return 0; 6099 } 6100 6101 static int virtnet_set_features(struct net_device *dev, 6102 netdev_features_t features) 6103 { 6104 struct virtnet_info *vi = netdev_priv(dev); 6105 u64 offloads; 6106 int err; 6107 6108 if ((dev->features ^ features) & NETIF_F_GRO_HW) { 6109 if (vi->xdp_enabled) 6110 return -EBUSY; 6111 6112 if (features & NETIF_F_GRO_HW) 6113 offloads = vi->guest_offloads_capable; 6114 else 6115 offloads = vi->guest_offloads_capable & 6116 ~GUEST_OFFLOAD_GRO_HW_MASK; 6117 6118 err = virtnet_set_guest_offloads(vi, offloads); 6119 if (err) 6120 return err; 6121 vi->guest_offloads = offloads; 6122 } 6123 6124 if ((dev->features ^ features) & NETIF_F_RXHASH) { 6125 if (features & NETIF_F_RXHASH) 6126 vi->rss_hdr->hash_types = cpu_to_le32(vi->rss_hash_types_saved); 6127 else 6128 vi->rss_hdr->hash_types = cpu_to_le32(VIRTIO_NET_HASH_REPORT_NONE); 6129 6130 if (!virtnet_commit_rss_command(vi)) 6131 return -EINVAL; 6132 } 6133 6134 return 0; 6135 } 6136 6137 static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue) 6138 { 6139 struct virtnet_info *priv = netdev_priv(dev); 6140 struct send_queue *sq = &priv->sq[txqueue]; 6141 struct netdev_queue *txq = netdev_get_tx_queue(dev, txqueue); 6142 6143 u64_stats_update_begin(&sq->stats.syncp); 6144 u64_stats_inc(&sq->stats.tx_timeouts); 6145 u64_stats_update_end(&sq->stats.syncp); 6146 6147 netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n", 6148 txqueue, sq->name, sq->vq->index, sq->vq->name, 6149 jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start))); 6150 } 6151 6152 static int virtnet_init_irq_moder(struct virtnet_info *vi) 6153 { 6154 u8 profile_flags = 0, coal_flags = 0; 6155 int ret, i; 6156 6157 profile_flags |= DIM_PROFILE_RX; 6158 coal_flags |= DIM_COALESCE_USEC | DIM_COALESCE_PKTS; 6159 ret = net_dim_init_irq_moder(vi->dev, profile_flags, coal_flags, 6160 DIM_CQ_PERIOD_MODE_START_FROM_EQE, 6161 0, virtnet_rx_dim_work, NULL); 6162 6163 if (ret) 6164 return ret; 6165 6166 for (i = 0; i < vi->max_queue_pairs; i++) 6167 net_dim_setting(vi->dev, &vi->rq[i].dim, false); 6168 6169 return 0; 6170 } 6171 6172 static void virtnet_free_irq_moder(struct virtnet_info *vi) 6173 { 6174 if (!virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) 6175 return; 6176 6177 rtnl_lock(); 6178 net_dim_free_irq_moder(vi->dev); 6179 rtnl_unlock(); 6180 } 6181 6182 static const struct net_device_ops virtnet_netdev = { 6183 .ndo_open = virtnet_open, 6184 .ndo_stop = virtnet_close, 6185 .ndo_start_xmit = start_xmit, 6186 .ndo_validate_addr = eth_validate_addr, 6187 .ndo_set_mac_address = virtnet_set_mac_address, 6188 .ndo_set_rx_mode = virtnet_set_rx_mode, 6189 .ndo_get_stats64 = virtnet_stats, 6190 .ndo_vlan_rx_add_vid = virtnet_vlan_rx_add_vid, 6191 .ndo_vlan_rx_kill_vid = virtnet_vlan_rx_kill_vid, 6192 .ndo_bpf = virtnet_xdp, 6193 .ndo_xdp_xmit = virtnet_xdp_xmit, 6194 .ndo_xsk_wakeup = virtnet_xsk_wakeup, 6195 .ndo_features_check = passthru_features_check, 6196 .ndo_get_phys_port_name = virtnet_get_phys_port_name, 6197 .ndo_set_features = virtnet_set_features, 6198 .ndo_tx_timeout = virtnet_tx_timeout, 6199 }; 6200 6201 static void virtnet_config_changed_work(struct work_struct *work) 6202 { 6203 struct virtnet_info *vi = 6204 container_of(work, struct virtnet_info, config_work); 6205 u16 v; 6206 6207 if (virtio_cread_feature(vi->vdev, VIRTIO_NET_F_STATUS, 6208 struct virtio_net_config, status, &v) < 0) 6209 return; 6210 6211 if (v & VIRTIO_NET_S_ANNOUNCE) { 6212 netdev_notify_peers(vi->dev); 6213 virtnet_ack_link_announce(vi); 6214 } 6215 6216 /* Ignore unknown (future) status bits */ 6217 v &= VIRTIO_NET_S_LINK_UP; 6218 6219 if (vi->status == v) 6220 return; 6221 6222 vi->status = v; 6223 6224 if (vi->status & VIRTIO_NET_S_LINK_UP) { 6225 virtnet_update_settings(vi); 6226 netif_carrier_on(vi->dev); 6227 netif_tx_wake_all_queues(vi->dev); 6228 } else { 6229 netif_carrier_off(vi->dev); 6230 netif_tx_stop_all_queues(vi->dev); 6231 } 6232 } 6233 6234 static void virtnet_config_changed(struct virtio_device *vdev) 6235 { 6236 struct virtnet_info *vi = vdev->priv; 6237 6238 schedule_work(&vi->config_work); 6239 } 6240 6241 static void virtnet_free_queues(struct virtnet_info *vi) 6242 { 6243 int i; 6244 6245 for (i = 0; i < vi->max_queue_pairs; i++) { 6246 __netif_napi_del(&vi->rq[i].napi); 6247 __netif_napi_del(&vi->sq[i].napi); 6248 } 6249 6250 /* We called __netif_napi_del(), 6251 * we need to respect an RCU grace period before freeing vi->rq 6252 */ 6253 synchronize_net(); 6254 6255 kfree(vi->rq); 6256 kfree(vi->sq); 6257 kfree(vi->ctrl); 6258 } 6259 6260 static void _free_receive_bufs(struct virtnet_info *vi) 6261 { 6262 struct bpf_prog *old_prog; 6263 int i; 6264 6265 for (i = 0; i < vi->max_queue_pairs; i++) { 6266 while (vi->rq[i].pages) 6267 __free_pages(get_a_page(&vi->rq[i], GFP_KERNEL), 0); 6268 6269 old_prog = rtnl_dereference(vi->rq[i].xdp_prog); 6270 RCU_INIT_POINTER(vi->rq[i].xdp_prog, NULL); 6271 if (old_prog) 6272 bpf_prog_put(old_prog); 6273 } 6274 } 6275 6276 static void free_receive_bufs(struct virtnet_info *vi) 6277 { 6278 rtnl_lock(); 6279 _free_receive_bufs(vi); 6280 rtnl_unlock(); 6281 } 6282 6283 static void free_receive_page_frags(struct virtnet_info *vi) 6284 { 6285 int i; 6286 for (i = 0; i < vi->max_queue_pairs; i++) 6287 if (vi->rq[i].alloc_frag.page) { 6288 if (vi->rq[i].last_dma) 6289 virtnet_rq_unmap(&vi->rq[i], vi->rq[i].last_dma, 0); 6290 put_page(vi->rq[i].alloc_frag.page); 6291 } 6292 } 6293 6294 static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf) 6295 { 6296 struct virtnet_info *vi = vq->vdev->priv; 6297 struct send_queue *sq; 6298 int i = vq2txq(vq); 6299 6300 sq = &vi->sq[i]; 6301 6302 switch (virtnet_xmit_ptr_unpack(&buf)) { 6303 case VIRTNET_XMIT_TYPE_SKB: 6304 case VIRTNET_XMIT_TYPE_SKB_ORPHAN: 6305 dev_kfree_skb(buf); 6306 break; 6307 6308 case VIRTNET_XMIT_TYPE_XDP: 6309 xdp_return_frame(buf); 6310 break; 6311 6312 case VIRTNET_XMIT_TYPE_XSK: 6313 xsk_tx_completed(sq->xsk_pool, 1); 6314 break; 6315 } 6316 } 6317 6318 static void virtnet_sq_free_unused_buf_done(struct virtqueue *vq) 6319 { 6320 struct virtnet_info *vi = vq->vdev->priv; 6321 int i = vq2txq(vq); 6322 6323 netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, i)); 6324 } 6325 6326 static void free_unused_bufs(struct virtnet_info *vi) 6327 { 6328 void *buf; 6329 int i; 6330 6331 for (i = 0; i < vi->max_queue_pairs; i++) { 6332 struct virtqueue *vq = vi->sq[i].vq; 6333 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) 6334 virtnet_sq_free_unused_buf(vq, buf); 6335 cond_resched(); 6336 } 6337 6338 for (i = 0; i < vi->max_queue_pairs; i++) { 6339 struct virtqueue *vq = vi->rq[i].vq; 6340 6341 while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) 6342 virtnet_rq_unmap_free_buf(vq, buf); 6343 cond_resched(); 6344 } 6345 } 6346 6347 static void virtnet_del_vqs(struct virtnet_info *vi) 6348 { 6349 struct virtio_device *vdev = vi->vdev; 6350 6351 virtnet_clean_affinity(vi); 6352 6353 vdev->config->del_vqs(vdev); 6354 6355 virtnet_free_queues(vi); 6356 } 6357 6358 /* How large should a single buffer be so a queue full of these can fit at 6359 * least one full packet? 6360 * Logic below assumes the mergeable buffer header is used. 6361 */ 6362 static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqueue *vq) 6363 { 6364 const unsigned int hdr_len = vi->hdr_len; 6365 unsigned int rq_size = virtqueue_get_vring_size(vq); 6366 unsigned int packet_len = vi->big_packets ? IP_MAX_MTU : vi->dev->max_mtu; 6367 unsigned int buf_len = hdr_len + ETH_HLEN + VLAN_HLEN + packet_len; 6368 unsigned int min_buf_len = DIV_ROUND_UP(buf_len, rq_size); 6369 6370 return max(max(min_buf_len, hdr_len) - hdr_len, 6371 (unsigned int)GOOD_PACKET_LEN); 6372 } 6373 6374 static int virtnet_find_vqs(struct virtnet_info *vi) 6375 { 6376 struct virtqueue_info *vqs_info; 6377 struct virtqueue **vqs; 6378 int ret = -ENOMEM; 6379 int total_vqs; 6380 bool *ctx; 6381 u16 i; 6382 6383 /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by 6384 * possible N-1 RX/TX queue pairs used in multiqueue mode, followed by 6385 * possible control vq. 6386 */ 6387 total_vqs = vi->max_queue_pairs * 2 + 6388 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VQ); 6389 6390 /* Allocate space for find_vqs parameters */ 6391 vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL); 6392 if (!vqs) 6393 goto err_vq; 6394 vqs_info = kcalloc(total_vqs, sizeof(*vqs_info), GFP_KERNEL); 6395 if (!vqs_info) 6396 goto err_vqs_info; 6397 if (!vi->big_packets || vi->mergeable_rx_bufs) { 6398 ctx = kcalloc(total_vqs, sizeof(*ctx), GFP_KERNEL); 6399 if (!ctx) 6400 goto err_ctx; 6401 } else { 6402 ctx = NULL; 6403 } 6404 6405 /* Parameters for control virtqueue, if any */ 6406 if (vi->has_cvq) { 6407 vqs_info[total_vqs - 1].name = "control"; 6408 } 6409 6410 /* Allocate/initialize parameters for send/receive virtqueues */ 6411 for (i = 0; i < vi->max_queue_pairs; i++) { 6412 vqs_info[rxq2vq(i)].callback = skb_recv_done; 6413 vqs_info[txq2vq(i)].callback = skb_xmit_done; 6414 sprintf(vi->rq[i].name, "input.%u", i); 6415 sprintf(vi->sq[i].name, "output.%u", i); 6416 vqs_info[rxq2vq(i)].name = vi->rq[i].name; 6417 vqs_info[txq2vq(i)].name = vi->sq[i].name; 6418 if (ctx) 6419 vqs_info[rxq2vq(i)].ctx = true; 6420 } 6421 6422 ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, vqs_info, NULL); 6423 if (ret) 6424 goto err_find; 6425 6426 if (vi->has_cvq) { 6427 vi->cvq = vqs[total_vqs - 1]; 6428 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_VLAN)) 6429 vi->dev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 6430 } 6431 6432 for (i = 0; i < vi->max_queue_pairs; i++) { 6433 vi->rq[i].vq = vqs[rxq2vq(i)]; 6434 vi->rq[i].min_buf_len = mergeable_min_buf_len(vi, vi->rq[i].vq); 6435 vi->sq[i].vq = vqs[txq2vq(i)]; 6436 } 6437 6438 /* run here: ret == 0. */ 6439 6440 6441 err_find: 6442 kfree(ctx); 6443 err_ctx: 6444 kfree(vqs_info); 6445 err_vqs_info: 6446 kfree(vqs); 6447 err_vq: 6448 return ret; 6449 } 6450 6451 static int virtnet_alloc_queues(struct virtnet_info *vi) 6452 { 6453 int i; 6454 6455 if (vi->has_cvq) { 6456 vi->ctrl = kzalloc(sizeof(*vi->ctrl), GFP_KERNEL); 6457 if (!vi->ctrl) 6458 goto err_ctrl; 6459 } else { 6460 vi->ctrl = NULL; 6461 } 6462 vi->sq = kcalloc(vi->max_queue_pairs, sizeof(*vi->sq), GFP_KERNEL); 6463 if (!vi->sq) 6464 goto err_sq; 6465 vi->rq = kcalloc(vi->max_queue_pairs, sizeof(*vi->rq), GFP_KERNEL); 6466 if (!vi->rq) 6467 goto err_rq; 6468 6469 INIT_DELAYED_WORK(&vi->refill, refill_work); 6470 for (i = 0; i < vi->max_queue_pairs; i++) { 6471 vi->rq[i].pages = NULL; 6472 netif_napi_add_config(vi->dev, &vi->rq[i].napi, virtnet_poll, 6473 i); 6474 vi->rq[i].napi.weight = napi_weight; 6475 netif_napi_add_tx_weight(vi->dev, &vi->sq[i].napi, 6476 virtnet_poll_tx, 6477 napi_tx ? napi_weight : 0); 6478 6479 sg_init_table(vi->rq[i].sg, ARRAY_SIZE(vi->rq[i].sg)); 6480 ewma_pkt_len_init(&vi->rq[i].mrg_avg_pkt_len); 6481 sg_init_table(vi->sq[i].sg, ARRAY_SIZE(vi->sq[i].sg)); 6482 6483 u64_stats_init(&vi->rq[i].stats.syncp); 6484 u64_stats_init(&vi->sq[i].stats.syncp); 6485 mutex_init(&vi->rq[i].dim_lock); 6486 } 6487 6488 return 0; 6489 6490 err_rq: 6491 kfree(vi->sq); 6492 err_sq: 6493 kfree(vi->ctrl); 6494 err_ctrl: 6495 return -ENOMEM; 6496 } 6497 6498 static int init_vqs(struct virtnet_info *vi) 6499 { 6500 int ret; 6501 6502 /* Allocate send & receive queues */ 6503 ret = virtnet_alloc_queues(vi); 6504 if (ret) 6505 goto err; 6506 6507 ret = virtnet_find_vqs(vi); 6508 if (ret) 6509 goto err_free; 6510 6511 cpus_read_lock(); 6512 virtnet_set_affinity(vi); 6513 cpus_read_unlock(); 6514 6515 return 0; 6516 6517 err_free: 6518 virtnet_free_queues(vi); 6519 err: 6520 return ret; 6521 } 6522 6523 #ifdef CONFIG_SYSFS 6524 static ssize_t mergeable_rx_buffer_size_show(struct netdev_rx_queue *queue, 6525 char *buf) 6526 { 6527 struct virtnet_info *vi = netdev_priv(queue->dev); 6528 unsigned int queue_index = get_netdev_rx_queue_index(queue); 6529 unsigned int headroom = virtnet_get_headroom(vi); 6530 unsigned int tailroom = headroom ? sizeof(struct skb_shared_info) : 0; 6531 struct ewma_pkt_len *avg; 6532 6533 BUG_ON(queue_index >= vi->max_queue_pairs); 6534 avg = &vi->rq[queue_index].mrg_avg_pkt_len; 6535 return sprintf(buf, "%u\n", 6536 get_mergeable_buf_len(&vi->rq[queue_index], avg, 6537 SKB_DATA_ALIGN(headroom + tailroom))); 6538 } 6539 6540 static struct rx_queue_attribute mergeable_rx_buffer_size_attribute = 6541 __ATTR_RO(mergeable_rx_buffer_size); 6542 6543 static struct attribute *virtio_net_mrg_rx_attrs[] = { 6544 &mergeable_rx_buffer_size_attribute.attr, 6545 NULL 6546 }; 6547 6548 static const struct attribute_group virtio_net_mrg_rx_group = { 6549 .name = "virtio_net", 6550 .attrs = virtio_net_mrg_rx_attrs 6551 }; 6552 #endif 6553 6554 static bool virtnet_fail_on_feature(struct virtio_device *vdev, 6555 unsigned int fbit, 6556 const char *fname, const char *dname) 6557 { 6558 if (!virtio_has_feature(vdev, fbit)) 6559 return false; 6560 6561 dev_err(&vdev->dev, "device advertises feature %s but not %s", 6562 fname, dname); 6563 6564 return true; 6565 } 6566 6567 #define VIRTNET_FAIL_ON(vdev, fbit, dbit) \ 6568 virtnet_fail_on_feature(vdev, fbit, #fbit, dbit) 6569 6570 static bool virtnet_validate_features(struct virtio_device *vdev) 6571 { 6572 if (!virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ) && 6573 (VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_RX, 6574 "VIRTIO_NET_F_CTRL_VQ") || 6575 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_VLAN, 6576 "VIRTIO_NET_F_CTRL_VQ") || 6577 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_GUEST_ANNOUNCE, 6578 "VIRTIO_NET_F_CTRL_VQ") || 6579 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_MQ, "VIRTIO_NET_F_CTRL_VQ") || 6580 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_CTRL_MAC_ADDR, 6581 "VIRTIO_NET_F_CTRL_VQ") || 6582 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, 6583 "VIRTIO_NET_F_CTRL_VQ") || 6584 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, 6585 "VIRTIO_NET_F_CTRL_VQ") || 6586 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL, 6587 "VIRTIO_NET_F_CTRL_VQ") || 6588 VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_VQ_NOTF_COAL, 6589 "VIRTIO_NET_F_CTRL_VQ"))) { 6590 return false; 6591 } 6592 6593 return true; 6594 } 6595 6596 #define MIN_MTU ETH_MIN_MTU 6597 #define MAX_MTU ETH_MAX_MTU 6598 6599 static int virtnet_validate(struct virtio_device *vdev) 6600 { 6601 if (!vdev->config->get) { 6602 dev_err(&vdev->dev, "%s failure: config access disabled\n", 6603 __func__); 6604 return -EINVAL; 6605 } 6606 6607 if (!virtnet_validate_features(vdev)) 6608 return -EINVAL; 6609 6610 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 6611 int mtu = virtio_cread16(vdev, 6612 offsetof(struct virtio_net_config, 6613 mtu)); 6614 if (mtu < MIN_MTU) 6615 __virtio_clear_bit(vdev, VIRTIO_NET_F_MTU); 6616 } 6617 6618 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY) && 6619 !virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 6620 dev_warn(&vdev->dev, "device advertises feature VIRTIO_NET_F_STANDBY but not VIRTIO_NET_F_MAC, disabling standby"); 6621 __virtio_clear_bit(vdev, VIRTIO_NET_F_STANDBY); 6622 } 6623 6624 return 0; 6625 } 6626 6627 static bool virtnet_check_guest_gso(const struct virtnet_info *vi) 6628 { 6629 return virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO4) || 6630 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_TSO6) || 6631 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_ECN) || 6632 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_UFO) || 6633 (virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO4) && 6634 virtio_has_feature(vi->vdev, VIRTIO_NET_F_GUEST_USO6)); 6635 } 6636 6637 static void virtnet_set_big_packets(struct virtnet_info *vi, const int mtu) 6638 { 6639 bool guest_gso = virtnet_check_guest_gso(vi); 6640 6641 /* If device can receive ANY guest GSO packets, regardless of mtu, 6642 * allocate packets of maximum size, otherwise limit it to only 6643 * mtu size worth only. 6644 */ 6645 if (mtu > ETH_DATA_LEN || guest_gso) { 6646 vi->big_packets = true; 6647 vi->big_packets_num_skbfrags = guest_gso ? MAX_SKB_FRAGS : DIV_ROUND_UP(mtu, PAGE_SIZE); 6648 } 6649 } 6650 6651 #define VIRTIO_NET_HASH_REPORT_MAX_TABLE 10 6652 static enum xdp_rss_hash_type 6653 virtnet_xdp_rss_type[VIRTIO_NET_HASH_REPORT_MAX_TABLE] = { 6654 [VIRTIO_NET_HASH_REPORT_NONE] = XDP_RSS_TYPE_NONE, 6655 [VIRTIO_NET_HASH_REPORT_IPv4] = XDP_RSS_TYPE_L3_IPV4, 6656 [VIRTIO_NET_HASH_REPORT_TCPv4] = XDP_RSS_TYPE_L4_IPV4_TCP, 6657 [VIRTIO_NET_HASH_REPORT_UDPv4] = XDP_RSS_TYPE_L4_IPV4_UDP, 6658 [VIRTIO_NET_HASH_REPORT_IPv6] = XDP_RSS_TYPE_L3_IPV6, 6659 [VIRTIO_NET_HASH_REPORT_TCPv6] = XDP_RSS_TYPE_L4_IPV6_TCP, 6660 [VIRTIO_NET_HASH_REPORT_UDPv6] = XDP_RSS_TYPE_L4_IPV6_UDP, 6661 [VIRTIO_NET_HASH_REPORT_IPv6_EX] = XDP_RSS_TYPE_L3_IPV6_EX, 6662 [VIRTIO_NET_HASH_REPORT_TCPv6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX, 6663 [VIRTIO_NET_HASH_REPORT_UDPv6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX 6664 }; 6665 6666 static int virtnet_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash, 6667 enum xdp_rss_hash_type *rss_type) 6668 { 6669 const struct xdp_buff *xdp = (void *)_ctx; 6670 struct virtio_net_hdr_v1_hash *hdr_hash; 6671 struct virtnet_info *vi; 6672 u16 hash_report; 6673 6674 if (!(xdp->rxq->dev->features & NETIF_F_RXHASH)) 6675 return -ENODATA; 6676 6677 vi = netdev_priv(xdp->rxq->dev); 6678 hdr_hash = (struct virtio_net_hdr_v1_hash *)(xdp->data - vi->hdr_len); 6679 hash_report = __le16_to_cpu(hdr_hash->hash_report); 6680 6681 if (hash_report >= VIRTIO_NET_HASH_REPORT_MAX_TABLE) 6682 hash_report = VIRTIO_NET_HASH_REPORT_NONE; 6683 6684 *rss_type = virtnet_xdp_rss_type[hash_report]; 6685 *hash = __le32_to_cpu(hdr_hash->hash_value); 6686 return 0; 6687 } 6688 6689 static const struct xdp_metadata_ops virtnet_xdp_metadata_ops = { 6690 .xmo_rx_hash = virtnet_xdp_rx_hash, 6691 }; 6692 6693 static int virtnet_probe(struct virtio_device *vdev) 6694 { 6695 int i, err = -ENOMEM; 6696 struct net_device *dev; 6697 struct virtnet_info *vi; 6698 u16 max_queue_pairs; 6699 int mtu = 0; 6700 6701 /* Find if host supports multiqueue/rss virtio_net device */ 6702 max_queue_pairs = 1; 6703 if (virtio_has_feature(vdev, VIRTIO_NET_F_MQ) || virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) 6704 max_queue_pairs = 6705 virtio_cread16(vdev, offsetof(struct virtio_net_config, max_virtqueue_pairs)); 6706 6707 /* We need at least 2 queue's */ 6708 if (max_queue_pairs < VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN || 6709 max_queue_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX || 6710 !virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 6711 max_queue_pairs = 1; 6712 6713 /* Allocate ourselves a network device with room for our info */ 6714 dev = alloc_etherdev_mq(sizeof(struct virtnet_info), max_queue_pairs); 6715 if (!dev) 6716 return -ENOMEM; 6717 6718 /* Set up network device as normal. */ 6719 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE | 6720 IFF_TX_SKB_NO_LINEAR; 6721 dev->netdev_ops = &virtnet_netdev; 6722 dev->stat_ops = &virtnet_stat_ops; 6723 dev->features = NETIF_F_HIGHDMA; 6724 6725 dev->ethtool_ops = &virtnet_ethtool_ops; 6726 SET_NETDEV_DEV(dev, &vdev->dev); 6727 6728 /* Do we support "hardware" checksums? */ 6729 if (virtio_has_feature(vdev, VIRTIO_NET_F_CSUM)) { 6730 /* This opens up the world of extra features. */ 6731 dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_SG; 6732 if (csum) 6733 dev->features |= NETIF_F_HW_CSUM | NETIF_F_SG; 6734 6735 if (virtio_has_feature(vdev, VIRTIO_NET_F_GSO)) { 6736 dev->hw_features |= NETIF_F_TSO 6737 | NETIF_F_TSO_ECN | NETIF_F_TSO6; 6738 } 6739 /* Individual feature bits: what can host handle? */ 6740 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO4)) 6741 dev->hw_features |= NETIF_F_TSO; 6742 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_TSO6)) 6743 dev->hw_features |= NETIF_F_TSO6; 6744 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_ECN)) 6745 dev->hw_features |= NETIF_F_TSO_ECN; 6746 if (virtio_has_feature(vdev, VIRTIO_NET_F_HOST_USO)) 6747 dev->hw_features |= NETIF_F_GSO_UDP_L4; 6748 6749 dev->features |= NETIF_F_GSO_ROBUST; 6750 6751 if (gso) 6752 dev->features |= dev->hw_features & NETIF_F_ALL_TSO; 6753 /* (!csum && gso) case will be fixed by register_netdev() */ 6754 } 6755 6756 /* 1. With VIRTIO_NET_F_GUEST_CSUM negotiation, the driver doesn't 6757 * need to calculate checksums for partially checksummed packets, 6758 * as they're considered valid by the upper layer. 6759 * 2. Without VIRTIO_NET_F_GUEST_CSUM negotiation, the driver only 6760 * receives fully checksummed packets. The device may assist in 6761 * validating these packets' checksums, so the driver won't have to. 6762 */ 6763 dev->features |= NETIF_F_RXCSUM; 6764 6765 if (virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO4) || 6766 virtio_has_feature(vdev, VIRTIO_NET_F_GUEST_TSO6)) 6767 dev->features |= NETIF_F_GRO_HW; 6768 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS)) 6769 dev->hw_features |= NETIF_F_GRO_HW; 6770 6771 dev->vlan_features = dev->features; 6772 dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | 6773 NETDEV_XDP_ACT_XSK_ZEROCOPY; 6774 6775 /* MTU range: 68 - 65535 */ 6776 dev->min_mtu = MIN_MTU; 6777 dev->max_mtu = MAX_MTU; 6778 6779 /* Configuration may specify what MAC to use. Otherwise random. */ 6780 if (virtio_has_feature(vdev, VIRTIO_NET_F_MAC)) { 6781 u8 addr[ETH_ALEN]; 6782 6783 virtio_cread_bytes(vdev, 6784 offsetof(struct virtio_net_config, mac), 6785 addr, ETH_ALEN); 6786 eth_hw_addr_set(dev, addr); 6787 } else { 6788 eth_hw_addr_random(dev); 6789 dev_info(&vdev->dev, "Assigned random MAC address %pM\n", 6790 dev->dev_addr); 6791 } 6792 6793 /* Set up our device-specific information */ 6794 vi = netdev_priv(dev); 6795 vi->dev = dev; 6796 vi->vdev = vdev; 6797 vdev->priv = vi; 6798 6799 INIT_WORK(&vi->config_work, virtnet_config_changed_work); 6800 INIT_WORK(&vi->rx_mode_work, virtnet_rx_mode_work); 6801 spin_lock_init(&vi->refill_lock); 6802 6803 if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) { 6804 vi->mergeable_rx_bufs = true; 6805 dev->xdp_features |= NETDEV_XDP_ACT_RX_SG; 6806 } 6807 6808 if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) 6809 vi->has_rss_hash_report = true; 6810 6811 if (virtio_has_feature(vdev, VIRTIO_NET_F_RSS)) { 6812 vi->has_rss = true; 6813 6814 vi->rss_indir_table_size = 6815 virtio_cread16(vdev, offsetof(struct virtio_net_config, 6816 rss_max_indirection_table_length)); 6817 } 6818 vi->rss_hdr = devm_kzalloc(&vdev->dev, virtnet_rss_hdr_size(vi), GFP_KERNEL); 6819 if (!vi->rss_hdr) { 6820 err = -ENOMEM; 6821 goto free; 6822 } 6823 6824 if (vi->has_rss || vi->has_rss_hash_report) { 6825 vi->rss_key_size = 6826 virtio_cread8(vdev, offsetof(struct virtio_net_config, rss_max_key_size)); 6827 if (vi->rss_key_size > VIRTIO_NET_RSS_MAX_KEY_SIZE) { 6828 dev_err(&vdev->dev, "rss_max_key_size=%u exceeds the limit %u.\n", 6829 vi->rss_key_size, VIRTIO_NET_RSS_MAX_KEY_SIZE); 6830 err = -EINVAL; 6831 goto free; 6832 } 6833 6834 vi->rss_hash_types_supported = 6835 virtio_cread32(vdev, offsetof(struct virtio_net_config, supported_hash_types)); 6836 vi->rss_hash_types_supported &= 6837 ~(VIRTIO_NET_RSS_HASH_TYPE_IP_EX | 6838 VIRTIO_NET_RSS_HASH_TYPE_TCP_EX | 6839 VIRTIO_NET_RSS_HASH_TYPE_UDP_EX); 6840 6841 dev->hw_features |= NETIF_F_RXHASH; 6842 dev->xdp_metadata_ops = &virtnet_xdp_metadata_ops; 6843 } 6844 6845 if (vi->has_rss_hash_report) 6846 vi->hdr_len = sizeof(struct virtio_net_hdr_v1_hash); 6847 else if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF) || 6848 virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 6849 vi->hdr_len = sizeof(struct virtio_net_hdr_mrg_rxbuf); 6850 else 6851 vi->hdr_len = sizeof(struct virtio_net_hdr); 6852 6853 if (virtio_has_feature(vdev, VIRTIO_F_ANY_LAYOUT) || 6854 virtio_has_feature(vdev, VIRTIO_F_VERSION_1)) 6855 vi->any_header_sg = true; 6856 6857 if (virtio_has_feature(vdev, VIRTIO_NET_F_CTRL_VQ)) 6858 vi->has_cvq = true; 6859 6860 mutex_init(&vi->cvq_lock); 6861 6862 if (virtio_has_feature(vdev, VIRTIO_NET_F_MTU)) { 6863 mtu = virtio_cread16(vdev, 6864 offsetof(struct virtio_net_config, 6865 mtu)); 6866 if (mtu < dev->min_mtu) { 6867 /* Should never trigger: MTU was previously validated 6868 * in virtnet_validate. 6869 */ 6870 dev_err(&vdev->dev, 6871 "device MTU appears to have changed it is now %d < %d", 6872 mtu, dev->min_mtu); 6873 err = -EINVAL; 6874 goto free; 6875 } 6876 6877 dev->mtu = mtu; 6878 dev->max_mtu = mtu; 6879 } 6880 6881 virtnet_set_big_packets(vi, mtu); 6882 6883 if (vi->any_header_sg) 6884 dev->needed_headroom = vi->hdr_len; 6885 6886 /* Enable multiqueue by default */ 6887 if (num_online_cpus() >= max_queue_pairs) 6888 vi->curr_queue_pairs = max_queue_pairs; 6889 else 6890 vi->curr_queue_pairs = num_online_cpus(); 6891 vi->max_queue_pairs = max_queue_pairs; 6892 6893 /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ 6894 err = init_vqs(vi); 6895 if (err) 6896 goto free; 6897 6898 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { 6899 vi->intr_coal_rx.max_usecs = 0; 6900 vi->intr_coal_tx.max_usecs = 0; 6901 vi->intr_coal_rx.max_packets = 0; 6902 6903 /* Keep the default values of the coalescing parameters 6904 * aligned with the default napi_tx state. 6905 */ 6906 if (vi->sq[0].napi.weight) 6907 vi->intr_coal_tx.max_packets = 1; 6908 else 6909 vi->intr_coal_tx.max_packets = 0; 6910 } 6911 6912 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_VQ_NOTF_COAL)) { 6913 /* The reason is the same as VIRTIO_NET_F_NOTF_COAL. */ 6914 for (i = 0; i < vi->max_queue_pairs; i++) 6915 if (vi->sq[i].napi.weight) 6916 vi->sq[i].intr_coal.max_packets = 1; 6917 6918 err = virtnet_init_irq_moder(vi); 6919 if (err) 6920 goto free; 6921 } 6922 6923 #ifdef CONFIG_SYSFS 6924 if (vi->mergeable_rx_bufs) 6925 dev->sysfs_rx_queue_group = &virtio_net_mrg_rx_group; 6926 #endif 6927 netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); 6928 netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); 6929 6930 virtnet_init_settings(dev); 6931 6932 if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { 6933 vi->failover = net_failover_create(vi->dev); 6934 if (IS_ERR(vi->failover)) { 6935 err = PTR_ERR(vi->failover); 6936 goto free_vqs; 6937 } 6938 } 6939 6940 if (vi->has_rss || vi->has_rss_hash_report) 6941 virtnet_init_default_rss(vi); 6942 6943 enable_rx_mode_work(vi); 6944 6945 /* serialize netdev register + virtio_device_ready() with ndo_open() */ 6946 rtnl_lock(); 6947 6948 err = register_netdevice(dev); 6949 if (err) { 6950 pr_debug("virtio_net: registering device failed\n"); 6951 rtnl_unlock(); 6952 goto free_failover; 6953 } 6954 6955 /* Disable config change notification until ndo_open. */ 6956 virtio_config_driver_disable(vi->vdev); 6957 6958 virtio_device_ready(vdev); 6959 6960 if (vi->has_rss || vi->has_rss_hash_report) { 6961 if (!virtnet_commit_rss_command(vi)) { 6962 dev_warn(&vdev->dev, "RSS disabled because committing failed.\n"); 6963 dev->hw_features &= ~NETIF_F_RXHASH; 6964 vi->has_rss_hash_report = false; 6965 vi->has_rss = false; 6966 } 6967 } 6968 6969 virtnet_set_queues(vi, vi->curr_queue_pairs); 6970 6971 /* a random MAC address has been assigned, notify the device. 6972 * We don't fail probe if VIRTIO_NET_F_CTRL_MAC_ADDR is not there 6973 * because many devices work fine without getting MAC explicitly 6974 */ 6975 if (!virtio_has_feature(vdev, VIRTIO_NET_F_MAC) && 6976 virtio_has_feature(vi->vdev, VIRTIO_NET_F_CTRL_MAC_ADDR)) { 6977 struct scatterlist sg; 6978 6979 sg_init_one(&sg, dev->dev_addr, dev->addr_len); 6980 if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_MAC, 6981 VIRTIO_NET_CTRL_MAC_ADDR_SET, &sg)) { 6982 pr_debug("virtio_net: setting MAC address failed\n"); 6983 rtnl_unlock(); 6984 err = -EINVAL; 6985 goto free_unregister_netdev; 6986 } 6987 } 6988 6989 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_DEVICE_STATS)) { 6990 struct virtio_net_stats_capabilities *stats_cap __free(kfree) = NULL; 6991 struct scatterlist sg; 6992 __le64 v; 6993 6994 stats_cap = kzalloc(sizeof(*stats_cap), GFP_KERNEL); 6995 if (!stats_cap) { 6996 rtnl_unlock(); 6997 err = -ENOMEM; 6998 goto free_unregister_netdev; 6999 } 7000 7001 sg_init_one(&sg, stats_cap, sizeof(*stats_cap)); 7002 7003 if (!virtnet_send_command_reply(vi, VIRTIO_NET_CTRL_STATS, 7004 VIRTIO_NET_CTRL_STATS_QUERY, 7005 NULL, &sg)) { 7006 pr_debug("virtio_net: fail to get stats capability\n"); 7007 rtnl_unlock(); 7008 err = -EINVAL; 7009 goto free_unregister_netdev; 7010 } 7011 7012 v = stats_cap->supported_stats_types[0]; 7013 vi->device_stats_cap = le64_to_cpu(v); 7014 } 7015 7016 /* Assume link up if device can't report link status, 7017 otherwise get link status from config. */ 7018 netif_carrier_off(dev); 7019 if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_STATUS)) { 7020 virtnet_config_changed_work(&vi->config_work); 7021 } else { 7022 vi->status = VIRTIO_NET_S_LINK_UP; 7023 virtnet_update_settings(vi); 7024 netif_carrier_on(dev); 7025 } 7026 7027 for (i = 0; i < ARRAY_SIZE(guest_offloads); i++) 7028 if (virtio_has_feature(vi->vdev, guest_offloads[i])) 7029 set_bit(guest_offloads[i], &vi->guest_offloads); 7030 vi->guest_offloads_capable = vi->guest_offloads; 7031 7032 rtnl_unlock(); 7033 7034 err = virtnet_cpu_notif_add(vi); 7035 if (err) { 7036 pr_debug("virtio_net: registering cpu notifier failed\n"); 7037 goto free_unregister_netdev; 7038 } 7039 7040 pr_debug("virtnet: registered device %s with %d RX and TX vq's\n", 7041 dev->name, max_queue_pairs); 7042 7043 return 0; 7044 7045 free_unregister_netdev: 7046 unregister_netdev(dev); 7047 free_failover: 7048 net_failover_destroy(vi->failover); 7049 free_vqs: 7050 virtio_reset_device(vdev); 7051 cancel_delayed_work_sync(&vi->refill); 7052 free_receive_page_frags(vi); 7053 virtnet_del_vqs(vi); 7054 free: 7055 free_netdev(dev); 7056 return err; 7057 } 7058 7059 static void remove_vq_common(struct virtnet_info *vi) 7060 { 7061 int i; 7062 7063 virtio_reset_device(vi->vdev); 7064 7065 /* Free unused buffers in both send and recv, if any. */ 7066 free_unused_bufs(vi); 7067 7068 /* 7069 * Rule of thumb is netdev_tx_reset_queue() should follow any 7070 * skb freeing not followed by netdev_tx_completed_queue() 7071 */ 7072 for (i = 0; i < vi->max_queue_pairs; i++) 7073 netdev_tx_reset_queue(netdev_get_tx_queue(vi->dev, i)); 7074 7075 free_receive_bufs(vi); 7076 7077 free_receive_page_frags(vi); 7078 7079 virtnet_del_vqs(vi); 7080 } 7081 7082 static void virtnet_remove(struct virtio_device *vdev) 7083 { 7084 struct virtnet_info *vi = vdev->priv; 7085 7086 virtnet_cpu_notif_remove(vi); 7087 7088 /* Make sure no work handler is accessing the device. */ 7089 flush_work(&vi->config_work); 7090 disable_rx_mode_work(vi); 7091 flush_work(&vi->rx_mode_work); 7092 7093 virtnet_free_irq_moder(vi); 7094 7095 unregister_netdev(vi->dev); 7096 7097 net_failover_destroy(vi->failover); 7098 7099 remove_vq_common(vi); 7100 7101 free_netdev(vi->dev); 7102 } 7103 7104 static __maybe_unused int virtnet_freeze(struct virtio_device *vdev) 7105 { 7106 struct virtnet_info *vi = vdev->priv; 7107 7108 virtnet_cpu_notif_remove(vi); 7109 virtnet_freeze_down(vdev); 7110 remove_vq_common(vi); 7111 7112 return 0; 7113 } 7114 7115 static __maybe_unused int virtnet_restore(struct virtio_device *vdev) 7116 { 7117 struct virtnet_info *vi = vdev->priv; 7118 int err; 7119 7120 err = virtnet_restore_up(vdev); 7121 if (err) 7122 return err; 7123 virtnet_set_queues(vi, vi->curr_queue_pairs); 7124 7125 err = virtnet_cpu_notif_add(vi); 7126 if (err) { 7127 virtnet_freeze_down(vdev); 7128 remove_vq_common(vi); 7129 return err; 7130 } 7131 7132 return 0; 7133 } 7134 7135 static struct virtio_device_id id_table[] = { 7136 { VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID }, 7137 { 0 }, 7138 }; 7139 7140 #define VIRTNET_FEATURES \ 7141 VIRTIO_NET_F_CSUM, VIRTIO_NET_F_GUEST_CSUM, \ 7142 VIRTIO_NET_F_MAC, \ 7143 VIRTIO_NET_F_HOST_TSO4, VIRTIO_NET_F_HOST_UFO, VIRTIO_NET_F_HOST_TSO6, \ 7144 VIRTIO_NET_F_HOST_ECN, VIRTIO_NET_F_GUEST_TSO4, VIRTIO_NET_F_GUEST_TSO6, \ 7145 VIRTIO_NET_F_GUEST_ECN, VIRTIO_NET_F_GUEST_UFO, \ 7146 VIRTIO_NET_F_HOST_USO, VIRTIO_NET_F_GUEST_USO4, VIRTIO_NET_F_GUEST_USO6, \ 7147 VIRTIO_NET_F_MRG_RXBUF, VIRTIO_NET_F_STATUS, VIRTIO_NET_F_CTRL_VQ, \ 7148 VIRTIO_NET_F_CTRL_RX, VIRTIO_NET_F_CTRL_VLAN, \ 7149 VIRTIO_NET_F_GUEST_ANNOUNCE, VIRTIO_NET_F_MQ, \ 7150 VIRTIO_NET_F_CTRL_MAC_ADDR, \ 7151 VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ 7152 VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ 7153 VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL, \ 7154 VIRTIO_NET_F_VQ_NOTF_COAL, \ 7155 VIRTIO_NET_F_GUEST_HDRLEN, VIRTIO_NET_F_DEVICE_STATS 7156 7157 static unsigned int features[] = { 7158 VIRTNET_FEATURES, 7159 }; 7160 7161 static unsigned int features_legacy[] = { 7162 VIRTNET_FEATURES, 7163 VIRTIO_NET_F_GSO, 7164 VIRTIO_F_ANY_LAYOUT, 7165 }; 7166 7167 static struct virtio_driver virtio_net_driver = { 7168 .feature_table = features, 7169 .feature_table_size = ARRAY_SIZE(features), 7170 .feature_table_legacy = features_legacy, 7171 .feature_table_size_legacy = ARRAY_SIZE(features_legacy), 7172 .driver.name = KBUILD_MODNAME, 7173 .id_table = id_table, 7174 .validate = virtnet_validate, 7175 .probe = virtnet_probe, 7176 .remove = virtnet_remove, 7177 .config_changed = virtnet_config_changed, 7178 #ifdef CONFIG_PM_SLEEP 7179 .freeze = virtnet_freeze, 7180 .restore = virtnet_restore, 7181 #endif 7182 }; 7183 7184 static __init int virtio_net_driver_init(void) 7185 { 7186 int ret; 7187 7188 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "virtio/net:online", 7189 virtnet_cpu_online, 7190 virtnet_cpu_down_prep); 7191 if (ret < 0) 7192 goto out; 7193 virtionet_online = ret; 7194 ret = cpuhp_setup_state_multi(CPUHP_VIRT_NET_DEAD, "virtio/net:dead", 7195 NULL, virtnet_cpu_dead); 7196 if (ret) 7197 goto err_dead; 7198 ret = register_virtio_driver(&virtio_net_driver); 7199 if (ret) 7200 goto err_virtio; 7201 return 0; 7202 err_virtio: 7203 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 7204 err_dead: 7205 cpuhp_remove_multi_state(virtionet_online); 7206 out: 7207 return ret; 7208 } 7209 module_init(virtio_net_driver_init); 7210 7211 static __exit void virtio_net_driver_exit(void) 7212 { 7213 unregister_virtio_driver(&virtio_net_driver); 7214 cpuhp_remove_multi_state(CPUHP_VIRT_NET_DEAD); 7215 cpuhp_remove_multi_state(virtionet_online); 7216 } 7217 module_exit(virtio_net_driver_exit); 7218 7219 MODULE_DEVICE_TABLE(virtio, id_table); 7220 MODULE_DESCRIPTION("Virtio network driver"); 7221 MODULE_LICENSE("GPL"); 7222