1 /* 2 * Copyright (c) 2009, Microsoft Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, see <http://www.gnu.org/licenses/>. 15 * 16 * Authors: 17 * Haiyang Zhang <haiyangz@microsoft.com> 18 * Hank Janssen <hjanssen@microsoft.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include <linux/init.h> 23 #include <linux/atomic.h> 24 #include <linux/module.h> 25 #include <linux/highmem.h> 26 #include <linux/device.h> 27 #include <linux/io.h> 28 #include <linux/delay.h> 29 #include <linux/netdevice.h> 30 #include <linux/inetdevice.h> 31 #include <linux/etherdevice.h> 32 #include <linux/skbuff.h> 33 #include <linux/if_vlan.h> 34 #include <linux/in.h> 35 #include <linux/slab.h> 36 #include <net/arp.h> 37 #include <net/route.h> 38 #include <net/sock.h> 39 #include <net/pkt_sched.h> 40 41 #include "hyperv_net.h" 42 43 44 #define RING_SIZE_MIN 64 45 #define LINKCHANGE_INT (2 * HZ) 46 #define NETVSC_HW_FEATURES (NETIF_F_RXCSUM | \ 47 NETIF_F_SG | \ 48 NETIF_F_TSO | \ 49 NETIF_F_TSO6 | \ 50 NETIF_F_HW_CSUM) 51 static int ring_size = 128; 52 module_param(ring_size, int, S_IRUGO); 53 MODULE_PARM_DESC(ring_size, "Ring buffer size (# of pages)"); 54 55 static int max_num_vrss_chns = 8; 56 57 static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | 58 NETIF_MSG_LINK | NETIF_MSG_IFUP | 59 NETIF_MSG_IFDOWN | NETIF_MSG_RX_ERR | 60 NETIF_MSG_TX_ERR; 61 62 static int debug = -1; 63 module_param(debug, int, S_IRUGO); 64 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 65 66 static void do_set_multicast(struct work_struct *w) 67 { 68 struct net_device_context *ndevctx = 69 container_of(w, struct net_device_context, work); 70 struct netvsc_device *nvdev; 71 struct rndis_device *rdev; 72 73 nvdev = hv_get_drvdata(ndevctx->device_ctx); 74 if (nvdev == NULL || nvdev->ndev == NULL) 75 return; 76 77 rdev = nvdev->extension; 78 if (rdev == NULL) 79 return; 80 81 if (nvdev->ndev->flags & IFF_PROMISC) 82 rndis_filter_set_packet_filter(rdev, 83 NDIS_PACKET_TYPE_PROMISCUOUS); 84 else 85 rndis_filter_set_packet_filter(rdev, 86 NDIS_PACKET_TYPE_BROADCAST | 87 NDIS_PACKET_TYPE_ALL_MULTICAST | 88 NDIS_PACKET_TYPE_DIRECTED); 89 } 90 91 static void netvsc_set_multicast_list(struct net_device *net) 92 { 93 struct net_device_context *net_device_ctx = netdev_priv(net); 94 95 schedule_work(&net_device_ctx->work); 96 } 97 98 static int netvsc_open(struct net_device *net) 99 { 100 struct net_device_context *net_device_ctx = netdev_priv(net); 101 struct hv_device *device_obj = net_device_ctx->device_ctx; 102 struct netvsc_device *nvdev; 103 struct rndis_device *rdev; 104 int ret = 0; 105 106 netif_carrier_off(net); 107 108 /* Open up the device */ 109 ret = rndis_filter_open(device_obj); 110 if (ret != 0) { 111 netdev_err(net, "unable to open device (ret %d).\n", ret); 112 return ret; 113 } 114 115 netif_tx_wake_all_queues(net); 116 117 nvdev = hv_get_drvdata(device_obj); 118 rdev = nvdev->extension; 119 if (!rdev->link_state) 120 netif_carrier_on(net); 121 122 return ret; 123 } 124 125 static int netvsc_close(struct net_device *net) 126 { 127 struct net_device_context *net_device_ctx = netdev_priv(net); 128 struct hv_device *device_obj = net_device_ctx->device_ctx; 129 struct netvsc_device *nvdev = hv_get_drvdata(device_obj); 130 int ret; 131 u32 aread, awrite, i, msec = 10, retry = 0, retry_max = 20; 132 struct vmbus_channel *chn; 133 134 netif_tx_disable(net); 135 136 /* Make sure netvsc_set_multicast_list doesn't re-enable filter! */ 137 cancel_work_sync(&net_device_ctx->work); 138 ret = rndis_filter_close(device_obj); 139 if (ret != 0) { 140 netdev_err(net, "unable to close device (ret %d).\n", ret); 141 return ret; 142 } 143 144 /* Ensure pending bytes in ring are read */ 145 while (true) { 146 aread = 0; 147 for (i = 0; i < nvdev->num_chn; i++) { 148 chn = nvdev->chn_table[i]; 149 if (!chn) 150 continue; 151 152 hv_get_ringbuffer_availbytes(&chn->inbound, &aread, 153 &awrite); 154 155 if (aread) 156 break; 157 158 hv_get_ringbuffer_availbytes(&chn->outbound, &aread, 159 &awrite); 160 161 if (aread) 162 break; 163 } 164 165 retry++; 166 if (retry > retry_max || aread == 0) 167 break; 168 169 msleep(msec); 170 171 if (msec < 1000) 172 msec *= 2; 173 } 174 175 if (aread) { 176 netdev_err(net, "Ring buffer not empty after closing rndis\n"); 177 ret = -ETIMEDOUT; 178 } 179 180 return ret; 181 } 182 183 static void *init_ppi_data(struct rndis_message *msg, u32 ppi_size, 184 int pkt_type) 185 { 186 struct rndis_packet *rndis_pkt; 187 struct rndis_per_packet_info *ppi; 188 189 rndis_pkt = &msg->msg.pkt; 190 rndis_pkt->data_offset += ppi_size; 191 192 ppi = (struct rndis_per_packet_info *)((void *)rndis_pkt + 193 rndis_pkt->per_pkt_info_offset + rndis_pkt->per_pkt_info_len); 194 195 ppi->size = ppi_size; 196 ppi->type = pkt_type; 197 ppi->ppi_offset = sizeof(struct rndis_per_packet_info); 198 199 rndis_pkt->per_pkt_info_len += ppi_size; 200 201 return ppi; 202 } 203 204 static u16 netvsc_select_queue(struct net_device *ndev, struct sk_buff *skb, 205 void *accel_priv, select_queue_fallback_t fallback) 206 { 207 struct net_device_context *net_device_ctx = netdev_priv(ndev); 208 struct hv_device *hdev = net_device_ctx->device_ctx; 209 struct netvsc_device *nvsc_dev = hv_get_drvdata(hdev); 210 u32 hash; 211 u16 q_idx = 0; 212 213 if (nvsc_dev == NULL || ndev->real_num_tx_queues <= 1) 214 return 0; 215 216 hash = skb_get_hash(skb); 217 q_idx = nvsc_dev->send_table[hash % VRSS_SEND_TAB_SIZE] % 218 ndev->real_num_tx_queues; 219 220 if (!nvsc_dev->chn_table[q_idx]) 221 q_idx = 0; 222 223 return q_idx; 224 } 225 226 static u32 fill_pg_buf(struct page *page, u32 offset, u32 len, 227 struct hv_page_buffer *pb) 228 { 229 int j = 0; 230 231 /* Deal with compund pages by ignoring unused part 232 * of the page. 233 */ 234 page += (offset >> PAGE_SHIFT); 235 offset &= ~PAGE_MASK; 236 237 while (len > 0) { 238 unsigned long bytes; 239 240 bytes = PAGE_SIZE - offset; 241 if (bytes > len) 242 bytes = len; 243 pb[j].pfn = page_to_pfn(page); 244 pb[j].offset = offset; 245 pb[j].len = bytes; 246 247 offset += bytes; 248 len -= bytes; 249 250 if (offset == PAGE_SIZE && len) { 251 page++; 252 offset = 0; 253 j++; 254 } 255 } 256 257 return j + 1; 258 } 259 260 static u32 init_page_array(void *hdr, u32 len, struct sk_buff *skb, 261 struct hv_netvsc_packet *packet, 262 struct hv_page_buffer **page_buf) 263 { 264 struct hv_page_buffer *pb = *page_buf; 265 u32 slots_used = 0; 266 char *data = skb->data; 267 int frags = skb_shinfo(skb)->nr_frags; 268 int i; 269 270 /* The packet is laid out thus: 271 * 1. hdr: RNDIS header and PPI 272 * 2. skb linear data 273 * 3. skb fragment data 274 */ 275 if (hdr != NULL) 276 slots_used += fill_pg_buf(virt_to_page(hdr), 277 offset_in_page(hdr), 278 len, &pb[slots_used]); 279 280 packet->rmsg_size = len; 281 packet->rmsg_pgcnt = slots_used; 282 283 slots_used += fill_pg_buf(virt_to_page(data), 284 offset_in_page(data), 285 skb_headlen(skb), &pb[slots_used]); 286 287 for (i = 0; i < frags; i++) { 288 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 289 290 slots_used += fill_pg_buf(skb_frag_page(frag), 291 frag->page_offset, 292 skb_frag_size(frag), &pb[slots_used]); 293 } 294 return slots_used; 295 } 296 297 static int count_skb_frag_slots(struct sk_buff *skb) 298 { 299 int i, frags = skb_shinfo(skb)->nr_frags; 300 int pages = 0; 301 302 for (i = 0; i < frags; i++) { 303 skb_frag_t *frag = skb_shinfo(skb)->frags + i; 304 unsigned long size = skb_frag_size(frag); 305 unsigned long offset = frag->page_offset; 306 307 /* Skip unused frames from start of page */ 308 offset &= ~PAGE_MASK; 309 pages += PFN_UP(offset + size); 310 } 311 return pages; 312 } 313 314 static int netvsc_get_slots(struct sk_buff *skb) 315 { 316 char *data = skb->data; 317 unsigned int offset = offset_in_page(data); 318 unsigned int len = skb_headlen(skb); 319 int slots; 320 int frag_slots; 321 322 slots = DIV_ROUND_UP(offset + len, PAGE_SIZE); 323 frag_slots = count_skb_frag_slots(skb); 324 return slots + frag_slots; 325 } 326 327 static u32 get_net_transport_info(struct sk_buff *skb, u32 *trans_off) 328 { 329 u32 ret_val = TRANSPORT_INFO_NOT_IP; 330 331 if ((eth_hdr(skb)->h_proto != htons(ETH_P_IP)) && 332 (eth_hdr(skb)->h_proto != htons(ETH_P_IPV6))) { 333 goto not_ip; 334 } 335 336 *trans_off = skb_transport_offset(skb); 337 338 if ((eth_hdr(skb)->h_proto == htons(ETH_P_IP))) { 339 struct iphdr *iphdr = ip_hdr(skb); 340 341 if (iphdr->protocol == IPPROTO_TCP) 342 ret_val = TRANSPORT_INFO_IPV4_TCP; 343 else if (iphdr->protocol == IPPROTO_UDP) 344 ret_val = TRANSPORT_INFO_IPV4_UDP; 345 } else { 346 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP) 347 ret_val = TRANSPORT_INFO_IPV6_TCP; 348 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP) 349 ret_val = TRANSPORT_INFO_IPV6_UDP; 350 } 351 352 not_ip: 353 return ret_val; 354 } 355 356 static int netvsc_start_xmit(struct sk_buff *skb, struct net_device *net) 357 { 358 struct net_device_context *net_device_ctx = netdev_priv(net); 359 struct hv_netvsc_packet *packet = NULL; 360 int ret; 361 unsigned int num_data_pgs; 362 struct rndis_message *rndis_msg; 363 struct rndis_packet *rndis_pkt; 364 u32 rndis_msg_size; 365 bool isvlan; 366 bool linear = false; 367 struct rndis_per_packet_info *ppi; 368 struct ndis_tcp_ip_checksum_info *csum_info; 369 struct ndis_tcp_lso_info *lso_info; 370 int hdr_offset; 371 u32 net_trans_info; 372 u32 hash; 373 u32 skb_length; 374 struct hv_page_buffer page_buf[MAX_PAGE_BUFFER_COUNT]; 375 struct hv_page_buffer *pb = page_buf; 376 struct netvsc_stats *tx_stats = this_cpu_ptr(net_device_ctx->tx_stats); 377 378 /* We will atmost need two pages to describe the rndis 379 * header. We can only transmit MAX_PAGE_BUFFER_COUNT number 380 * of pages in a single packet. If skb is scattered around 381 * more pages we try linearizing it. 382 */ 383 384 check_size: 385 skb_length = skb->len; 386 num_data_pgs = netvsc_get_slots(skb) + 2; 387 if (num_data_pgs > MAX_PAGE_BUFFER_COUNT && linear) { 388 net_alert_ratelimited("packet too big: %u pages (%u bytes)\n", 389 num_data_pgs, skb->len); 390 ret = -EFAULT; 391 goto drop; 392 } else if (num_data_pgs > MAX_PAGE_BUFFER_COUNT) { 393 if (skb_linearize(skb)) { 394 net_alert_ratelimited("failed to linearize skb\n"); 395 ret = -ENOMEM; 396 goto drop; 397 } 398 linear = true; 399 goto check_size; 400 } 401 402 /* 403 * Place the rndis header in the skb head room and 404 * the skb->cb will be used for hv_netvsc_packet 405 * structure. 406 */ 407 ret = skb_cow_head(skb, RNDIS_AND_PPI_SIZE); 408 if (ret) { 409 netdev_err(net, "unable to alloc hv_netvsc_packet\n"); 410 ret = -ENOMEM; 411 goto drop; 412 } 413 /* Use the skb control buffer for building up the packet */ 414 BUILD_BUG_ON(sizeof(struct hv_netvsc_packet) > 415 FIELD_SIZEOF(struct sk_buff, cb)); 416 packet = (struct hv_netvsc_packet *)skb->cb; 417 418 419 packet->q_idx = skb_get_queue_mapping(skb); 420 421 packet->total_data_buflen = skb->len; 422 423 rndis_msg = (struct rndis_message *)skb->head; 424 425 memset(rndis_msg, 0, RNDIS_AND_PPI_SIZE); 426 427 isvlan = skb->vlan_tci & VLAN_TAG_PRESENT; 428 429 /* Add the rndis header */ 430 rndis_msg->ndis_msg_type = RNDIS_MSG_PACKET; 431 rndis_msg->msg_len = packet->total_data_buflen; 432 rndis_pkt = &rndis_msg->msg.pkt; 433 rndis_pkt->data_offset = sizeof(struct rndis_packet); 434 rndis_pkt->data_len = packet->total_data_buflen; 435 rndis_pkt->per_pkt_info_offset = sizeof(struct rndis_packet); 436 437 rndis_msg_size = RNDIS_MESSAGE_SIZE(struct rndis_packet); 438 439 hash = skb_get_hash_raw(skb); 440 if (hash != 0 && net->real_num_tx_queues > 1) { 441 rndis_msg_size += NDIS_HASH_PPI_SIZE; 442 ppi = init_ppi_data(rndis_msg, NDIS_HASH_PPI_SIZE, 443 NBL_HASH_VALUE); 444 *(u32 *)((void *)ppi + ppi->ppi_offset) = hash; 445 } 446 447 if (isvlan) { 448 struct ndis_pkt_8021q_info *vlan; 449 450 rndis_msg_size += NDIS_VLAN_PPI_SIZE; 451 ppi = init_ppi_data(rndis_msg, NDIS_VLAN_PPI_SIZE, 452 IEEE_8021Q_INFO); 453 vlan = (struct ndis_pkt_8021q_info *)((void *)ppi + 454 ppi->ppi_offset); 455 vlan->vlanid = skb->vlan_tci & VLAN_VID_MASK; 456 vlan->pri = (skb->vlan_tci & VLAN_PRIO_MASK) >> 457 VLAN_PRIO_SHIFT; 458 } 459 460 net_trans_info = get_net_transport_info(skb, &hdr_offset); 461 if (net_trans_info == TRANSPORT_INFO_NOT_IP) 462 goto do_send; 463 464 /* 465 * Setup the sendside checksum offload only if this is not a 466 * GSO packet. 467 */ 468 if (skb_is_gso(skb)) 469 goto do_lso; 470 471 if ((skb->ip_summed == CHECKSUM_NONE) || 472 (skb->ip_summed == CHECKSUM_UNNECESSARY)) 473 goto do_send; 474 475 rndis_msg_size += NDIS_CSUM_PPI_SIZE; 476 ppi = init_ppi_data(rndis_msg, NDIS_CSUM_PPI_SIZE, 477 TCPIP_CHKSUM_PKTINFO); 478 479 csum_info = (struct ndis_tcp_ip_checksum_info *)((void *)ppi + 480 ppi->ppi_offset); 481 482 if (net_trans_info & (INFO_IPV4 << 16)) 483 csum_info->transmit.is_ipv4 = 1; 484 else 485 csum_info->transmit.is_ipv6 = 1; 486 487 if (net_trans_info & INFO_TCP) { 488 csum_info->transmit.tcp_checksum = 1; 489 csum_info->transmit.tcp_header_offset = hdr_offset; 490 } else if (net_trans_info & INFO_UDP) { 491 /* UDP checksum offload is not supported on ws2008r2. 492 * Furthermore, on ws2012 and ws2012r2, there are some 493 * issues with udp checksum offload from Linux guests. 494 * (these are host issues). 495 * For now compute the checksum here. 496 */ 497 struct udphdr *uh; 498 u16 udp_len; 499 500 ret = skb_cow_head(skb, 0); 501 if (ret) 502 goto drop; 503 504 uh = udp_hdr(skb); 505 udp_len = ntohs(uh->len); 506 uh->check = 0; 507 uh->check = csum_tcpudp_magic(ip_hdr(skb)->saddr, 508 ip_hdr(skb)->daddr, 509 udp_len, IPPROTO_UDP, 510 csum_partial(uh, udp_len, 0)); 511 if (uh->check == 0) 512 uh->check = CSUM_MANGLED_0; 513 514 csum_info->transmit.udp_checksum = 0; 515 } 516 goto do_send; 517 518 do_lso: 519 rndis_msg_size += NDIS_LSO_PPI_SIZE; 520 ppi = init_ppi_data(rndis_msg, NDIS_LSO_PPI_SIZE, 521 TCP_LARGESEND_PKTINFO); 522 523 lso_info = (struct ndis_tcp_lso_info *)((void *)ppi + 524 ppi->ppi_offset); 525 526 lso_info->lso_v2_transmit.type = NDIS_TCP_LARGE_SEND_OFFLOAD_V2_TYPE; 527 if (net_trans_info & (INFO_IPV4 << 16)) { 528 lso_info->lso_v2_transmit.ip_version = 529 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV4; 530 ip_hdr(skb)->tot_len = 0; 531 ip_hdr(skb)->check = 0; 532 tcp_hdr(skb)->check = 533 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 534 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 535 } else { 536 lso_info->lso_v2_transmit.ip_version = 537 NDIS_TCP_LARGE_SEND_OFFLOAD_IPV6; 538 ipv6_hdr(skb)->payload_len = 0; 539 tcp_hdr(skb)->check = 540 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 541 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0); 542 } 543 lso_info->lso_v2_transmit.tcp_header_offset = hdr_offset; 544 lso_info->lso_v2_transmit.mss = skb_shinfo(skb)->gso_size; 545 546 do_send: 547 /* Start filling in the page buffers with the rndis hdr */ 548 rndis_msg->msg_len += rndis_msg_size; 549 packet->total_data_buflen = rndis_msg->msg_len; 550 packet->page_buf_cnt = init_page_array(rndis_msg, rndis_msg_size, 551 skb, packet, &pb); 552 553 /* timestamp packet in software */ 554 skb_tx_timestamp(skb); 555 ret = netvsc_send(net_device_ctx->device_ctx, packet, 556 rndis_msg, &pb, skb); 557 558 drop: 559 if (ret == 0) { 560 u64_stats_update_begin(&tx_stats->syncp); 561 tx_stats->packets++; 562 tx_stats->bytes += skb_length; 563 u64_stats_update_end(&tx_stats->syncp); 564 } else { 565 if (ret != -EAGAIN) { 566 dev_kfree_skb_any(skb); 567 net->stats.tx_dropped++; 568 } 569 } 570 571 return (ret == -EAGAIN) ? NETDEV_TX_BUSY : NETDEV_TX_OK; 572 } 573 574 /* 575 * netvsc_linkstatus_callback - Link up/down notification 576 */ 577 void netvsc_linkstatus_callback(struct hv_device *device_obj, 578 struct rndis_message *resp) 579 { 580 struct rndis_indicate_status *indicate = &resp->msg.indicate_status; 581 struct net_device *net; 582 struct net_device_context *ndev_ctx; 583 struct netvsc_device *net_device; 584 struct netvsc_reconfig *event; 585 unsigned long flags; 586 587 /* Handle link change statuses only */ 588 if (indicate->status != RNDIS_STATUS_NETWORK_CHANGE && 589 indicate->status != RNDIS_STATUS_MEDIA_CONNECT && 590 indicate->status != RNDIS_STATUS_MEDIA_DISCONNECT) 591 return; 592 593 net_device = hv_get_drvdata(device_obj); 594 net = net_device->ndev; 595 596 if (!net || net->reg_state != NETREG_REGISTERED) 597 return; 598 599 ndev_ctx = netdev_priv(net); 600 601 event = kzalloc(sizeof(*event), GFP_ATOMIC); 602 if (!event) 603 return; 604 event->event = indicate->status; 605 606 spin_lock_irqsave(&ndev_ctx->lock, flags); 607 list_add_tail(&event->list, &ndev_ctx->reconfig_events); 608 spin_unlock_irqrestore(&ndev_ctx->lock, flags); 609 610 schedule_delayed_work(&ndev_ctx->dwork, 0); 611 } 612 613 /* 614 * netvsc_recv_callback - Callback when we receive a packet from the 615 * "wire" on the specified device. 616 */ 617 int netvsc_recv_callback(struct hv_device *device_obj, 618 struct hv_netvsc_packet *packet, 619 void **data, 620 struct ndis_tcp_ip_checksum_info *csum_info, 621 struct vmbus_channel *channel, 622 u16 vlan_tci) 623 { 624 struct net_device *net; 625 struct net_device_context *net_device_ctx; 626 struct sk_buff *skb; 627 struct netvsc_stats *rx_stats; 628 629 net = ((struct netvsc_device *)hv_get_drvdata(device_obj))->ndev; 630 if (!net || net->reg_state != NETREG_REGISTERED) { 631 return NVSP_STAT_FAIL; 632 } 633 net_device_ctx = netdev_priv(net); 634 rx_stats = this_cpu_ptr(net_device_ctx->rx_stats); 635 636 /* Allocate a skb - TODO direct I/O to pages? */ 637 skb = netdev_alloc_skb_ip_align(net, packet->total_data_buflen); 638 if (unlikely(!skb)) { 639 ++net->stats.rx_dropped; 640 return NVSP_STAT_FAIL; 641 } 642 643 /* 644 * Copy to skb. This copy is needed here since the memory pointed by 645 * hv_netvsc_packet cannot be deallocated 646 */ 647 memcpy(skb_put(skb, packet->total_data_buflen), *data, 648 packet->total_data_buflen); 649 650 skb->protocol = eth_type_trans(skb, net); 651 if (csum_info) { 652 /* We only look at the IP checksum here. 653 * Should we be dropping the packet if checksum 654 * failed? How do we deal with other checksums - TCP/UDP? 655 */ 656 if (csum_info->receive.ip_checksum_succeeded) 657 skb->ip_summed = CHECKSUM_UNNECESSARY; 658 else 659 skb->ip_summed = CHECKSUM_NONE; 660 } 661 662 if (vlan_tci & VLAN_TAG_PRESENT) 663 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 664 vlan_tci); 665 666 skb_record_rx_queue(skb, channel-> 667 offermsg.offer.sub_channel_index); 668 669 u64_stats_update_begin(&rx_stats->syncp); 670 rx_stats->packets++; 671 rx_stats->bytes += packet->total_data_buflen; 672 u64_stats_update_end(&rx_stats->syncp); 673 674 /* 675 * Pass the skb back up. Network stack will deallocate the skb when it 676 * is done. 677 * TODO - use NAPI? 678 */ 679 netif_rx(skb); 680 681 return 0; 682 } 683 684 static void netvsc_get_drvinfo(struct net_device *net, 685 struct ethtool_drvinfo *info) 686 { 687 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 688 strlcpy(info->fw_version, "N/A", sizeof(info->fw_version)); 689 } 690 691 static void netvsc_get_channels(struct net_device *net, 692 struct ethtool_channels *channel) 693 { 694 struct net_device_context *net_device_ctx = netdev_priv(net); 695 struct hv_device *dev = net_device_ctx->device_ctx; 696 struct netvsc_device *nvdev = hv_get_drvdata(dev); 697 698 if (nvdev) { 699 channel->max_combined = nvdev->max_chn; 700 channel->combined_count = nvdev->num_chn; 701 } 702 } 703 704 static int netvsc_set_channels(struct net_device *net, 705 struct ethtool_channels *channels) 706 { 707 struct net_device_context *net_device_ctx = netdev_priv(net); 708 struct hv_device *dev = net_device_ctx->device_ctx; 709 struct netvsc_device *nvdev = hv_get_drvdata(dev); 710 struct netvsc_device_info device_info; 711 u32 num_chn; 712 u32 max_chn; 713 int ret = 0; 714 bool recovering = false; 715 716 if (!nvdev || nvdev->destroy) 717 return -ENODEV; 718 719 num_chn = nvdev->num_chn; 720 max_chn = min_t(u32, nvdev->max_chn, num_online_cpus()); 721 722 if (nvdev->nvsp_version < NVSP_PROTOCOL_VERSION_5) { 723 pr_info("vRSS unsupported before NVSP Version 5\n"); 724 return -EINVAL; 725 } 726 727 /* We do not support rx, tx, or other */ 728 if (!channels || 729 channels->rx_count || 730 channels->tx_count || 731 channels->other_count || 732 (channels->combined_count < 1)) 733 return -EINVAL; 734 735 if (channels->combined_count > max_chn) { 736 pr_info("combined channels too high, using %d\n", max_chn); 737 channels->combined_count = max_chn; 738 } 739 740 ret = netvsc_close(net); 741 if (ret) 742 goto out; 743 744 do_set: 745 nvdev->start_remove = true; 746 rndis_filter_device_remove(dev); 747 748 nvdev->num_chn = channels->combined_count; 749 750 net_device_ctx->device_ctx = dev; 751 hv_set_drvdata(dev, net); 752 753 memset(&device_info, 0, sizeof(device_info)); 754 device_info.num_chn = nvdev->num_chn; /* passed to RNDIS */ 755 device_info.ring_size = ring_size; 756 device_info.max_num_vrss_chns = max_num_vrss_chns; 757 758 ret = rndis_filter_device_add(dev, &device_info); 759 if (ret) { 760 if (recovering) { 761 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret); 762 return ret; 763 } 764 goto recover; 765 } 766 767 nvdev = hv_get_drvdata(dev); 768 769 ret = netif_set_real_num_tx_queues(net, nvdev->num_chn); 770 if (ret) { 771 if (recovering) { 772 netdev_err(net, "could not set tx queue count (ret %d)\n", ret); 773 return ret; 774 } 775 goto recover; 776 } 777 778 ret = netif_set_real_num_rx_queues(net, nvdev->num_chn); 779 if (ret) { 780 if (recovering) { 781 netdev_err(net, "could not set rx queue count (ret %d)\n", ret); 782 return ret; 783 } 784 goto recover; 785 } 786 787 out: 788 netvsc_open(net); 789 790 return ret; 791 792 recover: 793 /* If the above failed, we attempt to recover through the same 794 * process but with the original number of channels. 795 */ 796 netdev_err(net, "could not set channels, recovering\n"); 797 recovering = true; 798 channels->combined_count = num_chn; 799 goto do_set; 800 } 801 802 static int netvsc_change_mtu(struct net_device *ndev, int mtu) 803 { 804 struct net_device_context *ndevctx = netdev_priv(ndev); 805 struct hv_device *hdev = ndevctx->device_ctx; 806 struct netvsc_device *nvdev = hv_get_drvdata(hdev); 807 struct netvsc_device_info device_info; 808 int limit = ETH_DATA_LEN; 809 int ret = 0; 810 811 if (nvdev == NULL || nvdev->destroy) 812 return -ENODEV; 813 814 if (nvdev->nvsp_version >= NVSP_PROTOCOL_VERSION_2) 815 limit = NETVSC_MTU - ETH_HLEN; 816 817 if (mtu < NETVSC_MTU_MIN || mtu > limit) 818 return -EINVAL; 819 820 ret = netvsc_close(ndev); 821 if (ret) 822 goto out; 823 824 nvdev->start_remove = true; 825 rndis_filter_device_remove(hdev); 826 827 ndev->mtu = mtu; 828 829 ndevctx->device_ctx = hdev; 830 hv_set_drvdata(hdev, ndev); 831 832 memset(&device_info, 0, sizeof(device_info)); 833 device_info.ring_size = ring_size; 834 device_info.num_chn = nvdev->num_chn; 835 device_info.max_num_vrss_chns = max_num_vrss_chns; 836 rndis_filter_device_add(hdev, &device_info); 837 838 out: 839 netvsc_open(ndev); 840 841 return ret; 842 } 843 844 static struct rtnl_link_stats64 *netvsc_get_stats64(struct net_device *net, 845 struct rtnl_link_stats64 *t) 846 { 847 struct net_device_context *ndev_ctx = netdev_priv(net); 848 int cpu; 849 850 for_each_possible_cpu(cpu) { 851 struct netvsc_stats *tx_stats = per_cpu_ptr(ndev_ctx->tx_stats, 852 cpu); 853 struct netvsc_stats *rx_stats = per_cpu_ptr(ndev_ctx->rx_stats, 854 cpu); 855 u64 tx_packets, tx_bytes, rx_packets, rx_bytes; 856 unsigned int start; 857 858 do { 859 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); 860 tx_packets = tx_stats->packets; 861 tx_bytes = tx_stats->bytes; 862 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); 863 864 do { 865 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); 866 rx_packets = rx_stats->packets; 867 rx_bytes = rx_stats->bytes; 868 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); 869 870 t->tx_bytes += tx_bytes; 871 t->tx_packets += tx_packets; 872 t->rx_bytes += rx_bytes; 873 t->rx_packets += rx_packets; 874 } 875 876 t->tx_dropped = net->stats.tx_dropped; 877 t->tx_errors = net->stats.tx_dropped; 878 879 t->rx_dropped = net->stats.rx_dropped; 880 t->rx_errors = net->stats.rx_errors; 881 882 return t; 883 } 884 885 static int netvsc_set_mac_addr(struct net_device *ndev, void *p) 886 { 887 struct net_device_context *ndevctx = netdev_priv(ndev); 888 struct hv_device *hdev = ndevctx->device_ctx; 889 struct sockaddr *addr = p; 890 char save_adr[ETH_ALEN]; 891 unsigned char save_aatype; 892 int err; 893 894 memcpy(save_adr, ndev->dev_addr, ETH_ALEN); 895 save_aatype = ndev->addr_assign_type; 896 897 err = eth_mac_addr(ndev, p); 898 if (err != 0) 899 return err; 900 901 err = rndis_filter_set_device_mac(hdev, addr->sa_data); 902 if (err != 0) { 903 /* roll back to saved MAC */ 904 memcpy(ndev->dev_addr, save_adr, ETH_ALEN); 905 ndev->addr_assign_type = save_aatype; 906 } 907 908 return err; 909 } 910 911 #ifdef CONFIG_NET_POLL_CONTROLLER 912 static void netvsc_poll_controller(struct net_device *net) 913 { 914 /* As netvsc_start_xmit() works synchronous we don't have to 915 * trigger anything here. 916 */ 917 } 918 #endif 919 920 static const struct ethtool_ops ethtool_ops = { 921 .get_drvinfo = netvsc_get_drvinfo, 922 .get_link = ethtool_op_get_link, 923 .get_channels = netvsc_get_channels, 924 .set_channels = netvsc_set_channels, 925 .get_ts_info = ethtool_op_get_ts_info, 926 }; 927 928 static const struct net_device_ops device_ops = { 929 .ndo_open = netvsc_open, 930 .ndo_stop = netvsc_close, 931 .ndo_start_xmit = netvsc_start_xmit, 932 .ndo_set_rx_mode = netvsc_set_multicast_list, 933 .ndo_change_mtu = netvsc_change_mtu, 934 .ndo_validate_addr = eth_validate_addr, 935 .ndo_set_mac_address = netvsc_set_mac_addr, 936 .ndo_select_queue = netvsc_select_queue, 937 .ndo_get_stats64 = netvsc_get_stats64, 938 #ifdef CONFIG_NET_POLL_CONTROLLER 939 .ndo_poll_controller = netvsc_poll_controller, 940 #endif 941 }; 942 943 /* 944 * Handle link status changes. For RNDIS_STATUS_NETWORK_CHANGE emulate link 945 * down/up sequence. In case of RNDIS_STATUS_MEDIA_CONNECT when carrier is 946 * present send GARP packet to network peers with netif_notify_peers(). 947 */ 948 static void netvsc_link_change(struct work_struct *w) 949 { 950 struct net_device_context *ndev_ctx; 951 struct net_device *net; 952 struct netvsc_device *net_device; 953 struct rndis_device *rdev; 954 struct netvsc_reconfig *event = NULL; 955 bool notify = false, reschedule = false; 956 unsigned long flags, next_reconfig, delay; 957 958 ndev_ctx = container_of(w, struct net_device_context, dwork.work); 959 net_device = hv_get_drvdata(ndev_ctx->device_ctx); 960 rdev = net_device->extension; 961 net = net_device->ndev; 962 963 next_reconfig = ndev_ctx->last_reconfig + LINKCHANGE_INT; 964 if (time_is_after_jiffies(next_reconfig)) { 965 /* link_watch only sends one notification with current state 966 * per second, avoid doing reconfig more frequently. Handle 967 * wrap around. 968 */ 969 delay = next_reconfig - jiffies; 970 delay = delay < LINKCHANGE_INT ? delay : LINKCHANGE_INT; 971 schedule_delayed_work(&ndev_ctx->dwork, delay); 972 return; 973 } 974 ndev_ctx->last_reconfig = jiffies; 975 976 spin_lock_irqsave(&ndev_ctx->lock, flags); 977 if (!list_empty(&ndev_ctx->reconfig_events)) { 978 event = list_first_entry(&ndev_ctx->reconfig_events, 979 struct netvsc_reconfig, list); 980 list_del(&event->list); 981 reschedule = !list_empty(&ndev_ctx->reconfig_events); 982 } 983 spin_unlock_irqrestore(&ndev_ctx->lock, flags); 984 985 if (!event) 986 return; 987 988 rtnl_lock(); 989 990 switch (event->event) { 991 /* Only the following events are possible due to the check in 992 * netvsc_linkstatus_callback() 993 */ 994 case RNDIS_STATUS_MEDIA_CONNECT: 995 if (rdev->link_state) { 996 rdev->link_state = false; 997 netif_carrier_on(net); 998 netif_tx_wake_all_queues(net); 999 } else { 1000 notify = true; 1001 } 1002 kfree(event); 1003 break; 1004 case RNDIS_STATUS_MEDIA_DISCONNECT: 1005 if (!rdev->link_state) { 1006 rdev->link_state = true; 1007 netif_carrier_off(net); 1008 netif_tx_stop_all_queues(net); 1009 } 1010 kfree(event); 1011 break; 1012 case RNDIS_STATUS_NETWORK_CHANGE: 1013 /* Only makes sense if carrier is present */ 1014 if (!rdev->link_state) { 1015 rdev->link_state = true; 1016 netif_carrier_off(net); 1017 netif_tx_stop_all_queues(net); 1018 event->event = RNDIS_STATUS_MEDIA_CONNECT; 1019 spin_lock_irqsave(&ndev_ctx->lock, flags); 1020 list_add_tail(&event->list, &ndev_ctx->reconfig_events); 1021 spin_unlock_irqrestore(&ndev_ctx->lock, flags); 1022 reschedule = true; 1023 } 1024 break; 1025 } 1026 1027 rtnl_unlock(); 1028 1029 if (notify) 1030 netdev_notify_peers(net); 1031 1032 /* link_watch only sends one notification with current state per 1033 * second, handle next reconfig event in 2 seconds. 1034 */ 1035 if (reschedule) 1036 schedule_delayed_work(&ndev_ctx->dwork, LINKCHANGE_INT); 1037 } 1038 1039 static void netvsc_free_netdev(struct net_device *netdev) 1040 { 1041 struct net_device_context *net_device_ctx = netdev_priv(netdev); 1042 1043 free_percpu(net_device_ctx->tx_stats); 1044 free_percpu(net_device_ctx->rx_stats); 1045 free_netdev(netdev); 1046 } 1047 1048 static int netvsc_probe(struct hv_device *dev, 1049 const struct hv_vmbus_device_id *dev_id) 1050 { 1051 struct net_device *net = NULL; 1052 struct net_device_context *net_device_ctx; 1053 struct netvsc_device_info device_info; 1054 struct netvsc_device *nvdev; 1055 int ret; 1056 1057 net = alloc_etherdev_mq(sizeof(struct net_device_context), 1058 num_online_cpus()); 1059 if (!net) 1060 return -ENOMEM; 1061 1062 netif_carrier_off(net); 1063 1064 net_device_ctx = netdev_priv(net); 1065 net_device_ctx->device_ctx = dev; 1066 net_device_ctx->msg_enable = netif_msg_init(debug, default_msg); 1067 if (netif_msg_probe(net_device_ctx)) 1068 netdev_dbg(net, "netvsc msg_enable: %d\n", 1069 net_device_ctx->msg_enable); 1070 1071 net_device_ctx->tx_stats = netdev_alloc_pcpu_stats(struct netvsc_stats); 1072 if (!net_device_ctx->tx_stats) { 1073 free_netdev(net); 1074 return -ENOMEM; 1075 } 1076 net_device_ctx->rx_stats = netdev_alloc_pcpu_stats(struct netvsc_stats); 1077 if (!net_device_ctx->rx_stats) { 1078 free_percpu(net_device_ctx->tx_stats); 1079 free_netdev(net); 1080 return -ENOMEM; 1081 } 1082 1083 hv_set_drvdata(dev, net); 1084 INIT_DELAYED_WORK(&net_device_ctx->dwork, netvsc_link_change); 1085 INIT_WORK(&net_device_ctx->work, do_set_multicast); 1086 1087 spin_lock_init(&net_device_ctx->lock); 1088 INIT_LIST_HEAD(&net_device_ctx->reconfig_events); 1089 1090 net->netdev_ops = &device_ops; 1091 1092 net->hw_features = NETVSC_HW_FEATURES; 1093 net->features = NETVSC_HW_FEATURES | NETIF_F_HW_VLAN_CTAG_TX; 1094 1095 net->ethtool_ops = ðtool_ops; 1096 SET_NETDEV_DEV(net, &dev->device); 1097 1098 /* We always need headroom for rndis header */ 1099 net->needed_headroom = RNDIS_AND_PPI_SIZE; 1100 1101 /* Notify the netvsc driver of the new device */ 1102 memset(&device_info, 0, sizeof(device_info)); 1103 device_info.ring_size = ring_size; 1104 device_info.max_num_vrss_chns = max_num_vrss_chns; 1105 ret = rndis_filter_device_add(dev, &device_info); 1106 if (ret != 0) { 1107 netdev_err(net, "unable to add netvsc device (ret %d)\n", ret); 1108 netvsc_free_netdev(net); 1109 hv_set_drvdata(dev, NULL); 1110 return ret; 1111 } 1112 memcpy(net->dev_addr, device_info.mac_adr, ETH_ALEN); 1113 1114 nvdev = hv_get_drvdata(dev); 1115 netif_set_real_num_tx_queues(net, nvdev->num_chn); 1116 netif_set_real_num_rx_queues(net, nvdev->num_chn); 1117 1118 ret = register_netdev(net); 1119 if (ret != 0) { 1120 pr_err("Unable to register netdev.\n"); 1121 rndis_filter_device_remove(dev); 1122 netvsc_free_netdev(net); 1123 } 1124 1125 return ret; 1126 } 1127 1128 static int netvsc_remove(struct hv_device *dev) 1129 { 1130 struct net_device *net; 1131 struct net_device_context *ndev_ctx; 1132 struct netvsc_device *net_device; 1133 1134 net_device = hv_get_drvdata(dev); 1135 net = net_device->ndev; 1136 1137 if (net == NULL) { 1138 dev_err(&dev->device, "No net device to remove\n"); 1139 return 0; 1140 } 1141 1142 net_device->start_remove = true; 1143 1144 ndev_ctx = netdev_priv(net); 1145 cancel_delayed_work_sync(&ndev_ctx->dwork); 1146 cancel_work_sync(&ndev_ctx->work); 1147 1148 /* Stop outbound asap */ 1149 netif_tx_disable(net); 1150 1151 unregister_netdev(net); 1152 1153 /* 1154 * Call to the vsc driver to let it know that the device is being 1155 * removed 1156 */ 1157 rndis_filter_device_remove(dev); 1158 1159 netvsc_free_netdev(net); 1160 return 0; 1161 } 1162 1163 static const struct hv_vmbus_device_id id_table[] = { 1164 /* Network guid */ 1165 { HV_NIC_GUID, }, 1166 { }, 1167 }; 1168 1169 MODULE_DEVICE_TABLE(vmbus, id_table); 1170 1171 /* The one and only one */ 1172 static struct hv_driver netvsc_drv = { 1173 .name = KBUILD_MODNAME, 1174 .id_table = id_table, 1175 .probe = netvsc_probe, 1176 .remove = netvsc_remove, 1177 }; 1178 1179 static void __exit netvsc_drv_exit(void) 1180 { 1181 vmbus_driver_unregister(&netvsc_drv); 1182 } 1183 1184 static int __init netvsc_drv_init(void) 1185 { 1186 if (ring_size < RING_SIZE_MIN) { 1187 ring_size = RING_SIZE_MIN; 1188 pr_info("Increased ring_size to %d (min allowed)\n", 1189 ring_size); 1190 } 1191 return vmbus_driver_register(&netvsc_drv); 1192 } 1193 1194 MODULE_LICENSE("GPL"); 1195 MODULE_DESCRIPTION("Microsoft Hyper-V network driver"); 1196 1197 module_init(netvsc_drv_init); 1198 module_exit(netvsc_drv_exit); 1199