1 /* 2 * Copyright (c) 2009, Microsoft Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, see <http://www.gnu.org/licenses/>. 15 * 16 * Authors: 17 * Haiyang Zhang <haiyangz@microsoft.com> 18 * Hank Janssen <hjanssen@microsoft.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include <linux/kernel.h> 23 #include <linux/sched.h> 24 #include <linux/wait.h> 25 #include <linux/mm.h> 26 #include <linux/delay.h> 27 #include <linux/io.h> 28 #include <linux/slab.h> 29 #include <linux/netdevice.h> 30 #include <linux/if_ether.h> 31 #include <linux/vmalloc.h> 32 #include <linux/rtnetlink.h> 33 #include <linux/prefetch.h> 34 35 #include <asm/sync_bitops.h> 36 37 #include "hyperv_net.h" 38 #include "netvsc_trace.h" 39 40 /* 41 * Switch the data path from the synthetic interface to the VF 42 * interface. 43 */ 44 void netvsc_switch_datapath(struct net_device *ndev, bool vf) 45 { 46 struct net_device_context *net_device_ctx = netdev_priv(ndev); 47 struct hv_device *dev = net_device_ctx->device_ctx; 48 struct netvsc_device *nv_dev = rtnl_dereference(net_device_ctx->nvdev); 49 struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt; 50 51 memset(init_pkt, 0, sizeof(struct nvsp_message)); 52 init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH; 53 if (vf) 54 init_pkt->msg.v4_msg.active_dp.active_datapath = 55 NVSP_DATAPATH_VF; 56 else 57 init_pkt->msg.v4_msg.active_dp.active_datapath = 58 NVSP_DATAPATH_SYNTHETIC; 59 60 trace_nvsp_send(ndev, init_pkt); 61 62 vmbus_sendpacket(dev->channel, init_pkt, 63 sizeof(struct nvsp_message), 64 (unsigned long)init_pkt, 65 VM_PKT_DATA_INBAND, 0); 66 } 67 68 /* Worker to setup sub channels on initial setup 69 * Initial hotplug event occurs in softirq context 70 * and can't wait for channels. 71 */ 72 static void netvsc_subchan_work(struct work_struct *w) 73 { 74 struct netvsc_device *nvdev = 75 container_of(w, struct netvsc_device, subchan_work); 76 struct rndis_device *rdev; 77 int i, ret; 78 79 /* Avoid deadlock with device removal already under RTNL */ 80 if (!rtnl_trylock()) { 81 schedule_work(w); 82 return; 83 } 84 85 rdev = nvdev->extension; 86 if (rdev) { 87 ret = rndis_set_subchannel(rdev->ndev, nvdev); 88 if (ret == 0) { 89 netif_device_attach(rdev->ndev); 90 } else { 91 /* fallback to only primary channel */ 92 for (i = 1; i < nvdev->num_chn; i++) 93 netif_napi_del(&nvdev->chan_table[i].napi); 94 95 nvdev->max_chn = 1; 96 nvdev->num_chn = 1; 97 } 98 } 99 100 rtnl_unlock(); 101 } 102 103 static struct netvsc_device *alloc_net_device(void) 104 { 105 struct netvsc_device *net_device; 106 107 net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL); 108 if (!net_device) 109 return NULL; 110 111 init_waitqueue_head(&net_device->wait_drain); 112 net_device->destroy = false; 113 114 net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT; 115 net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT; 116 117 init_completion(&net_device->channel_init_wait); 118 init_waitqueue_head(&net_device->subchan_open); 119 INIT_WORK(&net_device->subchan_work, netvsc_subchan_work); 120 121 return net_device; 122 } 123 124 static void free_netvsc_device(struct rcu_head *head) 125 { 126 struct netvsc_device *nvdev 127 = container_of(head, struct netvsc_device, rcu); 128 int i; 129 130 kfree(nvdev->extension); 131 vfree(nvdev->recv_buf); 132 vfree(nvdev->send_buf); 133 kfree(nvdev->send_section_map); 134 135 for (i = 0; i < VRSS_CHANNEL_MAX; i++) 136 vfree(nvdev->chan_table[i].mrc.slots); 137 138 kfree(nvdev); 139 } 140 141 static void free_netvsc_device_rcu(struct netvsc_device *nvdev) 142 { 143 call_rcu(&nvdev->rcu, free_netvsc_device); 144 } 145 146 static void netvsc_revoke_recv_buf(struct hv_device *device, 147 struct netvsc_device *net_device, 148 struct net_device *ndev) 149 { 150 struct nvsp_message *revoke_packet; 151 int ret; 152 153 /* 154 * If we got a section count, it means we received a 155 * SendReceiveBufferComplete msg (ie sent 156 * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need 157 * to send a revoke msg here 158 */ 159 if (net_device->recv_section_cnt) { 160 /* Send the revoke receive buffer */ 161 revoke_packet = &net_device->revoke_packet; 162 memset(revoke_packet, 0, sizeof(struct nvsp_message)); 163 164 revoke_packet->hdr.msg_type = 165 NVSP_MSG1_TYPE_REVOKE_RECV_BUF; 166 revoke_packet->msg.v1_msg. 167 revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 168 169 trace_nvsp_send(ndev, revoke_packet); 170 171 ret = vmbus_sendpacket(device->channel, 172 revoke_packet, 173 sizeof(struct nvsp_message), 174 (unsigned long)revoke_packet, 175 VM_PKT_DATA_INBAND, 0); 176 /* If the failure is because the channel is rescinded; 177 * ignore the failure since we cannot send on a rescinded 178 * channel. This would allow us to properly cleanup 179 * even when the channel is rescinded. 180 */ 181 if (device->channel->rescind) 182 ret = 0; 183 /* 184 * If we failed here, we might as well return and 185 * have a leak rather than continue and a bugchk 186 */ 187 if (ret != 0) { 188 netdev_err(ndev, "unable to send " 189 "revoke receive buffer to netvsp\n"); 190 return; 191 } 192 net_device->recv_section_cnt = 0; 193 } 194 } 195 196 static void netvsc_revoke_send_buf(struct hv_device *device, 197 struct netvsc_device *net_device, 198 struct net_device *ndev) 199 { 200 struct nvsp_message *revoke_packet; 201 int ret; 202 203 /* Deal with the send buffer we may have setup. 204 * If we got a send section size, it means we received a 205 * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent 206 * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need 207 * to send a revoke msg here 208 */ 209 if (net_device->send_section_cnt) { 210 /* Send the revoke receive buffer */ 211 revoke_packet = &net_device->revoke_packet; 212 memset(revoke_packet, 0, sizeof(struct nvsp_message)); 213 214 revoke_packet->hdr.msg_type = 215 NVSP_MSG1_TYPE_REVOKE_SEND_BUF; 216 revoke_packet->msg.v1_msg.revoke_send_buf.id = 217 NETVSC_SEND_BUFFER_ID; 218 219 trace_nvsp_send(ndev, revoke_packet); 220 221 ret = vmbus_sendpacket(device->channel, 222 revoke_packet, 223 sizeof(struct nvsp_message), 224 (unsigned long)revoke_packet, 225 VM_PKT_DATA_INBAND, 0); 226 227 /* If the failure is because the channel is rescinded; 228 * ignore the failure since we cannot send on a rescinded 229 * channel. This would allow us to properly cleanup 230 * even when the channel is rescinded. 231 */ 232 if (device->channel->rescind) 233 ret = 0; 234 235 /* If we failed here, we might as well return and 236 * have a leak rather than continue and a bugchk 237 */ 238 if (ret != 0) { 239 netdev_err(ndev, "unable to send " 240 "revoke send buffer to netvsp\n"); 241 return; 242 } 243 net_device->send_section_cnt = 0; 244 } 245 } 246 247 static void netvsc_teardown_recv_gpadl(struct hv_device *device, 248 struct netvsc_device *net_device, 249 struct net_device *ndev) 250 { 251 int ret; 252 253 if (net_device->recv_buf_gpadl_handle) { 254 ret = vmbus_teardown_gpadl(device->channel, 255 net_device->recv_buf_gpadl_handle); 256 257 /* If we failed here, we might as well return and have a leak 258 * rather than continue and a bugchk 259 */ 260 if (ret != 0) { 261 netdev_err(ndev, 262 "unable to teardown receive buffer's gpadl\n"); 263 return; 264 } 265 net_device->recv_buf_gpadl_handle = 0; 266 } 267 } 268 269 static void netvsc_teardown_send_gpadl(struct hv_device *device, 270 struct netvsc_device *net_device, 271 struct net_device *ndev) 272 { 273 int ret; 274 275 if (net_device->send_buf_gpadl_handle) { 276 ret = vmbus_teardown_gpadl(device->channel, 277 net_device->send_buf_gpadl_handle); 278 279 /* If we failed here, we might as well return and have a leak 280 * rather than continue and a bugchk 281 */ 282 if (ret != 0) { 283 netdev_err(ndev, 284 "unable to teardown send buffer's gpadl\n"); 285 return; 286 } 287 net_device->send_buf_gpadl_handle = 0; 288 } 289 } 290 291 int netvsc_alloc_recv_comp_ring(struct netvsc_device *net_device, u32 q_idx) 292 { 293 struct netvsc_channel *nvchan = &net_device->chan_table[q_idx]; 294 int node = cpu_to_node(nvchan->channel->target_cpu); 295 size_t size; 296 297 size = net_device->recv_completion_cnt * sizeof(struct recv_comp_data); 298 nvchan->mrc.slots = vzalloc_node(size, node); 299 if (!nvchan->mrc.slots) 300 nvchan->mrc.slots = vzalloc(size); 301 302 return nvchan->mrc.slots ? 0 : -ENOMEM; 303 } 304 305 static int netvsc_init_buf(struct hv_device *device, 306 struct netvsc_device *net_device, 307 const struct netvsc_device_info *device_info) 308 { 309 struct nvsp_1_message_send_receive_buffer_complete *resp; 310 struct net_device *ndev = hv_get_drvdata(device); 311 struct nvsp_message *init_packet; 312 unsigned int buf_size; 313 size_t map_words; 314 int ret = 0; 315 316 /* Get receive buffer area. */ 317 buf_size = device_info->recv_sections * device_info->recv_section_size; 318 buf_size = roundup(buf_size, PAGE_SIZE); 319 320 /* Legacy hosts only allow smaller receive buffer */ 321 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2) 322 buf_size = min_t(unsigned int, buf_size, 323 NETVSC_RECEIVE_BUFFER_SIZE_LEGACY); 324 325 net_device->recv_buf = vzalloc(buf_size); 326 if (!net_device->recv_buf) { 327 netdev_err(ndev, 328 "unable to allocate receive buffer of size %u\n", 329 buf_size); 330 ret = -ENOMEM; 331 goto cleanup; 332 } 333 334 net_device->recv_buf_size = buf_size; 335 336 /* 337 * Establish the gpadl handle for this buffer on this 338 * channel. Note: This call uses the vmbus connection rather 339 * than the channel to establish the gpadl handle. 340 */ 341 ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf, 342 buf_size, 343 &net_device->recv_buf_gpadl_handle); 344 if (ret != 0) { 345 netdev_err(ndev, 346 "unable to establish receive buffer's gpadl\n"); 347 goto cleanup; 348 } 349 350 /* Notify the NetVsp of the gpadl handle */ 351 init_packet = &net_device->channel_init_pkt; 352 memset(init_packet, 0, sizeof(struct nvsp_message)); 353 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF; 354 init_packet->msg.v1_msg.send_recv_buf. 355 gpadl_handle = net_device->recv_buf_gpadl_handle; 356 init_packet->msg.v1_msg. 357 send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 358 359 trace_nvsp_send(ndev, init_packet); 360 361 /* Send the gpadl notification request */ 362 ret = vmbus_sendpacket(device->channel, init_packet, 363 sizeof(struct nvsp_message), 364 (unsigned long)init_packet, 365 VM_PKT_DATA_INBAND, 366 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 367 if (ret != 0) { 368 netdev_err(ndev, 369 "unable to send receive buffer's gpadl to netvsp\n"); 370 goto cleanup; 371 } 372 373 wait_for_completion(&net_device->channel_init_wait); 374 375 /* Check the response */ 376 resp = &init_packet->msg.v1_msg.send_recv_buf_complete; 377 if (resp->status != NVSP_STAT_SUCCESS) { 378 netdev_err(ndev, 379 "Unable to complete receive buffer initialization with NetVsp - status %d\n", 380 resp->status); 381 ret = -EINVAL; 382 goto cleanup; 383 } 384 385 /* Parse the response */ 386 netdev_dbg(ndev, "Receive sections: %u sub_allocs: size %u count: %u\n", 387 resp->num_sections, resp->sections[0].sub_alloc_size, 388 resp->sections[0].num_sub_allocs); 389 390 /* There should only be one section for the entire receive buffer */ 391 if (resp->num_sections != 1 || resp->sections[0].offset != 0) { 392 ret = -EINVAL; 393 goto cleanup; 394 } 395 396 net_device->recv_section_size = resp->sections[0].sub_alloc_size; 397 net_device->recv_section_cnt = resp->sections[0].num_sub_allocs; 398 399 /* Setup receive completion ring */ 400 net_device->recv_completion_cnt 401 = round_up(net_device->recv_section_cnt + 1, 402 PAGE_SIZE / sizeof(u64)); 403 ret = netvsc_alloc_recv_comp_ring(net_device, 0); 404 if (ret) 405 goto cleanup; 406 407 /* Now setup the send buffer. */ 408 buf_size = device_info->send_sections * device_info->send_section_size; 409 buf_size = round_up(buf_size, PAGE_SIZE); 410 411 net_device->send_buf = vzalloc(buf_size); 412 if (!net_device->send_buf) { 413 netdev_err(ndev, "unable to allocate send buffer of size %u\n", 414 buf_size); 415 ret = -ENOMEM; 416 goto cleanup; 417 } 418 419 /* Establish the gpadl handle for this buffer on this 420 * channel. Note: This call uses the vmbus connection rather 421 * than the channel to establish the gpadl handle. 422 */ 423 ret = vmbus_establish_gpadl(device->channel, net_device->send_buf, 424 buf_size, 425 &net_device->send_buf_gpadl_handle); 426 if (ret != 0) { 427 netdev_err(ndev, 428 "unable to establish send buffer's gpadl\n"); 429 goto cleanup; 430 } 431 432 /* Notify the NetVsp of the gpadl handle */ 433 init_packet = &net_device->channel_init_pkt; 434 memset(init_packet, 0, sizeof(struct nvsp_message)); 435 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF; 436 init_packet->msg.v1_msg.send_send_buf.gpadl_handle = 437 net_device->send_buf_gpadl_handle; 438 init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID; 439 440 trace_nvsp_send(ndev, init_packet); 441 442 /* Send the gpadl notification request */ 443 ret = vmbus_sendpacket(device->channel, init_packet, 444 sizeof(struct nvsp_message), 445 (unsigned long)init_packet, 446 VM_PKT_DATA_INBAND, 447 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 448 if (ret != 0) { 449 netdev_err(ndev, 450 "unable to send send buffer's gpadl to netvsp\n"); 451 goto cleanup; 452 } 453 454 wait_for_completion(&net_device->channel_init_wait); 455 456 /* Check the response */ 457 if (init_packet->msg.v1_msg. 458 send_send_buf_complete.status != NVSP_STAT_SUCCESS) { 459 netdev_err(ndev, "Unable to complete send buffer " 460 "initialization with NetVsp - status %d\n", 461 init_packet->msg.v1_msg. 462 send_send_buf_complete.status); 463 ret = -EINVAL; 464 goto cleanup; 465 } 466 467 /* Parse the response */ 468 net_device->send_section_size = init_packet->msg. 469 v1_msg.send_send_buf_complete.section_size; 470 471 /* Section count is simply the size divided by the section size. */ 472 net_device->send_section_cnt = buf_size / net_device->send_section_size; 473 474 netdev_dbg(ndev, "Send section size: %d, Section count:%d\n", 475 net_device->send_section_size, net_device->send_section_cnt); 476 477 /* Setup state for managing the send buffer. */ 478 map_words = DIV_ROUND_UP(net_device->send_section_cnt, BITS_PER_LONG); 479 480 net_device->send_section_map = kcalloc(map_words, sizeof(ulong), GFP_KERNEL); 481 if (net_device->send_section_map == NULL) { 482 ret = -ENOMEM; 483 goto cleanup; 484 } 485 486 goto exit; 487 488 cleanup: 489 netvsc_revoke_recv_buf(device, net_device, ndev); 490 netvsc_revoke_send_buf(device, net_device, ndev); 491 netvsc_teardown_recv_gpadl(device, net_device, ndev); 492 netvsc_teardown_send_gpadl(device, net_device, ndev); 493 494 exit: 495 return ret; 496 } 497 498 /* Negotiate NVSP protocol version */ 499 static int negotiate_nvsp_ver(struct hv_device *device, 500 struct netvsc_device *net_device, 501 struct nvsp_message *init_packet, 502 u32 nvsp_ver) 503 { 504 struct net_device *ndev = hv_get_drvdata(device); 505 int ret; 506 507 memset(init_packet, 0, sizeof(struct nvsp_message)); 508 init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT; 509 init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver; 510 init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver; 511 trace_nvsp_send(ndev, init_packet); 512 513 /* Send the init request */ 514 ret = vmbus_sendpacket(device->channel, init_packet, 515 sizeof(struct nvsp_message), 516 (unsigned long)init_packet, 517 VM_PKT_DATA_INBAND, 518 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 519 520 if (ret != 0) 521 return ret; 522 523 wait_for_completion(&net_device->channel_init_wait); 524 525 if (init_packet->msg.init_msg.init_complete.status != 526 NVSP_STAT_SUCCESS) 527 return -EINVAL; 528 529 if (nvsp_ver == NVSP_PROTOCOL_VERSION_1) 530 return 0; 531 532 /* NVSPv2 or later: Send NDIS config */ 533 memset(init_packet, 0, sizeof(struct nvsp_message)); 534 init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG; 535 init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN; 536 init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1; 537 538 if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) { 539 init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1; 540 541 /* Teaming bit is needed to receive link speed updates */ 542 init_packet->msg.v2_msg.send_ndis_config.capability.teaming = 1; 543 } 544 545 trace_nvsp_send(ndev, init_packet); 546 547 ret = vmbus_sendpacket(device->channel, init_packet, 548 sizeof(struct nvsp_message), 549 (unsigned long)init_packet, 550 VM_PKT_DATA_INBAND, 0); 551 552 return ret; 553 } 554 555 static int netvsc_connect_vsp(struct hv_device *device, 556 struct netvsc_device *net_device, 557 const struct netvsc_device_info *device_info) 558 { 559 struct net_device *ndev = hv_get_drvdata(device); 560 static const u32 ver_list[] = { 561 NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, 562 NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5, 563 NVSP_PROTOCOL_VERSION_6, NVSP_PROTOCOL_VERSION_61 564 }; 565 struct nvsp_message *init_packet; 566 int ndis_version, i, ret; 567 568 init_packet = &net_device->channel_init_pkt; 569 570 /* Negotiate the latest NVSP protocol supported */ 571 for (i = ARRAY_SIZE(ver_list) - 1; i >= 0; i--) 572 if (negotiate_nvsp_ver(device, net_device, init_packet, 573 ver_list[i]) == 0) { 574 net_device->nvsp_version = ver_list[i]; 575 break; 576 } 577 578 if (i < 0) { 579 ret = -EPROTO; 580 goto cleanup; 581 } 582 583 pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version); 584 585 /* Send the ndis version */ 586 memset(init_packet, 0, sizeof(struct nvsp_message)); 587 588 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4) 589 ndis_version = 0x00060001; 590 else 591 ndis_version = 0x0006001e; 592 593 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER; 594 init_packet->msg.v1_msg. 595 send_ndis_ver.ndis_major_ver = 596 (ndis_version & 0xFFFF0000) >> 16; 597 init_packet->msg.v1_msg. 598 send_ndis_ver.ndis_minor_ver = 599 ndis_version & 0xFFFF; 600 601 trace_nvsp_send(ndev, init_packet); 602 603 /* Send the init request */ 604 ret = vmbus_sendpacket(device->channel, init_packet, 605 sizeof(struct nvsp_message), 606 (unsigned long)init_packet, 607 VM_PKT_DATA_INBAND, 0); 608 if (ret != 0) 609 goto cleanup; 610 611 612 ret = netvsc_init_buf(device, net_device, device_info); 613 614 cleanup: 615 return ret; 616 } 617 618 /* 619 * netvsc_device_remove - Callback when the root bus device is removed 620 */ 621 void netvsc_device_remove(struct hv_device *device) 622 { 623 struct net_device *ndev = hv_get_drvdata(device); 624 struct net_device_context *net_device_ctx = netdev_priv(ndev); 625 struct netvsc_device *net_device 626 = rtnl_dereference(net_device_ctx->nvdev); 627 int i; 628 629 /* 630 * Revoke receive buffer. If host is pre-Win2016 then tear down 631 * receive buffer GPADL. Do the same for send buffer. 632 */ 633 netvsc_revoke_recv_buf(device, net_device, ndev); 634 if (vmbus_proto_version < VERSION_WIN10) 635 netvsc_teardown_recv_gpadl(device, net_device, ndev); 636 637 netvsc_revoke_send_buf(device, net_device, ndev); 638 if (vmbus_proto_version < VERSION_WIN10) 639 netvsc_teardown_send_gpadl(device, net_device, ndev); 640 641 RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); 642 643 /* And disassociate NAPI context from device */ 644 for (i = 0; i < net_device->num_chn; i++) 645 netif_napi_del(&net_device->chan_table[i].napi); 646 647 /* 648 * At this point, no one should be accessing net_device 649 * except in here 650 */ 651 netdev_dbg(ndev, "net device safe to remove\n"); 652 653 /* Now, we can close the channel safely */ 654 vmbus_close(device->channel); 655 656 /* 657 * If host is Win2016 or higher then we do the GPADL tear down 658 * here after VMBus is closed. 659 */ 660 if (vmbus_proto_version >= VERSION_WIN10) { 661 netvsc_teardown_recv_gpadl(device, net_device, ndev); 662 netvsc_teardown_send_gpadl(device, net_device, ndev); 663 } 664 665 /* Release all resources */ 666 free_netvsc_device_rcu(net_device); 667 } 668 669 #define RING_AVAIL_PERCENT_HIWATER 20 670 #define RING_AVAIL_PERCENT_LOWATER 10 671 672 static inline void netvsc_free_send_slot(struct netvsc_device *net_device, 673 u32 index) 674 { 675 sync_change_bit(index, net_device->send_section_map); 676 } 677 678 static void netvsc_send_tx_complete(struct net_device *ndev, 679 struct netvsc_device *net_device, 680 struct vmbus_channel *channel, 681 const struct vmpacket_descriptor *desc, 682 int budget) 683 { 684 struct sk_buff *skb = (struct sk_buff *)(unsigned long)desc->trans_id; 685 struct net_device_context *ndev_ctx = netdev_priv(ndev); 686 u16 q_idx = 0; 687 int queue_sends; 688 689 /* Notify the layer above us */ 690 if (likely(skb)) { 691 const struct hv_netvsc_packet *packet 692 = (struct hv_netvsc_packet *)skb->cb; 693 u32 send_index = packet->send_buf_index; 694 struct netvsc_stats *tx_stats; 695 696 if (send_index != NETVSC_INVALID_INDEX) 697 netvsc_free_send_slot(net_device, send_index); 698 q_idx = packet->q_idx; 699 700 tx_stats = &net_device->chan_table[q_idx].tx_stats; 701 702 u64_stats_update_begin(&tx_stats->syncp); 703 tx_stats->packets += packet->total_packets; 704 tx_stats->bytes += packet->total_bytes; 705 u64_stats_update_end(&tx_stats->syncp); 706 707 napi_consume_skb(skb, budget); 708 } 709 710 queue_sends = 711 atomic_dec_return(&net_device->chan_table[q_idx].queue_sends); 712 713 if (unlikely(net_device->destroy)) { 714 if (queue_sends == 0) 715 wake_up(&net_device->wait_drain); 716 } else { 717 struct netdev_queue *txq = netdev_get_tx_queue(ndev, q_idx); 718 719 if (netif_tx_queue_stopped(txq) && 720 (hv_get_avail_to_write_percent(&channel->outbound) > 721 RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) { 722 netif_tx_wake_queue(txq); 723 ndev_ctx->eth_stats.wake_queue++; 724 } 725 } 726 } 727 728 static void netvsc_send_completion(struct net_device *ndev, 729 struct netvsc_device *net_device, 730 struct vmbus_channel *incoming_channel, 731 const struct vmpacket_descriptor *desc, 732 int budget) 733 { 734 const struct nvsp_message *nvsp_packet = hv_pkt_data(desc); 735 736 switch (nvsp_packet->hdr.msg_type) { 737 case NVSP_MSG_TYPE_INIT_COMPLETE: 738 case NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE: 739 case NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE: 740 case NVSP_MSG5_TYPE_SUBCHANNEL: 741 /* Copy the response back */ 742 memcpy(&net_device->channel_init_pkt, nvsp_packet, 743 sizeof(struct nvsp_message)); 744 complete(&net_device->channel_init_wait); 745 break; 746 747 case NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE: 748 netvsc_send_tx_complete(ndev, net_device, incoming_channel, 749 desc, budget); 750 break; 751 752 default: 753 netdev_err(ndev, 754 "Unknown send completion type %d received!!\n", 755 nvsp_packet->hdr.msg_type); 756 } 757 } 758 759 static u32 netvsc_get_next_send_section(struct netvsc_device *net_device) 760 { 761 unsigned long *map_addr = net_device->send_section_map; 762 unsigned int i; 763 764 for_each_clear_bit(i, map_addr, net_device->send_section_cnt) { 765 if (sync_test_and_set_bit(i, map_addr) == 0) 766 return i; 767 } 768 769 return NETVSC_INVALID_INDEX; 770 } 771 772 static void netvsc_copy_to_send_buf(struct netvsc_device *net_device, 773 unsigned int section_index, 774 u32 pend_size, 775 struct hv_netvsc_packet *packet, 776 struct rndis_message *rndis_msg, 777 struct hv_page_buffer *pb, 778 bool xmit_more) 779 { 780 char *start = net_device->send_buf; 781 char *dest = start + (section_index * net_device->send_section_size) 782 + pend_size; 783 int i; 784 u32 padding = 0; 785 u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : 786 packet->page_buf_cnt; 787 u32 remain; 788 789 /* Add padding */ 790 remain = packet->total_data_buflen & (net_device->pkt_align - 1); 791 if (xmit_more && remain) { 792 padding = net_device->pkt_align - remain; 793 rndis_msg->msg_len += padding; 794 packet->total_data_buflen += padding; 795 } 796 797 for (i = 0; i < page_count; i++) { 798 char *src = phys_to_virt(pb[i].pfn << PAGE_SHIFT); 799 u32 offset = pb[i].offset; 800 u32 len = pb[i].len; 801 802 memcpy(dest, (src + offset), len); 803 dest += len; 804 } 805 806 if (padding) 807 memset(dest, 0, padding); 808 } 809 810 static inline int netvsc_send_pkt( 811 struct hv_device *device, 812 struct hv_netvsc_packet *packet, 813 struct netvsc_device *net_device, 814 struct hv_page_buffer *pb, 815 struct sk_buff *skb) 816 { 817 struct nvsp_message nvmsg; 818 struct nvsp_1_message_send_rndis_packet *rpkt = 819 &nvmsg.msg.v1_msg.send_rndis_pkt; 820 struct netvsc_channel * const nvchan = 821 &net_device->chan_table[packet->q_idx]; 822 struct vmbus_channel *out_channel = nvchan->channel; 823 struct net_device *ndev = hv_get_drvdata(device); 824 struct net_device_context *ndev_ctx = netdev_priv(ndev); 825 struct netdev_queue *txq = netdev_get_tx_queue(ndev, packet->q_idx); 826 u64 req_id; 827 int ret; 828 u32 ring_avail = hv_get_avail_to_write_percent(&out_channel->outbound); 829 830 nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; 831 if (skb) 832 rpkt->channel_type = 0; /* 0 is RMC_DATA */ 833 else 834 rpkt->channel_type = 1; /* 1 is RMC_CONTROL */ 835 836 rpkt->send_buf_section_index = packet->send_buf_index; 837 if (packet->send_buf_index == NETVSC_INVALID_INDEX) 838 rpkt->send_buf_section_size = 0; 839 else 840 rpkt->send_buf_section_size = packet->total_data_buflen; 841 842 req_id = (ulong)skb; 843 844 if (out_channel->rescind) 845 return -ENODEV; 846 847 trace_nvsp_send_pkt(ndev, out_channel, rpkt); 848 849 if (packet->page_buf_cnt) { 850 if (packet->cp_partial) 851 pb += packet->rmsg_pgcnt; 852 853 ret = vmbus_sendpacket_pagebuffer(out_channel, 854 pb, packet->page_buf_cnt, 855 &nvmsg, sizeof(nvmsg), 856 req_id); 857 } else { 858 ret = vmbus_sendpacket(out_channel, 859 &nvmsg, sizeof(nvmsg), 860 req_id, VM_PKT_DATA_INBAND, 861 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 862 } 863 864 if (ret == 0) { 865 atomic_inc_return(&nvchan->queue_sends); 866 867 if (ring_avail < RING_AVAIL_PERCENT_LOWATER) { 868 netif_tx_stop_queue(txq); 869 ndev_ctx->eth_stats.stop_queue++; 870 } 871 } else if (ret == -EAGAIN) { 872 netif_tx_stop_queue(txq); 873 ndev_ctx->eth_stats.stop_queue++; 874 if (atomic_read(&nvchan->queue_sends) < 1) { 875 netif_tx_wake_queue(txq); 876 ndev_ctx->eth_stats.wake_queue++; 877 ret = -ENOSPC; 878 } 879 } else { 880 netdev_err(ndev, 881 "Unable to send packet pages %u len %u, ret %d\n", 882 packet->page_buf_cnt, packet->total_data_buflen, 883 ret); 884 } 885 886 return ret; 887 } 888 889 /* Move packet out of multi send data (msd), and clear msd */ 890 static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send, 891 struct sk_buff **msd_skb, 892 struct multi_send_data *msdp) 893 { 894 *msd_skb = msdp->skb; 895 *msd_send = msdp->pkt; 896 msdp->skb = NULL; 897 msdp->pkt = NULL; 898 msdp->count = 0; 899 } 900 901 /* RCU already held by caller */ 902 int netvsc_send(struct net_device *ndev, 903 struct hv_netvsc_packet *packet, 904 struct rndis_message *rndis_msg, 905 struct hv_page_buffer *pb, 906 struct sk_buff *skb) 907 { 908 struct net_device_context *ndev_ctx = netdev_priv(ndev); 909 struct netvsc_device *net_device 910 = rcu_dereference_bh(ndev_ctx->nvdev); 911 struct hv_device *device = ndev_ctx->device_ctx; 912 int ret = 0; 913 struct netvsc_channel *nvchan; 914 u32 pktlen = packet->total_data_buflen, msd_len = 0; 915 unsigned int section_index = NETVSC_INVALID_INDEX; 916 struct multi_send_data *msdp; 917 struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL; 918 struct sk_buff *msd_skb = NULL; 919 bool try_batch, xmit_more; 920 921 /* If device is rescinded, return error and packet will get dropped. */ 922 if (unlikely(!net_device || net_device->destroy)) 923 return -ENODEV; 924 925 nvchan = &net_device->chan_table[packet->q_idx]; 926 packet->send_buf_index = NETVSC_INVALID_INDEX; 927 packet->cp_partial = false; 928 929 /* Send control message directly without accessing msd (Multi-Send 930 * Data) field which may be changed during data packet processing. 931 */ 932 if (!skb) 933 return netvsc_send_pkt(device, packet, net_device, pb, skb); 934 935 /* batch packets in send buffer if possible */ 936 msdp = &nvchan->msd; 937 if (msdp->pkt) 938 msd_len = msdp->pkt->total_data_buflen; 939 940 try_batch = msd_len > 0 && msdp->count < net_device->max_pkt; 941 if (try_batch && msd_len + pktlen + net_device->pkt_align < 942 net_device->send_section_size) { 943 section_index = msdp->pkt->send_buf_index; 944 945 } else if (try_batch && msd_len + packet->rmsg_size < 946 net_device->send_section_size) { 947 section_index = msdp->pkt->send_buf_index; 948 packet->cp_partial = true; 949 950 } else if (pktlen + net_device->pkt_align < 951 net_device->send_section_size) { 952 section_index = netvsc_get_next_send_section(net_device); 953 if (unlikely(section_index == NETVSC_INVALID_INDEX)) { 954 ++ndev_ctx->eth_stats.tx_send_full; 955 } else { 956 move_pkt_msd(&msd_send, &msd_skb, msdp); 957 msd_len = 0; 958 } 959 } 960 961 /* Keep aggregating only if stack says more data is coming 962 * and not doing mixed modes send and not flow blocked 963 */ 964 xmit_more = skb->xmit_more && 965 !packet->cp_partial && 966 !netif_xmit_stopped(netdev_get_tx_queue(ndev, packet->q_idx)); 967 968 if (section_index != NETVSC_INVALID_INDEX) { 969 netvsc_copy_to_send_buf(net_device, 970 section_index, msd_len, 971 packet, rndis_msg, pb, xmit_more); 972 973 packet->send_buf_index = section_index; 974 975 if (packet->cp_partial) { 976 packet->page_buf_cnt -= packet->rmsg_pgcnt; 977 packet->total_data_buflen = msd_len + packet->rmsg_size; 978 } else { 979 packet->page_buf_cnt = 0; 980 packet->total_data_buflen += msd_len; 981 } 982 983 if (msdp->pkt) { 984 packet->total_packets += msdp->pkt->total_packets; 985 packet->total_bytes += msdp->pkt->total_bytes; 986 } 987 988 if (msdp->skb) 989 dev_consume_skb_any(msdp->skb); 990 991 if (xmit_more) { 992 msdp->skb = skb; 993 msdp->pkt = packet; 994 msdp->count++; 995 } else { 996 cur_send = packet; 997 msdp->skb = NULL; 998 msdp->pkt = NULL; 999 msdp->count = 0; 1000 } 1001 } else { 1002 move_pkt_msd(&msd_send, &msd_skb, msdp); 1003 cur_send = packet; 1004 } 1005 1006 if (msd_send) { 1007 int m_ret = netvsc_send_pkt(device, msd_send, net_device, 1008 NULL, msd_skb); 1009 1010 if (m_ret != 0) { 1011 netvsc_free_send_slot(net_device, 1012 msd_send->send_buf_index); 1013 dev_kfree_skb_any(msd_skb); 1014 } 1015 } 1016 1017 if (cur_send) 1018 ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb); 1019 1020 if (ret != 0 && section_index != NETVSC_INVALID_INDEX) 1021 netvsc_free_send_slot(net_device, section_index); 1022 1023 return ret; 1024 } 1025 1026 /* Send pending recv completions */ 1027 static int send_recv_completions(struct net_device *ndev, 1028 struct netvsc_device *nvdev, 1029 struct netvsc_channel *nvchan) 1030 { 1031 struct multi_recv_comp *mrc = &nvchan->mrc; 1032 struct recv_comp_msg { 1033 struct nvsp_message_header hdr; 1034 u32 status; 1035 } __packed; 1036 struct recv_comp_msg msg = { 1037 .hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE, 1038 }; 1039 int ret; 1040 1041 while (mrc->first != mrc->next) { 1042 const struct recv_comp_data *rcd 1043 = mrc->slots + mrc->first; 1044 1045 msg.status = rcd->status; 1046 ret = vmbus_sendpacket(nvchan->channel, &msg, sizeof(msg), 1047 rcd->tid, VM_PKT_COMP, 0); 1048 if (unlikely(ret)) { 1049 struct net_device_context *ndev_ctx = netdev_priv(ndev); 1050 1051 ++ndev_ctx->eth_stats.rx_comp_busy; 1052 return ret; 1053 } 1054 1055 if (++mrc->first == nvdev->recv_completion_cnt) 1056 mrc->first = 0; 1057 } 1058 1059 /* receive completion ring has been emptied */ 1060 if (unlikely(nvdev->destroy)) 1061 wake_up(&nvdev->wait_drain); 1062 1063 return 0; 1064 } 1065 1066 /* Count how many receive completions are outstanding */ 1067 static void recv_comp_slot_avail(const struct netvsc_device *nvdev, 1068 const struct multi_recv_comp *mrc, 1069 u32 *filled, u32 *avail) 1070 { 1071 u32 count = nvdev->recv_completion_cnt; 1072 1073 if (mrc->next >= mrc->first) 1074 *filled = mrc->next - mrc->first; 1075 else 1076 *filled = (count - mrc->first) + mrc->next; 1077 1078 *avail = count - *filled - 1; 1079 } 1080 1081 /* Add receive complete to ring to send to host. */ 1082 static void enq_receive_complete(struct net_device *ndev, 1083 struct netvsc_device *nvdev, u16 q_idx, 1084 u64 tid, u32 status) 1085 { 1086 struct netvsc_channel *nvchan = &nvdev->chan_table[q_idx]; 1087 struct multi_recv_comp *mrc = &nvchan->mrc; 1088 struct recv_comp_data *rcd; 1089 u32 filled, avail; 1090 1091 recv_comp_slot_avail(nvdev, mrc, &filled, &avail); 1092 1093 if (unlikely(filled > NAPI_POLL_WEIGHT)) { 1094 send_recv_completions(ndev, nvdev, nvchan); 1095 recv_comp_slot_avail(nvdev, mrc, &filled, &avail); 1096 } 1097 1098 if (unlikely(!avail)) { 1099 netdev_err(ndev, "Recv_comp full buf q:%hd, tid:%llx\n", 1100 q_idx, tid); 1101 return; 1102 } 1103 1104 rcd = mrc->slots + mrc->next; 1105 rcd->tid = tid; 1106 rcd->status = status; 1107 1108 if (++mrc->next == nvdev->recv_completion_cnt) 1109 mrc->next = 0; 1110 } 1111 1112 static int netvsc_receive(struct net_device *ndev, 1113 struct netvsc_device *net_device, 1114 struct vmbus_channel *channel, 1115 const struct vmpacket_descriptor *desc, 1116 const struct nvsp_message *nvsp) 1117 { 1118 struct net_device_context *net_device_ctx = netdev_priv(ndev); 1119 const struct vmtransfer_page_packet_header *vmxferpage_packet 1120 = container_of(desc, const struct vmtransfer_page_packet_header, d); 1121 u16 q_idx = channel->offermsg.offer.sub_channel_index; 1122 char *recv_buf = net_device->recv_buf; 1123 u32 status = NVSP_STAT_SUCCESS; 1124 int i; 1125 int count = 0; 1126 1127 /* Make sure this is a valid nvsp packet */ 1128 if (unlikely(nvsp->hdr.msg_type != NVSP_MSG1_TYPE_SEND_RNDIS_PKT)) { 1129 netif_err(net_device_ctx, rx_err, ndev, 1130 "Unknown nvsp packet type received %u\n", 1131 nvsp->hdr.msg_type); 1132 return 0; 1133 } 1134 1135 if (unlikely(vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID)) { 1136 netif_err(net_device_ctx, rx_err, ndev, 1137 "Invalid xfer page set id - expecting %x got %x\n", 1138 NETVSC_RECEIVE_BUFFER_ID, 1139 vmxferpage_packet->xfer_pageset_id); 1140 return 0; 1141 } 1142 1143 count = vmxferpage_packet->range_cnt; 1144 1145 /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ 1146 for (i = 0; i < count; i++) { 1147 u32 offset = vmxferpage_packet->ranges[i].byte_offset; 1148 u32 buflen = vmxferpage_packet->ranges[i].byte_count; 1149 void *data; 1150 int ret; 1151 1152 if (unlikely(offset + buflen > net_device->recv_buf_size)) { 1153 status = NVSP_STAT_FAIL; 1154 netif_err(net_device_ctx, rx_err, ndev, 1155 "Packet offset:%u + len:%u too big\n", 1156 offset, buflen); 1157 1158 continue; 1159 } 1160 1161 data = recv_buf + offset; 1162 1163 trace_rndis_recv(ndev, q_idx, data); 1164 1165 /* Pass it to the upper layer */ 1166 ret = rndis_filter_receive(ndev, net_device, 1167 channel, data, buflen); 1168 1169 if (unlikely(ret != NVSP_STAT_SUCCESS)) 1170 status = NVSP_STAT_FAIL; 1171 } 1172 1173 enq_receive_complete(ndev, net_device, q_idx, 1174 vmxferpage_packet->d.trans_id, status); 1175 1176 return count; 1177 } 1178 1179 static void netvsc_send_table(struct net_device *ndev, 1180 const struct nvsp_message *nvmsg) 1181 { 1182 struct net_device_context *net_device_ctx = netdev_priv(ndev); 1183 u32 count, *tab; 1184 int i; 1185 1186 count = nvmsg->msg.v5_msg.send_table.count; 1187 if (count != VRSS_SEND_TAB_SIZE) { 1188 netdev_err(ndev, "Received wrong send-table size:%u\n", count); 1189 return; 1190 } 1191 1192 tab = (u32 *)((unsigned long)&nvmsg->msg.v5_msg.send_table + 1193 nvmsg->msg.v5_msg.send_table.offset); 1194 1195 for (i = 0; i < count; i++) 1196 net_device_ctx->tx_table[i] = tab[i]; 1197 } 1198 1199 static void netvsc_send_vf(struct net_device *ndev, 1200 const struct nvsp_message *nvmsg) 1201 { 1202 struct net_device_context *net_device_ctx = netdev_priv(ndev); 1203 1204 net_device_ctx->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; 1205 net_device_ctx->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; 1206 } 1207 1208 static void netvsc_receive_inband(struct net_device *ndev, 1209 const struct nvsp_message *nvmsg) 1210 { 1211 switch (nvmsg->hdr.msg_type) { 1212 case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE: 1213 netvsc_send_table(ndev, nvmsg); 1214 break; 1215 1216 case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION: 1217 netvsc_send_vf(ndev, nvmsg); 1218 break; 1219 } 1220 } 1221 1222 static int netvsc_process_raw_pkt(struct hv_device *device, 1223 struct vmbus_channel *channel, 1224 struct netvsc_device *net_device, 1225 struct net_device *ndev, 1226 const struct vmpacket_descriptor *desc, 1227 int budget) 1228 { 1229 const struct nvsp_message *nvmsg = hv_pkt_data(desc); 1230 1231 trace_nvsp_recv(ndev, channel, nvmsg); 1232 1233 switch (desc->type) { 1234 case VM_PKT_COMP: 1235 netvsc_send_completion(ndev, net_device, channel, 1236 desc, budget); 1237 break; 1238 1239 case VM_PKT_DATA_USING_XFER_PAGES: 1240 return netvsc_receive(ndev, net_device, channel, 1241 desc, nvmsg); 1242 break; 1243 1244 case VM_PKT_DATA_INBAND: 1245 netvsc_receive_inband(ndev, nvmsg); 1246 break; 1247 1248 default: 1249 netdev_err(ndev, "unhandled packet type %d, tid %llx\n", 1250 desc->type, desc->trans_id); 1251 break; 1252 } 1253 1254 return 0; 1255 } 1256 1257 static struct hv_device *netvsc_channel_to_device(struct vmbus_channel *channel) 1258 { 1259 struct vmbus_channel *primary = channel->primary_channel; 1260 1261 return primary ? primary->device_obj : channel->device_obj; 1262 } 1263 1264 /* Network processing softirq 1265 * Process data in incoming ring buffer from host 1266 * Stops when ring is empty or budget is met or exceeded. 1267 */ 1268 int netvsc_poll(struct napi_struct *napi, int budget) 1269 { 1270 struct netvsc_channel *nvchan 1271 = container_of(napi, struct netvsc_channel, napi); 1272 struct netvsc_device *net_device = nvchan->net_device; 1273 struct vmbus_channel *channel = nvchan->channel; 1274 struct hv_device *device = netvsc_channel_to_device(channel); 1275 struct net_device *ndev = hv_get_drvdata(device); 1276 int work_done = 0; 1277 int ret; 1278 1279 /* If starting a new interval */ 1280 if (!nvchan->desc) 1281 nvchan->desc = hv_pkt_iter_first(channel); 1282 1283 while (nvchan->desc && work_done < budget) { 1284 work_done += netvsc_process_raw_pkt(device, channel, net_device, 1285 ndev, nvchan->desc, budget); 1286 nvchan->desc = hv_pkt_iter_next(channel, nvchan->desc); 1287 } 1288 1289 /* Send any pending receive completions */ 1290 ret = send_recv_completions(ndev, net_device, nvchan); 1291 1292 /* If it did not exhaust NAPI budget this time 1293 * and not doing busy poll 1294 * then re-enable host interrupts 1295 * and reschedule if ring is not empty 1296 * or sending receive completion failed. 1297 */ 1298 if (work_done < budget && 1299 napi_complete_done(napi, work_done) && 1300 (ret || hv_end_read(&channel->inbound)) && 1301 napi_schedule_prep(napi)) { 1302 hv_begin_read(&channel->inbound); 1303 __napi_schedule(napi); 1304 } 1305 1306 /* Driver may overshoot since multiple packets per descriptor */ 1307 return min(work_done, budget); 1308 } 1309 1310 /* Call back when data is available in host ring buffer. 1311 * Processing is deferred until network softirq (NAPI) 1312 */ 1313 void netvsc_channel_cb(void *context) 1314 { 1315 struct netvsc_channel *nvchan = context; 1316 struct vmbus_channel *channel = nvchan->channel; 1317 struct hv_ring_buffer_info *rbi = &channel->inbound; 1318 1319 /* preload first vmpacket descriptor */ 1320 prefetch(hv_get_ring_buffer(rbi) + rbi->priv_read_index); 1321 1322 if (napi_schedule_prep(&nvchan->napi)) { 1323 /* disable interupts from host */ 1324 hv_begin_read(rbi); 1325 1326 __napi_schedule_irqoff(&nvchan->napi); 1327 } 1328 } 1329 1330 /* 1331 * netvsc_device_add - Callback when the device belonging to this 1332 * driver is added 1333 */ 1334 struct netvsc_device *netvsc_device_add(struct hv_device *device, 1335 const struct netvsc_device_info *device_info) 1336 { 1337 int i, ret = 0; 1338 struct netvsc_device *net_device; 1339 struct net_device *ndev = hv_get_drvdata(device); 1340 struct net_device_context *net_device_ctx = netdev_priv(ndev); 1341 1342 net_device = alloc_net_device(); 1343 if (!net_device) 1344 return ERR_PTR(-ENOMEM); 1345 1346 for (i = 0; i < VRSS_SEND_TAB_SIZE; i++) 1347 net_device_ctx->tx_table[i] = 0; 1348 1349 /* Because the device uses NAPI, all the interrupt batching and 1350 * control is done via Net softirq, not the channel handling 1351 */ 1352 set_channel_read_mode(device->channel, HV_CALL_ISR); 1353 1354 /* If we're reopening the device we may have multiple queues, fill the 1355 * chn_table with the default channel to use it before subchannels are 1356 * opened. 1357 * Initialize the channel state before we open; 1358 * we can be interrupted as soon as we open the channel. 1359 */ 1360 1361 for (i = 0; i < VRSS_CHANNEL_MAX; i++) { 1362 struct netvsc_channel *nvchan = &net_device->chan_table[i]; 1363 1364 nvchan->channel = device->channel; 1365 nvchan->net_device = net_device; 1366 u64_stats_init(&nvchan->tx_stats.syncp); 1367 u64_stats_init(&nvchan->rx_stats.syncp); 1368 } 1369 1370 /* Enable NAPI handler before init callbacks */ 1371 netif_napi_add(ndev, &net_device->chan_table[0].napi, 1372 netvsc_poll, NAPI_POLL_WEIGHT); 1373 1374 /* Open the channel */ 1375 ret = vmbus_open(device->channel, netvsc_ring_bytes, 1376 netvsc_ring_bytes, NULL, 0, 1377 netvsc_channel_cb, net_device->chan_table); 1378 1379 if (ret != 0) { 1380 netdev_err(ndev, "unable to open channel: %d\n", ret); 1381 goto cleanup; 1382 } 1383 1384 /* Channel is opened */ 1385 netdev_dbg(ndev, "hv_netvsc channel opened successfully\n"); 1386 1387 napi_enable(&net_device->chan_table[0].napi); 1388 1389 /* Connect with the NetVsp */ 1390 ret = netvsc_connect_vsp(device, net_device, device_info); 1391 if (ret != 0) { 1392 netdev_err(ndev, 1393 "unable to connect to NetVSP - %d\n", ret); 1394 goto close; 1395 } 1396 1397 /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is 1398 * populated. 1399 */ 1400 rcu_assign_pointer(net_device_ctx->nvdev, net_device); 1401 1402 return net_device; 1403 1404 close: 1405 RCU_INIT_POINTER(net_device_ctx->nvdev, NULL); 1406 napi_disable(&net_device->chan_table[0].napi); 1407 1408 /* Now, we can close the channel safely */ 1409 vmbus_close(device->channel); 1410 1411 cleanup: 1412 netif_napi_del(&net_device->chan_table[0].napi); 1413 free_netvsc_device(&net_device->rcu); 1414 1415 return ERR_PTR(ret); 1416 } 1417