1 /* 2 * Copyright (c) 2009, Microsoft Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * You should have received a copy of the GNU General Public License along with 14 * this program; if not, see <http://www.gnu.org/licenses/>. 15 * 16 * Authors: 17 * Haiyang Zhang <haiyangz@microsoft.com> 18 * Hank Janssen <hjanssen@microsoft.com> 19 */ 20 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 21 22 #include <linux/kernel.h> 23 #include <linux/sched.h> 24 #include <linux/wait.h> 25 #include <linux/mm.h> 26 #include <linux/delay.h> 27 #include <linux/io.h> 28 #include <linux/slab.h> 29 #include <linux/netdevice.h> 30 #include <linux/if_ether.h> 31 #include <linux/vmalloc.h> 32 #include <asm/sync_bitops.h> 33 34 #include "hyperv_net.h" 35 36 /* 37 * Switch the data path from the synthetic interface to the VF 38 * interface. 39 */ 40 void netvsc_switch_datapath(struct net_device *ndev, bool vf) 41 { 42 struct net_device_context *net_device_ctx = netdev_priv(ndev); 43 struct hv_device *dev = net_device_ctx->device_ctx; 44 struct netvsc_device *nv_dev = net_device_ctx->nvdev; 45 struct nvsp_message *init_pkt = &nv_dev->channel_init_pkt; 46 47 memset(init_pkt, 0, sizeof(struct nvsp_message)); 48 init_pkt->hdr.msg_type = NVSP_MSG4_TYPE_SWITCH_DATA_PATH; 49 if (vf) 50 init_pkt->msg.v4_msg.active_dp.active_datapath = 51 NVSP_DATAPATH_VF; 52 else 53 init_pkt->msg.v4_msg.active_dp.active_datapath = 54 NVSP_DATAPATH_SYNTHETIC; 55 56 vmbus_sendpacket(dev->channel, init_pkt, 57 sizeof(struct nvsp_message), 58 (unsigned long)init_pkt, 59 VM_PKT_DATA_INBAND, 0); 60 } 61 62 63 static struct netvsc_device *alloc_net_device(void) 64 { 65 struct netvsc_device *net_device; 66 67 net_device = kzalloc(sizeof(struct netvsc_device), GFP_KERNEL); 68 if (!net_device) 69 return NULL; 70 71 net_device->cb_buffer = kzalloc(NETVSC_PACKET_SIZE, GFP_KERNEL); 72 if (!net_device->cb_buffer) { 73 kfree(net_device); 74 return NULL; 75 } 76 77 init_waitqueue_head(&net_device->wait_drain); 78 net_device->destroy = false; 79 atomic_set(&net_device->open_cnt, 0); 80 atomic_set(&net_device->vf_use_cnt, 0); 81 net_device->max_pkt = RNDIS_MAX_PKT_DEFAULT; 82 net_device->pkt_align = RNDIS_PKT_ALIGN_DEFAULT; 83 84 net_device->vf_netdev = NULL; 85 net_device->vf_inject = false; 86 87 return net_device; 88 } 89 90 static void free_netvsc_device(struct netvsc_device *nvdev) 91 { 92 kfree(nvdev->cb_buffer); 93 kfree(nvdev); 94 } 95 96 static struct netvsc_device *get_outbound_net_device(struct hv_device *device) 97 { 98 struct net_device *ndev = hv_get_drvdata(device); 99 struct net_device_context *net_device_ctx = netdev_priv(ndev); 100 struct netvsc_device *net_device = net_device_ctx->nvdev; 101 102 if (net_device && net_device->destroy) 103 net_device = NULL; 104 105 return net_device; 106 } 107 108 static struct netvsc_device *get_inbound_net_device(struct hv_device *device) 109 { 110 struct net_device *ndev = hv_get_drvdata(device); 111 struct net_device_context *net_device_ctx = netdev_priv(ndev); 112 struct netvsc_device *net_device = net_device_ctx->nvdev; 113 114 if (!net_device) 115 goto get_in_err; 116 117 if (net_device->destroy && 118 atomic_read(&net_device->num_outstanding_sends) == 0) 119 net_device = NULL; 120 121 get_in_err: 122 return net_device; 123 } 124 125 126 static int netvsc_destroy_buf(struct hv_device *device) 127 { 128 struct nvsp_message *revoke_packet; 129 int ret = 0; 130 struct net_device *ndev = hv_get_drvdata(device); 131 struct net_device_context *net_device_ctx = netdev_priv(ndev); 132 struct netvsc_device *net_device = net_device_ctx->nvdev; 133 134 /* 135 * If we got a section count, it means we received a 136 * SendReceiveBufferComplete msg (ie sent 137 * NvspMessage1TypeSendReceiveBuffer msg) therefore, we need 138 * to send a revoke msg here 139 */ 140 if (net_device->recv_section_cnt) { 141 /* Send the revoke receive buffer */ 142 revoke_packet = &net_device->revoke_packet; 143 memset(revoke_packet, 0, sizeof(struct nvsp_message)); 144 145 revoke_packet->hdr.msg_type = 146 NVSP_MSG1_TYPE_REVOKE_RECV_BUF; 147 revoke_packet->msg.v1_msg. 148 revoke_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 149 150 ret = vmbus_sendpacket(device->channel, 151 revoke_packet, 152 sizeof(struct nvsp_message), 153 (unsigned long)revoke_packet, 154 VM_PKT_DATA_INBAND, 0); 155 /* 156 * If we failed here, we might as well return and 157 * have a leak rather than continue and a bugchk 158 */ 159 if (ret != 0) { 160 netdev_err(ndev, "unable to send " 161 "revoke receive buffer to netvsp\n"); 162 return ret; 163 } 164 } 165 166 /* Teardown the gpadl on the vsp end */ 167 if (net_device->recv_buf_gpadl_handle) { 168 ret = vmbus_teardown_gpadl(device->channel, 169 net_device->recv_buf_gpadl_handle); 170 171 /* If we failed here, we might as well return and have a leak 172 * rather than continue and a bugchk 173 */ 174 if (ret != 0) { 175 netdev_err(ndev, 176 "unable to teardown receive buffer's gpadl\n"); 177 return ret; 178 } 179 net_device->recv_buf_gpadl_handle = 0; 180 } 181 182 if (net_device->recv_buf) { 183 /* Free up the receive buffer */ 184 vfree(net_device->recv_buf); 185 net_device->recv_buf = NULL; 186 } 187 188 if (net_device->recv_section) { 189 net_device->recv_section_cnt = 0; 190 kfree(net_device->recv_section); 191 net_device->recv_section = NULL; 192 } 193 194 /* Deal with the send buffer we may have setup. 195 * If we got a send section size, it means we received a 196 * NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE msg (ie sent 197 * NVSP_MSG1_TYPE_SEND_SEND_BUF msg) therefore, we need 198 * to send a revoke msg here 199 */ 200 if (net_device->send_section_size) { 201 /* Send the revoke receive buffer */ 202 revoke_packet = &net_device->revoke_packet; 203 memset(revoke_packet, 0, sizeof(struct nvsp_message)); 204 205 revoke_packet->hdr.msg_type = 206 NVSP_MSG1_TYPE_REVOKE_SEND_BUF; 207 revoke_packet->msg.v1_msg.revoke_send_buf.id = 208 NETVSC_SEND_BUFFER_ID; 209 210 ret = vmbus_sendpacket(device->channel, 211 revoke_packet, 212 sizeof(struct nvsp_message), 213 (unsigned long)revoke_packet, 214 VM_PKT_DATA_INBAND, 0); 215 /* If we failed here, we might as well return and 216 * have a leak rather than continue and a bugchk 217 */ 218 if (ret != 0) { 219 netdev_err(ndev, "unable to send " 220 "revoke send buffer to netvsp\n"); 221 return ret; 222 } 223 } 224 /* Teardown the gpadl on the vsp end */ 225 if (net_device->send_buf_gpadl_handle) { 226 ret = vmbus_teardown_gpadl(device->channel, 227 net_device->send_buf_gpadl_handle); 228 229 /* If we failed here, we might as well return and have a leak 230 * rather than continue and a bugchk 231 */ 232 if (ret != 0) { 233 netdev_err(ndev, 234 "unable to teardown send buffer's gpadl\n"); 235 return ret; 236 } 237 net_device->send_buf_gpadl_handle = 0; 238 } 239 if (net_device->send_buf) { 240 /* Free up the send buffer */ 241 vfree(net_device->send_buf); 242 net_device->send_buf = NULL; 243 } 244 kfree(net_device->send_section_map); 245 246 return ret; 247 } 248 249 static int netvsc_init_buf(struct hv_device *device) 250 { 251 int ret = 0; 252 unsigned long t; 253 struct netvsc_device *net_device; 254 struct nvsp_message *init_packet; 255 struct net_device *ndev; 256 int node; 257 258 net_device = get_outbound_net_device(device); 259 if (!net_device) 260 return -ENODEV; 261 ndev = hv_get_drvdata(device); 262 263 node = cpu_to_node(device->channel->target_cpu); 264 net_device->recv_buf = vzalloc_node(net_device->recv_buf_size, node); 265 if (!net_device->recv_buf) 266 net_device->recv_buf = vzalloc(net_device->recv_buf_size); 267 268 if (!net_device->recv_buf) { 269 netdev_err(ndev, "unable to allocate receive " 270 "buffer of size %d\n", net_device->recv_buf_size); 271 ret = -ENOMEM; 272 goto cleanup; 273 } 274 275 /* 276 * Establish the gpadl handle for this buffer on this 277 * channel. Note: This call uses the vmbus connection rather 278 * than the channel to establish the gpadl handle. 279 */ 280 ret = vmbus_establish_gpadl(device->channel, net_device->recv_buf, 281 net_device->recv_buf_size, 282 &net_device->recv_buf_gpadl_handle); 283 if (ret != 0) { 284 netdev_err(ndev, 285 "unable to establish receive buffer's gpadl\n"); 286 goto cleanup; 287 } 288 289 290 /* Notify the NetVsp of the gpadl handle */ 291 init_packet = &net_device->channel_init_pkt; 292 293 memset(init_packet, 0, sizeof(struct nvsp_message)); 294 295 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_RECV_BUF; 296 init_packet->msg.v1_msg.send_recv_buf. 297 gpadl_handle = net_device->recv_buf_gpadl_handle; 298 init_packet->msg.v1_msg. 299 send_recv_buf.id = NETVSC_RECEIVE_BUFFER_ID; 300 301 /* Send the gpadl notification request */ 302 ret = vmbus_sendpacket(device->channel, init_packet, 303 sizeof(struct nvsp_message), 304 (unsigned long)init_packet, 305 VM_PKT_DATA_INBAND, 306 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 307 if (ret != 0) { 308 netdev_err(ndev, 309 "unable to send receive buffer's gpadl to netvsp\n"); 310 goto cleanup; 311 } 312 313 t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ); 314 BUG_ON(t == 0); 315 316 317 /* Check the response */ 318 if (init_packet->msg.v1_msg. 319 send_recv_buf_complete.status != NVSP_STAT_SUCCESS) { 320 netdev_err(ndev, "Unable to complete receive buffer " 321 "initialization with NetVsp - status %d\n", 322 init_packet->msg.v1_msg. 323 send_recv_buf_complete.status); 324 ret = -EINVAL; 325 goto cleanup; 326 } 327 328 /* Parse the response */ 329 330 net_device->recv_section_cnt = init_packet->msg. 331 v1_msg.send_recv_buf_complete.num_sections; 332 333 net_device->recv_section = kmemdup( 334 init_packet->msg.v1_msg.send_recv_buf_complete.sections, 335 net_device->recv_section_cnt * 336 sizeof(struct nvsp_1_receive_buffer_section), 337 GFP_KERNEL); 338 if (net_device->recv_section == NULL) { 339 ret = -EINVAL; 340 goto cleanup; 341 } 342 343 /* 344 * For 1st release, there should only be 1 section that represents the 345 * entire receive buffer 346 */ 347 if (net_device->recv_section_cnt != 1 || 348 net_device->recv_section->offset != 0) { 349 ret = -EINVAL; 350 goto cleanup; 351 } 352 353 /* Now setup the send buffer. 354 */ 355 net_device->send_buf = vzalloc_node(net_device->send_buf_size, node); 356 if (!net_device->send_buf) 357 net_device->send_buf = vzalloc(net_device->send_buf_size); 358 if (!net_device->send_buf) { 359 netdev_err(ndev, "unable to allocate send " 360 "buffer of size %d\n", net_device->send_buf_size); 361 ret = -ENOMEM; 362 goto cleanup; 363 } 364 365 /* Establish the gpadl handle for this buffer on this 366 * channel. Note: This call uses the vmbus connection rather 367 * than the channel to establish the gpadl handle. 368 */ 369 ret = vmbus_establish_gpadl(device->channel, net_device->send_buf, 370 net_device->send_buf_size, 371 &net_device->send_buf_gpadl_handle); 372 if (ret != 0) { 373 netdev_err(ndev, 374 "unable to establish send buffer's gpadl\n"); 375 goto cleanup; 376 } 377 378 /* Notify the NetVsp of the gpadl handle */ 379 init_packet = &net_device->channel_init_pkt; 380 memset(init_packet, 0, sizeof(struct nvsp_message)); 381 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_SEND_BUF; 382 init_packet->msg.v1_msg.send_send_buf.gpadl_handle = 383 net_device->send_buf_gpadl_handle; 384 init_packet->msg.v1_msg.send_send_buf.id = NETVSC_SEND_BUFFER_ID; 385 386 /* Send the gpadl notification request */ 387 ret = vmbus_sendpacket(device->channel, init_packet, 388 sizeof(struct nvsp_message), 389 (unsigned long)init_packet, 390 VM_PKT_DATA_INBAND, 391 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 392 if (ret != 0) { 393 netdev_err(ndev, 394 "unable to send send buffer's gpadl to netvsp\n"); 395 goto cleanup; 396 } 397 398 t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ); 399 BUG_ON(t == 0); 400 401 /* Check the response */ 402 if (init_packet->msg.v1_msg. 403 send_send_buf_complete.status != NVSP_STAT_SUCCESS) { 404 netdev_err(ndev, "Unable to complete send buffer " 405 "initialization with NetVsp - status %d\n", 406 init_packet->msg.v1_msg. 407 send_send_buf_complete.status); 408 ret = -EINVAL; 409 goto cleanup; 410 } 411 412 /* Parse the response */ 413 net_device->send_section_size = init_packet->msg. 414 v1_msg.send_send_buf_complete.section_size; 415 416 /* Section count is simply the size divided by the section size. 417 */ 418 net_device->send_section_cnt = 419 net_device->send_buf_size/net_device->send_section_size; 420 421 dev_info(&device->device, "Send section size: %d, Section count:%d\n", 422 net_device->send_section_size, net_device->send_section_cnt); 423 424 /* Setup state for managing the send buffer. */ 425 net_device->map_words = DIV_ROUND_UP(net_device->send_section_cnt, 426 BITS_PER_LONG); 427 428 net_device->send_section_map = 429 kzalloc(net_device->map_words * sizeof(ulong), GFP_KERNEL); 430 if (net_device->send_section_map == NULL) { 431 ret = -ENOMEM; 432 goto cleanup; 433 } 434 435 goto exit; 436 437 cleanup: 438 netvsc_destroy_buf(device); 439 440 exit: 441 return ret; 442 } 443 444 445 /* Negotiate NVSP protocol version */ 446 static int negotiate_nvsp_ver(struct hv_device *device, 447 struct netvsc_device *net_device, 448 struct nvsp_message *init_packet, 449 u32 nvsp_ver) 450 { 451 struct net_device *ndev = hv_get_drvdata(device); 452 int ret; 453 unsigned long t; 454 455 memset(init_packet, 0, sizeof(struct nvsp_message)); 456 init_packet->hdr.msg_type = NVSP_MSG_TYPE_INIT; 457 init_packet->msg.init_msg.init.min_protocol_ver = nvsp_ver; 458 init_packet->msg.init_msg.init.max_protocol_ver = nvsp_ver; 459 460 /* Send the init request */ 461 ret = vmbus_sendpacket(device->channel, init_packet, 462 sizeof(struct nvsp_message), 463 (unsigned long)init_packet, 464 VM_PKT_DATA_INBAND, 465 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED); 466 467 if (ret != 0) 468 return ret; 469 470 t = wait_for_completion_timeout(&net_device->channel_init_wait, 5*HZ); 471 472 if (t == 0) 473 return -ETIMEDOUT; 474 475 if (init_packet->msg.init_msg.init_complete.status != 476 NVSP_STAT_SUCCESS) 477 return -EINVAL; 478 479 if (nvsp_ver == NVSP_PROTOCOL_VERSION_1) 480 return 0; 481 482 /* NVSPv2 or later: Send NDIS config */ 483 memset(init_packet, 0, sizeof(struct nvsp_message)); 484 init_packet->hdr.msg_type = NVSP_MSG2_TYPE_SEND_NDIS_CONFIG; 485 init_packet->msg.v2_msg.send_ndis_config.mtu = ndev->mtu + ETH_HLEN; 486 init_packet->msg.v2_msg.send_ndis_config.capability.ieee8021q = 1; 487 488 if (nvsp_ver >= NVSP_PROTOCOL_VERSION_5) 489 init_packet->msg.v2_msg.send_ndis_config.capability.sriov = 1; 490 491 ret = vmbus_sendpacket(device->channel, init_packet, 492 sizeof(struct nvsp_message), 493 (unsigned long)init_packet, 494 VM_PKT_DATA_INBAND, 0); 495 496 return ret; 497 } 498 499 static int netvsc_connect_vsp(struct hv_device *device) 500 { 501 int ret; 502 struct netvsc_device *net_device; 503 struct nvsp_message *init_packet; 504 int ndis_version; 505 u32 ver_list[] = { NVSP_PROTOCOL_VERSION_1, NVSP_PROTOCOL_VERSION_2, 506 NVSP_PROTOCOL_VERSION_4, NVSP_PROTOCOL_VERSION_5 }; 507 int i, num_ver = 4; /* number of different NVSP versions */ 508 509 net_device = get_outbound_net_device(device); 510 if (!net_device) 511 return -ENODEV; 512 513 init_packet = &net_device->channel_init_pkt; 514 515 /* Negotiate the latest NVSP protocol supported */ 516 for (i = num_ver - 1; i >= 0; i--) 517 if (negotiate_nvsp_ver(device, net_device, init_packet, 518 ver_list[i]) == 0) { 519 net_device->nvsp_version = ver_list[i]; 520 break; 521 } 522 523 if (i < 0) { 524 ret = -EPROTO; 525 goto cleanup; 526 } 527 528 pr_debug("Negotiated NVSP version:%x\n", net_device->nvsp_version); 529 530 /* Send the ndis version */ 531 memset(init_packet, 0, sizeof(struct nvsp_message)); 532 533 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_4) 534 ndis_version = 0x00060001; 535 else 536 ndis_version = 0x0006001e; 537 538 init_packet->hdr.msg_type = NVSP_MSG1_TYPE_SEND_NDIS_VER; 539 init_packet->msg.v1_msg. 540 send_ndis_ver.ndis_major_ver = 541 (ndis_version & 0xFFFF0000) >> 16; 542 init_packet->msg.v1_msg. 543 send_ndis_ver.ndis_minor_ver = 544 ndis_version & 0xFFFF; 545 546 /* Send the init request */ 547 ret = vmbus_sendpacket(device->channel, init_packet, 548 sizeof(struct nvsp_message), 549 (unsigned long)init_packet, 550 VM_PKT_DATA_INBAND, 0); 551 if (ret != 0) 552 goto cleanup; 553 554 /* Post the big receive buffer to NetVSP */ 555 if (net_device->nvsp_version <= NVSP_PROTOCOL_VERSION_2) 556 net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE_LEGACY; 557 else 558 net_device->recv_buf_size = NETVSC_RECEIVE_BUFFER_SIZE; 559 net_device->send_buf_size = NETVSC_SEND_BUFFER_SIZE; 560 561 ret = netvsc_init_buf(device); 562 563 cleanup: 564 return ret; 565 } 566 567 static void netvsc_disconnect_vsp(struct hv_device *device) 568 { 569 netvsc_destroy_buf(device); 570 } 571 572 /* 573 * netvsc_device_remove - Callback when the root bus device is removed 574 */ 575 int netvsc_device_remove(struct hv_device *device) 576 { 577 struct net_device *ndev = hv_get_drvdata(device); 578 struct net_device_context *net_device_ctx = netdev_priv(ndev); 579 struct netvsc_device *net_device = net_device_ctx->nvdev; 580 581 netvsc_disconnect_vsp(device); 582 583 net_device_ctx->nvdev = NULL; 584 585 /* 586 * At this point, no one should be accessing net_device 587 * except in here 588 */ 589 dev_notice(&device->device, "net device safe to remove\n"); 590 591 /* Now, we can close the channel safely */ 592 vmbus_close(device->channel); 593 594 /* Release all resources */ 595 vfree(net_device->sub_cb_buf); 596 free_netvsc_device(net_device); 597 return 0; 598 } 599 600 601 #define RING_AVAIL_PERCENT_HIWATER 20 602 #define RING_AVAIL_PERCENT_LOWATER 10 603 604 /* 605 * Get the percentage of available bytes to write in the ring. 606 * The return value is in range from 0 to 100. 607 */ 608 static inline u32 hv_ringbuf_avail_percent( 609 struct hv_ring_buffer_info *ring_info) 610 { 611 u32 avail_read, avail_write; 612 613 hv_get_ringbuffer_availbytes(ring_info, &avail_read, &avail_write); 614 615 return avail_write * 100 / ring_info->ring_datasize; 616 } 617 618 static inline void netvsc_free_send_slot(struct netvsc_device *net_device, 619 u32 index) 620 { 621 sync_change_bit(index, net_device->send_section_map); 622 } 623 624 static void netvsc_send_completion(struct netvsc_device *net_device, 625 struct vmbus_channel *incoming_channel, 626 struct hv_device *device, 627 struct vmpacket_descriptor *packet) 628 { 629 struct nvsp_message *nvsp_packet; 630 struct hv_netvsc_packet *nvsc_packet; 631 struct net_device *ndev = hv_get_drvdata(device); 632 struct net_device_context *net_device_ctx = netdev_priv(ndev); 633 u32 send_index; 634 struct sk_buff *skb; 635 636 nvsp_packet = (struct nvsp_message *)((unsigned long)packet + 637 (packet->offset8 << 3)); 638 639 if ((nvsp_packet->hdr.msg_type == NVSP_MSG_TYPE_INIT_COMPLETE) || 640 (nvsp_packet->hdr.msg_type == 641 NVSP_MSG1_TYPE_SEND_RECV_BUF_COMPLETE) || 642 (nvsp_packet->hdr.msg_type == 643 NVSP_MSG1_TYPE_SEND_SEND_BUF_COMPLETE) || 644 (nvsp_packet->hdr.msg_type == 645 NVSP_MSG5_TYPE_SUBCHANNEL)) { 646 /* Copy the response back */ 647 memcpy(&net_device->channel_init_pkt, nvsp_packet, 648 sizeof(struct nvsp_message)); 649 complete(&net_device->channel_init_wait); 650 } else if (nvsp_packet->hdr.msg_type == 651 NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE) { 652 int num_outstanding_sends; 653 u16 q_idx = 0; 654 struct vmbus_channel *channel = device->channel; 655 int queue_sends; 656 657 /* Get the send context */ 658 skb = (struct sk_buff *)(unsigned long)packet->trans_id; 659 660 /* Notify the layer above us */ 661 if (skb) { 662 nvsc_packet = (struct hv_netvsc_packet *) skb->cb; 663 send_index = nvsc_packet->send_buf_index; 664 if (send_index != NETVSC_INVALID_INDEX) 665 netvsc_free_send_slot(net_device, send_index); 666 q_idx = nvsc_packet->q_idx; 667 channel = incoming_channel; 668 dev_kfree_skb_any(skb); 669 } 670 671 num_outstanding_sends = 672 atomic_dec_return(&net_device->num_outstanding_sends); 673 queue_sends = atomic_dec_return(&net_device-> 674 queue_sends[q_idx]); 675 676 if (net_device->destroy && num_outstanding_sends == 0) 677 wake_up(&net_device->wait_drain); 678 679 if (netif_tx_queue_stopped(netdev_get_tx_queue(ndev, q_idx)) && 680 !net_device_ctx->start_remove && 681 (hv_ringbuf_avail_percent(&channel->outbound) > 682 RING_AVAIL_PERCENT_HIWATER || queue_sends < 1)) 683 netif_tx_wake_queue(netdev_get_tx_queue( 684 ndev, q_idx)); 685 } else { 686 netdev_err(ndev, "Unknown send completion packet type- " 687 "%d received!!\n", nvsp_packet->hdr.msg_type); 688 } 689 690 } 691 692 static u32 netvsc_get_next_send_section(struct netvsc_device *net_device) 693 { 694 unsigned long index; 695 u32 max_words = net_device->map_words; 696 unsigned long *map_addr = (unsigned long *)net_device->send_section_map; 697 u32 section_cnt = net_device->send_section_cnt; 698 int ret_val = NETVSC_INVALID_INDEX; 699 int i; 700 int prev_val; 701 702 for (i = 0; i < max_words; i++) { 703 if (!~(map_addr[i])) 704 continue; 705 index = ffz(map_addr[i]); 706 prev_val = sync_test_and_set_bit(index, &map_addr[i]); 707 if (prev_val) 708 continue; 709 if ((index + (i * BITS_PER_LONG)) >= section_cnt) 710 break; 711 ret_val = (index + (i * BITS_PER_LONG)); 712 break; 713 } 714 return ret_val; 715 } 716 717 static u32 netvsc_copy_to_send_buf(struct netvsc_device *net_device, 718 unsigned int section_index, 719 u32 pend_size, 720 struct hv_netvsc_packet *packet, 721 struct rndis_message *rndis_msg, 722 struct hv_page_buffer **pb, 723 struct sk_buff *skb) 724 { 725 char *start = net_device->send_buf; 726 char *dest = start + (section_index * net_device->send_section_size) 727 + pend_size; 728 int i; 729 bool is_data_pkt = (skb != NULL) ? true : false; 730 bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 731 u32 msg_size = 0; 732 u32 padding = 0; 733 u32 remain = packet->total_data_buflen % net_device->pkt_align; 734 u32 page_count = packet->cp_partial ? packet->rmsg_pgcnt : 735 packet->page_buf_cnt; 736 737 /* Add padding */ 738 if (is_data_pkt && xmit_more && remain && 739 !packet->cp_partial) { 740 padding = net_device->pkt_align - remain; 741 rndis_msg->msg_len += padding; 742 packet->total_data_buflen += padding; 743 } 744 745 for (i = 0; i < page_count; i++) { 746 char *src = phys_to_virt((*pb)[i].pfn << PAGE_SHIFT); 747 u32 offset = (*pb)[i].offset; 748 u32 len = (*pb)[i].len; 749 750 memcpy(dest, (src + offset), len); 751 msg_size += len; 752 dest += len; 753 } 754 755 if (padding) { 756 memset(dest, 0, padding); 757 msg_size += padding; 758 } 759 760 return msg_size; 761 } 762 763 static inline int netvsc_send_pkt( 764 struct hv_device *device, 765 struct hv_netvsc_packet *packet, 766 struct netvsc_device *net_device, 767 struct hv_page_buffer **pb, 768 struct sk_buff *skb) 769 { 770 struct nvsp_message nvmsg; 771 u16 q_idx = packet->q_idx; 772 struct vmbus_channel *out_channel = net_device->chn_table[q_idx]; 773 struct net_device *ndev = hv_get_drvdata(device); 774 u64 req_id; 775 int ret; 776 struct hv_page_buffer *pgbuf; 777 u32 ring_avail = hv_ringbuf_avail_percent(&out_channel->outbound); 778 bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 779 780 nvmsg.hdr.msg_type = NVSP_MSG1_TYPE_SEND_RNDIS_PKT; 781 if (skb != NULL) { 782 /* 0 is RMC_DATA; */ 783 nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 0; 784 } else { 785 /* 1 is RMC_CONTROL; */ 786 nvmsg.msg.v1_msg.send_rndis_pkt.channel_type = 1; 787 } 788 789 nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_index = 790 packet->send_buf_index; 791 if (packet->send_buf_index == NETVSC_INVALID_INDEX) 792 nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 0; 793 else 794 nvmsg.msg.v1_msg.send_rndis_pkt.send_buf_section_size = 795 packet->total_data_buflen; 796 797 req_id = (ulong)skb; 798 799 if (out_channel->rescind) 800 return -ENODEV; 801 802 /* 803 * It is possible that once we successfully place this packet 804 * on the ringbuffer, we may stop the queue. In that case, we want 805 * to notify the host independent of the xmit_more flag. We don't 806 * need to be precise here; in the worst case we may signal the host 807 * unnecessarily. 808 */ 809 if (ring_avail < (RING_AVAIL_PERCENT_LOWATER + 1)) 810 xmit_more = false; 811 812 if (packet->page_buf_cnt) { 813 pgbuf = packet->cp_partial ? (*pb) + 814 packet->rmsg_pgcnt : (*pb); 815 ret = vmbus_sendpacket_pagebuffer_ctl(out_channel, 816 pgbuf, 817 packet->page_buf_cnt, 818 &nvmsg, 819 sizeof(struct nvsp_message), 820 req_id, 821 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED, 822 !xmit_more); 823 } else { 824 ret = vmbus_sendpacket_ctl(out_channel, &nvmsg, 825 sizeof(struct nvsp_message), 826 req_id, 827 VM_PKT_DATA_INBAND, 828 VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED, 829 !xmit_more); 830 } 831 832 if (ret == 0) { 833 atomic_inc(&net_device->num_outstanding_sends); 834 atomic_inc(&net_device->queue_sends[q_idx]); 835 836 if (ring_avail < RING_AVAIL_PERCENT_LOWATER) { 837 netif_tx_stop_queue(netdev_get_tx_queue(ndev, q_idx)); 838 839 if (atomic_read(&net_device-> 840 queue_sends[q_idx]) < 1) 841 netif_tx_wake_queue(netdev_get_tx_queue( 842 ndev, q_idx)); 843 } 844 } else if (ret == -EAGAIN) { 845 netif_tx_stop_queue(netdev_get_tx_queue( 846 ndev, q_idx)); 847 if (atomic_read(&net_device->queue_sends[q_idx]) < 1) { 848 netif_tx_wake_queue(netdev_get_tx_queue( 849 ndev, q_idx)); 850 ret = -ENOSPC; 851 } 852 } else { 853 netdev_err(ndev, "Unable to send packet %p ret %d\n", 854 packet, ret); 855 } 856 857 return ret; 858 } 859 860 /* Move packet out of multi send data (msd), and clear msd */ 861 static inline void move_pkt_msd(struct hv_netvsc_packet **msd_send, 862 struct sk_buff **msd_skb, 863 struct multi_send_data *msdp) 864 { 865 *msd_skb = msdp->skb; 866 *msd_send = msdp->pkt; 867 msdp->skb = NULL; 868 msdp->pkt = NULL; 869 msdp->count = 0; 870 } 871 872 int netvsc_send(struct hv_device *device, 873 struct hv_netvsc_packet *packet, 874 struct rndis_message *rndis_msg, 875 struct hv_page_buffer **pb, 876 struct sk_buff *skb) 877 { 878 struct netvsc_device *net_device; 879 int ret = 0, m_ret = 0; 880 struct vmbus_channel *out_channel; 881 u16 q_idx = packet->q_idx; 882 u32 pktlen = packet->total_data_buflen, msd_len = 0; 883 unsigned int section_index = NETVSC_INVALID_INDEX; 884 struct multi_send_data *msdp; 885 struct hv_netvsc_packet *msd_send = NULL, *cur_send = NULL; 886 struct sk_buff *msd_skb = NULL; 887 bool try_batch; 888 bool xmit_more = (skb != NULL) ? skb->xmit_more : false; 889 890 net_device = get_outbound_net_device(device); 891 if (!net_device) 892 return -ENODEV; 893 894 out_channel = net_device->chn_table[q_idx]; 895 896 packet->send_buf_index = NETVSC_INVALID_INDEX; 897 packet->cp_partial = false; 898 899 /* Send control message directly without accessing msd (Multi-Send 900 * Data) field which may be changed during data packet processing. 901 */ 902 if (!skb) { 903 cur_send = packet; 904 goto send_now; 905 } 906 907 msdp = &net_device->msd[q_idx]; 908 909 /* batch packets in send buffer if possible */ 910 if (msdp->pkt) 911 msd_len = msdp->pkt->total_data_buflen; 912 913 try_batch = (skb != NULL) && msd_len > 0 && msdp->count < 914 net_device->max_pkt; 915 916 if (try_batch && msd_len + pktlen + net_device->pkt_align < 917 net_device->send_section_size) { 918 section_index = msdp->pkt->send_buf_index; 919 920 } else if (try_batch && msd_len + packet->rmsg_size < 921 net_device->send_section_size) { 922 section_index = msdp->pkt->send_buf_index; 923 packet->cp_partial = true; 924 925 } else if ((skb != NULL) && pktlen + net_device->pkt_align < 926 net_device->send_section_size) { 927 section_index = netvsc_get_next_send_section(net_device); 928 if (section_index != NETVSC_INVALID_INDEX) { 929 move_pkt_msd(&msd_send, &msd_skb, msdp); 930 msd_len = 0; 931 } 932 } 933 934 if (section_index != NETVSC_INVALID_INDEX) { 935 netvsc_copy_to_send_buf(net_device, 936 section_index, msd_len, 937 packet, rndis_msg, pb, skb); 938 939 packet->send_buf_index = section_index; 940 941 if (packet->cp_partial) { 942 packet->page_buf_cnt -= packet->rmsg_pgcnt; 943 packet->total_data_buflen = msd_len + packet->rmsg_size; 944 } else { 945 packet->page_buf_cnt = 0; 946 packet->total_data_buflen += msd_len; 947 } 948 949 if (msdp->skb) 950 dev_kfree_skb_any(msdp->skb); 951 952 if (xmit_more && !packet->cp_partial) { 953 msdp->skb = skb; 954 msdp->pkt = packet; 955 msdp->count++; 956 } else { 957 cur_send = packet; 958 msdp->skb = NULL; 959 msdp->pkt = NULL; 960 msdp->count = 0; 961 } 962 } else { 963 move_pkt_msd(&msd_send, &msd_skb, msdp); 964 cur_send = packet; 965 } 966 967 if (msd_send) { 968 m_ret = netvsc_send_pkt(device, msd_send, net_device, 969 NULL, msd_skb); 970 971 if (m_ret != 0) { 972 netvsc_free_send_slot(net_device, 973 msd_send->send_buf_index); 974 dev_kfree_skb_any(msd_skb); 975 } 976 } 977 978 send_now: 979 if (cur_send) 980 ret = netvsc_send_pkt(device, cur_send, net_device, pb, skb); 981 982 if (ret != 0 && section_index != NETVSC_INVALID_INDEX) 983 netvsc_free_send_slot(net_device, section_index); 984 985 return ret; 986 } 987 988 static void netvsc_send_recv_completion(struct hv_device *device, 989 struct vmbus_channel *channel, 990 struct netvsc_device *net_device, 991 u64 transaction_id, u32 status) 992 { 993 struct nvsp_message recvcompMessage; 994 int retries = 0; 995 int ret; 996 struct net_device *ndev = hv_get_drvdata(device); 997 998 recvcompMessage.hdr.msg_type = 999 NVSP_MSG1_TYPE_SEND_RNDIS_PKT_COMPLETE; 1000 1001 recvcompMessage.msg.v1_msg.send_rndis_pkt_complete.status = status; 1002 1003 retry_send_cmplt: 1004 /* Send the completion */ 1005 ret = vmbus_sendpacket(channel, &recvcompMessage, 1006 sizeof(struct nvsp_message), transaction_id, 1007 VM_PKT_COMP, 0); 1008 if (ret == 0) { 1009 /* success */ 1010 /* no-op */ 1011 } else if (ret == -EAGAIN) { 1012 /* no more room...wait a bit and attempt to retry 3 times */ 1013 retries++; 1014 netdev_err(ndev, "unable to send receive completion pkt" 1015 " (tid %llx)...retrying %d\n", transaction_id, retries); 1016 1017 if (retries < 4) { 1018 udelay(100); 1019 goto retry_send_cmplt; 1020 } else { 1021 netdev_err(ndev, "unable to send receive " 1022 "completion pkt (tid %llx)...give up retrying\n", 1023 transaction_id); 1024 } 1025 } else { 1026 netdev_err(ndev, "unable to send receive " 1027 "completion pkt - %llx\n", transaction_id); 1028 } 1029 } 1030 1031 static void netvsc_receive(struct netvsc_device *net_device, 1032 struct vmbus_channel *channel, 1033 struct hv_device *device, 1034 struct vmpacket_descriptor *packet) 1035 { 1036 struct vmtransfer_page_packet_header *vmxferpage_packet; 1037 struct nvsp_message *nvsp_packet; 1038 struct hv_netvsc_packet nv_pkt; 1039 struct hv_netvsc_packet *netvsc_packet = &nv_pkt; 1040 u32 status = NVSP_STAT_SUCCESS; 1041 int i; 1042 int count = 0; 1043 struct net_device *ndev = hv_get_drvdata(device); 1044 void *data; 1045 1046 /* 1047 * All inbound packets other than send completion should be xfer page 1048 * packet 1049 */ 1050 if (packet->type != VM_PKT_DATA_USING_XFER_PAGES) { 1051 netdev_err(ndev, "Unknown packet type received - %d\n", 1052 packet->type); 1053 return; 1054 } 1055 1056 nvsp_packet = (struct nvsp_message *)((unsigned long)packet + 1057 (packet->offset8 << 3)); 1058 1059 /* Make sure this is a valid nvsp packet */ 1060 if (nvsp_packet->hdr.msg_type != 1061 NVSP_MSG1_TYPE_SEND_RNDIS_PKT) { 1062 netdev_err(ndev, "Unknown nvsp packet type received-" 1063 " %d\n", nvsp_packet->hdr.msg_type); 1064 return; 1065 } 1066 1067 vmxferpage_packet = (struct vmtransfer_page_packet_header *)packet; 1068 1069 if (vmxferpage_packet->xfer_pageset_id != NETVSC_RECEIVE_BUFFER_ID) { 1070 netdev_err(ndev, "Invalid xfer page set id - " 1071 "expecting %x got %x\n", NETVSC_RECEIVE_BUFFER_ID, 1072 vmxferpage_packet->xfer_pageset_id); 1073 return; 1074 } 1075 1076 count = vmxferpage_packet->range_cnt; 1077 1078 /* Each range represents 1 RNDIS pkt that contains 1 ethernet frame */ 1079 for (i = 0; i < count; i++) { 1080 /* Initialize the netvsc packet */ 1081 data = (void *)((unsigned long)net_device-> 1082 recv_buf + vmxferpage_packet->ranges[i].byte_offset); 1083 netvsc_packet->total_data_buflen = 1084 vmxferpage_packet->ranges[i].byte_count; 1085 1086 /* Pass it to the upper layer */ 1087 status = rndis_filter_receive(device, netvsc_packet, &data, 1088 channel); 1089 1090 } 1091 1092 netvsc_send_recv_completion(device, channel, net_device, 1093 vmxferpage_packet->d.trans_id, status); 1094 } 1095 1096 1097 static void netvsc_send_table(struct hv_device *hdev, 1098 struct nvsp_message *nvmsg) 1099 { 1100 struct netvsc_device *nvscdev; 1101 struct net_device *ndev = hv_get_drvdata(hdev); 1102 int i; 1103 u32 count, *tab; 1104 1105 nvscdev = get_outbound_net_device(hdev); 1106 if (!nvscdev) 1107 return; 1108 1109 count = nvmsg->msg.v5_msg.send_table.count; 1110 if (count != VRSS_SEND_TAB_SIZE) { 1111 netdev_err(ndev, "Received wrong send-table size:%u\n", count); 1112 return; 1113 } 1114 1115 tab = (u32 *)((unsigned long)&nvmsg->msg.v5_msg.send_table + 1116 nvmsg->msg.v5_msg.send_table.offset); 1117 1118 for (i = 0; i < count; i++) 1119 nvscdev->send_table[i] = tab[i]; 1120 } 1121 1122 static void netvsc_send_vf(struct netvsc_device *nvdev, 1123 struct nvsp_message *nvmsg) 1124 { 1125 nvdev->vf_alloc = nvmsg->msg.v4_msg.vf_assoc.allocated; 1126 nvdev->vf_serial = nvmsg->msg.v4_msg.vf_assoc.serial; 1127 } 1128 1129 static inline void netvsc_receive_inband(struct hv_device *hdev, 1130 struct netvsc_device *nvdev, 1131 struct nvsp_message *nvmsg) 1132 { 1133 switch (nvmsg->hdr.msg_type) { 1134 case NVSP_MSG5_TYPE_SEND_INDIRECTION_TABLE: 1135 netvsc_send_table(hdev, nvmsg); 1136 break; 1137 1138 case NVSP_MSG4_TYPE_SEND_VF_ASSOCIATION: 1139 netvsc_send_vf(nvdev, nvmsg); 1140 break; 1141 } 1142 } 1143 1144 void netvsc_channel_cb(void *context) 1145 { 1146 int ret; 1147 struct vmbus_channel *channel = (struct vmbus_channel *)context; 1148 struct hv_device *device; 1149 struct netvsc_device *net_device; 1150 u32 bytes_recvd; 1151 u64 request_id; 1152 struct vmpacket_descriptor *desc; 1153 unsigned char *buffer; 1154 int bufferlen = NETVSC_PACKET_SIZE; 1155 struct net_device *ndev; 1156 struct nvsp_message *nvmsg; 1157 1158 if (channel->primary_channel != NULL) 1159 device = channel->primary_channel->device_obj; 1160 else 1161 device = channel->device_obj; 1162 1163 net_device = get_inbound_net_device(device); 1164 if (!net_device) 1165 return; 1166 ndev = hv_get_drvdata(device); 1167 buffer = get_per_channel_state(channel); 1168 1169 do { 1170 ret = vmbus_recvpacket_raw(channel, buffer, bufferlen, 1171 &bytes_recvd, &request_id); 1172 if (ret == 0) { 1173 if (bytes_recvd > 0) { 1174 desc = (struct vmpacket_descriptor *)buffer; 1175 nvmsg = (struct nvsp_message *)((unsigned long) 1176 desc + (desc->offset8 << 3)); 1177 switch (desc->type) { 1178 case VM_PKT_COMP: 1179 netvsc_send_completion(net_device, 1180 channel, 1181 device, desc); 1182 break; 1183 1184 case VM_PKT_DATA_USING_XFER_PAGES: 1185 netvsc_receive(net_device, channel, 1186 device, desc); 1187 break; 1188 1189 case VM_PKT_DATA_INBAND: 1190 netvsc_receive_inband(device, 1191 net_device, 1192 nvmsg); 1193 break; 1194 1195 default: 1196 netdev_err(ndev, 1197 "unhandled packet type %d, " 1198 "tid %llx len %d\n", 1199 desc->type, request_id, 1200 bytes_recvd); 1201 break; 1202 } 1203 1204 } else { 1205 /* 1206 * We are done for this pass. 1207 */ 1208 break; 1209 } 1210 1211 } else if (ret == -ENOBUFS) { 1212 if (bufferlen > NETVSC_PACKET_SIZE) 1213 kfree(buffer); 1214 /* Handle large packet */ 1215 buffer = kmalloc(bytes_recvd, GFP_ATOMIC); 1216 if (buffer == NULL) { 1217 /* Try again next time around */ 1218 netdev_err(ndev, 1219 "unable to allocate buffer of size " 1220 "(%d)!!\n", bytes_recvd); 1221 break; 1222 } 1223 1224 bufferlen = bytes_recvd; 1225 } 1226 } while (1); 1227 1228 if (bufferlen > NETVSC_PACKET_SIZE) 1229 kfree(buffer); 1230 return; 1231 } 1232 1233 /* 1234 * netvsc_device_add - Callback when the device belonging to this 1235 * driver is added 1236 */ 1237 int netvsc_device_add(struct hv_device *device, void *additional_info) 1238 { 1239 int i, ret = 0; 1240 int ring_size = 1241 ((struct netvsc_device_info *)additional_info)->ring_size; 1242 struct netvsc_device *net_device; 1243 struct net_device *ndev = hv_get_drvdata(device); 1244 struct net_device_context *net_device_ctx = netdev_priv(ndev); 1245 1246 net_device = alloc_net_device(); 1247 if (!net_device) 1248 return -ENOMEM; 1249 1250 net_device->ring_size = ring_size; 1251 1252 /* Initialize the NetVSC channel extension */ 1253 init_completion(&net_device->channel_init_wait); 1254 1255 set_per_channel_state(device->channel, net_device->cb_buffer); 1256 1257 /* Open the channel */ 1258 ret = vmbus_open(device->channel, ring_size * PAGE_SIZE, 1259 ring_size * PAGE_SIZE, NULL, 0, 1260 netvsc_channel_cb, device->channel); 1261 1262 if (ret != 0) { 1263 netdev_err(ndev, "unable to open channel: %d\n", ret); 1264 goto cleanup; 1265 } 1266 1267 /* Channel is opened */ 1268 pr_info("hv_netvsc channel opened successfully\n"); 1269 1270 /* If we're reopening the device we may have multiple queues, fill the 1271 * chn_table with the default channel to use it before subchannels are 1272 * opened. 1273 */ 1274 for (i = 0; i < VRSS_CHANNEL_MAX; i++) 1275 net_device->chn_table[i] = device->channel; 1276 1277 /* Writing nvdev pointer unlocks netvsc_send(), make sure chn_table is 1278 * populated. 1279 */ 1280 wmb(); 1281 1282 net_device_ctx->nvdev = net_device; 1283 1284 /* Connect with the NetVsp */ 1285 ret = netvsc_connect_vsp(device); 1286 if (ret != 0) { 1287 netdev_err(ndev, 1288 "unable to connect to NetVSP - %d\n", ret); 1289 goto close; 1290 } 1291 1292 return ret; 1293 1294 close: 1295 /* Now, we can close the channel safely */ 1296 vmbus_close(device->channel); 1297 1298 cleanup: 1299 free_netvsc_device(net_device); 1300 1301 return ret; 1302 } 1303