1 #include <linux/etherdevice.h> 2 #include <linux/if_macvlan.h> 3 #include <linux/interrupt.h> 4 #include <linux/nsproxy.h> 5 #include <linux/compat.h> 6 #include <linux/if_tun.h> 7 #include <linux/module.h> 8 #include <linux/skbuff.h> 9 #include <linux/cache.h> 10 #include <linux/sched.h> 11 #include <linux/types.h> 12 #include <linux/slab.h> 13 #include <linux/init.h> 14 #include <linux/wait.h> 15 #include <linux/cdev.h> 16 #include <linux/fs.h> 17 18 #include <net/net_namespace.h> 19 #include <net/rtnetlink.h> 20 #include <net/sock.h> 21 #include <linux/virtio_net.h> 22 23 /* 24 * A macvtap queue is the central object of this driver, it connects 25 * an open character device to a macvlan interface. There can be 26 * multiple queues on one interface, which map back to queues 27 * implemented in hardware on the underlying device. 28 * 29 * macvtap_proto is used to allocate queues through the sock allocation 30 * mechanism. 31 * 32 * TODO: multiqueue support is currently not implemented, even though 33 * macvtap is basically prepared for that. We will need to add this 34 * here as well as in virtio-net and qemu to get line rate on 10gbit 35 * adapters from a guest. 36 */ 37 struct macvtap_queue { 38 struct sock sk; 39 struct socket sock; 40 struct socket_wq wq; 41 int vnet_hdr_sz; 42 struct macvlan_dev __rcu *vlan; 43 struct file *file; 44 unsigned int flags; 45 }; 46 47 static struct proto macvtap_proto = { 48 .name = "macvtap", 49 .owner = THIS_MODULE, 50 .obj_size = sizeof (struct macvtap_queue), 51 }; 52 53 /* 54 * Variables for dealing with macvtaps device numbers. 55 */ 56 static dev_t macvtap_major; 57 #define MACVTAP_NUM_DEVS (1U << MINORBITS) 58 static DEFINE_MUTEX(minor_lock); 59 static DEFINE_IDR(minor_idr); 60 61 #define GOODCOPY_LEN 128 62 static struct class *macvtap_class; 63 static struct cdev macvtap_cdev; 64 65 static const struct proto_ops macvtap_socket_ops; 66 67 /* 68 * RCU usage: 69 * The macvtap_queue and the macvlan_dev are loosely coupled, the 70 * pointers from one to the other can only be read while rcu_read_lock 71 * or macvtap_lock is held. 72 * 73 * Both the file and the macvlan_dev hold a reference on the macvtap_queue 74 * through sock_hold(&q->sk). When the macvlan_dev goes away first, 75 * q->vlan becomes inaccessible. When the files gets closed, 76 * macvtap_get_queue() fails. 77 * 78 * There may still be references to the struct sock inside of the 79 * queue from outbound SKBs, but these never reference back to the 80 * file or the dev. The data structure is freed through __sk_free 81 * when both our references and any pending SKBs are gone. 82 */ 83 static DEFINE_SPINLOCK(macvtap_lock); 84 85 /* 86 * get_slot: return a [unused/occupied] slot in vlan->taps[]: 87 * - if 'q' is NULL, return the first empty slot; 88 * - otherwise, return the slot this pointer occupies. 89 */ 90 static int get_slot(struct macvlan_dev *vlan, struct macvtap_queue *q) 91 { 92 int i; 93 94 for (i = 0; i < MAX_MACVTAP_QUEUES; i++) { 95 if (rcu_dereference(vlan->taps[i]) == q) 96 return i; 97 } 98 99 /* Should never happen */ 100 BUG_ON(1); 101 } 102 103 static int macvtap_set_queue(struct net_device *dev, struct file *file, 104 struct macvtap_queue *q) 105 { 106 struct macvlan_dev *vlan = netdev_priv(dev); 107 int index; 108 int err = -EBUSY; 109 110 spin_lock(&macvtap_lock); 111 if (vlan->numvtaps == MAX_MACVTAP_QUEUES) 112 goto out; 113 114 err = 0; 115 index = get_slot(vlan, NULL); 116 rcu_assign_pointer(q->vlan, vlan); 117 rcu_assign_pointer(vlan->taps[index], q); 118 sock_hold(&q->sk); 119 120 q->file = file; 121 file->private_data = q; 122 123 vlan->numvtaps++; 124 125 out: 126 spin_unlock(&macvtap_lock); 127 return err; 128 } 129 130 /* 131 * The file owning the queue got closed, give up both 132 * the reference that the files holds as well as the 133 * one from the macvlan_dev if that still exists. 134 * 135 * Using the spinlock makes sure that we don't get 136 * to the queue again after destroying it. 137 */ 138 static void macvtap_put_queue(struct macvtap_queue *q) 139 { 140 struct macvlan_dev *vlan; 141 142 spin_lock(&macvtap_lock); 143 vlan = rcu_dereference_protected(q->vlan, 144 lockdep_is_held(&macvtap_lock)); 145 if (vlan) { 146 int index = get_slot(vlan, q); 147 148 rcu_assign_pointer(vlan->taps[index], NULL); 149 rcu_assign_pointer(q->vlan, NULL); 150 sock_put(&q->sk); 151 --vlan->numvtaps; 152 } 153 154 spin_unlock(&macvtap_lock); 155 156 synchronize_rcu(); 157 sock_put(&q->sk); 158 } 159 160 /* 161 * Select a queue based on the rxq of the device on which this packet 162 * arrived. If the incoming device is not mq, calculate a flow hash 163 * to select a queue. If all fails, find the first available queue. 164 * Cache vlan->numvtaps since it can become zero during the execution 165 * of this function. 166 */ 167 static struct macvtap_queue *macvtap_get_queue(struct net_device *dev, 168 struct sk_buff *skb) 169 { 170 struct macvlan_dev *vlan = netdev_priv(dev); 171 struct macvtap_queue *tap = NULL; 172 int numvtaps = vlan->numvtaps; 173 __u32 rxq; 174 175 if (!numvtaps) 176 goto out; 177 178 if (likely(skb_rx_queue_recorded(skb))) { 179 rxq = skb_get_rx_queue(skb); 180 181 while (unlikely(rxq >= numvtaps)) 182 rxq -= numvtaps; 183 184 tap = rcu_dereference(vlan->taps[rxq]); 185 if (tap) 186 goto out; 187 } 188 189 /* Check if we can use flow to select a queue */ 190 rxq = skb_get_rxhash(skb); 191 if (rxq) { 192 tap = rcu_dereference(vlan->taps[rxq % numvtaps]); 193 if (tap) 194 goto out; 195 } 196 197 /* Everything failed - find first available queue */ 198 for (rxq = 0; rxq < MAX_MACVTAP_QUEUES; rxq++) { 199 tap = rcu_dereference(vlan->taps[rxq]); 200 if (tap) 201 break; 202 } 203 204 out: 205 return tap; 206 } 207 208 /* 209 * The net_device is going away, give up the reference 210 * that it holds on all queues and safely set the pointer 211 * from the queues to NULL. 212 */ 213 static void macvtap_del_queues(struct net_device *dev) 214 { 215 struct macvlan_dev *vlan = netdev_priv(dev); 216 struct macvtap_queue *q, *qlist[MAX_MACVTAP_QUEUES]; 217 int i, j = 0; 218 219 /* macvtap_put_queue can free some slots, so go through all slots */ 220 spin_lock(&macvtap_lock); 221 for (i = 0; i < MAX_MACVTAP_QUEUES && vlan->numvtaps; i++) { 222 q = rcu_dereference_protected(vlan->taps[i], 223 lockdep_is_held(&macvtap_lock)); 224 if (q) { 225 qlist[j++] = q; 226 rcu_assign_pointer(vlan->taps[i], NULL); 227 rcu_assign_pointer(q->vlan, NULL); 228 vlan->numvtaps--; 229 } 230 } 231 BUG_ON(vlan->numvtaps != 0); 232 /* guarantee that any future macvtap_set_queue will fail */ 233 vlan->numvtaps = MAX_MACVTAP_QUEUES; 234 spin_unlock(&macvtap_lock); 235 236 synchronize_rcu(); 237 238 for (--j; j >= 0; j--) 239 sock_put(&qlist[j]->sk); 240 } 241 242 /* 243 * Forward happens for data that gets sent from one macvlan 244 * endpoint to another one in bridge mode. We just take 245 * the skb and put it into the receive queue. 246 */ 247 static int macvtap_forward(struct net_device *dev, struct sk_buff *skb) 248 { 249 struct macvtap_queue *q = macvtap_get_queue(dev, skb); 250 if (!q) 251 goto drop; 252 253 if (skb_queue_len(&q->sk.sk_receive_queue) >= dev->tx_queue_len) 254 goto drop; 255 256 skb_queue_tail(&q->sk.sk_receive_queue, skb); 257 wake_up_interruptible_poll(sk_sleep(&q->sk), POLLIN | POLLRDNORM | POLLRDBAND); 258 return NET_RX_SUCCESS; 259 260 drop: 261 kfree_skb(skb); 262 return NET_RX_DROP; 263 } 264 265 /* 266 * Receive is for data from the external interface (lowerdev), 267 * in case of macvtap, we can treat that the same way as 268 * forward, which macvlan cannot. 269 */ 270 static int macvtap_receive(struct sk_buff *skb) 271 { 272 skb_push(skb, ETH_HLEN); 273 return macvtap_forward(skb->dev, skb); 274 } 275 276 static int macvtap_get_minor(struct macvlan_dev *vlan) 277 { 278 int retval = -ENOMEM; 279 int id; 280 281 mutex_lock(&minor_lock); 282 if (idr_pre_get(&minor_idr, GFP_KERNEL) == 0) 283 goto exit; 284 285 retval = idr_get_new_above(&minor_idr, vlan, 1, &id); 286 if (retval < 0) { 287 if (retval == -EAGAIN) 288 retval = -ENOMEM; 289 goto exit; 290 } 291 if (id < MACVTAP_NUM_DEVS) { 292 vlan->minor = id; 293 } else { 294 printk(KERN_ERR "too many macvtap devices\n"); 295 retval = -EINVAL; 296 idr_remove(&minor_idr, id); 297 } 298 exit: 299 mutex_unlock(&minor_lock); 300 return retval; 301 } 302 303 static void macvtap_free_minor(struct macvlan_dev *vlan) 304 { 305 mutex_lock(&minor_lock); 306 if (vlan->minor) { 307 idr_remove(&minor_idr, vlan->minor); 308 vlan->minor = 0; 309 } 310 mutex_unlock(&minor_lock); 311 } 312 313 static struct net_device *dev_get_by_macvtap_minor(int minor) 314 { 315 struct net_device *dev = NULL; 316 struct macvlan_dev *vlan; 317 318 mutex_lock(&minor_lock); 319 vlan = idr_find(&minor_idr, minor); 320 if (vlan) { 321 dev = vlan->dev; 322 dev_hold(dev); 323 } 324 mutex_unlock(&minor_lock); 325 return dev; 326 } 327 328 static int macvtap_newlink(struct net *src_net, 329 struct net_device *dev, 330 struct nlattr *tb[], 331 struct nlattr *data[]) 332 { 333 /* Don't put anything that may fail after macvlan_common_newlink 334 * because we can't undo what it does. 335 */ 336 return macvlan_common_newlink(src_net, dev, tb, data, 337 macvtap_receive, macvtap_forward); 338 } 339 340 static void macvtap_dellink(struct net_device *dev, 341 struct list_head *head) 342 { 343 macvtap_del_queues(dev); 344 macvlan_dellink(dev, head); 345 } 346 347 static void macvtap_setup(struct net_device *dev) 348 { 349 macvlan_common_setup(dev); 350 dev->tx_queue_len = TUN_READQ_SIZE; 351 } 352 353 static struct rtnl_link_ops macvtap_link_ops __read_mostly = { 354 .kind = "macvtap", 355 .setup = macvtap_setup, 356 .newlink = macvtap_newlink, 357 .dellink = macvtap_dellink, 358 }; 359 360 361 static void macvtap_sock_write_space(struct sock *sk) 362 { 363 wait_queue_head_t *wqueue; 364 365 if (!sock_writeable(sk) || 366 !test_and_clear_bit(SOCK_ASYNC_NOSPACE, &sk->sk_socket->flags)) 367 return; 368 369 wqueue = sk_sleep(sk); 370 if (wqueue && waitqueue_active(wqueue)) 371 wake_up_interruptible_poll(wqueue, POLLOUT | POLLWRNORM | POLLWRBAND); 372 } 373 374 static void macvtap_sock_destruct(struct sock *sk) 375 { 376 skb_queue_purge(&sk->sk_receive_queue); 377 } 378 379 static int macvtap_open(struct inode *inode, struct file *file) 380 { 381 struct net *net = current->nsproxy->net_ns; 382 struct net_device *dev = dev_get_by_macvtap_minor(iminor(inode)); 383 struct macvtap_queue *q; 384 int err; 385 386 err = -ENODEV; 387 if (!dev) 388 goto out; 389 390 err = -ENOMEM; 391 q = (struct macvtap_queue *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, 392 &macvtap_proto); 393 if (!q) 394 goto out; 395 396 q->sock.wq = &q->wq; 397 init_waitqueue_head(&q->wq.wait); 398 q->sock.type = SOCK_RAW; 399 q->sock.state = SS_CONNECTED; 400 q->sock.file = file; 401 q->sock.ops = &macvtap_socket_ops; 402 sock_init_data(&q->sock, &q->sk); 403 q->sk.sk_write_space = macvtap_sock_write_space; 404 q->sk.sk_destruct = macvtap_sock_destruct; 405 q->flags = IFF_VNET_HDR | IFF_NO_PI | IFF_TAP; 406 q->vnet_hdr_sz = sizeof(struct virtio_net_hdr); 407 408 /* 409 * so far only KVM virtio_net uses macvtap, enable zero copy between 410 * guest kernel and host kernel when lower device supports zerocopy 411 * 412 * The macvlan supports zerocopy iff the lower device supports zero 413 * copy so we don't have to look at the lower device directly. 414 */ 415 if ((dev->features & NETIF_F_HIGHDMA) && (dev->features & NETIF_F_SG)) 416 sock_set_flag(&q->sk, SOCK_ZEROCOPY); 417 418 err = macvtap_set_queue(dev, file, q); 419 if (err) 420 sock_put(&q->sk); 421 422 out: 423 if (dev) 424 dev_put(dev); 425 426 return err; 427 } 428 429 static int macvtap_release(struct inode *inode, struct file *file) 430 { 431 struct macvtap_queue *q = file->private_data; 432 macvtap_put_queue(q); 433 return 0; 434 } 435 436 static unsigned int macvtap_poll(struct file *file, poll_table * wait) 437 { 438 struct macvtap_queue *q = file->private_data; 439 unsigned int mask = POLLERR; 440 441 if (!q) 442 goto out; 443 444 mask = 0; 445 poll_wait(file, &q->wq.wait, wait); 446 447 if (!skb_queue_empty(&q->sk.sk_receive_queue)) 448 mask |= POLLIN | POLLRDNORM; 449 450 if (sock_writeable(&q->sk) || 451 (!test_and_set_bit(SOCK_ASYNC_NOSPACE, &q->sock.flags) && 452 sock_writeable(&q->sk))) 453 mask |= POLLOUT | POLLWRNORM; 454 455 out: 456 return mask; 457 } 458 459 static inline struct sk_buff *macvtap_alloc_skb(struct sock *sk, size_t prepad, 460 size_t len, size_t linear, 461 int noblock, int *err) 462 { 463 struct sk_buff *skb; 464 465 /* Under a page? Don't bother with paged skb. */ 466 if (prepad + len < PAGE_SIZE || !linear) 467 linear = len; 468 469 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 470 err); 471 if (!skb) 472 return NULL; 473 474 skb_reserve(skb, prepad); 475 skb_put(skb, linear); 476 skb->data_len = len - linear; 477 skb->len += len - linear; 478 479 return skb; 480 } 481 482 /* set skb frags from iovec, this can move to core network code for reuse */ 483 static int zerocopy_sg_from_iovec(struct sk_buff *skb, const struct iovec *from, 484 int offset, size_t count) 485 { 486 int len = iov_length(from, count) - offset; 487 int copy = skb_headlen(skb); 488 int size, offset1 = 0; 489 int i = 0; 490 491 /* Skip over from offset */ 492 while (count && (offset >= from->iov_len)) { 493 offset -= from->iov_len; 494 ++from; 495 --count; 496 } 497 498 /* copy up to skb headlen */ 499 while (count && (copy > 0)) { 500 size = min_t(unsigned int, copy, from->iov_len - offset); 501 if (copy_from_user(skb->data + offset1, from->iov_base + offset, 502 size)) 503 return -EFAULT; 504 if (copy > size) { 505 ++from; 506 --count; 507 } 508 copy -= size; 509 offset1 += size; 510 offset = 0; 511 } 512 513 if (len == offset1) 514 return 0; 515 516 while (count--) { 517 struct page *page[MAX_SKB_FRAGS]; 518 int num_pages; 519 unsigned long base; 520 521 len = from->iov_len - offset1; 522 if (!len) { 523 offset1 = 0; 524 ++from; 525 continue; 526 } 527 base = (unsigned long)from->iov_base + offset1; 528 size = ((base & ~PAGE_MASK) + len + ~PAGE_MASK) >> PAGE_SHIFT; 529 num_pages = get_user_pages_fast(base, size, 0, &page[i]); 530 if ((num_pages != size) || 531 (num_pages > MAX_SKB_FRAGS - skb_shinfo(skb)->nr_frags)) 532 /* put_page is in skb free */ 533 return -EFAULT; 534 skb->data_len += len; 535 skb->len += len; 536 skb->truesize += len; 537 atomic_add(len, &skb->sk->sk_wmem_alloc); 538 while (len) { 539 int off = base & ~PAGE_MASK; 540 int size = min_t(int, len, PAGE_SIZE - off); 541 __skb_fill_page_desc(skb, i, page[i], off, size); 542 skb_shinfo(skb)->nr_frags++; 543 /* increase sk_wmem_alloc */ 544 base += size; 545 len -= size; 546 i++; 547 } 548 offset1 = 0; 549 ++from; 550 } 551 return 0; 552 } 553 554 /* 555 * macvtap_skb_from_vnet_hdr and macvtap_skb_to_vnet_hdr should 556 * be shared with the tun/tap driver. 557 */ 558 static int macvtap_skb_from_vnet_hdr(struct sk_buff *skb, 559 struct virtio_net_hdr *vnet_hdr) 560 { 561 unsigned short gso_type = 0; 562 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 563 switch (vnet_hdr->gso_type & ~VIRTIO_NET_HDR_GSO_ECN) { 564 case VIRTIO_NET_HDR_GSO_TCPV4: 565 gso_type = SKB_GSO_TCPV4; 566 break; 567 case VIRTIO_NET_HDR_GSO_TCPV6: 568 gso_type = SKB_GSO_TCPV6; 569 break; 570 case VIRTIO_NET_HDR_GSO_UDP: 571 gso_type = SKB_GSO_UDP; 572 break; 573 default: 574 return -EINVAL; 575 } 576 577 if (vnet_hdr->gso_type & VIRTIO_NET_HDR_GSO_ECN) 578 gso_type |= SKB_GSO_TCP_ECN; 579 580 if (vnet_hdr->gso_size == 0) 581 return -EINVAL; 582 } 583 584 if (vnet_hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) { 585 if (!skb_partial_csum_set(skb, vnet_hdr->csum_start, 586 vnet_hdr->csum_offset)) 587 return -EINVAL; 588 } 589 590 if (vnet_hdr->gso_type != VIRTIO_NET_HDR_GSO_NONE) { 591 skb_shinfo(skb)->gso_size = vnet_hdr->gso_size; 592 skb_shinfo(skb)->gso_type = gso_type; 593 594 /* Header must be checked, and gso_segs computed. */ 595 skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; 596 skb_shinfo(skb)->gso_segs = 0; 597 } 598 return 0; 599 } 600 601 static int macvtap_skb_to_vnet_hdr(const struct sk_buff *skb, 602 struct virtio_net_hdr *vnet_hdr) 603 { 604 memset(vnet_hdr, 0, sizeof(*vnet_hdr)); 605 606 if (skb_is_gso(skb)) { 607 struct skb_shared_info *sinfo = skb_shinfo(skb); 608 609 /* This is a hint as to how much should be linear. */ 610 vnet_hdr->hdr_len = skb_headlen(skb); 611 vnet_hdr->gso_size = sinfo->gso_size; 612 if (sinfo->gso_type & SKB_GSO_TCPV4) 613 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4; 614 else if (sinfo->gso_type & SKB_GSO_TCPV6) 615 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6; 616 else if (sinfo->gso_type & SKB_GSO_UDP) 617 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_UDP; 618 else 619 BUG(); 620 if (sinfo->gso_type & SKB_GSO_TCP_ECN) 621 vnet_hdr->gso_type |= VIRTIO_NET_HDR_GSO_ECN; 622 } else 623 vnet_hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE; 624 625 if (skb->ip_summed == CHECKSUM_PARTIAL) { 626 vnet_hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM; 627 vnet_hdr->csum_start = skb_checksum_start_offset(skb); 628 vnet_hdr->csum_offset = skb->csum_offset; 629 } else if (skb->ip_summed == CHECKSUM_UNNECESSARY) { 630 vnet_hdr->flags = VIRTIO_NET_HDR_F_DATA_VALID; 631 } /* else everything is zero */ 632 633 return 0; 634 } 635 636 637 /* Get packet from user space buffer */ 638 static ssize_t macvtap_get_user(struct macvtap_queue *q, struct msghdr *m, 639 const struct iovec *iv, unsigned long total_len, 640 size_t count, int noblock) 641 { 642 struct sk_buff *skb; 643 struct macvlan_dev *vlan; 644 unsigned long len = total_len; 645 int err; 646 struct virtio_net_hdr vnet_hdr = { 0 }; 647 int vnet_hdr_len = 0; 648 int copylen; 649 bool zerocopy = false; 650 651 if (q->flags & IFF_VNET_HDR) { 652 vnet_hdr_len = q->vnet_hdr_sz; 653 654 err = -EINVAL; 655 if (len < vnet_hdr_len) 656 goto err; 657 len -= vnet_hdr_len; 658 659 err = memcpy_fromiovecend((void *)&vnet_hdr, iv, 0, 660 sizeof(vnet_hdr)); 661 if (err < 0) 662 goto err; 663 if ((vnet_hdr.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 664 vnet_hdr.csum_start + vnet_hdr.csum_offset + 2 > 665 vnet_hdr.hdr_len) 666 vnet_hdr.hdr_len = vnet_hdr.csum_start + 667 vnet_hdr.csum_offset + 2; 668 err = -EINVAL; 669 if (vnet_hdr.hdr_len > len) 670 goto err; 671 } 672 673 err = -EINVAL; 674 if (unlikely(len < ETH_HLEN)) 675 goto err; 676 677 if (m && m->msg_control && sock_flag(&q->sk, SOCK_ZEROCOPY)) 678 zerocopy = true; 679 680 if (zerocopy) { 681 /* There are 256 bytes to be copied in skb, so there is enough 682 * room for skb expand head in case it is used. 683 * The rest buffer is mapped from userspace. 684 */ 685 copylen = vnet_hdr.hdr_len; 686 if (!copylen) 687 copylen = GOODCOPY_LEN; 688 } else 689 copylen = len; 690 691 skb = macvtap_alloc_skb(&q->sk, NET_IP_ALIGN, copylen, 692 vnet_hdr.hdr_len, noblock, &err); 693 if (!skb) 694 goto err; 695 696 if (zerocopy) { 697 err = zerocopy_sg_from_iovec(skb, iv, vnet_hdr_len, count); 698 skb_shinfo(skb)->tx_flags |= SKBTX_DEV_ZEROCOPY; 699 } else 700 err = skb_copy_datagram_from_iovec(skb, 0, iv, vnet_hdr_len, 701 len); 702 if (err) 703 goto err_kfree; 704 705 skb_set_network_header(skb, ETH_HLEN); 706 skb_reset_mac_header(skb); 707 skb->protocol = eth_hdr(skb)->h_proto; 708 709 if (vnet_hdr_len) { 710 err = macvtap_skb_from_vnet_hdr(skb, &vnet_hdr); 711 if (err) 712 goto err_kfree; 713 } 714 715 rcu_read_lock_bh(); 716 vlan = rcu_dereference_bh(q->vlan); 717 /* copy skb_ubuf_info for callback when skb has no error */ 718 if (zerocopy) 719 skb_shinfo(skb)->destructor_arg = m->msg_control; 720 if (vlan) 721 macvlan_start_xmit(skb, vlan->dev); 722 else 723 kfree_skb(skb); 724 rcu_read_unlock_bh(); 725 726 return total_len; 727 728 err_kfree: 729 kfree_skb(skb); 730 731 err: 732 rcu_read_lock_bh(); 733 vlan = rcu_dereference_bh(q->vlan); 734 if (vlan) 735 vlan->dev->stats.tx_dropped++; 736 rcu_read_unlock_bh(); 737 738 return err; 739 } 740 741 static ssize_t macvtap_aio_write(struct kiocb *iocb, const struct iovec *iv, 742 unsigned long count, loff_t pos) 743 { 744 struct file *file = iocb->ki_filp; 745 ssize_t result = -ENOLINK; 746 struct macvtap_queue *q = file->private_data; 747 748 result = macvtap_get_user(q, NULL, iv, iov_length(iv, count), count, 749 file->f_flags & O_NONBLOCK); 750 return result; 751 } 752 753 /* Put packet to the user space buffer */ 754 static ssize_t macvtap_put_user(struct macvtap_queue *q, 755 const struct sk_buff *skb, 756 const struct iovec *iv, int len) 757 { 758 struct macvlan_dev *vlan; 759 int ret; 760 int vnet_hdr_len = 0; 761 762 if (q->flags & IFF_VNET_HDR) { 763 struct virtio_net_hdr vnet_hdr; 764 vnet_hdr_len = q->vnet_hdr_sz; 765 if ((len -= vnet_hdr_len) < 0) 766 return -EINVAL; 767 768 ret = macvtap_skb_to_vnet_hdr(skb, &vnet_hdr); 769 if (ret) 770 return ret; 771 772 if (memcpy_toiovecend(iv, (void *)&vnet_hdr, 0, sizeof(vnet_hdr))) 773 return -EFAULT; 774 } 775 776 len = min_t(int, skb->len, len); 777 778 ret = skb_copy_datagram_const_iovec(skb, 0, iv, vnet_hdr_len, len); 779 780 rcu_read_lock_bh(); 781 vlan = rcu_dereference_bh(q->vlan); 782 if (vlan) 783 macvlan_count_rx(vlan, len, ret == 0, 0); 784 rcu_read_unlock_bh(); 785 786 return ret ? ret : (len + vnet_hdr_len); 787 } 788 789 static ssize_t macvtap_do_read(struct macvtap_queue *q, struct kiocb *iocb, 790 const struct iovec *iv, unsigned long len, 791 int noblock) 792 { 793 DECLARE_WAITQUEUE(wait, current); 794 struct sk_buff *skb; 795 ssize_t ret = 0; 796 797 add_wait_queue(sk_sleep(&q->sk), &wait); 798 while (len) { 799 current->state = TASK_INTERRUPTIBLE; 800 801 /* Read frames from the queue */ 802 skb = skb_dequeue(&q->sk.sk_receive_queue); 803 if (!skb) { 804 if (noblock) { 805 ret = -EAGAIN; 806 break; 807 } 808 if (signal_pending(current)) { 809 ret = -ERESTARTSYS; 810 break; 811 } 812 /* Nothing to read, let's sleep */ 813 schedule(); 814 continue; 815 } 816 ret = macvtap_put_user(q, skb, iv, len); 817 kfree_skb(skb); 818 break; 819 } 820 821 current->state = TASK_RUNNING; 822 remove_wait_queue(sk_sleep(&q->sk), &wait); 823 return ret; 824 } 825 826 static ssize_t macvtap_aio_read(struct kiocb *iocb, const struct iovec *iv, 827 unsigned long count, loff_t pos) 828 { 829 struct file *file = iocb->ki_filp; 830 struct macvtap_queue *q = file->private_data; 831 ssize_t len, ret = 0; 832 833 len = iov_length(iv, count); 834 if (len < 0) { 835 ret = -EINVAL; 836 goto out; 837 } 838 839 ret = macvtap_do_read(q, iocb, iv, len, file->f_flags & O_NONBLOCK); 840 ret = min_t(ssize_t, ret, len); /* XXX copied from tun.c. Why? */ 841 out: 842 return ret; 843 } 844 845 /* 846 * provide compatibility with generic tun/tap interface 847 */ 848 static long macvtap_ioctl(struct file *file, unsigned int cmd, 849 unsigned long arg) 850 { 851 struct macvtap_queue *q = file->private_data; 852 struct macvlan_dev *vlan; 853 void __user *argp = (void __user *)arg; 854 struct ifreq __user *ifr = argp; 855 unsigned int __user *up = argp; 856 unsigned int u; 857 int __user *sp = argp; 858 int s; 859 int ret; 860 861 switch (cmd) { 862 case TUNSETIFF: 863 /* ignore the name, just look at flags */ 864 if (get_user(u, &ifr->ifr_flags)) 865 return -EFAULT; 866 867 ret = 0; 868 if ((u & ~IFF_VNET_HDR) != (IFF_NO_PI | IFF_TAP)) 869 ret = -EINVAL; 870 else 871 q->flags = u; 872 873 return ret; 874 875 case TUNGETIFF: 876 rcu_read_lock_bh(); 877 vlan = rcu_dereference_bh(q->vlan); 878 if (vlan) 879 dev_hold(vlan->dev); 880 rcu_read_unlock_bh(); 881 882 if (!vlan) 883 return -ENOLINK; 884 885 ret = 0; 886 if (copy_to_user(&ifr->ifr_name, vlan->dev->name, IFNAMSIZ) || 887 put_user(q->flags, &ifr->ifr_flags)) 888 ret = -EFAULT; 889 dev_put(vlan->dev); 890 return ret; 891 892 case TUNGETFEATURES: 893 if (put_user(IFF_TAP | IFF_NO_PI | IFF_VNET_HDR, up)) 894 return -EFAULT; 895 return 0; 896 897 case TUNSETSNDBUF: 898 if (get_user(u, up)) 899 return -EFAULT; 900 901 q->sk.sk_sndbuf = u; 902 return 0; 903 904 case TUNGETVNETHDRSZ: 905 s = q->vnet_hdr_sz; 906 if (put_user(s, sp)) 907 return -EFAULT; 908 return 0; 909 910 case TUNSETVNETHDRSZ: 911 if (get_user(s, sp)) 912 return -EFAULT; 913 if (s < (int)sizeof(struct virtio_net_hdr)) 914 return -EINVAL; 915 916 q->vnet_hdr_sz = s; 917 return 0; 918 919 case TUNSETOFFLOAD: 920 /* let the user check for future flags */ 921 if (arg & ~(TUN_F_CSUM | TUN_F_TSO4 | TUN_F_TSO6 | 922 TUN_F_TSO_ECN | TUN_F_UFO)) 923 return -EINVAL; 924 925 /* TODO: only accept frames with the features that 926 got enabled for forwarded frames */ 927 if (!(q->flags & IFF_VNET_HDR)) 928 return -EINVAL; 929 return 0; 930 931 default: 932 return -EINVAL; 933 } 934 } 935 936 #ifdef CONFIG_COMPAT 937 static long macvtap_compat_ioctl(struct file *file, unsigned int cmd, 938 unsigned long arg) 939 { 940 return macvtap_ioctl(file, cmd, (unsigned long)compat_ptr(arg)); 941 } 942 #endif 943 944 static const struct file_operations macvtap_fops = { 945 .owner = THIS_MODULE, 946 .open = macvtap_open, 947 .release = macvtap_release, 948 .aio_read = macvtap_aio_read, 949 .aio_write = macvtap_aio_write, 950 .poll = macvtap_poll, 951 .llseek = no_llseek, 952 .unlocked_ioctl = macvtap_ioctl, 953 #ifdef CONFIG_COMPAT 954 .compat_ioctl = macvtap_compat_ioctl, 955 #endif 956 }; 957 958 static int macvtap_sendmsg(struct kiocb *iocb, struct socket *sock, 959 struct msghdr *m, size_t total_len) 960 { 961 struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock); 962 return macvtap_get_user(q, m, m->msg_iov, total_len, m->msg_iovlen, 963 m->msg_flags & MSG_DONTWAIT); 964 } 965 966 static int macvtap_recvmsg(struct kiocb *iocb, struct socket *sock, 967 struct msghdr *m, size_t total_len, 968 int flags) 969 { 970 struct macvtap_queue *q = container_of(sock, struct macvtap_queue, sock); 971 int ret; 972 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC)) 973 return -EINVAL; 974 ret = macvtap_do_read(q, iocb, m->msg_iov, total_len, 975 flags & MSG_DONTWAIT); 976 if (ret > total_len) { 977 m->msg_flags |= MSG_TRUNC; 978 ret = flags & MSG_TRUNC ? ret : total_len; 979 } 980 return ret; 981 } 982 983 /* Ops structure to mimic raw sockets with tun */ 984 static const struct proto_ops macvtap_socket_ops = { 985 .sendmsg = macvtap_sendmsg, 986 .recvmsg = macvtap_recvmsg, 987 }; 988 989 /* Get an underlying socket object from tun file. Returns error unless file is 990 * attached to a device. The returned object works like a packet socket, it 991 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for 992 * holding a reference to the file for as long as the socket is in use. */ 993 struct socket *macvtap_get_socket(struct file *file) 994 { 995 struct macvtap_queue *q; 996 if (file->f_op != &macvtap_fops) 997 return ERR_PTR(-EINVAL); 998 q = file->private_data; 999 if (!q) 1000 return ERR_PTR(-EBADFD); 1001 return &q->sock; 1002 } 1003 EXPORT_SYMBOL_GPL(macvtap_get_socket); 1004 1005 static int macvtap_device_event(struct notifier_block *unused, 1006 unsigned long event, void *ptr) 1007 { 1008 struct net_device *dev = ptr; 1009 struct macvlan_dev *vlan; 1010 struct device *classdev; 1011 dev_t devt; 1012 int err; 1013 1014 if (dev->rtnl_link_ops != &macvtap_link_ops) 1015 return NOTIFY_DONE; 1016 1017 vlan = netdev_priv(dev); 1018 1019 switch (event) { 1020 case NETDEV_REGISTER: 1021 /* Create the device node here after the network device has 1022 * been registered but before register_netdevice has 1023 * finished running. 1024 */ 1025 err = macvtap_get_minor(vlan); 1026 if (err) 1027 return notifier_from_errno(err); 1028 1029 devt = MKDEV(MAJOR(macvtap_major), vlan->minor); 1030 classdev = device_create(macvtap_class, &dev->dev, devt, 1031 dev, "tap%d", dev->ifindex); 1032 if (IS_ERR(classdev)) { 1033 macvtap_free_minor(vlan); 1034 return notifier_from_errno(PTR_ERR(classdev)); 1035 } 1036 break; 1037 case NETDEV_UNREGISTER: 1038 devt = MKDEV(MAJOR(macvtap_major), vlan->minor); 1039 device_destroy(macvtap_class, devt); 1040 macvtap_free_minor(vlan); 1041 break; 1042 } 1043 1044 return NOTIFY_DONE; 1045 } 1046 1047 static struct notifier_block macvtap_notifier_block __read_mostly = { 1048 .notifier_call = macvtap_device_event, 1049 }; 1050 1051 static int macvtap_init(void) 1052 { 1053 int err; 1054 1055 err = alloc_chrdev_region(&macvtap_major, 0, 1056 MACVTAP_NUM_DEVS, "macvtap"); 1057 if (err) 1058 goto out1; 1059 1060 cdev_init(&macvtap_cdev, &macvtap_fops); 1061 err = cdev_add(&macvtap_cdev, macvtap_major, MACVTAP_NUM_DEVS); 1062 if (err) 1063 goto out2; 1064 1065 macvtap_class = class_create(THIS_MODULE, "macvtap"); 1066 if (IS_ERR(macvtap_class)) { 1067 err = PTR_ERR(macvtap_class); 1068 goto out3; 1069 } 1070 1071 err = register_netdevice_notifier(&macvtap_notifier_block); 1072 if (err) 1073 goto out4; 1074 1075 err = macvlan_link_register(&macvtap_link_ops); 1076 if (err) 1077 goto out5; 1078 1079 return 0; 1080 1081 out5: 1082 unregister_netdevice_notifier(&macvtap_notifier_block); 1083 out4: 1084 class_unregister(macvtap_class); 1085 out3: 1086 cdev_del(&macvtap_cdev); 1087 out2: 1088 unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS); 1089 out1: 1090 return err; 1091 } 1092 module_init(macvtap_init); 1093 1094 static void macvtap_exit(void) 1095 { 1096 rtnl_link_unregister(&macvtap_link_ops); 1097 unregister_netdevice_notifier(&macvtap_notifier_block); 1098 class_unregister(macvtap_class); 1099 cdev_del(&macvtap_cdev); 1100 unregister_chrdev_region(macvtap_major, MACVTAP_NUM_DEVS); 1101 } 1102 module_exit(macvtap_exit); 1103 1104 MODULE_ALIAS_RTNL_LINK("macvtap"); 1105 MODULE_AUTHOR("Arnd Bergmann <arnd@arndb.de>"); 1106 MODULE_LICENSE("GPL"); 1107