1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TUN - Universal TUN/TAP device driver. 4 * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com> 5 * 6 * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $ 7 */ 8 9 /* 10 * Changes: 11 * 12 * Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14 13 * Add TUNSETLINK ioctl to set the link encapsulation 14 * 15 * Mark Smith <markzzzsmith@yahoo.com.au> 16 * Use eth_random_addr() for tap MAC address. 17 * 18 * Harald Roelle <harald.roelle@ifi.lmu.de> 2004/04/20 19 * Fixes in packet dropping, queue length setting and queue wakeup. 20 * Increased default tx queue length. 21 * Added ethtool API. 22 * Minor cleanups 23 * 24 * Daniel Podlejski <underley@underley.eu.org> 25 * Modifications for 2.3.99-pre5 kernel. 26 */ 27 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #define DRV_NAME "tun" 31 #define DRV_VERSION "1.6" 32 #define DRV_DESCRIPTION "Universal TUN/TAP device driver" 33 #define DRV_COPYRIGHT "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>" 34 35 #include <linux/module.h> 36 #include <linux/errno.h> 37 #include <linux/kernel.h> 38 #include <linux/sched/signal.h> 39 #include <linux/major.h> 40 #include <linux/slab.h> 41 #include <linux/poll.h> 42 #include <linux/fcntl.h> 43 #include <linux/init.h> 44 #include <linux/skbuff.h> 45 #include <linux/netdevice.h> 46 #include <linux/etherdevice.h> 47 #include <linux/miscdevice.h> 48 #include <linux/ethtool.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/compat.h> 51 #include <linux/if.h> 52 #include <linux/if_arp.h> 53 #include <linux/if_ether.h> 54 #include <linux/if_tun.h> 55 #include <linux/if_vlan.h> 56 #include <linux/crc32.h> 57 #include <linux/math.h> 58 #include <linux/nsproxy.h> 59 #include <linux/virtio_net.h> 60 #include <linux/rcupdate.h> 61 #include <net/net_namespace.h> 62 #include <net/netns/generic.h> 63 #include <net/rtnetlink.h> 64 #include <net/sock.h> 65 #include <net/xdp.h> 66 #include <net/ip_tunnels.h> 67 #include <linux/seq_file.h> 68 #include <linux/uio.h> 69 #include <linux/skb_array.h> 70 #include <linux/bpf.h> 71 #include <linux/bpf_trace.h> 72 #include <linux/mutex.h> 73 #include <linux/ieee802154.h> 74 #include <uapi/linux/if_ltalk.h> 75 #include <uapi/linux/if_fddi.h> 76 #include <uapi/linux/if_hippi.h> 77 #include <uapi/linux/if_fc.h> 78 #include <net/ax25.h> 79 #include <net/rose.h> 80 #include <net/6lowpan.h> 81 #include <net/rps.h> 82 83 #include <linux/uaccess.h> 84 #include <linux/proc_fs.h> 85 86 static void tun_default_link_ksettings(struct net_device *dev, 87 struct ethtool_link_ksettings *cmd); 88 89 #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 90 91 /* TUN device flags */ 92 93 /* IFF_ATTACH_QUEUE is never stored in device flags, 94 * overload it to mean fasync when stored there. 95 */ 96 #define TUN_FASYNC IFF_ATTACH_QUEUE 97 /* High bits in flags field are unused. */ 98 #define TUN_VNET_LE 0x80000000 99 #define TUN_VNET_BE 0x40000000 100 101 #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \ 102 IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS) 103 104 #define GOODCOPY_LEN 128 105 106 #define FLT_EXACT_COUNT 8 107 struct tap_filter { 108 unsigned int count; /* Number of addrs. Zero means disabled */ 109 u32 mask[2]; /* Mask of the hashed addrs */ 110 unsigned char addr[FLT_EXACT_COUNT][ETH_ALEN]; 111 }; 112 113 /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal 114 * to max number of VCPUs in guest. */ 115 #define MAX_TAP_QUEUES 256 116 #define MAX_TAP_FLOWS 4096 117 118 #define TUN_FLOW_EXPIRE (3 * HZ) 119 120 /* A tun_file connects an open character device to a tuntap netdevice. It 121 * also contains all socket related structures (except sock_fprog and tap_filter) 122 * to serve as one transmit queue for tuntap device. The sock_fprog and 123 * tap_filter were kept in tun_struct since they were used for filtering for the 124 * netdevice not for a specific queue (at least I didn't see the requirement for 125 * this). 126 * 127 * RCU usage: 128 * The tun_file and tun_struct are loosely coupled, the pointer from one to the 129 * other can only be read while rcu_read_lock or rtnl_lock is held. 130 */ 131 struct tun_file { 132 struct sock sk; 133 struct socket socket; 134 struct tun_struct __rcu *tun; 135 struct fasync_struct *fasync; 136 /* only used for fasnyc */ 137 unsigned int flags; 138 union { 139 u16 queue_index; 140 unsigned int ifindex; 141 }; 142 struct napi_struct napi; 143 bool napi_enabled; 144 bool napi_frags_enabled; 145 struct mutex napi_mutex; /* Protects access to the above napi */ 146 struct list_head next; 147 struct tun_struct *detached; 148 struct ptr_ring tx_ring; 149 struct xdp_rxq_info xdp_rxq; 150 }; 151 152 struct tun_page { 153 struct page *page; 154 int count; 155 }; 156 157 struct tun_flow_entry { 158 struct hlist_node hash_link; 159 struct rcu_head rcu; 160 struct tun_struct *tun; 161 162 u32 rxhash; 163 u32 rps_rxhash; 164 int queue_index; 165 unsigned long updated ____cacheline_aligned_in_smp; 166 }; 167 168 #define TUN_NUM_FLOW_ENTRIES 1024 169 #define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1) 170 171 struct tun_prog { 172 struct rcu_head rcu; 173 struct bpf_prog *prog; 174 }; 175 176 /* Since the socket were moved to tun_file, to preserve the behavior of persist 177 * device, socket filter, sndbuf and vnet header size were restore when the 178 * file were attached to a persist device. 179 */ 180 struct tun_struct { 181 struct tun_file __rcu *tfiles[MAX_TAP_QUEUES]; 182 unsigned int numqueues; 183 unsigned int flags; 184 kuid_t owner; 185 kgid_t group; 186 187 struct net_device *dev; 188 netdev_features_t set_features; 189 #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \ 190 NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4) 191 192 int align; 193 int vnet_hdr_sz; 194 int sndbuf; 195 struct tap_filter txflt; 196 struct sock_fprog fprog; 197 /* protected by rtnl lock */ 198 bool filter_attached; 199 u32 msg_enable; 200 spinlock_t lock; 201 struct hlist_head flows[TUN_NUM_FLOW_ENTRIES]; 202 struct timer_list flow_gc_timer; 203 unsigned long ageing_time; 204 unsigned int numdisabled; 205 struct list_head disabled; 206 void *security; 207 u32 flow_count; 208 u32 rx_batched; 209 atomic_long_t rx_frame_errors; 210 struct bpf_prog __rcu *xdp_prog; 211 struct tun_prog __rcu *steering_prog; 212 struct tun_prog __rcu *filter_prog; 213 struct ethtool_link_ksettings link_ksettings; 214 /* init args */ 215 struct file *file; 216 struct ifreq *ifr; 217 }; 218 219 struct veth { 220 __be16 h_vlan_proto; 221 __be16 h_vlan_TCI; 222 }; 223 224 static void tun_flow_init(struct tun_struct *tun); 225 static void tun_flow_uninit(struct tun_struct *tun); 226 227 static int tun_napi_receive(struct napi_struct *napi, int budget) 228 { 229 struct tun_file *tfile = container_of(napi, struct tun_file, napi); 230 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 231 struct sk_buff_head process_queue; 232 struct sk_buff *skb; 233 int received = 0; 234 235 __skb_queue_head_init(&process_queue); 236 237 spin_lock(&queue->lock); 238 skb_queue_splice_tail_init(queue, &process_queue); 239 spin_unlock(&queue->lock); 240 241 while (received < budget && (skb = __skb_dequeue(&process_queue))) { 242 napi_gro_receive(napi, skb); 243 ++received; 244 } 245 246 if (!skb_queue_empty(&process_queue)) { 247 spin_lock(&queue->lock); 248 skb_queue_splice(&process_queue, queue); 249 spin_unlock(&queue->lock); 250 } 251 252 return received; 253 } 254 255 static int tun_napi_poll(struct napi_struct *napi, int budget) 256 { 257 unsigned int received; 258 259 received = tun_napi_receive(napi, budget); 260 261 if (received < budget) 262 napi_complete_done(napi, received); 263 264 return received; 265 } 266 267 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, 268 bool napi_en, bool napi_frags) 269 { 270 tfile->napi_enabled = napi_en; 271 tfile->napi_frags_enabled = napi_en && napi_frags; 272 if (napi_en) { 273 netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll); 274 napi_enable(&tfile->napi); 275 } 276 } 277 278 static void tun_napi_enable(struct tun_file *tfile) 279 { 280 if (tfile->napi_enabled) 281 napi_enable(&tfile->napi); 282 } 283 284 static void tun_napi_disable(struct tun_file *tfile) 285 { 286 if (tfile->napi_enabled) 287 napi_disable(&tfile->napi); 288 } 289 290 static void tun_napi_del(struct tun_file *tfile) 291 { 292 if (tfile->napi_enabled) 293 netif_napi_del(&tfile->napi); 294 } 295 296 static bool tun_napi_frags_enabled(const struct tun_file *tfile) 297 { 298 return tfile->napi_frags_enabled; 299 } 300 301 #ifdef CONFIG_TUN_VNET_CROSS_LE 302 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) 303 { 304 return tun->flags & TUN_VNET_BE ? false : 305 virtio_legacy_is_little_endian(); 306 } 307 308 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) 309 { 310 int be = !!(tun->flags & TUN_VNET_BE); 311 312 if (put_user(be, argp)) 313 return -EFAULT; 314 315 return 0; 316 } 317 318 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) 319 { 320 int be; 321 322 if (get_user(be, argp)) 323 return -EFAULT; 324 325 if (be) 326 tun->flags |= TUN_VNET_BE; 327 else 328 tun->flags &= ~TUN_VNET_BE; 329 330 return 0; 331 } 332 #else 333 static inline bool tun_legacy_is_little_endian(struct tun_struct *tun) 334 { 335 return virtio_legacy_is_little_endian(); 336 } 337 338 static long tun_get_vnet_be(struct tun_struct *tun, int __user *argp) 339 { 340 return -EINVAL; 341 } 342 343 static long tun_set_vnet_be(struct tun_struct *tun, int __user *argp) 344 { 345 return -EINVAL; 346 } 347 #endif /* CONFIG_TUN_VNET_CROSS_LE */ 348 349 static inline bool tun_is_little_endian(struct tun_struct *tun) 350 { 351 return tun->flags & TUN_VNET_LE || 352 tun_legacy_is_little_endian(tun); 353 } 354 355 static inline u16 tun16_to_cpu(struct tun_struct *tun, __virtio16 val) 356 { 357 return __virtio16_to_cpu(tun_is_little_endian(tun), val); 358 } 359 360 static inline __virtio16 cpu_to_tun16(struct tun_struct *tun, u16 val) 361 { 362 return __cpu_to_virtio16(tun_is_little_endian(tun), val); 363 } 364 365 static inline u32 tun_hashfn(u32 rxhash) 366 { 367 return rxhash & TUN_MASK_FLOW_ENTRIES; 368 } 369 370 static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash) 371 { 372 struct tun_flow_entry *e; 373 374 hlist_for_each_entry_rcu(e, head, hash_link) { 375 if (e->rxhash == rxhash) 376 return e; 377 } 378 return NULL; 379 } 380 381 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun, 382 struct hlist_head *head, 383 u32 rxhash, u16 queue_index) 384 { 385 struct tun_flow_entry *e = kmalloc(sizeof(*e), GFP_ATOMIC); 386 387 if (e) { 388 netif_info(tun, tx_queued, tun->dev, 389 "create flow: hash %u index %u\n", 390 rxhash, queue_index); 391 e->updated = jiffies; 392 e->rxhash = rxhash; 393 e->rps_rxhash = 0; 394 e->queue_index = queue_index; 395 e->tun = tun; 396 hlist_add_head_rcu(&e->hash_link, head); 397 ++tun->flow_count; 398 } 399 return e; 400 } 401 402 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e) 403 { 404 netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n", 405 e->rxhash, e->queue_index); 406 hlist_del_rcu(&e->hash_link); 407 kfree_rcu(e, rcu); 408 --tun->flow_count; 409 } 410 411 static void tun_flow_flush(struct tun_struct *tun) 412 { 413 int i; 414 415 spin_lock_bh(&tun->lock); 416 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 417 struct tun_flow_entry *e; 418 struct hlist_node *n; 419 420 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) 421 tun_flow_delete(tun, e); 422 } 423 spin_unlock_bh(&tun->lock); 424 } 425 426 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index) 427 { 428 int i; 429 430 spin_lock_bh(&tun->lock); 431 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 432 struct tun_flow_entry *e; 433 struct hlist_node *n; 434 435 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 436 if (e->queue_index == queue_index) 437 tun_flow_delete(tun, e); 438 } 439 } 440 spin_unlock_bh(&tun->lock); 441 } 442 443 static void tun_flow_cleanup(struct timer_list *t) 444 { 445 struct tun_struct *tun = from_timer(tun, t, flow_gc_timer); 446 unsigned long delay = tun->ageing_time; 447 unsigned long next_timer = jiffies + delay; 448 unsigned long count = 0; 449 int i; 450 451 spin_lock(&tun->lock); 452 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 453 struct tun_flow_entry *e; 454 struct hlist_node *n; 455 456 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 457 unsigned long this_timer; 458 459 this_timer = e->updated + delay; 460 if (time_before_eq(this_timer, jiffies)) { 461 tun_flow_delete(tun, e); 462 continue; 463 } 464 count++; 465 if (time_before(this_timer, next_timer)) 466 next_timer = this_timer; 467 } 468 } 469 470 if (count) 471 mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer)); 472 spin_unlock(&tun->lock); 473 } 474 475 static void tun_flow_update(struct tun_struct *tun, u32 rxhash, 476 struct tun_file *tfile) 477 { 478 struct hlist_head *head; 479 struct tun_flow_entry *e; 480 unsigned long delay = tun->ageing_time; 481 u16 queue_index = tfile->queue_index; 482 483 head = &tun->flows[tun_hashfn(rxhash)]; 484 485 rcu_read_lock(); 486 487 e = tun_flow_find(head, rxhash); 488 if (likely(e)) { 489 /* TODO: keep queueing to old queue until it's empty? */ 490 if (READ_ONCE(e->queue_index) != queue_index) 491 WRITE_ONCE(e->queue_index, queue_index); 492 if (e->updated != jiffies) 493 e->updated = jiffies; 494 sock_rps_record_flow_hash(e->rps_rxhash); 495 } else { 496 spin_lock_bh(&tun->lock); 497 if (!tun_flow_find(head, rxhash) && 498 tun->flow_count < MAX_TAP_FLOWS) 499 tun_flow_create(tun, head, rxhash, queue_index); 500 501 if (!timer_pending(&tun->flow_gc_timer)) 502 mod_timer(&tun->flow_gc_timer, 503 round_jiffies_up(jiffies + delay)); 504 spin_unlock_bh(&tun->lock); 505 } 506 507 rcu_read_unlock(); 508 } 509 510 /* Save the hash received in the stack receive path and update the 511 * flow_hash table accordingly. 512 */ 513 static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash) 514 { 515 if (unlikely(e->rps_rxhash != hash)) 516 e->rps_rxhash = hash; 517 } 518 519 /* We try to identify a flow through its rxhash. The reason that 520 * we do not check rxq no. is because some cards(e.g 82599), chooses 521 * the rxq based on the txq where the last packet of the flow comes. As 522 * the userspace application move between processors, we may get a 523 * different rxq no. here. 524 */ 525 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb) 526 { 527 struct tun_flow_entry *e; 528 u32 txq, numqueues; 529 530 numqueues = READ_ONCE(tun->numqueues); 531 532 txq = __skb_get_hash_symmetric(skb); 533 e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq); 534 if (e) { 535 tun_flow_save_rps_rxhash(e, txq); 536 txq = e->queue_index; 537 } else { 538 txq = reciprocal_scale(txq, numqueues); 539 } 540 541 return txq; 542 } 543 544 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb) 545 { 546 struct tun_prog *prog; 547 u32 numqueues; 548 u16 ret = 0; 549 550 numqueues = READ_ONCE(tun->numqueues); 551 if (!numqueues) 552 return 0; 553 554 prog = rcu_dereference(tun->steering_prog); 555 if (prog) 556 ret = bpf_prog_run_clear_cb(prog->prog, skb); 557 558 return ret % numqueues; 559 } 560 561 static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb, 562 struct net_device *sb_dev) 563 { 564 struct tun_struct *tun = netdev_priv(dev); 565 u16 ret; 566 567 rcu_read_lock(); 568 if (rcu_dereference(tun->steering_prog)) 569 ret = tun_ebpf_select_queue(tun, skb); 570 else 571 ret = tun_automq_select_queue(tun, skb); 572 rcu_read_unlock(); 573 574 return ret; 575 } 576 577 static inline bool tun_not_capable(struct tun_struct *tun) 578 { 579 const struct cred *cred = current_cred(); 580 struct net *net = dev_net(tun->dev); 581 582 return ((uid_valid(tun->owner) && !uid_eq(cred->euid, tun->owner)) || 583 (gid_valid(tun->group) && !in_egroup_p(tun->group))) && 584 !ns_capable(net->user_ns, CAP_NET_ADMIN); 585 } 586 587 static void tun_set_real_num_queues(struct tun_struct *tun) 588 { 589 netif_set_real_num_tx_queues(tun->dev, tun->numqueues); 590 netif_set_real_num_rx_queues(tun->dev, tun->numqueues); 591 } 592 593 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile) 594 { 595 tfile->detached = tun; 596 list_add_tail(&tfile->next, &tun->disabled); 597 ++tun->numdisabled; 598 } 599 600 static struct tun_struct *tun_enable_queue(struct tun_file *tfile) 601 { 602 struct tun_struct *tun = tfile->detached; 603 604 tfile->detached = NULL; 605 list_del_init(&tfile->next); 606 --tun->numdisabled; 607 return tun; 608 } 609 610 void tun_ptr_free(void *ptr) 611 { 612 if (!ptr) 613 return; 614 if (tun_is_xdp_frame(ptr)) { 615 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 616 617 xdp_return_frame(xdpf); 618 } else { 619 __skb_array_destroy_skb(ptr); 620 } 621 } 622 EXPORT_SYMBOL_GPL(tun_ptr_free); 623 624 static void tun_queue_purge(struct tun_file *tfile) 625 { 626 void *ptr; 627 628 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL) 629 tun_ptr_free(ptr); 630 631 skb_queue_purge(&tfile->sk.sk_write_queue); 632 skb_queue_purge(&tfile->sk.sk_error_queue); 633 } 634 635 static void __tun_detach(struct tun_file *tfile, bool clean) 636 { 637 struct tun_file *ntfile; 638 struct tun_struct *tun; 639 640 tun = rtnl_dereference(tfile->tun); 641 642 if (tun && clean) { 643 if (!tfile->detached) 644 tun_napi_disable(tfile); 645 tun_napi_del(tfile); 646 } 647 648 if (tun && !tfile->detached) { 649 u16 index = tfile->queue_index; 650 BUG_ON(index >= tun->numqueues); 651 652 rcu_assign_pointer(tun->tfiles[index], 653 tun->tfiles[tun->numqueues - 1]); 654 ntfile = rtnl_dereference(tun->tfiles[index]); 655 ntfile->queue_index = index; 656 ntfile->xdp_rxq.queue_index = index; 657 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], 658 NULL); 659 660 --tun->numqueues; 661 if (clean) { 662 RCU_INIT_POINTER(tfile->tun, NULL); 663 sock_put(&tfile->sk); 664 } else { 665 tun_disable_queue(tun, tfile); 666 tun_napi_disable(tfile); 667 } 668 669 synchronize_net(); 670 tun_flow_delete_by_queue(tun, tun->numqueues + 1); 671 /* Drop read queue */ 672 tun_queue_purge(tfile); 673 tun_set_real_num_queues(tun); 674 } else if (tfile->detached && clean) { 675 tun = tun_enable_queue(tfile); 676 sock_put(&tfile->sk); 677 } 678 679 if (clean) { 680 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) { 681 netif_carrier_off(tun->dev); 682 683 if (!(tun->flags & IFF_PERSIST) && 684 tun->dev->reg_state == NETREG_REGISTERED) 685 unregister_netdevice(tun->dev); 686 } 687 if (tun) 688 xdp_rxq_info_unreg(&tfile->xdp_rxq); 689 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free); 690 } 691 } 692 693 static void tun_detach(struct tun_file *tfile, bool clean) 694 { 695 struct tun_struct *tun; 696 struct net_device *dev; 697 698 rtnl_lock(); 699 tun = rtnl_dereference(tfile->tun); 700 dev = tun ? tun->dev : NULL; 701 __tun_detach(tfile, clean); 702 if (dev) 703 netdev_state_change(dev); 704 rtnl_unlock(); 705 706 if (clean) 707 sock_put(&tfile->sk); 708 } 709 710 static void tun_detach_all(struct net_device *dev) 711 { 712 struct tun_struct *tun = netdev_priv(dev); 713 struct tun_file *tfile, *tmp; 714 int i, n = tun->numqueues; 715 716 for (i = 0; i < n; i++) { 717 tfile = rtnl_dereference(tun->tfiles[i]); 718 BUG_ON(!tfile); 719 tun_napi_disable(tfile); 720 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 721 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 722 RCU_INIT_POINTER(tfile->tun, NULL); 723 --tun->numqueues; 724 } 725 list_for_each_entry(tfile, &tun->disabled, next) { 726 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 727 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 728 RCU_INIT_POINTER(tfile->tun, NULL); 729 } 730 BUG_ON(tun->numqueues != 0); 731 732 synchronize_net(); 733 for (i = 0; i < n; i++) { 734 tfile = rtnl_dereference(tun->tfiles[i]); 735 tun_napi_del(tfile); 736 /* Drop read queue */ 737 tun_queue_purge(tfile); 738 xdp_rxq_info_unreg(&tfile->xdp_rxq); 739 sock_put(&tfile->sk); 740 } 741 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { 742 tun_napi_del(tfile); 743 tun_enable_queue(tfile); 744 tun_queue_purge(tfile); 745 xdp_rxq_info_unreg(&tfile->xdp_rxq); 746 sock_put(&tfile->sk); 747 } 748 BUG_ON(tun->numdisabled != 0); 749 750 if (tun->flags & IFF_PERSIST) 751 module_put(THIS_MODULE); 752 } 753 754 static int tun_attach(struct tun_struct *tun, struct file *file, 755 bool skip_filter, bool napi, bool napi_frags, 756 bool publish_tun) 757 { 758 struct tun_file *tfile = file->private_data; 759 struct net_device *dev = tun->dev; 760 int err; 761 762 err = security_tun_dev_attach(tfile->socket.sk, tun->security); 763 if (err < 0) 764 goto out; 765 766 err = -EINVAL; 767 if (rtnl_dereference(tfile->tun) && !tfile->detached) 768 goto out; 769 770 err = -EBUSY; 771 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1) 772 goto out; 773 774 err = -E2BIG; 775 if (!tfile->detached && 776 tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES) 777 goto out; 778 779 err = 0; 780 781 /* Re-attach the filter to persist device */ 782 if (!skip_filter && (tun->filter_attached == true)) { 783 lock_sock(tfile->socket.sk); 784 err = sk_attach_filter(&tun->fprog, tfile->socket.sk); 785 release_sock(tfile->socket.sk); 786 if (!err) 787 goto out; 788 } 789 790 if (!tfile->detached && 791 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len, 792 GFP_KERNEL, tun_ptr_free)) { 793 err = -ENOMEM; 794 goto out; 795 } 796 797 tfile->queue_index = tun->numqueues; 798 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; 799 800 if (tfile->detached) { 801 /* Re-attach detached tfile, updating XDP queue_index */ 802 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq)); 803 804 if (tfile->xdp_rxq.queue_index != tfile->queue_index) 805 tfile->xdp_rxq.queue_index = tfile->queue_index; 806 } else { 807 /* Setup XDP RX-queue info, for new tfile getting attached */ 808 err = xdp_rxq_info_reg(&tfile->xdp_rxq, 809 tun->dev, tfile->queue_index, 0); 810 if (err < 0) 811 goto out; 812 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, 813 MEM_TYPE_PAGE_SHARED, NULL); 814 if (err < 0) { 815 xdp_rxq_info_unreg(&tfile->xdp_rxq); 816 goto out; 817 } 818 err = 0; 819 } 820 821 if (tfile->detached) { 822 tun_enable_queue(tfile); 823 tun_napi_enable(tfile); 824 } else { 825 sock_hold(&tfile->sk); 826 tun_napi_init(tun, tfile, napi, napi_frags); 827 } 828 829 if (rtnl_dereference(tun->xdp_prog)) 830 sock_set_flag(&tfile->sk, SOCK_XDP); 831 832 /* device is allowed to go away first, so no need to hold extra 833 * refcnt. 834 */ 835 836 /* Publish tfile->tun and tun->tfiles only after we've fully 837 * initialized tfile; otherwise we risk using half-initialized 838 * object. 839 */ 840 if (publish_tun) 841 rcu_assign_pointer(tfile->tun, tun); 842 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile); 843 tun->numqueues++; 844 tun_set_real_num_queues(tun); 845 out: 846 return err; 847 } 848 849 static struct tun_struct *tun_get(struct tun_file *tfile) 850 { 851 struct tun_struct *tun; 852 853 rcu_read_lock(); 854 tun = rcu_dereference(tfile->tun); 855 if (tun) 856 dev_hold(tun->dev); 857 rcu_read_unlock(); 858 859 return tun; 860 } 861 862 static void tun_put(struct tun_struct *tun) 863 { 864 dev_put(tun->dev); 865 } 866 867 /* TAP filtering */ 868 static void addr_hash_set(u32 *mask, const u8 *addr) 869 { 870 int n = ether_crc(ETH_ALEN, addr) >> 26; 871 mask[n >> 5] |= (1 << (n & 31)); 872 } 873 874 static unsigned int addr_hash_test(const u32 *mask, const u8 *addr) 875 { 876 int n = ether_crc(ETH_ALEN, addr) >> 26; 877 return mask[n >> 5] & (1 << (n & 31)); 878 } 879 880 static int update_filter(struct tap_filter *filter, void __user *arg) 881 { 882 struct { u8 u[ETH_ALEN]; } *addr; 883 struct tun_filter uf; 884 int err, alen, n, nexact; 885 886 if (copy_from_user(&uf, arg, sizeof(uf))) 887 return -EFAULT; 888 889 if (!uf.count) { 890 /* Disabled */ 891 filter->count = 0; 892 return 0; 893 } 894 895 alen = ETH_ALEN * uf.count; 896 addr = memdup_user(arg + sizeof(uf), alen); 897 if (IS_ERR(addr)) 898 return PTR_ERR(addr); 899 900 /* The filter is updated without holding any locks. Which is 901 * perfectly safe. We disable it first and in the worst 902 * case we'll accept a few undesired packets. */ 903 filter->count = 0; 904 wmb(); 905 906 /* Use first set of addresses as an exact filter */ 907 for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++) 908 memcpy(filter->addr[n], addr[n].u, ETH_ALEN); 909 910 nexact = n; 911 912 /* Remaining multicast addresses are hashed, 913 * unicast will leave the filter disabled. */ 914 memset(filter->mask, 0, sizeof(filter->mask)); 915 for (; n < uf.count; n++) { 916 if (!is_multicast_ether_addr(addr[n].u)) { 917 err = 0; /* no filter */ 918 goto free_addr; 919 } 920 addr_hash_set(filter->mask, addr[n].u); 921 } 922 923 /* For ALLMULTI just set the mask to all ones. 924 * This overrides the mask populated above. */ 925 if ((uf.flags & TUN_FLT_ALLMULTI)) 926 memset(filter->mask, ~0, sizeof(filter->mask)); 927 928 /* Now enable the filter */ 929 wmb(); 930 filter->count = nexact; 931 932 /* Return the number of exact filters */ 933 err = nexact; 934 free_addr: 935 kfree(addr); 936 return err; 937 } 938 939 /* Returns: 0 - drop, !=0 - accept */ 940 static int run_filter(struct tap_filter *filter, const struct sk_buff *skb) 941 { 942 /* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect 943 * at this point. */ 944 struct ethhdr *eh = (struct ethhdr *) skb->data; 945 int i; 946 947 /* Exact match */ 948 for (i = 0; i < filter->count; i++) 949 if (ether_addr_equal(eh->h_dest, filter->addr[i])) 950 return 1; 951 952 /* Inexact match (multicast only) */ 953 if (is_multicast_ether_addr(eh->h_dest)) 954 return addr_hash_test(filter->mask, eh->h_dest); 955 956 return 0; 957 } 958 959 /* 960 * Checks whether the packet is accepted or not. 961 * Returns: 0 - drop, !=0 - accept 962 */ 963 static int check_filter(struct tap_filter *filter, const struct sk_buff *skb) 964 { 965 if (!filter->count) 966 return 1; 967 968 return run_filter(filter, skb); 969 } 970 971 /* Network device part of the driver */ 972 973 static const struct ethtool_ops tun_ethtool_ops; 974 975 static int tun_net_init(struct net_device *dev) 976 { 977 struct tun_struct *tun = netdev_priv(dev); 978 struct ifreq *ifr = tun->ifr; 979 int err; 980 981 spin_lock_init(&tun->lock); 982 983 err = security_tun_dev_alloc_security(&tun->security); 984 if (err < 0) 985 return err; 986 987 tun_flow_init(tun); 988 989 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 990 dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | 991 TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | 992 NETIF_F_HW_VLAN_STAG_TX; 993 dev->features = dev->hw_features; 994 dev->vlan_features = dev->features & 995 ~(NETIF_F_HW_VLAN_CTAG_TX | 996 NETIF_F_HW_VLAN_STAG_TX); 997 dev->lltx = true; 998 999 tun->flags = (tun->flags & ~TUN_FEATURES) | 1000 (ifr->ifr_flags & TUN_FEATURES); 1001 1002 INIT_LIST_HEAD(&tun->disabled); 1003 err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI, 1004 ifr->ifr_flags & IFF_NAPI_FRAGS, false); 1005 if (err < 0) { 1006 tun_flow_uninit(tun); 1007 security_tun_dev_free_security(tun->security); 1008 return err; 1009 } 1010 return 0; 1011 } 1012 1013 /* Net device detach from fd. */ 1014 static void tun_net_uninit(struct net_device *dev) 1015 { 1016 tun_detach_all(dev); 1017 } 1018 1019 /* Net device open. */ 1020 static int tun_net_open(struct net_device *dev) 1021 { 1022 netif_tx_start_all_queues(dev); 1023 1024 return 0; 1025 } 1026 1027 /* Net device close. */ 1028 static int tun_net_close(struct net_device *dev) 1029 { 1030 netif_tx_stop_all_queues(dev); 1031 return 0; 1032 } 1033 1034 /* Net device start xmit */ 1035 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb) 1036 { 1037 #ifdef CONFIG_RPS 1038 if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) { 1039 /* Select queue was not called for the skbuff, so we extract the 1040 * RPS hash and save it into the flow_table here. 1041 */ 1042 struct tun_flow_entry *e; 1043 __u32 rxhash; 1044 1045 rxhash = __skb_get_hash_symmetric(skb); 1046 e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash); 1047 if (e) 1048 tun_flow_save_rps_rxhash(e, rxhash); 1049 } 1050 #endif 1051 } 1052 1053 static unsigned int run_ebpf_filter(struct tun_struct *tun, 1054 struct sk_buff *skb, 1055 int len) 1056 { 1057 struct tun_prog *prog = rcu_dereference(tun->filter_prog); 1058 1059 if (prog) 1060 len = bpf_prog_run_clear_cb(prog->prog, skb); 1061 1062 return len; 1063 } 1064 1065 /* Net device start xmit */ 1066 static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) 1067 { 1068 struct tun_struct *tun = netdev_priv(dev); 1069 enum skb_drop_reason drop_reason; 1070 int txq = skb->queue_mapping; 1071 struct netdev_queue *queue; 1072 struct tun_file *tfile; 1073 int len = skb->len; 1074 1075 rcu_read_lock(); 1076 tfile = rcu_dereference(tun->tfiles[txq]); 1077 1078 /* Drop packet if interface is not attached */ 1079 if (!tfile) { 1080 drop_reason = SKB_DROP_REASON_DEV_READY; 1081 goto drop; 1082 } 1083 1084 if (!rcu_dereference(tun->steering_prog)) 1085 tun_automq_xmit(tun, skb); 1086 1087 netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len); 1088 1089 /* Drop if the filter does not like it. 1090 * This is a noop if the filter is disabled. 1091 * Filter can be enabled only for the TAP devices. */ 1092 if (!check_filter(&tun->txflt, skb)) { 1093 drop_reason = SKB_DROP_REASON_TAP_TXFILTER; 1094 goto drop; 1095 } 1096 1097 if (tfile->socket.sk->sk_filter && 1098 sk_filter(tfile->socket.sk, skb)) { 1099 drop_reason = SKB_DROP_REASON_SOCKET_FILTER; 1100 goto drop; 1101 } 1102 1103 len = run_ebpf_filter(tun, skb, len); 1104 if (len == 0) { 1105 drop_reason = SKB_DROP_REASON_TAP_FILTER; 1106 goto drop; 1107 } 1108 1109 if (pskb_trim(skb, len)) { 1110 drop_reason = SKB_DROP_REASON_NOMEM; 1111 goto drop; 1112 } 1113 1114 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) { 1115 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT; 1116 goto drop; 1117 } 1118 1119 skb_tx_timestamp(skb); 1120 1121 /* Orphan the skb - required as we might hang on to it 1122 * for indefinite time. 1123 */ 1124 skb_orphan(skb); 1125 1126 nf_reset_ct(skb); 1127 1128 if (ptr_ring_produce(&tfile->tx_ring, skb)) { 1129 drop_reason = SKB_DROP_REASON_FULL_RING; 1130 goto drop; 1131 } 1132 1133 /* dev->lltx requires to do our own update of trans_start */ 1134 queue = netdev_get_tx_queue(dev, txq); 1135 txq_trans_cond_update(queue); 1136 1137 /* Notify and wake up reader process */ 1138 if (tfile->flags & TUN_FASYNC) 1139 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1140 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1141 1142 rcu_read_unlock(); 1143 return NETDEV_TX_OK; 1144 1145 drop: 1146 dev_core_stats_tx_dropped_inc(dev); 1147 skb_tx_error(skb); 1148 kfree_skb_reason(skb, drop_reason); 1149 rcu_read_unlock(); 1150 return NET_XMIT_DROP; 1151 } 1152 1153 static void tun_net_mclist(struct net_device *dev) 1154 { 1155 /* 1156 * This callback is supposed to deal with mc filter in 1157 * _rx_ path and has nothing to do with the _tx_ path. 1158 * In rx path we always accept everything userspace gives us. 1159 */ 1160 } 1161 1162 static netdev_features_t tun_net_fix_features(struct net_device *dev, 1163 netdev_features_t features) 1164 { 1165 struct tun_struct *tun = netdev_priv(dev); 1166 1167 return (features & tun->set_features) | (features & ~TUN_USER_FEATURES); 1168 } 1169 1170 static void tun_set_headroom(struct net_device *dev, int new_hr) 1171 { 1172 struct tun_struct *tun = netdev_priv(dev); 1173 1174 if (new_hr < NET_SKB_PAD) 1175 new_hr = NET_SKB_PAD; 1176 1177 tun->align = new_hr; 1178 } 1179 1180 static void 1181 tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1182 { 1183 struct tun_struct *tun = netdev_priv(dev); 1184 1185 dev_get_tstats64(dev, stats); 1186 1187 stats->rx_frame_errors += 1188 (unsigned long)atomic_long_read(&tun->rx_frame_errors); 1189 } 1190 1191 static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1192 struct netlink_ext_ack *extack) 1193 { 1194 struct tun_struct *tun = netdev_priv(dev); 1195 struct tun_file *tfile; 1196 struct bpf_prog *old_prog; 1197 int i; 1198 1199 old_prog = rtnl_dereference(tun->xdp_prog); 1200 rcu_assign_pointer(tun->xdp_prog, prog); 1201 if (old_prog) 1202 bpf_prog_put(old_prog); 1203 1204 for (i = 0; i < tun->numqueues; i++) { 1205 tfile = rtnl_dereference(tun->tfiles[i]); 1206 if (prog) 1207 sock_set_flag(&tfile->sk, SOCK_XDP); 1208 else 1209 sock_reset_flag(&tfile->sk, SOCK_XDP); 1210 } 1211 list_for_each_entry(tfile, &tun->disabled, next) { 1212 if (prog) 1213 sock_set_flag(&tfile->sk, SOCK_XDP); 1214 else 1215 sock_reset_flag(&tfile->sk, SOCK_XDP); 1216 } 1217 1218 return 0; 1219 } 1220 1221 static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1222 { 1223 switch (xdp->command) { 1224 case XDP_SETUP_PROG: 1225 return tun_xdp_set(dev, xdp->prog, xdp->extack); 1226 default: 1227 return -EINVAL; 1228 } 1229 } 1230 1231 static int tun_net_change_carrier(struct net_device *dev, bool new_carrier) 1232 { 1233 if (new_carrier) { 1234 struct tun_struct *tun = netdev_priv(dev); 1235 1236 if (!tun->numqueues) 1237 return -EPERM; 1238 1239 netif_carrier_on(dev); 1240 } else { 1241 netif_carrier_off(dev); 1242 } 1243 return 0; 1244 } 1245 1246 static const struct net_device_ops tun_netdev_ops = { 1247 .ndo_init = tun_net_init, 1248 .ndo_uninit = tun_net_uninit, 1249 .ndo_open = tun_net_open, 1250 .ndo_stop = tun_net_close, 1251 .ndo_start_xmit = tun_net_xmit, 1252 .ndo_fix_features = tun_net_fix_features, 1253 .ndo_select_queue = tun_select_queue, 1254 .ndo_set_rx_headroom = tun_set_headroom, 1255 .ndo_get_stats64 = tun_net_get_stats64, 1256 .ndo_change_carrier = tun_net_change_carrier, 1257 }; 1258 1259 static void __tun_xdp_flush_tfile(struct tun_file *tfile) 1260 { 1261 /* Notify and wake up reader process */ 1262 if (tfile->flags & TUN_FASYNC) 1263 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1264 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1265 } 1266 1267 static int tun_xdp_xmit(struct net_device *dev, int n, 1268 struct xdp_frame **frames, u32 flags) 1269 { 1270 struct tun_struct *tun = netdev_priv(dev); 1271 struct tun_file *tfile; 1272 u32 numqueues; 1273 int nxmit = 0; 1274 int i; 1275 1276 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1277 return -EINVAL; 1278 1279 rcu_read_lock(); 1280 1281 resample: 1282 numqueues = READ_ONCE(tun->numqueues); 1283 if (!numqueues) { 1284 rcu_read_unlock(); 1285 return -ENXIO; /* Caller will free/return all frames */ 1286 } 1287 1288 tfile = rcu_dereference(tun->tfiles[smp_processor_id() % 1289 numqueues]); 1290 if (unlikely(!tfile)) 1291 goto resample; 1292 1293 spin_lock(&tfile->tx_ring.producer_lock); 1294 for (i = 0; i < n; i++) { 1295 struct xdp_frame *xdp = frames[i]; 1296 /* Encode the XDP flag into lowest bit for consumer to differ 1297 * XDP buffer from sk_buff. 1298 */ 1299 void *frame = tun_xdp_to_ptr(xdp); 1300 1301 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { 1302 dev_core_stats_tx_dropped_inc(dev); 1303 break; 1304 } 1305 nxmit++; 1306 } 1307 spin_unlock(&tfile->tx_ring.producer_lock); 1308 1309 if (flags & XDP_XMIT_FLUSH) 1310 __tun_xdp_flush_tfile(tfile); 1311 1312 rcu_read_unlock(); 1313 return nxmit; 1314 } 1315 1316 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp) 1317 { 1318 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 1319 int nxmit; 1320 1321 if (unlikely(!frame)) 1322 return -EOVERFLOW; 1323 1324 nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1325 if (!nxmit) 1326 xdp_return_frame_rx_napi(frame); 1327 return nxmit; 1328 } 1329 1330 static const struct net_device_ops tap_netdev_ops = { 1331 .ndo_init = tun_net_init, 1332 .ndo_uninit = tun_net_uninit, 1333 .ndo_open = tun_net_open, 1334 .ndo_stop = tun_net_close, 1335 .ndo_start_xmit = tun_net_xmit, 1336 .ndo_fix_features = tun_net_fix_features, 1337 .ndo_set_rx_mode = tun_net_mclist, 1338 .ndo_set_mac_address = eth_mac_addr, 1339 .ndo_validate_addr = eth_validate_addr, 1340 .ndo_select_queue = tun_select_queue, 1341 .ndo_features_check = passthru_features_check, 1342 .ndo_set_rx_headroom = tun_set_headroom, 1343 .ndo_bpf = tun_xdp, 1344 .ndo_xdp_xmit = tun_xdp_xmit, 1345 .ndo_change_carrier = tun_net_change_carrier, 1346 }; 1347 1348 static void tun_flow_init(struct tun_struct *tun) 1349 { 1350 int i; 1351 1352 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) 1353 INIT_HLIST_HEAD(&tun->flows[i]); 1354 1355 tun->ageing_time = TUN_FLOW_EXPIRE; 1356 timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0); 1357 mod_timer(&tun->flow_gc_timer, 1358 round_jiffies_up(jiffies + tun->ageing_time)); 1359 } 1360 1361 static void tun_flow_uninit(struct tun_struct *tun) 1362 { 1363 del_timer_sync(&tun->flow_gc_timer); 1364 tun_flow_flush(tun); 1365 } 1366 1367 #define MIN_MTU 68 1368 #define MAX_MTU 65535 1369 1370 /* Initialize net device. */ 1371 static void tun_net_initialize(struct net_device *dev) 1372 { 1373 struct tun_struct *tun = netdev_priv(dev); 1374 1375 switch (tun->flags & TUN_TYPE_MASK) { 1376 case IFF_TUN: 1377 dev->netdev_ops = &tun_netdev_ops; 1378 dev->header_ops = &ip_tunnel_header_ops; 1379 1380 /* Point-to-Point TUN Device */ 1381 dev->hard_header_len = 0; 1382 dev->addr_len = 0; 1383 dev->mtu = 1500; 1384 1385 /* Zero header length */ 1386 dev->type = ARPHRD_NONE; 1387 dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; 1388 break; 1389 1390 case IFF_TAP: 1391 dev->netdev_ops = &tap_netdev_ops; 1392 /* Ethernet TAP Device */ 1393 ether_setup(dev); 1394 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1395 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1396 1397 eth_hw_addr_random(dev); 1398 1399 /* Currently tun does not support XDP, only tap does. */ 1400 dev->xdp_features = NETDEV_XDP_ACT_BASIC | 1401 NETDEV_XDP_ACT_REDIRECT | 1402 NETDEV_XDP_ACT_NDO_XMIT; 1403 1404 break; 1405 } 1406 1407 dev->min_mtu = MIN_MTU; 1408 dev->max_mtu = MAX_MTU - dev->hard_header_len; 1409 } 1410 1411 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile) 1412 { 1413 struct sock *sk = tfile->socket.sk; 1414 1415 return (tun->dev->flags & IFF_UP) && sock_writeable(sk); 1416 } 1417 1418 /* Character device part */ 1419 1420 /* Poll */ 1421 static __poll_t tun_chr_poll(struct file *file, poll_table *wait) 1422 { 1423 struct tun_file *tfile = file->private_data; 1424 struct tun_struct *tun = tun_get(tfile); 1425 struct sock *sk; 1426 __poll_t mask = 0; 1427 1428 if (!tun) 1429 return EPOLLERR; 1430 1431 sk = tfile->socket.sk; 1432 1433 poll_wait(file, sk_sleep(sk), wait); 1434 1435 if (!ptr_ring_empty(&tfile->tx_ring)) 1436 mask |= EPOLLIN | EPOLLRDNORM; 1437 1438 /* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to 1439 * guarantee EPOLLOUT to be raised by either here or 1440 * tun_sock_write_space(). Then process could get notification 1441 * after it writes to a down device and meets -EIO. 1442 */ 1443 if (tun_sock_writeable(tun, tfile) || 1444 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) && 1445 tun_sock_writeable(tun, tfile))) 1446 mask |= EPOLLOUT | EPOLLWRNORM; 1447 1448 if (tun->dev->reg_state != NETREG_REGISTERED) 1449 mask = EPOLLERR; 1450 1451 tun_put(tun); 1452 return mask; 1453 } 1454 1455 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, 1456 size_t len, 1457 const struct iov_iter *it) 1458 { 1459 struct sk_buff *skb; 1460 size_t linear; 1461 int err; 1462 int i; 1463 1464 if (it->nr_segs > MAX_SKB_FRAGS + 1 || 1465 len > (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN)) 1466 return ERR_PTR(-EMSGSIZE); 1467 1468 local_bh_disable(); 1469 skb = napi_get_frags(&tfile->napi); 1470 local_bh_enable(); 1471 if (!skb) 1472 return ERR_PTR(-ENOMEM); 1473 1474 linear = iov_iter_single_seg_count(it); 1475 err = __skb_grow(skb, linear); 1476 if (err) 1477 goto free; 1478 1479 skb->len = len; 1480 skb->data_len = len - linear; 1481 skb->truesize += skb->data_len; 1482 1483 for (i = 1; i < it->nr_segs; i++) { 1484 const struct iovec *iov = iter_iov(it); 1485 size_t fragsz = iov->iov_len; 1486 struct page *page; 1487 void *frag; 1488 1489 if (fragsz == 0 || fragsz > PAGE_SIZE) { 1490 err = -EINVAL; 1491 goto free; 1492 } 1493 frag = netdev_alloc_frag(fragsz); 1494 if (!frag) { 1495 err = -ENOMEM; 1496 goto free; 1497 } 1498 page = virt_to_head_page(frag); 1499 skb_fill_page_desc(skb, i - 1, page, 1500 frag - page_address(page), fragsz); 1501 } 1502 1503 return skb; 1504 free: 1505 /* frees skb and all frags allocated with napi_alloc_frag() */ 1506 napi_free_frags(&tfile->napi); 1507 return ERR_PTR(err); 1508 } 1509 1510 /* prepad is the amount to reserve at front. len is length after that. 1511 * linear is a hint as to how much to copy (usually headers). */ 1512 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, 1513 size_t prepad, size_t len, 1514 size_t linear, int noblock) 1515 { 1516 struct sock *sk = tfile->socket.sk; 1517 struct sk_buff *skb; 1518 int err; 1519 1520 /* Under a page? Don't bother with paged skb. */ 1521 if (prepad + len < PAGE_SIZE) 1522 linear = len; 1523 1524 if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) 1525 linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); 1526 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 1527 &err, PAGE_ALLOC_COSTLY_ORDER); 1528 if (!skb) 1529 return ERR_PTR(err); 1530 1531 skb_reserve(skb, prepad); 1532 skb_put(skb, linear); 1533 skb->data_len = len - linear; 1534 skb->len += len - linear; 1535 1536 return skb; 1537 } 1538 1539 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile, 1540 struct sk_buff *skb, int more) 1541 { 1542 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 1543 struct sk_buff_head process_queue; 1544 u32 rx_batched = tun->rx_batched; 1545 bool rcv = false; 1546 1547 if (!rx_batched || (!more && skb_queue_empty(queue))) { 1548 local_bh_disable(); 1549 skb_record_rx_queue(skb, tfile->queue_index); 1550 netif_receive_skb(skb); 1551 local_bh_enable(); 1552 return; 1553 } 1554 1555 spin_lock(&queue->lock); 1556 if (!more || skb_queue_len(queue) == rx_batched) { 1557 __skb_queue_head_init(&process_queue); 1558 skb_queue_splice_tail_init(queue, &process_queue); 1559 rcv = true; 1560 } else { 1561 __skb_queue_tail(queue, skb); 1562 } 1563 spin_unlock(&queue->lock); 1564 1565 if (rcv) { 1566 struct sk_buff *nskb; 1567 1568 local_bh_disable(); 1569 while ((nskb = __skb_dequeue(&process_queue))) { 1570 skb_record_rx_queue(nskb, tfile->queue_index); 1571 netif_receive_skb(nskb); 1572 } 1573 skb_record_rx_queue(skb, tfile->queue_index); 1574 netif_receive_skb(skb); 1575 local_bh_enable(); 1576 } 1577 } 1578 1579 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile, 1580 int len, int noblock, bool zerocopy) 1581 { 1582 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 1583 return false; 1584 1585 if (tfile->socket.sk->sk_sndbuf != INT_MAX) 1586 return false; 1587 1588 if (!noblock) 1589 return false; 1590 1591 if (zerocopy) 1592 return false; 1593 1594 if (SKB_DATA_ALIGN(len + TUN_RX_PAD + XDP_PACKET_HEADROOM) + 1595 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE) 1596 return false; 1597 1598 return true; 1599 } 1600 1601 static struct sk_buff *__tun_build_skb(struct tun_file *tfile, 1602 struct page_frag *alloc_frag, char *buf, 1603 int buflen, int len, int pad) 1604 { 1605 struct sk_buff *skb = build_skb(buf, buflen); 1606 1607 if (!skb) 1608 return ERR_PTR(-ENOMEM); 1609 1610 skb_reserve(skb, pad); 1611 skb_put(skb, len); 1612 skb_set_owner_w(skb, tfile->socket.sk); 1613 1614 get_page(alloc_frag->page); 1615 alloc_frag->offset += buflen; 1616 1617 return skb; 1618 } 1619 1620 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, 1621 struct xdp_buff *xdp, u32 act) 1622 { 1623 int err; 1624 1625 switch (act) { 1626 case XDP_REDIRECT: 1627 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); 1628 if (err) { 1629 dev_core_stats_rx_dropped_inc(tun->dev); 1630 return err; 1631 } 1632 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data); 1633 break; 1634 case XDP_TX: 1635 err = tun_xdp_tx(tun->dev, xdp); 1636 if (err < 0) { 1637 dev_core_stats_rx_dropped_inc(tun->dev); 1638 return err; 1639 } 1640 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data); 1641 break; 1642 case XDP_PASS: 1643 break; 1644 default: 1645 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); 1646 fallthrough; 1647 case XDP_ABORTED: 1648 trace_xdp_exception(tun->dev, xdp_prog, act); 1649 fallthrough; 1650 case XDP_DROP: 1651 dev_core_stats_rx_dropped_inc(tun->dev); 1652 break; 1653 } 1654 1655 return act; 1656 } 1657 1658 static struct sk_buff *tun_build_skb(struct tun_struct *tun, 1659 struct tun_file *tfile, 1660 struct iov_iter *from, 1661 struct virtio_net_hdr *hdr, 1662 int len, int *skb_xdp) 1663 { 1664 struct page_frag *alloc_frag = ¤t->task_frag; 1665 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 1666 struct bpf_prog *xdp_prog; 1667 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1668 char *buf; 1669 size_t copied; 1670 int pad = TUN_RX_PAD; 1671 int err = 0; 1672 1673 rcu_read_lock(); 1674 xdp_prog = rcu_dereference(tun->xdp_prog); 1675 if (xdp_prog) 1676 pad += XDP_PACKET_HEADROOM; 1677 buflen += SKB_DATA_ALIGN(len + pad); 1678 rcu_read_unlock(); 1679 1680 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES); 1681 if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL))) 1682 return ERR_PTR(-ENOMEM); 1683 1684 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1685 copied = copy_page_from_iter(alloc_frag->page, 1686 alloc_frag->offset + pad, 1687 len, from); 1688 if (copied != len) 1689 return ERR_PTR(-EFAULT); 1690 1691 /* There's a small window that XDP may be set after the check 1692 * of xdp_prog above, this should be rare and for simplicity 1693 * we do XDP on skb in case the headroom is not enough. 1694 */ 1695 if (hdr->gso_type || !xdp_prog) { 1696 *skb_xdp = 1; 1697 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, 1698 pad); 1699 } 1700 1701 *skb_xdp = 0; 1702 1703 local_bh_disable(); 1704 rcu_read_lock(); 1705 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 1706 xdp_prog = rcu_dereference(tun->xdp_prog); 1707 if (xdp_prog) { 1708 struct xdp_buff xdp; 1709 u32 act; 1710 1711 xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq); 1712 xdp_prepare_buff(&xdp, buf, pad, len, false); 1713 1714 act = bpf_prog_run_xdp(xdp_prog, &xdp); 1715 if (act == XDP_REDIRECT || act == XDP_TX) { 1716 get_page(alloc_frag->page); 1717 alloc_frag->offset += buflen; 1718 } 1719 err = tun_xdp_act(tun, xdp_prog, &xdp, act); 1720 if (err < 0) { 1721 if (act == XDP_REDIRECT || act == XDP_TX) 1722 put_page(alloc_frag->page); 1723 goto out; 1724 } 1725 1726 if (err == XDP_REDIRECT) 1727 xdp_do_flush(); 1728 if (err != XDP_PASS) 1729 goto out; 1730 1731 pad = xdp.data - xdp.data_hard_start; 1732 len = xdp.data_end - xdp.data; 1733 } 1734 bpf_net_ctx_clear(bpf_net_ctx); 1735 rcu_read_unlock(); 1736 local_bh_enable(); 1737 1738 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad); 1739 1740 out: 1741 bpf_net_ctx_clear(bpf_net_ctx); 1742 rcu_read_unlock(); 1743 local_bh_enable(); 1744 return NULL; 1745 } 1746 1747 /* Get packet from user space buffer */ 1748 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 1749 void *msg_control, struct iov_iter *from, 1750 int noblock, bool more) 1751 { 1752 struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) }; 1753 struct sk_buff *skb; 1754 size_t total_len = iov_iter_count(from); 1755 size_t len = total_len, align = tun->align, linear; 1756 struct virtio_net_hdr gso = { 0 }; 1757 int good_linear; 1758 int copylen; 1759 bool zerocopy = false; 1760 int err; 1761 u32 rxhash = 0; 1762 int skb_xdp = 1; 1763 bool frags = tun_napi_frags_enabled(tfile); 1764 enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 1765 1766 if (!(tun->flags & IFF_NO_PI)) { 1767 if (len < sizeof(pi)) 1768 return -EINVAL; 1769 len -= sizeof(pi); 1770 1771 if (!copy_from_iter_full(&pi, sizeof(pi), from)) 1772 return -EFAULT; 1773 } 1774 1775 if (tun->flags & IFF_VNET_HDR) { 1776 int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 1777 1778 if (len < vnet_hdr_sz) 1779 return -EINVAL; 1780 len -= vnet_hdr_sz; 1781 1782 if (!copy_from_iter_full(&gso, sizeof(gso), from)) 1783 return -EFAULT; 1784 1785 if ((gso.flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && 1786 tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2 > tun16_to_cpu(tun, gso.hdr_len)) 1787 gso.hdr_len = cpu_to_tun16(tun, tun16_to_cpu(tun, gso.csum_start) + tun16_to_cpu(tun, gso.csum_offset) + 2); 1788 1789 if (tun16_to_cpu(tun, gso.hdr_len) > len) 1790 return -EINVAL; 1791 iov_iter_advance(from, vnet_hdr_sz - sizeof(gso)); 1792 } 1793 1794 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) { 1795 align += NET_IP_ALIGN; 1796 if (unlikely(len < ETH_HLEN || 1797 (gso.hdr_len && tun16_to_cpu(tun, gso.hdr_len) < ETH_HLEN))) 1798 return -EINVAL; 1799 } 1800 1801 good_linear = SKB_MAX_HEAD(align); 1802 1803 if (msg_control) { 1804 struct iov_iter i = *from; 1805 1806 /* There are 256 bytes to be copied in skb, so there is 1807 * enough room for skb expand head in case it is used. 1808 * The rest of the buffer is mapped from userspace. 1809 */ 1810 copylen = gso.hdr_len ? tun16_to_cpu(tun, gso.hdr_len) : GOODCOPY_LEN; 1811 if (copylen > good_linear) 1812 copylen = good_linear; 1813 linear = copylen; 1814 iov_iter_advance(&i, copylen); 1815 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS) 1816 zerocopy = true; 1817 } 1818 1819 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) { 1820 /* For the packet that is not easy to be processed 1821 * (e.g gso or jumbo packet), we will do it at after 1822 * skb was created with generic XDP routine. 1823 */ 1824 skb = tun_build_skb(tun, tfile, from, &gso, len, &skb_xdp); 1825 err = PTR_ERR_OR_ZERO(skb); 1826 if (err) 1827 goto drop; 1828 if (!skb) 1829 return total_len; 1830 } else { 1831 if (!zerocopy) { 1832 copylen = len; 1833 if (tun16_to_cpu(tun, gso.hdr_len) > good_linear) 1834 linear = good_linear; 1835 else 1836 linear = tun16_to_cpu(tun, gso.hdr_len); 1837 } 1838 1839 if (frags) { 1840 mutex_lock(&tfile->napi_mutex); 1841 skb = tun_napi_alloc_frags(tfile, copylen, from); 1842 /* tun_napi_alloc_frags() enforces a layout for the skb. 1843 * If zerocopy is enabled, then this layout will be 1844 * overwritten by zerocopy_sg_from_iter(). 1845 */ 1846 zerocopy = false; 1847 } else { 1848 if (!linear) 1849 linear = min_t(size_t, good_linear, copylen); 1850 1851 skb = tun_alloc_skb(tfile, align, copylen, linear, 1852 noblock); 1853 } 1854 1855 err = PTR_ERR_OR_ZERO(skb); 1856 if (err) 1857 goto drop; 1858 1859 if (zerocopy) 1860 err = zerocopy_sg_from_iter(skb, from); 1861 else 1862 err = skb_copy_datagram_from_iter(skb, 0, from, len); 1863 1864 if (err) { 1865 err = -EFAULT; 1866 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT; 1867 goto drop; 1868 } 1869 } 1870 1871 if (virtio_net_hdr_to_skb(skb, &gso, tun_is_little_endian(tun))) { 1872 atomic_long_inc(&tun->rx_frame_errors); 1873 err = -EINVAL; 1874 goto free_skb; 1875 } 1876 1877 switch (tun->flags & TUN_TYPE_MASK) { 1878 case IFF_TUN: 1879 if (tun->flags & IFF_NO_PI) { 1880 u8 ip_version = skb->len ? (skb->data[0] >> 4) : 0; 1881 1882 switch (ip_version) { 1883 case 4: 1884 pi.proto = htons(ETH_P_IP); 1885 break; 1886 case 6: 1887 pi.proto = htons(ETH_P_IPV6); 1888 break; 1889 default: 1890 err = -EINVAL; 1891 goto drop; 1892 } 1893 } 1894 1895 skb_reset_mac_header(skb); 1896 skb->protocol = pi.proto; 1897 skb->dev = tun->dev; 1898 break; 1899 case IFF_TAP: 1900 if (frags && !pskb_may_pull(skb, ETH_HLEN)) { 1901 err = -ENOMEM; 1902 drop_reason = SKB_DROP_REASON_HDR_TRUNC; 1903 goto drop; 1904 } 1905 skb->protocol = eth_type_trans(skb, tun->dev); 1906 break; 1907 } 1908 1909 /* copy skb_ubuf_info for callback when skb has no error */ 1910 if (zerocopy) { 1911 skb_zcopy_init(skb, msg_control); 1912 } else if (msg_control) { 1913 struct ubuf_info *uarg = msg_control; 1914 uarg->ops->complete(NULL, uarg, false); 1915 } 1916 1917 skb_reset_network_header(skb); 1918 skb_probe_transport_header(skb); 1919 skb_record_rx_queue(skb, tfile->queue_index); 1920 1921 if (skb_xdp) { 1922 struct bpf_prog *xdp_prog; 1923 int ret; 1924 1925 local_bh_disable(); 1926 rcu_read_lock(); 1927 xdp_prog = rcu_dereference(tun->xdp_prog); 1928 if (xdp_prog) { 1929 ret = do_xdp_generic(xdp_prog, &skb); 1930 if (ret != XDP_PASS) { 1931 rcu_read_unlock(); 1932 local_bh_enable(); 1933 goto unlock_frags; 1934 } 1935 } 1936 rcu_read_unlock(); 1937 local_bh_enable(); 1938 } 1939 1940 /* Compute the costly rx hash only if needed for flow updates. 1941 * We may get a very small possibility of OOO during switching, not 1942 * worth to optimize. 1943 */ 1944 if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 && 1945 !tfile->detached) 1946 rxhash = __skb_get_hash_symmetric(skb); 1947 1948 rcu_read_lock(); 1949 if (unlikely(!(tun->dev->flags & IFF_UP))) { 1950 err = -EIO; 1951 rcu_read_unlock(); 1952 drop_reason = SKB_DROP_REASON_DEV_READY; 1953 goto drop; 1954 } 1955 1956 if (frags) { 1957 u32 headlen; 1958 1959 /* Exercise flow dissector code path. */ 1960 skb_push(skb, ETH_HLEN); 1961 headlen = eth_get_headlen(tun->dev, skb->data, 1962 skb_headlen(skb)); 1963 1964 if (unlikely(headlen > skb_headlen(skb))) { 1965 WARN_ON_ONCE(1); 1966 err = -ENOMEM; 1967 dev_core_stats_rx_dropped_inc(tun->dev); 1968 napi_busy: 1969 napi_free_frags(&tfile->napi); 1970 rcu_read_unlock(); 1971 mutex_unlock(&tfile->napi_mutex); 1972 return err; 1973 } 1974 1975 if (likely(napi_schedule_prep(&tfile->napi))) { 1976 local_bh_disable(); 1977 napi_gro_frags(&tfile->napi); 1978 napi_complete(&tfile->napi); 1979 local_bh_enable(); 1980 } else { 1981 err = -EBUSY; 1982 goto napi_busy; 1983 } 1984 mutex_unlock(&tfile->napi_mutex); 1985 } else if (tfile->napi_enabled) { 1986 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 1987 int queue_len; 1988 1989 spin_lock_bh(&queue->lock); 1990 1991 if (unlikely(tfile->detached)) { 1992 spin_unlock_bh(&queue->lock); 1993 rcu_read_unlock(); 1994 err = -EBUSY; 1995 goto free_skb; 1996 } 1997 1998 __skb_queue_tail(queue, skb); 1999 queue_len = skb_queue_len(queue); 2000 spin_unlock(&queue->lock); 2001 2002 if (!more || queue_len > NAPI_POLL_WEIGHT) 2003 napi_schedule(&tfile->napi); 2004 2005 local_bh_enable(); 2006 } else if (!IS_ENABLED(CONFIG_4KSTACKS)) { 2007 tun_rx_batched(tun, tfile, skb, more); 2008 } else { 2009 netif_rx(skb); 2010 } 2011 rcu_read_unlock(); 2012 2013 preempt_disable(); 2014 dev_sw_netstats_rx_add(tun->dev, len); 2015 preempt_enable(); 2016 2017 if (rxhash) 2018 tun_flow_update(tun, rxhash, tfile); 2019 2020 return total_len; 2021 2022 drop: 2023 if (err != -EAGAIN) 2024 dev_core_stats_rx_dropped_inc(tun->dev); 2025 2026 free_skb: 2027 if (!IS_ERR_OR_NULL(skb)) 2028 kfree_skb_reason(skb, drop_reason); 2029 2030 unlock_frags: 2031 if (frags) { 2032 tfile->napi.skb = NULL; 2033 mutex_unlock(&tfile->napi_mutex); 2034 } 2035 2036 return err ?: total_len; 2037 } 2038 2039 static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from) 2040 { 2041 struct file *file = iocb->ki_filp; 2042 struct tun_file *tfile = file->private_data; 2043 struct tun_struct *tun = tun_get(tfile); 2044 ssize_t result; 2045 int noblock = 0; 2046 2047 if (!tun) 2048 return -EBADFD; 2049 2050 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT)) 2051 noblock = 1; 2052 2053 result = tun_get_user(tun, tfile, NULL, from, noblock, false); 2054 2055 tun_put(tun); 2056 return result; 2057 } 2058 2059 static ssize_t tun_put_user_xdp(struct tun_struct *tun, 2060 struct tun_file *tfile, 2061 struct xdp_frame *xdp_frame, 2062 struct iov_iter *iter) 2063 { 2064 int vnet_hdr_sz = 0; 2065 size_t size = xdp_frame->len; 2066 size_t ret; 2067 2068 if (tun->flags & IFF_VNET_HDR) { 2069 struct virtio_net_hdr gso = { 0 }; 2070 2071 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2072 if (unlikely(iov_iter_count(iter) < vnet_hdr_sz)) 2073 return -EINVAL; 2074 if (unlikely(copy_to_iter(&gso, sizeof(gso), iter) != 2075 sizeof(gso))) 2076 return -EFAULT; 2077 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso)); 2078 } 2079 2080 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz; 2081 2082 preempt_disable(); 2083 dev_sw_netstats_tx_add(tun->dev, 1, ret); 2084 preempt_enable(); 2085 2086 return ret; 2087 } 2088 2089 /* Put packet to the user space buffer */ 2090 static ssize_t tun_put_user(struct tun_struct *tun, 2091 struct tun_file *tfile, 2092 struct sk_buff *skb, 2093 struct iov_iter *iter) 2094 { 2095 struct tun_pi pi = { 0, skb->protocol }; 2096 ssize_t total; 2097 int vlan_offset = 0; 2098 int vlan_hlen = 0; 2099 int vnet_hdr_sz = 0; 2100 2101 if (skb_vlan_tag_present(skb)) 2102 vlan_hlen = VLAN_HLEN; 2103 2104 if (tun->flags & IFF_VNET_HDR) 2105 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2106 2107 total = skb->len + vlan_hlen + vnet_hdr_sz; 2108 2109 if (!(tun->flags & IFF_NO_PI)) { 2110 if (iov_iter_count(iter) < sizeof(pi)) 2111 return -EINVAL; 2112 2113 total += sizeof(pi); 2114 if (iov_iter_count(iter) < total) { 2115 /* Packet will be striped */ 2116 pi.flags |= TUN_PKT_STRIP; 2117 } 2118 2119 if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi)) 2120 return -EFAULT; 2121 } 2122 2123 if (vnet_hdr_sz) { 2124 struct virtio_net_hdr gso; 2125 2126 if (iov_iter_count(iter) < vnet_hdr_sz) 2127 return -EINVAL; 2128 2129 if (virtio_net_hdr_from_skb(skb, &gso, 2130 tun_is_little_endian(tun), true, 2131 vlan_hlen)) { 2132 struct skb_shared_info *sinfo = skb_shinfo(skb); 2133 2134 if (net_ratelimit()) { 2135 netdev_err(tun->dev, "unexpected GSO type: 0x%x, gso_size %d, hdr_len %d\n", 2136 sinfo->gso_type, tun16_to_cpu(tun, gso.gso_size), 2137 tun16_to_cpu(tun, gso.hdr_len)); 2138 print_hex_dump(KERN_ERR, "tun: ", 2139 DUMP_PREFIX_NONE, 2140 16, 1, skb->head, 2141 min((int)tun16_to_cpu(tun, gso.hdr_len), 64), true); 2142 } 2143 WARN_ON_ONCE(1); 2144 return -EINVAL; 2145 } 2146 2147 if (copy_to_iter(&gso, sizeof(gso), iter) != sizeof(gso)) 2148 return -EFAULT; 2149 2150 iov_iter_advance(iter, vnet_hdr_sz - sizeof(gso)); 2151 } 2152 2153 if (vlan_hlen) { 2154 int ret; 2155 struct veth veth; 2156 2157 veth.h_vlan_proto = skb->vlan_proto; 2158 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 2159 2160 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto); 2161 2162 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset); 2163 if (ret || !iov_iter_count(iter)) 2164 goto done; 2165 2166 ret = copy_to_iter(&veth, sizeof(veth), iter); 2167 if (ret != sizeof(veth) || !iov_iter_count(iter)) 2168 goto done; 2169 } 2170 2171 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset); 2172 2173 done: 2174 /* caller is in process context, */ 2175 preempt_disable(); 2176 dev_sw_netstats_tx_add(tun->dev, 1, skb->len + vlan_hlen); 2177 preempt_enable(); 2178 2179 return total; 2180 } 2181 2182 static void *tun_ring_recv(struct tun_file *tfile, int noblock, int *err) 2183 { 2184 DECLARE_WAITQUEUE(wait, current); 2185 void *ptr = NULL; 2186 int error = 0; 2187 2188 ptr = ptr_ring_consume(&tfile->tx_ring); 2189 if (ptr) 2190 goto out; 2191 if (noblock) { 2192 error = -EAGAIN; 2193 goto out; 2194 } 2195 2196 add_wait_queue(&tfile->socket.wq.wait, &wait); 2197 2198 while (1) { 2199 set_current_state(TASK_INTERRUPTIBLE); 2200 ptr = ptr_ring_consume(&tfile->tx_ring); 2201 if (ptr) 2202 break; 2203 if (signal_pending(current)) { 2204 error = -ERESTARTSYS; 2205 break; 2206 } 2207 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) { 2208 error = -EFAULT; 2209 break; 2210 } 2211 2212 schedule(); 2213 } 2214 2215 __set_current_state(TASK_RUNNING); 2216 remove_wait_queue(&tfile->socket.wq.wait, &wait); 2217 2218 out: 2219 *err = error; 2220 return ptr; 2221 } 2222 2223 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, 2224 struct iov_iter *to, 2225 int noblock, void *ptr) 2226 { 2227 ssize_t ret; 2228 int err; 2229 2230 if (!iov_iter_count(to)) { 2231 tun_ptr_free(ptr); 2232 return 0; 2233 } 2234 2235 if (!ptr) { 2236 /* Read frames from ring */ 2237 ptr = tun_ring_recv(tfile, noblock, &err); 2238 if (!ptr) 2239 return err; 2240 } 2241 2242 if (tun_is_xdp_frame(ptr)) { 2243 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2244 2245 ret = tun_put_user_xdp(tun, tfile, xdpf, to); 2246 xdp_return_frame(xdpf); 2247 } else { 2248 struct sk_buff *skb = ptr; 2249 2250 ret = tun_put_user(tun, tfile, skb, to); 2251 if (unlikely(ret < 0)) 2252 kfree_skb(skb); 2253 else 2254 consume_skb(skb); 2255 } 2256 2257 return ret; 2258 } 2259 2260 static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 2261 { 2262 struct file *file = iocb->ki_filp; 2263 struct tun_file *tfile = file->private_data; 2264 struct tun_struct *tun = tun_get(tfile); 2265 ssize_t len = iov_iter_count(to), ret; 2266 int noblock = 0; 2267 2268 if (!tun) 2269 return -EBADFD; 2270 2271 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT)) 2272 noblock = 1; 2273 2274 ret = tun_do_read(tun, tfile, to, noblock, NULL); 2275 ret = min_t(ssize_t, ret, len); 2276 if (ret > 0) 2277 iocb->ki_pos = ret; 2278 tun_put(tun); 2279 return ret; 2280 } 2281 2282 static void tun_prog_free(struct rcu_head *rcu) 2283 { 2284 struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu); 2285 2286 bpf_prog_destroy(prog->prog); 2287 kfree(prog); 2288 } 2289 2290 static int __tun_set_ebpf(struct tun_struct *tun, 2291 struct tun_prog __rcu **prog_p, 2292 struct bpf_prog *prog) 2293 { 2294 struct tun_prog *old, *new = NULL; 2295 2296 if (prog) { 2297 new = kmalloc(sizeof(*new), GFP_KERNEL); 2298 if (!new) 2299 return -ENOMEM; 2300 new->prog = prog; 2301 } 2302 2303 spin_lock_bh(&tun->lock); 2304 old = rcu_dereference_protected(*prog_p, 2305 lockdep_is_held(&tun->lock)); 2306 rcu_assign_pointer(*prog_p, new); 2307 spin_unlock_bh(&tun->lock); 2308 2309 if (old) 2310 call_rcu(&old->rcu, tun_prog_free); 2311 2312 return 0; 2313 } 2314 2315 static void tun_free_netdev(struct net_device *dev) 2316 { 2317 struct tun_struct *tun = netdev_priv(dev); 2318 2319 BUG_ON(!(list_empty(&tun->disabled))); 2320 2321 tun_flow_uninit(tun); 2322 security_tun_dev_free_security(tun->security); 2323 __tun_set_ebpf(tun, &tun->steering_prog, NULL); 2324 __tun_set_ebpf(tun, &tun->filter_prog, NULL); 2325 } 2326 2327 static void tun_setup(struct net_device *dev) 2328 { 2329 struct tun_struct *tun = netdev_priv(dev); 2330 2331 tun->owner = INVALID_UID; 2332 tun->group = INVALID_GID; 2333 tun_default_link_ksettings(dev, &tun->link_ksettings); 2334 2335 dev->ethtool_ops = &tun_ethtool_ops; 2336 dev->needs_free_netdev = true; 2337 dev->priv_destructor = tun_free_netdev; 2338 /* We prefer our own queue length */ 2339 dev->tx_queue_len = TUN_READQ_SIZE; 2340 } 2341 2342 /* Trivial set of netlink ops to allow deleting tun or tap 2343 * device with netlink. 2344 */ 2345 static int tun_validate(struct nlattr *tb[], struct nlattr *data[], 2346 struct netlink_ext_ack *extack) 2347 { 2348 NL_SET_ERR_MSG(extack, 2349 "tun/tap creation via rtnetlink is not supported."); 2350 return -EOPNOTSUPP; 2351 } 2352 2353 static size_t tun_get_size(const struct net_device *dev) 2354 { 2355 BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t)); 2356 BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t)); 2357 2358 return nla_total_size(sizeof(uid_t)) + /* OWNER */ 2359 nla_total_size(sizeof(gid_t)) + /* GROUP */ 2360 nla_total_size(sizeof(u8)) + /* TYPE */ 2361 nla_total_size(sizeof(u8)) + /* PI */ 2362 nla_total_size(sizeof(u8)) + /* VNET_HDR */ 2363 nla_total_size(sizeof(u8)) + /* PERSIST */ 2364 nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */ 2365 nla_total_size(sizeof(u32)) + /* NUM_QUEUES */ 2366 nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */ 2367 0; 2368 } 2369 2370 static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev) 2371 { 2372 struct tun_struct *tun = netdev_priv(dev); 2373 2374 if (nla_put_u8(skb, IFLA_TUN_TYPE, tun->flags & TUN_TYPE_MASK)) 2375 goto nla_put_failure; 2376 if (uid_valid(tun->owner) && 2377 nla_put_u32(skb, IFLA_TUN_OWNER, 2378 from_kuid_munged(current_user_ns(), tun->owner))) 2379 goto nla_put_failure; 2380 if (gid_valid(tun->group) && 2381 nla_put_u32(skb, IFLA_TUN_GROUP, 2382 from_kgid_munged(current_user_ns(), tun->group))) 2383 goto nla_put_failure; 2384 if (nla_put_u8(skb, IFLA_TUN_PI, !(tun->flags & IFF_NO_PI))) 2385 goto nla_put_failure; 2386 if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(tun->flags & IFF_VNET_HDR))) 2387 goto nla_put_failure; 2388 if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(tun->flags & IFF_PERSIST))) 2389 goto nla_put_failure; 2390 if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE, 2391 !!(tun->flags & IFF_MULTI_QUEUE))) 2392 goto nla_put_failure; 2393 if (tun->flags & IFF_MULTI_QUEUE) { 2394 if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, tun->numqueues)) 2395 goto nla_put_failure; 2396 if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES, 2397 tun->numdisabled)) 2398 goto nla_put_failure; 2399 } 2400 2401 return 0; 2402 2403 nla_put_failure: 2404 return -EMSGSIZE; 2405 } 2406 2407 static struct rtnl_link_ops tun_link_ops __read_mostly = { 2408 .kind = DRV_NAME, 2409 .priv_size = sizeof(struct tun_struct), 2410 .setup = tun_setup, 2411 .validate = tun_validate, 2412 .get_size = tun_get_size, 2413 .fill_info = tun_fill_info, 2414 }; 2415 2416 static void tun_sock_write_space(struct sock *sk) 2417 { 2418 struct tun_file *tfile; 2419 wait_queue_head_t *wqueue; 2420 2421 if (!sock_writeable(sk)) 2422 return; 2423 2424 if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) 2425 return; 2426 2427 wqueue = sk_sleep(sk); 2428 if (wqueue && waitqueue_active(wqueue)) 2429 wake_up_interruptible_sync_poll(wqueue, EPOLLOUT | 2430 EPOLLWRNORM | EPOLLWRBAND); 2431 2432 tfile = container_of(sk, struct tun_file, sk); 2433 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT); 2434 } 2435 2436 static void tun_put_page(struct tun_page *tpage) 2437 { 2438 if (tpage->page) 2439 __page_frag_cache_drain(tpage->page, tpage->count); 2440 } 2441 2442 static int tun_xdp_one(struct tun_struct *tun, 2443 struct tun_file *tfile, 2444 struct xdp_buff *xdp, int *flush, 2445 struct tun_page *tpage) 2446 { 2447 unsigned int datasize = xdp->data_end - xdp->data; 2448 struct tun_xdp_hdr *hdr = xdp->data_hard_start; 2449 struct virtio_net_hdr *gso = &hdr->gso; 2450 struct bpf_prog *xdp_prog; 2451 struct sk_buff *skb = NULL; 2452 struct sk_buff_head *queue; 2453 u32 rxhash = 0, act; 2454 int buflen = hdr->buflen; 2455 int ret = 0; 2456 bool skb_xdp = false; 2457 struct page *page; 2458 2459 if (unlikely(datasize < ETH_HLEN)) 2460 return -EINVAL; 2461 2462 xdp_prog = rcu_dereference(tun->xdp_prog); 2463 if (xdp_prog) { 2464 if (gso->gso_type) { 2465 skb_xdp = true; 2466 goto build; 2467 } 2468 2469 xdp_init_buff(xdp, buflen, &tfile->xdp_rxq); 2470 xdp_set_data_meta_invalid(xdp); 2471 2472 act = bpf_prog_run_xdp(xdp_prog, xdp); 2473 ret = tun_xdp_act(tun, xdp_prog, xdp, act); 2474 if (ret < 0) { 2475 put_page(virt_to_head_page(xdp->data)); 2476 return ret; 2477 } 2478 2479 switch (ret) { 2480 case XDP_REDIRECT: 2481 *flush = true; 2482 fallthrough; 2483 case XDP_TX: 2484 return 0; 2485 case XDP_PASS: 2486 break; 2487 default: 2488 page = virt_to_head_page(xdp->data); 2489 if (tpage->page == page) { 2490 ++tpage->count; 2491 } else { 2492 tun_put_page(tpage); 2493 tpage->page = page; 2494 tpage->count = 1; 2495 } 2496 return 0; 2497 } 2498 } 2499 2500 build: 2501 skb = build_skb(xdp->data_hard_start, buflen); 2502 if (!skb) { 2503 ret = -ENOMEM; 2504 goto out; 2505 } 2506 2507 skb_reserve(skb, xdp->data - xdp->data_hard_start); 2508 skb_put(skb, xdp->data_end - xdp->data); 2509 2510 if (virtio_net_hdr_to_skb(skb, gso, tun_is_little_endian(tun))) { 2511 atomic_long_inc(&tun->rx_frame_errors); 2512 kfree_skb(skb); 2513 ret = -EINVAL; 2514 goto out; 2515 } 2516 2517 skb->protocol = eth_type_trans(skb, tun->dev); 2518 skb_reset_network_header(skb); 2519 skb_probe_transport_header(skb); 2520 skb_record_rx_queue(skb, tfile->queue_index); 2521 2522 if (skb_xdp) { 2523 ret = do_xdp_generic(xdp_prog, &skb); 2524 if (ret != XDP_PASS) { 2525 ret = 0; 2526 goto out; 2527 } 2528 } 2529 2530 if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 && 2531 !tfile->detached) 2532 rxhash = __skb_get_hash_symmetric(skb); 2533 2534 if (tfile->napi_enabled) { 2535 queue = &tfile->sk.sk_write_queue; 2536 spin_lock(&queue->lock); 2537 2538 if (unlikely(tfile->detached)) { 2539 spin_unlock(&queue->lock); 2540 kfree_skb(skb); 2541 return -EBUSY; 2542 } 2543 2544 __skb_queue_tail(queue, skb); 2545 spin_unlock(&queue->lock); 2546 ret = 1; 2547 } else { 2548 netif_receive_skb(skb); 2549 ret = 0; 2550 } 2551 2552 /* No need to disable preemption here since this function is 2553 * always called with bh disabled 2554 */ 2555 dev_sw_netstats_rx_add(tun->dev, datasize); 2556 2557 if (rxhash) 2558 tun_flow_update(tun, rxhash, tfile); 2559 2560 out: 2561 return ret; 2562 } 2563 2564 static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) 2565 { 2566 int ret, i; 2567 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2568 struct tun_struct *tun = tun_get(tfile); 2569 struct tun_msg_ctl *ctl = m->msg_control; 2570 struct xdp_buff *xdp; 2571 2572 if (!tun) 2573 return -EBADFD; 2574 2575 if (m->msg_controllen == sizeof(struct tun_msg_ctl) && 2576 ctl && ctl->type == TUN_MSG_PTR) { 2577 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 2578 struct tun_page tpage; 2579 int n = ctl->num; 2580 int flush = 0, queued = 0; 2581 2582 memset(&tpage, 0, sizeof(tpage)); 2583 2584 local_bh_disable(); 2585 rcu_read_lock(); 2586 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 2587 2588 for (i = 0; i < n; i++) { 2589 xdp = &((struct xdp_buff *)ctl->ptr)[i]; 2590 ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage); 2591 if (ret > 0) 2592 queued += ret; 2593 } 2594 2595 if (flush) 2596 xdp_do_flush(); 2597 2598 if (tfile->napi_enabled && queued > 0) 2599 napi_schedule(&tfile->napi); 2600 2601 bpf_net_ctx_clear(bpf_net_ctx); 2602 rcu_read_unlock(); 2603 local_bh_enable(); 2604 2605 tun_put_page(&tpage); 2606 2607 ret = total_len; 2608 goto out; 2609 } 2610 2611 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter, 2612 m->msg_flags & MSG_DONTWAIT, 2613 m->msg_flags & MSG_MORE); 2614 out: 2615 tun_put(tun); 2616 return ret; 2617 } 2618 2619 static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len, 2620 int flags) 2621 { 2622 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2623 struct tun_struct *tun = tun_get(tfile); 2624 void *ptr = m->msg_control; 2625 int ret; 2626 2627 if (!tun) { 2628 ret = -EBADFD; 2629 goto out_free; 2630 } 2631 2632 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) { 2633 ret = -EINVAL; 2634 goto out_put_tun; 2635 } 2636 if (flags & MSG_ERRQUEUE) { 2637 ret = sock_recv_errqueue(sock->sk, m, total_len, 2638 SOL_PACKET, TUN_TX_TIMESTAMP); 2639 goto out; 2640 } 2641 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr); 2642 if (ret > (ssize_t)total_len) { 2643 m->msg_flags |= MSG_TRUNC; 2644 ret = flags & MSG_TRUNC ? ret : total_len; 2645 } 2646 out: 2647 tun_put(tun); 2648 return ret; 2649 2650 out_put_tun: 2651 tun_put(tun); 2652 out_free: 2653 tun_ptr_free(ptr); 2654 return ret; 2655 } 2656 2657 static int tun_ptr_peek_len(void *ptr) 2658 { 2659 if (likely(ptr)) { 2660 if (tun_is_xdp_frame(ptr)) { 2661 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2662 2663 return xdpf->len; 2664 } 2665 return __skb_array_len_with_tag(ptr); 2666 } else { 2667 return 0; 2668 } 2669 } 2670 2671 static int tun_peek_len(struct socket *sock) 2672 { 2673 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2674 struct tun_struct *tun; 2675 int ret = 0; 2676 2677 tun = tun_get(tfile); 2678 if (!tun) 2679 return 0; 2680 2681 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len); 2682 tun_put(tun); 2683 2684 return ret; 2685 } 2686 2687 /* Ops structure to mimic raw sockets with tun */ 2688 static const struct proto_ops tun_socket_ops = { 2689 .peek_len = tun_peek_len, 2690 .sendmsg = tun_sendmsg, 2691 .recvmsg = tun_recvmsg, 2692 }; 2693 2694 static struct proto tun_proto = { 2695 .name = "tun", 2696 .owner = THIS_MODULE, 2697 .obj_size = sizeof(struct tun_file), 2698 }; 2699 2700 static int tun_flags(struct tun_struct *tun) 2701 { 2702 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP); 2703 } 2704 2705 static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr, 2706 char *buf) 2707 { 2708 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2709 return sysfs_emit(buf, "0x%x\n", tun_flags(tun)); 2710 } 2711 2712 static ssize_t owner_show(struct device *dev, struct device_attribute *attr, 2713 char *buf) 2714 { 2715 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2716 return uid_valid(tun->owner)? 2717 sysfs_emit(buf, "%u\n", 2718 from_kuid_munged(current_user_ns(), tun->owner)) : 2719 sysfs_emit(buf, "-1\n"); 2720 } 2721 2722 static ssize_t group_show(struct device *dev, struct device_attribute *attr, 2723 char *buf) 2724 { 2725 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2726 return gid_valid(tun->group) ? 2727 sysfs_emit(buf, "%u\n", 2728 from_kgid_munged(current_user_ns(), tun->group)) : 2729 sysfs_emit(buf, "-1\n"); 2730 } 2731 2732 static DEVICE_ATTR_RO(tun_flags); 2733 static DEVICE_ATTR_RO(owner); 2734 static DEVICE_ATTR_RO(group); 2735 2736 static struct attribute *tun_dev_attrs[] = { 2737 &dev_attr_tun_flags.attr, 2738 &dev_attr_owner.attr, 2739 &dev_attr_group.attr, 2740 NULL 2741 }; 2742 2743 static const struct attribute_group tun_attr_group = { 2744 .attrs = tun_dev_attrs 2745 }; 2746 2747 static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) 2748 { 2749 struct tun_struct *tun; 2750 struct tun_file *tfile = file->private_data; 2751 struct net_device *dev; 2752 int err; 2753 2754 if (tfile->detached) 2755 return -EINVAL; 2756 2757 if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) { 2758 if (!capable(CAP_NET_ADMIN)) 2759 return -EPERM; 2760 2761 if (!(ifr->ifr_flags & IFF_NAPI) || 2762 (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP) 2763 return -EINVAL; 2764 } 2765 2766 dev = __dev_get_by_name(net, ifr->ifr_name); 2767 if (dev) { 2768 if (ifr->ifr_flags & IFF_TUN_EXCL) 2769 return -EBUSY; 2770 if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops) 2771 tun = netdev_priv(dev); 2772 else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops) 2773 tun = netdev_priv(dev); 2774 else 2775 return -EINVAL; 2776 2777 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) != 2778 !!(tun->flags & IFF_MULTI_QUEUE)) 2779 return -EINVAL; 2780 2781 if (tun_not_capable(tun)) 2782 return -EPERM; 2783 err = security_tun_dev_open(tun->security); 2784 if (err < 0) 2785 return err; 2786 2787 err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER, 2788 ifr->ifr_flags & IFF_NAPI, 2789 ifr->ifr_flags & IFF_NAPI_FRAGS, true); 2790 if (err < 0) 2791 return err; 2792 2793 if (tun->flags & IFF_MULTI_QUEUE && 2794 (tun->numqueues + tun->numdisabled > 1)) { 2795 /* One or more queue has already been attached, no need 2796 * to initialize the device again. 2797 */ 2798 netdev_state_change(dev); 2799 return 0; 2800 } 2801 2802 tun->flags = (tun->flags & ~TUN_FEATURES) | 2803 (ifr->ifr_flags & TUN_FEATURES); 2804 2805 netdev_state_change(dev); 2806 } else { 2807 char *name; 2808 unsigned long flags = 0; 2809 int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ? 2810 MAX_TAP_QUEUES : 1; 2811 2812 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 2813 return -EPERM; 2814 err = security_tun_dev_create(); 2815 if (err < 0) 2816 return err; 2817 2818 /* Set dev type */ 2819 if (ifr->ifr_flags & IFF_TUN) { 2820 /* TUN device */ 2821 flags |= IFF_TUN; 2822 name = "tun%d"; 2823 } else if (ifr->ifr_flags & IFF_TAP) { 2824 /* TAP device */ 2825 flags |= IFF_TAP; 2826 name = "tap%d"; 2827 } else 2828 return -EINVAL; 2829 2830 if (*ifr->ifr_name) 2831 name = ifr->ifr_name; 2832 2833 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name, 2834 NET_NAME_UNKNOWN, tun_setup, queues, 2835 queues); 2836 2837 if (!dev) 2838 return -ENOMEM; 2839 2840 dev_net_set(dev, net); 2841 dev->rtnl_link_ops = &tun_link_ops; 2842 dev->ifindex = tfile->ifindex; 2843 dev->sysfs_groups[0] = &tun_attr_group; 2844 2845 tun = netdev_priv(dev); 2846 tun->dev = dev; 2847 tun->flags = flags; 2848 tun->txflt.count = 0; 2849 tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr); 2850 2851 tun->align = NET_SKB_PAD; 2852 tun->filter_attached = false; 2853 tun->sndbuf = tfile->socket.sk->sk_sndbuf; 2854 tun->rx_batched = 0; 2855 RCU_INIT_POINTER(tun->steering_prog, NULL); 2856 2857 tun->ifr = ifr; 2858 tun->file = file; 2859 2860 tun_net_initialize(dev); 2861 2862 err = register_netdevice(tun->dev); 2863 if (err < 0) { 2864 free_netdev(dev); 2865 return err; 2866 } 2867 /* free_netdev() won't check refcnt, to avoid race 2868 * with dev_put() we need publish tun after registration. 2869 */ 2870 rcu_assign_pointer(tfile->tun, tun); 2871 } 2872 2873 if (ifr->ifr_flags & IFF_NO_CARRIER) 2874 netif_carrier_off(tun->dev); 2875 else 2876 netif_carrier_on(tun->dev); 2877 2878 /* Make sure persistent devices do not get stuck in 2879 * xoff state. 2880 */ 2881 if (netif_running(tun->dev)) 2882 netif_tx_wake_all_queues(tun->dev); 2883 2884 strcpy(ifr->ifr_name, tun->dev->name); 2885 return 0; 2886 } 2887 2888 static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr) 2889 { 2890 strcpy(ifr->ifr_name, tun->dev->name); 2891 2892 ifr->ifr_flags = tun_flags(tun); 2893 2894 } 2895 2896 /* This is like a cut-down ethtool ops, except done via tun fd so no 2897 * privs required. */ 2898 static int set_offload(struct tun_struct *tun, unsigned long arg) 2899 { 2900 netdev_features_t features = 0; 2901 2902 if (arg & TUN_F_CSUM) { 2903 features |= NETIF_F_HW_CSUM; 2904 arg &= ~TUN_F_CSUM; 2905 2906 if (arg & (TUN_F_TSO4|TUN_F_TSO6)) { 2907 if (arg & TUN_F_TSO_ECN) { 2908 features |= NETIF_F_TSO_ECN; 2909 arg &= ~TUN_F_TSO_ECN; 2910 } 2911 if (arg & TUN_F_TSO4) 2912 features |= NETIF_F_TSO; 2913 if (arg & TUN_F_TSO6) 2914 features |= NETIF_F_TSO6; 2915 arg &= ~(TUN_F_TSO4|TUN_F_TSO6); 2916 } 2917 2918 arg &= ~TUN_F_UFO; 2919 2920 /* TODO: for now USO4 and USO6 should work simultaneously */ 2921 if (arg & TUN_F_USO4 && arg & TUN_F_USO6) { 2922 features |= NETIF_F_GSO_UDP_L4; 2923 arg &= ~(TUN_F_USO4 | TUN_F_USO6); 2924 } 2925 } 2926 2927 /* This gives the user a way to test for new features in future by 2928 * trying to set them. */ 2929 if (arg) 2930 return -EINVAL; 2931 2932 tun->set_features = features; 2933 tun->dev->wanted_features &= ~TUN_USER_FEATURES; 2934 tun->dev->wanted_features |= features; 2935 netdev_update_features(tun->dev); 2936 2937 return 0; 2938 } 2939 2940 static void tun_detach_filter(struct tun_struct *tun, int n) 2941 { 2942 int i; 2943 struct tun_file *tfile; 2944 2945 for (i = 0; i < n; i++) { 2946 tfile = rtnl_dereference(tun->tfiles[i]); 2947 lock_sock(tfile->socket.sk); 2948 sk_detach_filter(tfile->socket.sk); 2949 release_sock(tfile->socket.sk); 2950 } 2951 2952 tun->filter_attached = false; 2953 } 2954 2955 static int tun_attach_filter(struct tun_struct *tun) 2956 { 2957 int i, ret = 0; 2958 struct tun_file *tfile; 2959 2960 for (i = 0; i < tun->numqueues; i++) { 2961 tfile = rtnl_dereference(tun->tfiles[i]); 2962 lock_sock(tfile->socket.sk); 2963 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); 2964 release_sock(tfile->socket.sk); 2965 if (ret) { 2966 tun_detach_filter(tun, i); 2967 return ret; 2968 } 2969 } 2970 2971 tun->filter_attached = true; 2972 return ret; 2973 } 2974 2975 static void tun_set_sndbuf(struct tun_struct *tun) 2976 { 2977 struct tun_file *tfile; 2978 int i; 2979 2980 for (i = 0; i < tun->numqueues; i++) { 2981 tfile = rtnl_dereference(tun->tfiles[i]); 2982 tfile->socket.sk->sk_sndbuf = tun->sndbuf; 2983 } 2984 } 2985 2986 static int tun_set_queue(struct file *file, struct ifreq *ifr) 2987 { 2988 struct tun_file *tfile = file->private_data; 2989 struct tun_struct *tun; 2990 int ret = 0; 2991 2992 rtnl_lock(); 2993 2994 if (ifr->ifr_flags & IFF_ATTACH_QUEUE) { 2995 tun = tfile->detached; 2996 if (!tun) { 2997 ret = -EINVAL; 2998 goto unlock; 2999 } 3000 ret = security_tun_dev_attach_queue(tun->security); 3001 if (ret < 0) 3002 goto unlock; 3003 ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI, 3004 tun->flags & IFF_NAPI_FRAGS, true); 3005 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) { 3006 tun = rtnl_dereference(tfile->tun); 3007 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached) 3008 ret = -EINVAL; 3009 else 3010 __tun_detach(tfile, false); 3011 } else 3012 ret = -EINVAL; 3013 3014 if (ret >= 0) 3015 netdev_state_change(tun->dev); 3016 3017 unlock: 3018 rtnl_unlock(); 3019 return ret; 3020 } 3021 3022 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p, 3023 void __user *data) 3024 { 3025 struct bpf_prog *prog; 3026 int fd; 3027 3028 if (copy_from_user(&fd, data, sizeof(fd))) 3029 return -EFAULT; 3030 3031 if (fd == -1) { 3032 prog = NULL; 3033 } else { 3034 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER); 3035 if (IS_ERR(prog)) 3036 return PTR_ERR(prog); 3037 } 3038 3039 return __tun_set_ebpf(tun, prog_p, prog); 3040 } 3041 3042 /* Return correct value for tun->dev->addr_len based on tun->dev->type. */ 3043 static unsigned char tun_get_addr_len(unsigned short type) 3044 { 3045 switch (type) { 3046 case ARPHRD_IP6GRE: 3047 case ARPHRD_TUNNEL6: 3048 return sizeof(struct in6_addr); 3049 case ARPHRD_IPGRE: 3050 case ARPHRD_TUNNEL: 3051 case ARPHRD_SIT: 3052 return 4; 3053 case ARPHRD_ETHER: 3054 return ETH_ALEN; 3055 case ARPHRD_IEEE802154: 3056 case ARPHRD_IEEE802154_MONITOR: 3057 return IEEE802154_EXTENDED_ADDR_LEN; 3058 case ARPHRD_PHONET_PIPE: 3059 case ARPHRD_PPP: 3060 case ARPHRD_NONE: 3061 return 0; 3062 case ARPHRD_6LOWPAN: 3063 return EUI64_ADDR_LEN; 3064 case ARPHRD_FDDI: 3065 return FDDI_K_ALEN; 3066 case ARPHRD_HIPPI: 3067 return HIPPI_ALEN; 3068 case ARPHRD_IEEE802: 3069 return FC_ALEN; 3070 case ARPHRD_ROSE: 3071 return ROSE_ADDR_LEN; 3072 case ARPHRD_NETROM: 3073 return AX25_ADDR_LEN; 3074 case ARPHRD_LOCALTLK: 3075 return LTALK_ALEN; 3076 default: 3077 return 0; 3078 } 3079 } 3080 3081 static long __tun_chr_ioctl(struct file *file, unsigned int cmd, 3082 unsigned long arg, int ifreq_len) 3083 { 3084 struct tun_file *tfile = file->private_data; 3085 struct net *net = sock_net(&tfile->sk); 3086 struct tun_struct *tun; 3087 void __user* argp = (void __user*)arg; 3088 unsigned int carrier; 3089 struct ifreq ifr; 3090 kuid_t owner; 3091 kgid_t group; 3092 int ifindex; 3093 int sndbuf; 3094 int vnet_hdr_sz; 3095 int le; 3096 int ret; 3097 bool do_notify = false; 3098 3099 if (cmd == TUNSETIFF || cmd == TUNSETQUEUE || 3100 (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) { 3101 if (copy_from_user(&ifr, argp, ifreq_len)) 3102 return -EFAULT; 3103 } else { 3104 memset(&ifr, 0, sizeof(ifr)); 3105 } 3106 if (cmd == TUNGETFEATURES) { 3107 /* Currently this just means: "what IFF flags are valid?". 3108 * This is needed because we never checked for invalid flags on 3109 * TUNSETIFF. 3110 */ 3111 return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER | 3112 TUN_FEATURES, (unsigned int __user*)argp); 3113 } else if (cmd == TUNSETQUEUE) { 3114 return tun_set_queue(file, &ifr); 3115 } else if (cmd == SIOCGSKNS) { 3116 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 3117 return -EPERM; 3118 return open_related_ns(&net->ns, get_net_ns); 3119 } 3120 3121 rtnl_lock(); 3122 3123 tun = tun_get(tfile); 3124 if (cmd == TUNSETIFF) { 3125 ret = -EEXIST; 3126 if (tun) 3127 goto unlock; 3128 3129 ifr.ifr_name[IFNAMSIZ-1] = '\0'; 3130 3131 ret = tun_set_iff(net, file, &ifr); 3132 3133 if (ret) 3134 goto unlock; 3135 3136 if (copy_to_user(argp, &ifr, ifreq_len)) 3137 ret = -EFAULT; 3138 goto unlock; 3139 } 3140 if (cmd == TUNSETIFINDEX) { 3141 ret = -EPERM; 3142 if (tun) 3143 goto unlock; 3144 3145 ret = -EFAULT; 3146 if (copy_from_user(&ifindex, argp, sizeof(ifindex))) 3147 goto unlock; 3148 ret = -EINVAL; 3149 if (ifindex < 0) 3150 goto unlock; 3151 ret = 0; 3152 tfile->ifindex = ifindex; 3153 goto unlock; 3154 } 3155 3156 ret = -EBADFD; 3157 if (!tun) 3158 goto unlock; 3159 3160 netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd); 3161 3162 net = dev_net(tun->dev); 3163 ret = 0; 3164 switch (cmd) { 3165 case TUNGETIFF: 3166 tun_get_iff(tun, &ifr); 3167 3168 if (tfile->detached) 3169 ifr.ifr_flags |= IFF_DETACH_QUEUE; 3170 if (!tfile->socket.sk->sk_filter) 3171 ifr.ifr_flags |= IFF_NOFILTER; 3172 3173 if (copy_to_user(argp, &ifr, ifreq_len)) 3174 ret = -EFAULT; 3175 break; 3176 3177 case TUNSETNOCSUM: 3178 /* Disable/Enable checksum */ 3179 3180 /* [unimplemented] */ 3181 netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n", 3182 arg ? "disabled" : "enabled"); 3183 break; 3184 3185 case TUNSETPERSIST: 3186 /* Disable/Enable persist mode. Keep an extra reference to the 3187 * module to prevent the module being unprobed. 3188 */ 3189 if (arg && !(tun->flags & IFF_PERSIST)) { 3190 tun->flags |= IFF_PERSIST; 3191 __module_get(THIS_MODULE); 3192 do_notify = true; 3193 } 3194 if (!arg && (tun->flags & IFF_PERSIST)) { 3195 tun->flags &= ~IFF_PERSIST; 3196 module_put(THIS_MODULE); 3197 do_notify = true; 3198 } 3199 3200 netif_info(tun, drv, tun->dev, "persist %s\n", 3201 arg ? "enabled" : "disabled"); 3202 break; 3203 3204 case TUNSETOWNER: 3205 /* Set owner of the device */ 3206 owner = make_kuid(current_user_ns(), arg); 3207 if (!uid_valid(owner)) { 3208 ret = -EINVAL; 3209 break; 3210 } 3211 tun->owner = owner; 3212 do_notify = true; 3213 netif_info(tun, drv, tun->dev, "owner set to %u\n", 3214 from_kuid(&init_user_ns, tun->owner)); 3215 break; 3216 3217 case TUNSETGROUP: 3218 /* Set group of the device */ 3219 group = make_kgid(current_user_ns(), arg); 3220 if (!gid_valid(group)) { 3221 ret = -EINVAL; 3222 break; 3223 } 3224 tun->group = group; 3225 do_notify = true; 3226 netif_info(tun, drv, tun->dev, "group set to %u\n", 3227 from_kgid(&init_user_ns, tun->group)); 3228 break; 3229 3230 case TUNSETLINK: 3231 /* Only allow setting the type when the interface is down */ 3232 if (tun->dev->flags & IFF_UP) { 3233 netif_info(tun, drv, tun->dev, 3234 "Linktype set failed because interface is up\n"); 3235 ret = -EBUSY; 3236 } else { 3237 ret = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE, 3238 tun->dev); 3239 ret = notifier_to_errno(ret); 3240 if (ret) { 3241 netif_info(tun, drv, tun->dev, 3242 "Refused to change device type\n"); 3243 break; 3244 } 3245 tun->dev->type = (int) arg; 3246 tun->dev->addr_len = tun_get_addr_len(tun->dev->type); 3247 netif_info(tun, drv, tun->dev, "linktype set to %d\n", 3248 tun->dev->type); 3249 call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE, 3250 tun->dev); 3251 } 3252 break; 3253 3254 case TUNSETDEBUG: 3255 tun->msg_enable = (u32)arg; 3256 break; 3257 3258 case TUNSETOFFLOAD: 3259 ret = set_offload(tun, arg); 3260 break; 3261 3262 case TUNSETTXFILTER: 3263 /* Can be set only for TAPs */ 3264 ret = -EINVAL; 3265 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3266 break; 3267 ret = update_filter(&tun->txflt, (void __user *)arg); 3268 break; 3269 3270 case SIOCGIFHWADDR: 3271 /* Get hw address */ 3272 dev_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name); 3273 if (copy_to_user(argp, &ifr, ifreq_len)) 3274 ret = -EFAULT; 3275 break; 3276 3277 case SIOCSIFHWADDR: 3278 /* Set hw address */ 3279 ret = dev_set_mac_address_user(tun->dev, &ifr.ifr_hwaddr, NULL); 3280 break; 3281 3282 case TUNGETSNDBUF: 3283 sndbuf = tfile->socket.sk->sk_sndbuf; 3284 if (copy_to_user(argp, &sndbuf, sizeof(sndbuf))) 3285 ret = -EFAULT; 3286 break; 3287 3288 case TUNSETSNDBUF: 3289 if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) { 3290 ret = -EFAULT; 3291 break; 3292 } 3293 if (sndbuf <= 0) { 3294 ret = -EINVAL; 3295 break; 3296 } 3297 3298 tun->sndbuf = sndbuf; 3299 tun_set_sndbuf(tun); 3300 break; 3301 3302 case TUNGETVNETHDRSZ: 3303 vnet_hdr_sz = tun->vnet_hdr_sz; 3304 if (copy_to_user(argp, &vnet_hdr_sz, sizeof(vnet_hdr_sz))) 3305 ret = -EFAULT; 3306 break; 3307 3308 case TUNSETVNETHDRSZ: 3309 if (copy_from_user(&vnet_hdr_sz, argp, sizeof(vnet_hdr_sz))) { 3310 ret = -EFAULT; 3311 break; 3312 } 3313 if (vnet_hdr_sz < (int)sizeof(struct virtio_net_hdr)) { 3314 ret = -EINVAL; 3315 break; 3316 } 3317 3318 tun->vnet_hdr_sz = vnet_hdr_sz; 3319 break; 3320 3321 case TUNGETVNETLE: 3322 le = !!(tun->flags & TUN_VNET_LE); 3323 if (put_user(le, (int __user *)argp)) 3324 ret = -EFAULT; 3325 break; 3326 3327 case TUNSETVNETLE: 3328 if (get_user(le, (int __user *)argp)) { 3329 ret = -EFAULT; 3330 break; 3331 } 3332 if (le) 3333 tun->flags |= TUN_VNET_LE; 3334 else 3335 tun->flags &= ~TUN_VNET_LE; 3336 break; 3337 3338 case TUNGETVNETBE: 3339 ret = tun_get_vnet_be(tun, argp); 3340 break; 3341 3342 case TUNSETVNETBE: 3343 ret = tun_set_vnet_be(tun, argp); 3344 break; 3345 3346 case TUNATTACHFILTER: 3347 /* Can be set only for TAPs */ 3348 ret = -EINVAL; 3349 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3350 break; 3351 ret = -EFAULT; 3352 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog))) 3353 break; 3354 3355 ret = tun_attach_filter(tun); 3356 break; 3357 3358 case TUNDETACHFILTER: 3359 /* Can be set only for TAPs */ 3360 ret = -EINVAL; 3361 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3362 break; 3363 ret = 0; 3364 tun_detach_filter(tun, tun->numqueues); 3365 break; 3366 3367 case TUNGETFILTER: 3368 ret = -EINVAL; 3369 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3370 break; 3371 ret = -EFAULT; 3372 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog))) 3373 break; 3374 ret = 0; 3375 break; 3376 3377 case TUNSETSTEERINGEBPF: 3378 ret = tun_set_ebpf(tun, &tun->steering_prog, argp); 3379 break; 3380 3381 case TUNSETFILTEREBPF: 3382 ret = tun_set_ebpf(tun, &tun->filter_prog, argp); 3383 break; 3384 3385 case TUNSETCARRIER: 3386 ret = -EFAULT; 3387 if (copy_from_user(&carrier, argp, sizeof(carrier))) 3388 goto unlock; 3389 3390 ret = tun_net_change_carrier(tun->dev, (bool)carrier); 3391 break; 3392 3393 case TUNGETDEVNETNS: 3394 ret = -EPERM; 3395 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 3396 goto unlock; 3397 ret = open_related_ns(&net->ns, get_net_ns); 3398 break; 3399 3400 default: 3401 ret = -EINVAL; 3402 break; 3403 } 3404 3405 if (do_notify) 3406 netdev_state_change(tun->dev); 3407 3408 unlock: 3409 rtnl_unlock(); 3410 if (tun) 3411 tun_put(tun); 3412 return ret; 3413 } 3414 3415 static long tun_chr_ioctl(struct file *file, 3416 unsigned int cmd, unsigned long arg) 3417 { 3418 return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq)); 3419 } 3420 3421 #ifdef CONFIG_COMPAT 3422 static long tun_chr_compat_ioctl(struct file *file, 3423 unsigned int cmd, unsigned long arg) 3424 { 3425 switch (cmd) { 3426 case TUNSETIFF: 3427 case TUNGETIFF: 3428 case TUNSETTXFILTER: 3429 case TUNGETSNDBUF: 3430 case TUNSETSNDBUF: 3431 case SIOCGIFHWADDR: 3432 case SIOCSIFHWADDR: 3433 arg = (unsigned long)compat_ptr(arg); 3434 break; 3435 default: 3436 arg = (compat_ulong_t)arg; 3437 break; 3438 } 3439 3440 /* 3441 * compat_ifreq is shorter than ifreq, so we must not access beyond 3442 * the end of that structure. All fields that are used in this 3443 * driver are compatible though, we don't need to convert the 3444 * contents. 3445 */ 3446 return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq)); 3447 } 3448 #endif /* CONFIG_COMPAT */ 3449 3450 static int tun_chr_fasync(int fd, struct file *file, int on) 3451 { 3452 struct tun_file *tfile = file->private_data; 3453 int ret; 3454 3455 if (on) { 3456 ret = file_f_owner_allocate(file); 3457 if (ret) 3458 goto out; 3459 } 3460 3461 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0) 3462 goto out; 3463 3464 if (on) { 3465 __f_setown(file, task_pid(current), PIDTYPE_TGID, 0); 3466 tfile->flags |= TUN_FASYNC; 3467 } else 3468 tfile->flags &= ~TUN_FASYNC; 3469 ret = 0; 3470 out: 3471 return ret; 3472 } 3473 3474 static int tun_chr_open(struct inode *inode, struct file * file) 3475 { 3476 struct net *net = current->nsproxy->net_ns; 3477 struct tun_file *tfile; 3478 3479 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, 3480 &tun_proto, 0); 3481 if (!tfile) 3482 return -ENOMEM; 3483 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) { 3484 sk_free(&tfile->sk); 3485 return -ENOMEM; 3486 } 3487 3488 mutex_init(&tfile->napi_mutex); 3489 RCU_INIT_POINTER(tfile->tun, NULL); 3490 tfile->flags = 0; 3491 tfile->ifindex = 0; 3492 3493 init_waitqueue_head(&tfile->socket.wq.wait); 3494 3495 tfile->socket.file = file; 3496 tfile->socket.ops = &tun_socket_ops; 3497 3498 sock_init_data_uid(&tfile->socket, &tfile->sk, current_fsuid()); 3499 3500 tfile->sk.sk_write_space = tun_sock_write_space; 3501 tfile->sk.sk_sndbuf = INT_MAX; 3502 3503 file->private_data = tfile; 3504 INIT_LIST_HEAD(&tfile->next); 3505 3506 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY); 3507 3508 /* tun groks IOCB_NOWAIT just fine, mark it as such */ 3509 file->f_mode |= FMODE_NOWAIT; 3510 return 0; 3511 } 3512 3513 static int tun_chr_close(struct inode *inode, struct file *file) 3514 { 3515 struct tun_file *tfile = file->private_data; 3516 3517 tun_detach(tfile, true); 3518 3519 return 0; 3520 } 3521 3522 #ifdef CONFIG_PROC_FS 3523 static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file) 3524 { 3525 struct tun_file *tfile = file->private_data; 3526 struct tun_struct *tun; 3527 struct ifreq ifr; 3528 3529 memset(&ifr, 0, sizeof(ifr)); 3530 3531 rtnl_lock(); 3532 tun = tun_get(tfile); 3533 if (tun) 3534 tun_get_iff(tun, &ifr); 3535 rtnl_unlock(); 3536 3537 if (tun) 3538 tun_put(tun); 3539 3540 seq_printf(m, "iff:\t%s\n", ifr.ifr_name); 3541 } 3542 #endif 3543 3544 static const struct file_operations tun_fops = { 3545 .owner = THIS_MODULE, 3546 .read_iter = tun_chr_read_iter, 3547 .write_iter = tun_chr_write_iter, 3548 .poll = tun_chr_poll, 3549 .unlocked_ioctl = tun_chr_ioctl, 3550 #ifdef CONFIG_COMPAT 3551 .compat_ioctl = tun_chr_compat_ioctl, 3552 #endif 3553 .open = tun_chr_open, 3554 .release = tun_chr_close, 3555 .fasync = tun_chr_fasync, 3556 #ifdef CONFIG_PROC_FS 3557 .show_fdinfo = tun_chr_show_fdinfo, 3558 #endif 3559 }; 3560 3561 static struct miscdevice tun_miscdev = { 3562 .minor = TUN_MINOR, 3563 .name = "tun", 3564 .nodename = "net/tun", 3565 .fops = &tun_fops, 3566 }; 3567 3568 /* ethtool interface */ 3569 3570 static void tun_default_link_ksettings(struct net_device *dev, 3571 struct ethtool_link_ksettings *cmd) 3572 { 3573 ethtool_link_ksettings_zero_link_mode(cmd, supported); 3574 ethtool_link_ksettings_zero_link_mode(cmd, advertising); 3575 cmd->base.speed = SPEED_10000; 3576 cmd->base.duplex = DUPLEX_FULL; 3577 cmd->base.port = PORT_TP; 3578 cmd->base.phy_address = 0; 3579 cmd->base.autoneg = AUTONEG_DISABLE; 3580 } 3581 3582 static int tun_get_link_ksettings(struct net_device *dev, 3583 struct ethtool_link_ksettings *cmd) 3584 { 3585 struct tun_struct *tun = netdev_priv(dev); 3586 3587 memcpy(cmd, &tun->link_ksettings, sizeof(*cmd)); 3588 return 0; 3589 } 3590 3591 static int tun_set_link_ksettings(struct net_device *dev, 3592 const struct ethtool_link_ksettings *cmd) 3593 { 3594 struct tun_struct *tun = netdev_priv(dev); 3595 3596 memcpy(&tun->link_ksettings, cmd, sizeof(*cmd)); 3597 return 0; 3598 } 3599 3600 static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 3601 { 3602 struct tun_struct *tun = netdev_priv(dev); 3603 3604 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 3605 strscpy(info->version, DRV_VERSION, sizeof(info->version)); 3606 3607 switch (tun->flags & TUN_TYPE_MASK) { 3608 case IFF_TUN: 3609 strscpy(info->bus_info, "tun", sizeof(info->bus_info)); 3610 break; 3611 case IFF_TAP: 3612 strscpy(info->bus_info, "tap", sizeof(info->bus_info)); 3613 break; 3614 } 3615 } 3616 3617 static u32 tun_get_msglevel(struct net_device *dev) 3618 { 3619 struct tun_struct *tun = netdev_priv(dev); 3620 3621 return tun->msg_enable; 3622 } 3623 3624 static void tun_set_msglevel(struct net_device *dev, u32 value) 3625 { 3626 struct tun_struct *tun = netdev_priv(dev); 3627 3628 tun->msg_enable = value; 3629 } 3630 3631 static int tun_get_coalesce(struct net_device *dev, 3632 struct ethtool_coalesce *ec, 3633 struct kernel_ethtool_coalesce *kernel_coal, 3634 struct netlink_ext_ack *extack) 3635 { 3636 struct tun_struct *tun = netdev_priv(dev); 3637 3638 ec->rx_max_coalesced_frames = tun->rx_batched; 3639 3640 return 0; 3641 } 3642 3643 static int tun_set_coalesce(struct net_device *dev, 3644 struct ethtool_coalesce *ec, 3645 struct kernel_ethtool_coalesce *kernel_coal, 3646 struct netlink_ext_ack *extack) 3647 { 3648 struct tun_struct *tun = netdev_priv(dev); 3649 3650 if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT) 3651 tun->rx_batched = NAPI_POLL_WEIGHT; 3652 else 3653 tun->rx_batched = ec->rx_max_coalesced_frames; 3654 3655 return 0; 3656 } 3657 3658 static void tun_get_channels(struct net_device *dev, 3659 struct ethtool_channels *channels) 3660 { 3661 struct tun_struct *tun = netdev_priv(dev); 3662 3663 channels->combined_count = tun->numqueues; 3664 channels->max_combined = tun->flags & IFF_MULTI_QUEUE ? MAX_TAP_QUEUES : 1; 3665 } 3666 3667 static const struct ethtool_ops tun_ethtool_ops = { 3668 .supported_coalesce_params = ETHTOOL_COALESCE_RX_MAX_FRAMES, 3669 .get_drvinfo = tun_get_drvinfo, 3670 .get_msglevel = tun_get_msglevel, 3671 .set_msglevel = tun_set_msglevel, 3672 .get_link = ethtool_op_get_link, 3673 .get_channels = tun_get_channels, 3674 .get_ts_info = ethtool_op_get_ts_info, 3675 .get_coalesce = tun_get_coalesce, 3676 .set_coalesce = tun_set_coalesce, 3677 .get_link_ksettings = tun_get_link_ksettings, 3678 .set_link_ksettings = tun_set_link_ksettings, 3679 }; 3680 3681 static int tun_queue_resize(struct tun_struct *tun) 3682 { 3683 struct net_device *dev = tun->dev; 3684 struct tun_file *tfile; 3685 struct ptr_ring **rings; 3686 int n = tun->numqueues + tun->numdisabled; 3687 int ret, i; 3688 3689 rings = kmalloc_array(n, sizeof(*rings), GFP_KERNEL); 3690 if (!rings) 3691 return -ENOMEM; 3692 3693 for (i = 0; i < tun->numqueues; i++) { 3694 tfile = rtnl_dereference(tun->tfiles[i]); 3695 rings[i] = &tfile->tx_ring; 3696 } 3697 list_for_each_entry(tfile, &tun->disabled, next) 3698 rings[i++] = &tfile->tx_ring; 3699 3700 ret = ptr_ring_resize_multiple(rings, n, 3701 dev->tx_queue_len, GFP_KERNEL, 3702 tun_ptr_free); 3703 3704 kfree(rings); 3705 return ret; 3706 } 3707 3708 static int tun_device_event(struct notifier_block *unused, 3709 unsigned long event, void *ptr) 3710 { 3711 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 3712 struct tun_struct *tun = netdev_priv(dev); 3713 int i; 3714 3715 if (dev->rtnl_link_ops != &tun_link_ops) 3716 return NOTIFY_DONE; 3717 3718 switch (event) { 3719 case NETDEV_CHANGE_TX_QUEUE_LEN: 3720 if (tun_queue_resize(tun)) 3721 return NOTIFY_BAD; 3722 break; 3723 case NETDEV_UP: 3724 for (i = 0; i < tun->numqueues; i++) { 3725 struct tun_file *tfile; 3726 3727 tfile = rtnl_dereference(tun->tfiles[i]); 3728 tfile->socket.sk->sk_write_space(tfile->socket.sk); 3729 } 3730 break; 3731 default: 3732 break; 3733 } 3734 3735 return NOTIFY_DONE; 3736 } 3737 3738 static struct notifier_block tun_notifier_block __read_mostly = { 3739 .notifier_call = tun_device_event, 3740 }; 3741 3742 static int __init tun_init(void) 3743 { 3744 int ret = 0; 3745 3746 pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION); 3747 3748 ret = rtnl_link_register(&tun_link_ops); 3749 if (ret) { 3750 pr_err("Can't register link_ops\n"); 3751 goto err_linkops; 3752 } 3753 3754 ret = misc_register(&tun_miscdev); 3755 if (ret) { 3756 pr_err("Can't register misc device %d\n", TUN_MINOR); 3757 goto err_misc; 3758 } 3759 3760 ret = register_netdevice_notifier(&tun_notifier_block); 3761 if (ret) { 3762 pr_err("Can't register netdevice notifier\n"); 3763 goto err_notifier; 3764 } 3765 3766 return 0; 3767 3768 err_notifier: 3769 misc_deregister(&tun_miscdev); 3770 err_misc: 3771 rtnl_link_unregister(&tun_link_ops); 3772 err_linkops: 3773 return ret; 3774 } 3775 3776 static void __exit tun_cleanup(void) 3777 { 3778 misc_deregister(&tun_miscdev); 3779 rtnl_link_unregister(&tun_link_ops); 3780 unregister_netdevice_notifier(&tun_notifier_block); 3781 } 3782 3783 /* Get an underlying socket object from tun file. Returns error unless file is 3784 * attached to a device. The returned object works like a packet socket, it 3785 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for 3786 * holding a reference to the file for as long as the socket is in use. */ 3787 struct socket *tun_get_socket(struct file *file) 3788 { 3789 struct tun_file *tfile; 3790 if (file->f_op != &tun_fops) 3791 return ERR_PTR(-EINVAL); 3792 tfile = file->private_data; 3793 if (!tfile) 3794 return ERR_PTR(-EBADFD); 3795 return &tfile->socket; 3796 } 3797 EXPORT_SYMBOL_GPL(tun_get_socket); 3798 3799 struct ptr_ring *tun_get_tx_ring(struct file *file) 3800 { 3801 struct tun_file *tfile; 3802 3803 if (file->f_op != &tun_fops) 3804 return ERR_PTR(-EINVAL); 3805 tfile = file->private_data; 3806 if (!tfile) 3807 return ERR_PTR(-EBADFD); 3808 return &tfile->tx_ring; 3809 } 3810 EXPORT_SYMBOL_GPL(tun_get_tx_ring); 3811 3812 module_init(tun_init); 3813 module_exit(tun_cleanup); 3814 MODULE_DESCRIPTION(DRV_DESCRIPTION); 3815 MODULE_AUTHOR(DRV_COPYRIGHT); 3816 MODULE_LICENSE("GPL"); 3817 MODULE_ALIAS_MISCDEV(TUN_MINOR); 3818 MODULE_ALIAS("devname:net/tun"); 3819