1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * TUN - Universal TUN/TAP device driver. 4 * Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com> 5 * 6 * $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $ 7 */ 8 9 /* 10 * Changes: 11 * 12 * Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14 13 * Add TUNSETLINK ioctl to set the link encapsulation 14 * 15 * Mark Smith <markzzzsmith@yahoo.com.au> 16 * Use eth_random_addr() for tap MAC address. 17 * 18 * Harald Roelle <harald.roelle@ifi.lmu.de> 2004/04/20 19 * Fixes in packet dropping, queue length setting and queue wakeup. 20 * Increased default tx queue length. 21 * Added ethtool API. 22 * Minor cleanups 23 * 24 * Daniel Podlejski <underley@underley.eu.org> 25 * Modifications for 2.3.99-pre5 kernel. 26 */ 27 28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 29 30 #define DRV_NAME "tun" 31 #define DRV_VERSION "1.6" 32 #define DRV_DESCRIPTION "Universal TUN/TAP device driver" 33 #define DRV_COPYRIGHT "(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>" 34 35 #include <linux/module.h> 36 #include <linux/errno.h> 37 #include <linux/kernel.h> 38 #include <linux/sched/signal.h> 39 #include <linux/major.h> 40 #include <linux/slab.h> 41 #include <linux/poll.h> 42 #include <linux/fcntl.h> 43 #include <linux/init.h> 44 #include <linux/skbuff.h> 45 #include <linux/netdevice.h> 46 #include <linux/etherdevice.h> 47 #include <linux/miscdevice.h> 48 #include <linux/ethtool.h> 49 #include <linux/rtnetlink.h> 50 #include <linux/compat.h> 51 #include <linux/if.h> 52 #include <linux/if_arp.h> 53 #include <linux/if_ether.h> 54 #include <linux/if_tun.h> 55 #include <linux/if_vlan.h> 56 #include <linux/crc32.h> 57 #include <linux/math.h> 58 #include <linux/nsproxy.h> 59 #include <linux/virtio_net.h> 60 #include <linux/rcupdate.h> 61 #include <net/net_namespace.h> 62 #include <net/netns/generic.h> 63 #include <net/rtnetlink.h> 64 #include <net/sock.h> 65 #include <net/xdp.h> 66 #include <net/ip_tunnels.h> 67 #include <linux/seq_file.h> 68 #include <linux/uio.h> 69 #include <linux/skb_array.h> 70 #include <linux/bpf.h> 71 #include <linux/bpf_trace.h> 72 #include <linux/mutex.h> 73 #include <linux/ieee802154.h> 74 #include <uapi/linux/if_ltalk.h> 75 #include <uapi/linux/if_fddi.h> 76 #include <uapi/linux/if_hippi.h> 77 #include <uapi/linux/if_fc.h> 78 #include <net/ax25.h> 79 #include <net/rose.h> 80 #include <net/6lowpan.h> 81 #include <net/rps.h> 82 83 #include <linux/uaccess.h> 84 #include <linux/proc_fs.h> 85 86 #include "tun_vnet.h" 87 88 static void tun_default_link_ksettings(struct net_device *dev, 89 struct ethtool_link_ksettings *cmd); 90 91 #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD) 92 93 /* TUN device flags */ 94 95 /* IFF_ATTACH_QUEUE is never stored in device flags, 96 * overload it to mean fasync when stored there. 97 */ 98 #define TUN_FASYNC IFF_ATTACH_QUEUE 99 100 #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \ 101 IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS | \ 102 IFF_BACKPRESSURE) 103 104 #define GOODCOPY_LEN 128 105 106 #define FLT_EXACT_COUNT 8 107 struct tap_filter { 108 unsigned int count; /* Number of addrs. Zero means disabled */ 109 u32 mask[2]; /* Mask of the hashed addrs */ 110 unsigned char addr[FLT_EXACT_COUNT][ETH_ALEN]; 111 }; 112 113 /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal 114 * to max number of VCPUs in guest. */ 115 #define MAX_TAP_QUEUES 256 116 #define MAX_TAP_FLOWS 4096 117 118 #define TUN_FLOW_EXPIRE (3 * HZ) 119 120 /* A tun_file connects an open character device to a tuntap netdevice. It 121 * also contains all socket related structures (except sock_fprog and tap_filter) 122 * to serve as one transmit queue for tuntap device. The sock_fprog and 123 * tap_filter were kept in tun_struct since they were used for filtering for the 124 * netdevice not for a specific queue (at least I didn't see the requirement for 125 * this). 126 * 127 * RCU usage: 128 * The tun_file and tun_struct are loosely coupled, the pointer from one to the 129 * other can only be read while rcu_read_lock or rtnl_lock is held. 130 */ 131 struct tun_file { 132 struct sock sk; 133 struct socket socket; 134 struct tun_struct __rcu *tun; 135 struct fasync_struct *fasync; 136 /* only used for fasnyc */ 137 unsigned int flags; 138 union { 139 u16 queue_index; 140 unsigned int ifindex; 141 }; 142 struct napi_struct napi; 143 bool napi_enabled; 144 bool napi_frags_enabled; 145 struct mutex napi_mutex; /* Protects access to the above napi */ 146 struct list_head next; 147 struct tun_struct *detached; 148 struct ptr_ring tx_ring; 149 /* Protected by tx_ring.consumer_lock */ 150 int cons_cnt; 151 struct xdp_rxq_info xdp_rxq; 152 }; 153 154 struct tun_page { 155 struct page *page; 156 int count; 157 }; 158 159 struct tun_flow_entry { 160 struct hlist_node hash_link; 161 struct rcu_head rcu; 162 struct tun_struct *tun; 163 164 u32 rxhash; 165 u32 rps_rxhash; 166 int queue_index; 167 unsigned long updated ____cacheline_aligned_in_smp; 168 }; 169 170 #define TUN_NUM_FLOW_ENTRIES 1024 171 #define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1) 172 173 struct tun_prog { 174 struct rcu_head rcu; 175 struct bpf_prog *prog; 176 }; 177 178 /* Since the socket were moved to tun_file, to preserve the behavior of persist 179 * device, socket filter, sndbuf and vnet header size were restore when the 180 * file were attached to a persist device. 181 */ 182 struct tun_struct { 183 struct tun_file __rcu *tfiles[MAX_TAP_QUEUES]; 184 unsigned int numqueues; 185 unsigned int flags; 186 kuid_t owner; 187 kgid_t group; 188 189 struct net_device *dev; 190 netdev_features_t set_features; 191 #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \ 192 NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4 | \ 193 NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_UDP_TUNNEL_CSUM) 194 195 int align; 196 int vnet_hdr_sz; 197 int sndbuf; 198 struct tap_filter txflt; 199 struct sock_fprog fprog; 200 /* protected by rtnl lock */ 201 bool filter_attached; 202 u32 msg_enable; 203 spinlock_t lock; 204 struct hlist_head flows[TUN_NUM_FLOW_ENTRIES]; 205 struct timer_list flow_gc_timer; 206 unsigned long ageing_time; 207 unsigned int numdisabled; 208 struct list_head disabled; 209 void *security; 210 u32 flow_count; 211 u32 rx_batched; 212 atomic_long_t rx_frame_errors; 213 struct bpf_prog __rcu *xdp_prog; 214 struct tun_prog __rcu *steering_prog; 215 struct tun_prog __rcu *filter_prog; 216 struct ethtool_link_ksettings link_ksettings; 217 /* init args */ 218 struct file *file; 219 struct ifreq *ifr; 220 }; 221 222 struct veth { 223 __be16 h_vlan_proto; 224 __be16 h_vlan_TCI; 225 }; 226 227 static void tun_flow_init(struct tun_struct *tun); 228 static void tun_flow_uninit(struct tun_struct *tun); 229 230 static int tun_napi_receive(struct napi_struct *napi, int budget) 231 { 232 struct tun_file *tfile = container_of(napi, struct tun_file, napi); 233 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 234 struct sk_buff_head process_queue; 235 struct sk_buff *skb; 236 int received = 0; 237 238 __skb_queue_head_init(&process_queue); 239 240 spin_lock(&queue->lock); 241 skb_queue_splice_tail_init(queue, &process_queue); 242 spin_unlock(&queue->lock); 243 244 while (received < budget && (skb = __skb_dequeue(&process_queue))) { 245 napi_gro_receive(napi, skb); 246 ++received; 247 } 248 249 if (!skb_queue_empty(&process_queue)) { 250 spin_lock(&queue->lock); 251 skb_queue_splice(&process_queue, queue); 252 spin_unlock(&queue->lock); 253 } 254 255 return received; 256 } 257 258 static int tun_napi_poll(struct napi_struct *napi, int budget) 259 { 260 unsigned int received; 261 262 received = tun_napi_receive(napi, budget); 263 264 if (received < budget) 265 napi_complete_done(napi, received); 266 267 return received; 268 } 269 270 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile, 271 bool napi_en, bool napi_frags) 272 { 273 tfile->napi_enabled = napi_en; 274 tfile->napi_frags_enabled = napi_en && napi_frags; 275 if (napi_en) { 276 netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll); 277 napi_enable(&tfile->napi); 278 } 279 } 280 281 static void tun_napi_enable(struct tun_file *tfile) 282 { 283 if (tfile->napi_enabled) 284 napi_enable(&tfile->napi); 285 } 286 287 static void tun_napi_disable(struct tun_file *tfile) 288 { 289 if (tfile->napi_enabled) 290 napi_disable(&tfile->napi); 291 } 292 293 static void tun_napi_del(struct tun_file *tfile) 294 { 295 if (tfile->napi_enabled) 296 netif_napi_del(&tfile->napi); 297 } 298 299 static bool tun_napi_frags_enabled(const struct tun_file *tfile) 300 { 301 return tfile->napi_frags_enabled; 302 } 303 304 static inline u32 tun_hashfn(u32 rxhash) 305 { 306 return rxhash & TUN_MASK_FLOW_ENTRIES; 307 } 308 309 static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash) 310 { 311 struct tun_flow_entry *e; 312 313 hlist_for_each_entry_rcu(e, head, hash_link) { 314 if (e->rxhash == rxhash) 315 return e; 316 } 317 return NULL; 318 } 319 320 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun, 321 struct hlist_head *head, 322 u32 rxhash, u16 queue_index) 323 { 324 struct tun_flow_entry *e = kmalloc_obj(*e, GFP_ATOMIC); 325 326 if (e) { 327 netif_info(tun, tx_queued, tun->dev, 328 "create flow: hash %u index %u\n", 329 rxhash, queue_index); 330 e->updated = jiffies; 331 e->rxhash = rxhash; 332 e->rps_rxhash = 0; 333 e->queue_index = queue_index; 334 e->tun = tun; 335 hlist_add_head_rcu(&e->hash_link, head); 336 ++tun->flow_count; 337 } 338 return e; 339 } 340 341 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e) 342 { 343 netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n", 344 e->rxhash, e->queue_index); 345 hlist_del_rcu(&e->hash_link); 346 kfree_rcu(e, rcu); 347 --tun->flow_count; 348 } 349 350 static void tun_flow_flush(struct tun_struct *tun) 351 { 352 int i; 353 354 spin_lock_bh(&tun->lock); 355 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 356 struct tun_flow_entry *e; 357 struct hlist_node *n; 358 359 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) 360 tun_flow_delete(tun, e); 361 } 362 spin_unlock_bh(&tun->lock); 363 } 364 365 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index) 366 { 367 int i; 368 369 spin_lock_bh(&tun->lock); 370 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 371 struct tun_flow_entry *e; 372 struct hlist_node *n; 373 374 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 375 if (e->queue_index == queue_index) 376 tun_flow_delete(tun, e); 377 } 378 } 379 spin_unlock_bh(&tun->lock); 380 } 381 382 static void tun_flow_cleanup(struct timer_list *t) 383 { 384 struct tun_struct *tun = timer_container_of(tun, t, flow_gc_timer); 385 unsigned long delay = tun->ageing_time; 386 unsigned long next_timer = jiffies + delay; 387 unsigned long count = 0; 388 int i; 389 390 spin_lock(&tun->lock); 391 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) { 392 struct tun_flow_entry *e; 393 struct hlist_node *n; 394 395 hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) { 396 unsigned long this_timer; 397 398 this_timer = e->updated + delay; 399 if (time_before_eq(this_timer, jiffies)) { 400 tun_flow_delete(tun, e); 401 continue; 402 } 403 count++; 404 if (time_before(this_timer, next_timer)) 405 next_timer = this_timer; 406 } 407 } 408 409 if (count) 410 mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer)); 411 spin_unlock(&tun->lock); 412 } 413 414 static void tun_flow_update(struct tun_struct *tun, u32 rxhash, 415 struct tun_file *tfile) 416 { 417 struct hlist_head *head; 418 struct tun_flow_entry *e; 419 unsigned long delay = tun->ageing_time; 420 u16 queue_index = tfile->queue_index; 421 422 head = &tun->flows[tun_hashfn(rxhash)]; 423 424 rcu_read_lock(); 425 426 e = tun_flow_find(head, rxhash); 427 if (likely(e)) { 428 /* TODO: keep queueing to old queue until it's empty? */ 429 if (READ_ONCE(e->queue_index) != queue_index) 430 WRITE_ONCE(e->queue_index, queue_index); 431 if (e->updated != jiffies) 432 e->updated = jiffies; 433 sock_rps_record_flow_hash(e->rps_rxhash); 434 } else { 435 spin_lock_bh(&tun->lock); 436 if (!tun_flow_find(head, rxhash) && 437 tun->flow_count < MAX_TAP_FLOWS) 438 tun_flow_create(tun, head, rxhash, queue_index); 439 440 if (!timer_pending(&tun->flow_gc_timer)) 441 mod_timer(&tun->flow_gc_timer, 442 round_jiffies_up(jiffies + delay)); 443 spin_unlock_bh(&tun->lock); 444 } 445 446 rcu_read_unlock(); 447 } 448 449 /* Save the hash received in the stack receive path and update the 450 * flow_hash table accordingly. 451 */ 452 static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash) 453 { 454 if (unlikely(e->rps_rxhash != hash)) 455 e->rps_rxhash = hash; 456 } 457 458 /* We try to identify a flow through its rxhash. The reason that 459 * we do not check rxq no. is because some cards(e.g 82599), chooses 460 * the rxq based on the txq where the last packet of the flow comes. As 461 * the userspace application move between processors, we may get a 462 * different rxq no. here. 463 */ 464 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb) 465 { 466 struct tun_flow_entry *e; 467 u32 txq, numqueues; 468 469 numqueues = READ_ONCE(tun->numqueues); 470 471 txq = __skb_get_hash_symmetric(skb); 472 e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq); 473 if (e) { 474 tun_flow_save_rps_rxhash(e, txq); 475 txq = e->queue_index; 476 } else { 477 txq = reciprocal_scale(txq, numqueues); 478 } 479 480 return txq; 481 } 482 483 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb) 484 { 485 struct tun_prog *prog; 486 u32 numqueues; 487 u16 ret = 0; 488 489 numqueues = READ_ONCE(tun->numqueues); 490 if (!numqueues) 491 return 0; 492 493 prog = rcu_dereference(tun->steering_prog); 494 if (prog) 495 ret = bpf_prog_run_clear_cb(prog->prog, skb); 496 497 return ret % numqueues; 498 } 499 500 static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb, 501 struct net_device *sb_dev) 502 { 503 struct tun_struct *tun = netdev_priv(dev); 504 u16 ret; 505 506 rcu_read_lock(); 507 if (rcu_dereference(tun->steering_prog)) 508 ret = tun_ebpf_select_queue(tun, skb); 509 else 510 ret = tun_automq_select_queue(tun, skb); 511 rcu_read_unlock(); 512 513 return ret; 514 } 515 516 static inline bool tun_permitted(struct tun_struct *tun) 517 { 518 const struct cred *cred = current_cred(); 519 struct net *net = dev_net(tun->dev); 520 521 /* If both uid and gid are set, *both* must match */ 522 if ((uid_valid(tun->owner) || gid_valid(tun->group)) && 523 (!uid_valid(tun->owner) || uid_eq(cred->euid, tun->owner)) && 524 (!gid_valid(tun->group) || in_egroup_p(tun->group))) 525 return true; 526 527 return ns_capable(net->user_ns, CAP_NET_ADMIN); 528 } 529 530 static void tun_set_real_num_queues(struct tun_struct *tun) 531 { 532 netif_set_real_num_tx_queues(tun->dev, tun->numqueues); 533 netif_set_real_num_rx_queues(tun->dev, tun->numqueues); 534 } 535 536 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile) 537 { 538 tfile->detached = tun; 539 list_add_tail(&tfile->next, &tun->disabled); 540 WRITE_ONCE(tun->numdisabled, tun->numdisabled + 1); 541 } 542 543 static struct tun_struct *tun_enable_queue(struct tun_file *tfile) 544 { 545 struct tun_struct *tun = tfile->detached; 546 547 tfile->detached = NULL; 548 list_del_init(&tfile->next); 549 WRITE_ONCE(tun->numdisabled, tun->numdisabled - 1); 550 return tun; 551 } 552 553 void tun_ptr_free(void *ptr) 554 { 555 if (!ptr) 556 return; 557 if (tun_is_xdp_frame(ptr)) { 558 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 559 560 xdp_return_frame(xdpf); 561 } else { 562 __skb_array_destroy_skb(ptr); 563 } 564 } 565 EXPORT_SYMBOL_GPL(tun_ptr_free); 566 567 static void tun_queue_purge(struct tun_file *tfile) 568 { 569 void *ptr; 570 571 while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL) 572 tun_ptr_free(ptr); 573 574 skb_queue_purge(&tfile->sk.sk_write_queue); 575 skb_queue_purge(&tfile->sk.sk_error_queue); 576 } 577 578 static void __tun_detach(struct tun_file *tfile, bool clean) 579 { 580 struct tun_file *ntfile; 581 struct tun_struct *tun; 582 583 tun = rtnl_dereference(tfile->tun); 584 585 if (tun && clean) { 586 if (!tfile->detached) 587 tun_napi_disable(tfile); 588 tun_napi_del(tfile); 589 } 590 591 if (tun && !tfile->detached) { 592 u16 index = tfile->queue_index; 593 BUG_ON(index >= tun->numqueues); 594 595 spin_lock(&tfile->tx_ring.consumer_lock); 596 rcu_assign_pointer(tun->tfiles[index], 597 tun->tfiles[tun->numqueues - 1]); 598 spin_unlock(&tfile->tx_ring.consumer_lock); 599 ntfile = rtnl_dereference(tun->tfiles[index]); 600 spin_lock(&ntfile->tx_ring.consumer_lock); 601 ntfile->queue_index = index; 602 ntfile->xdp_rxq.queue_index = index; 603 ntfile->cons_cnt = 0; 604 spin_unlock(&ntfile->tx_ring.consumer_lock); 605 rcu_assign_pointer(tun->tfiles[tun->numqueues - 1], 606 NULL); 607 608 WRITE_ONCE(tun->numqueues, tun->numqueues - 1); 609 if (clean) { 610 RCU_INIT_POINTER(tfile->tun, NULL); 611 sock_put(&tfile->sk); 612 } else { 613 tun_disable_queue(tun, tfile); 614 tun_napi_disable(tfile); 615 } 616 617 synchronize_net(); 618 tun_flow_delete_by_queue(tun, tun->numqueues); 619 /* Drop read queue */ 620 tun_queue_purge(tfile); 621 spin_lock_bh(&ntfile->tx_ring.consumer_lock); 622 spin_lock(&ntfile->tx_ring.producer_lock); 623 ntfile->cons_cnt = 0; 624 if (netif_running(tun->dev) && 625 __ptr_ring_empty(&ntfile->tx_ring)) 626 netif_wake_subqueue(tun->dev, index); 627 spin_unlock(&ntfile->tx_ring.producer_lock); 628 spin_unlock_bh(&ntfile->tx_ring.consumer_lock); 629 tun_set_real_num_queues(tun); 630 } else if (tfile->detached && clean) { 631 tun = tun_enable_queue(tfile); 632 sock_put(&tfile->sk); 633 } 634 635 if (clean) { 636 if (tun && tun->numqueues == 0 && tun->numdisabled == 0) { 637 netif_carrier_off(tun->dev); 638 639 if (!(tun->flags & IFF_PERSIST) && 640 tun->dev->reg_state == NETREG_REGISTERED) 641 unregister_netdevice(tun->dev); 642 } 643 if (tun) 644 xdp_rxq_info_unreg(&tfile->xdp_rxq); 645 ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free); 646 } 647 } 648 649 static void tun_detach(struct tun_file *tfile, bool clean) 650 { 651 struct tun_struct *tun; 652 struct net_device *dev; 653 654 rtnl_lock(); 655 tun = rtnl_dereference(tfile->tun); 656 dev = tun ? tun->dev : NULL; 657 __tun_detach(tfile, clean); 658 if (dev) 659 netdev_state_change(dev); 660 rtnl_unlock(); 661 662 if (clean) 663 sock_put(&tfile->sk); 664 } 665 666 static void tun_detach_all(struct net_device *dev) 667 { 668 struct tun_struct *tun = netdev_priv(dev); 669 struct tun_file *tfile, *tmp; 670 int i, n = tun->numqueues; 671 672 for (i = 0; i < n; i++) { 673 tfile = rtnl_dereference(tun->tfiles[i]); 674 BUG_ON(!tfile); 675 tun_napi_disable(tfile); 676 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 677 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 678 RCU_INIT_POINTER(tfile->tun, NULL); 679 WRITE_ONCE(tun->numqueues, tun->numqueues - 1); 680 } 681 list_for_each_entry(tfile, &tun->disabled, next) { 682 tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN; 683 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 684 RCU_INIT_POINTER(tfile->tun, NULL); 685 } 686 BUG_ON(tun->numqueues != 0); 687 688 synchronize_net(); 689 for (i = 0; i < n; i++) { 690 tfile = rtnl_dereference(tun->tfiles[i]); 691 mutex_lock(&tfile->napi_mutex); 692 tun_napi_del(tfile); 693 mutex_unlock(&tfile->napi_mutex); 694 /* Drop read queue */ 695 tun_queue_purge(tfile); 696 xdp_rxq_info_unreg(&tfile->xdp_rxq); 697 sock_put(&tfile->sk); 698 } 699 list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) { 700 mutex_lock(&tfile->napi_mutex); 701 tun_napi_del(tfile); 702 mutex_unlock(&tfile->napi_mutex); 703 tun_enable_queue(tfile); 704 tun_queue_purge(tfile); 705 xdp_rxq_info_unreg(&tfile->xdp_rxq); 706 sock_put(&tfile->sk); 707 } 708 BUG_ON(tun->numdisabled != 0); 709 710 if (tun->flags & IFF_PERSIST) 711 module_put(THIS_MODULE); 712 } 713 714 static void tun_force_wake_queue(struct tun_struct *tun, 715 struct tun_file *tfile) 716 { 717 /* Ensure that the producer can not stop the 718 * queue concurrently by taking locks. 719 */ 720 spin_lock_bh(&tfile->tx_ring.consumer_lock); 721 spin_lock(&tfile->tx_ring.producer_lock); 722 tfile->cons_cnt = 0; 723 /* Tested under the locks that tun_net_close() takes, so this can not 724 * undo its stop. tun_net_open() wakes the queues of a device that 725 * comes back up. 726 */ 727 if (netif_running(tun->dev)) 728 netif_wake_subqueue(tun->dev, tfile->queue_index); 729 spin_unlock(&tfile->tx_ring.producer_lock); 730 spin_unlock_bh(&tfile->tx_ring.consumer_lock); 731 } 732 733 static int tun_attach(struct tun_struct *tun, struct file *file, 734 bool skip_filter, bool napi, bool napi_frags, 735 bool publish_tun) 736 { 737 struct tun_file *tfile = file->private_data; 738 struct net_device *dev = tun->dev; 739 int err; 740 741 err = security_tun_dev_attach(tfile->socket.sk, tun->security); 742 if (err < 0) 743 goto out; 744 745 err = -EINVAL; 746 if (rtnl_dereference(tfile->tun) && !tfile->detached) 747 goto out; 748 749 err = -EBUSY; 750 if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1) 751 goto out; 752 753 err = -E2BIG; 754 if (!tfile->detached && 755 tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES) 756 goto out; 757 758 err = 0; 759 760 /* Re-attach the filter to persist device */ 761 if (!skip_filter && (tun->filter_attached == true)) { 762 lock_sock(tfile->socket.sk); 763 err = sk_attach_filter(&tun->fprog, tfile->socket.sk); 764 release_sock(tfile->socket.sk); 765 if (!err) 766 goto out; 767 } 768 769 if (!tfile->detached && 770 ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len, 771 GFP_KERNEL, tun_ptr_free)) { 772 err = -ENOMEM; 773 goto out; 774 } 775 776 spin_lock(&tfile->tx_ring.consumer_lock); 777 tfile->queue_index = tun->numqueues; 778 spin_unlock(&tfile->tx_ring.consumer_lock); 779 tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN; 780 tun_force_wake_queue(tun, tfile); 781 782 if (tfile->detached) { 783 /* Re-attach detached tfile, updating XDP queue_index */ 784 WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq)); 785 786 if (tfile->xdp_rxq.queue_index != tfile->queue_index) 787 tfile->xdp_rxq.queue_index = tfile->queue_index; 788 } else { 789 /* Setup XDP RX-queue info, for new tfile getting attached */ 790 err = xdp_rxq_info_reg(&tfile->xdp_rxq, 791 tun->dev, tfile->queue_index, 0); 792 if (err < 0) 793 goto out; 794 err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq, 795 MEM_TYPE_PAGE_SHARED, NULL); 796 if (err < 0) { 797 xdp_rxq_info_unreg(&tfile->xdp_rxq); 798 goto out; 799 } 800 err = 0; 801 } 802 803 if (tfile->detached) { 804 tun_enable_queue(tfile); 805 tun_napi_enable(tfile); 806 } else { 807 sock_hold(&tfile->sk); 808 tun_napi_init(tun, tfile, napi, napi_frags); 809 } 810 811 if (rtnl_dereference(tun->xdp_prog)) 812 sock_set_flag(&tfile->sk, SOCK_XDP); 813 814 /* device is allowed to go away first, so no need to hold extra 815 * refcnt. 816 */ 817 818 /* Publish tfile->tun and tun->tfiles only after we've fully 819 * initialized tfile; otherwise we risk using half-initialized 820 * object. 821 */ 822 if (publish_tun) 823 rcu_assign_pointer(tfile->tun, tun); 824 rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile); 825 WRITE_ONCE(tun->numqueues, tun->numqueues + 1); 826 tun_set_real_num_queues(tun); 827 out: 828 return err; 829 } 830 831 static struct tun_struct *tun_get(struct tun_file *tfile) 832 { 833 struct tun_struct *tun; 834 835 rcu_read_lock(); 836 tun = rcu_dereference(tfile->tun); 837 if (tun) 838 dev_hold(tun->dev); 839 rcu_read_unlock(); 840 841 return tun; 842 } 843 844 static void tun_put(struct tun_struct *tun) 845 { 846 dev_put(tun->dev); 847 } 848 849 /* TAP filtering */ 850 static void addr_hash_set(u32 *mask, const u8 *addr) 851 { 852 int n = ether_crc(ETH_ALEN, addr) >> 26; 853 mask[n >> 5] |= (1 << (n & 31)); 854 } 855 856 static unsigned int addr_hash_test(const u32 *mask, const u8 *addr) 857 { 858 int n = ether_crc(ETH_ALEN, addr) >> 26; 859 return mask[n >> 5] & (1 << (n & 31)); 860 } 861 862 static int update_filter(struct tap_filter *filter, void __user *arg) 863 { 864 struct { u8 u[ETH_ALEN]; } *addr; 865 struct tun_filter uf; 866 int err, alen, n, nexact; 867 868 if (copy_from_user(&uf, arg, sizeof(uf))) 869 return -EFAULT; 870 871 if (!uf.count) { 872 /* Disabled */ 873 filter->count = 0; 874 return 0; 875 } 876 877 alen = ETH_ALEN * uf.count; 878 addr = memdup_user(arg + sizeof(uf), alen); 879 if (IS_ERR(addr)) 880 return PTR_ERR(addr); 881 882 /* The filter is updated without holding any locks. Which is 883 * perfectly safe. We disable it first and in the worst 884 * case we'll accept a few undesired packets. */ 885 filter->count = 0; 886 wmb(); 887 888 /* Use first set of addresses as an exact filter */ 889 for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++) 890 memcpy(filter->addr[n], addr[n].u, ETH_ALEN); 891 892 nexact = n; 893 894 /* Remaining multicast addresses are hashed, 895 * unicast will leave the filter disabled. */ 896 memset(filter->mask, 0, sizeof(filter->mask)); 897 for (; n < uf.count; n++) { 898 if (!is_multicast_ether_addr(addr[n].u)) { 899 err = 0; /* no filter */ 900 goto free_addr; 901 } 902 addr_hash_set(filter->mask, addr[n].u); 903 } 904 905 /* For ALLMULTI just set the mask to all ones. 906 * This overrides the mask populated above. */ 907 if ((uf.flags & TUN_FLT_ALLMULTI)) 908 memset(filter->mask, ~0, sizeof(filter->mask)); 909 910 /* Now enable the filter */ 911 wmb(); 912 filter->count = nexact; 913 914 /* Return the number of exact filters */ 915 err = nexact; 916 free_addr: 917 kfree(addr); 918 return err; 919 } 920 921 /* Returns: 0 - drop, !=0 - accept */ 922 static int run_filter(struct tap_filter *filter, const struct sk_buff *skb) 923 { 924 /* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect 925 * at this point. */ 926 struct ethhdr *eh = (struct ethhdr *) skb->data; 927 int i; 928 929 /* Exact match */ 930 for (i = 0; i < filter->count; i++) 931 if (ether_addr_equal(eh->h_dest, filter->addr[i])) 932 return 1; 933 934 /* Inexact match (multicast only) */ 935 if (is_multicast_ether_addr(eh->h_dest)) 936 return addr_hash_test(filter->mask, eh->h_dest); 937 938 return 0; 939 } 940 941 /* 942 * Checks whether the packet is accepted or not. 943 * Returns: 0 - drop, !=0 - accept 944 */ 945 static int check_filter(struct tap_filter *filter, const struct sk_buff *skb) 946 { 947 if (!filter->count) 948 return 1; 949 950 return run_filter(filter, skb); 951 } 952 953 /* Network device part of the driver */ 954 955 static const struct ethtool_ops tun_ethtool_ops; 956 957 static int tun_net_init(struct net_device *dev) 958 { 959 struct tun_struct *tun = netdev_priv(dev); 960 struct ifreq *ifr = tun->ifr; 961 int err; 962 963 spin_lock_init(&tun->lock); 964 965 err = security_tun_dev_alloc_security(&tun->security); 966 if (err < 0) 967 return err; 968 969 tun_flow_init(tun); 970 971 dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS; 972 dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST | 973 TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX | 974 NETIF_F_HW_VLAN_STAG_TX; 975 dev->hw_enc_features = dev->hw_features; 976 dev->features = dev->hw_features; 977 dev->vlan_features = dev->features & 978 ~(NETIF_F_HW_VLAN_CTAG_TX | 979 NETIF_F_HW_VLAN_STAG_TX); 980 dev->lltx = true; 981 982 tun->flags = (tun->flags & ~TUN_FEATURES) | 983 (ifr->ifr_flags & TUN_FEATURES); 984 985 INIT_LIST_HEAD(&tun->disabled); 986 err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI, 987 ifr->ifr_flags & IFF_NAPI_FRAGS, false); 988 if (err < 0) { 989 tun_flow_uninit(tun); 990 security_tun_dev_free_security(tun->security); 991 return err; 992 } 993 return 0; 994 } 995 996 /* Net device detach from fd. */ 997 static void tun_net_uninit(struct net_device *dev) 998 { 999 tun_detach_all(dev); 1000 } 1001 1002 /* Net device open. */ 1003 static int tun_net_open(struct net_device *dev) 1004 { 1005 netif_tx_start_all_queues(dev); 1006 1007 return 0; 1008 } 1009 1010 /* Net device close. */ 1011 static int tun_net_close(struct net_device *dev) 1012 { 1013 struct tun_struct *tun = netdev_priv(dev); 1014 struct tun_file *tfile; 1015 int i; 1016 1017 /* netif_running() is already false: take both ring locks to keep the 1018 * wake sites out, so the stop below is the last write to 1019 * __QUEUE_STATE_DRV_XOFF. 1020 */ 1021 for (i = 0; i < tun->numqueues; i++) { 1022 tfile = rtnl_dereference(tun->tfiles[i]); 1023 1024 spin_lock_bh(&tfile->tx_ring.consumer_lock); 1025 spin_lock(&tfile->tx_ring.producer_lock); 1026 tfile->cons_cnt = 0; 1027 spin_unlock(&tfile->tx_ring.producer_lock); 1028 spin_unlock_bh(&tfile->tx_ring.consumer_lock); 1029 } 1030 1031 netif_tx_stop_all_queues(dev); 1032 return 0; 1033 } 1034 1035 /* Net device start xmit */ 1036 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb) 1037 { 1038 #ifdef CONFIG_RPS 1039 if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) { 1040 /* Select queue was not called for the skbuff, so we extract the 1041 * RPS hash and save it into the flow_table here. 1042 */ 1043 struct tun_flow_entry *e; 1044 __u32 rxhash; 1045 1046 rxhash = __skb_get_hash_symmetric(skb); 1047 e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash); 1048 if (e) 1049 tun_flow_save_rps_rxhash(e, rxhash); 1050 } 1051 #endif 1052 } 1053 1054 static unsigned int run_ebpf_filter(struct tun_struct *tun, 1055 struct sk_buff *skb, 1056 int len) 1057 { 1058 struct tun_prog *prog = rcu_dereference(tun->filter_prog); 1059 1060 if (prog) 1061 len = bpf_prog_run_clear_cb(prog->prog, skb); 1062 1063 return len; 1064 } 1065 1066 /* Net device start xmit */ 1067 static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev) 1068 { 1069 enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 1070 struct tun_struct *tun = netdev_priv(dev); 1071 int txq = skb->queue_mapping; 1072 struct netdev_queue *queue; 1073 struct tun_file *tfile; 1074 int len = skb->len; 1075 int ret; 1076 1077 rcu_read_lock(); 1078 tfile = rcu_dereference(tun->tfiles[txq]); 1079 1080 /* Drop packet if interface is not attached */ 1081 if (!tfile) { 1082 drop_reason = SKB_DROP_REASON_DEV_READY; 1083 goto drop; 1084 } 1085 1086 if (!rcu_dereference(tun->steering_prog)) 1087 tun_automq_xmit(tun, skb); 1088 1089 netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len); 1090 1091 /* Drop if the filter does not like it. 1092 * This is a noop if the filter is disabled. 1093 * Filter can be enabled only for the TAP devices. */ 1094 if (!check_filter(&tun->txflt, skb)) { 1095 drop_reason = SKB_DROP_REASON_TAP_TXFILTER; 1096 goto drop; 1097 } 1098 1099 if (tfile->socket.sk->sk_filter) { 1100 drop_reason = sk_filter_reason(tfile->socket.sk, skb); 1101 if (drop_reason) 1102 goto drop; 1103 } 1104 1105 len = run_ebpf_filter(tun, skb, len); 1106 if (len == 0) { 1107 drop_reason = SKB_DROP_REASON_TAP_FILTER; 1108 goto drop; 1109 } 1110 1111 if (pskb_trim(skb, len)) { 1112 drop_reason = SKB_DROP_REASON_NOMEM; 1113 goto drop; 1114 } 1115 1116 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) { 1117 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT; 1118 goto drop; 1119 } 1120 1121 skb_tx_timestamp(skb); 1122 1123 /* Orphan the skb - required as we might hang on to it 1124 * for indefinite time. 1125 */ 1126 skb_orphan(skb); 1127 1128 nf_reset_ct(skb); 1129 1130 queue = netdev_get_tx_queue(dev, txq); 1131 1132 spin_lock(&tfile->tx_ring.producer_lock); 1133 ret = __ptr_ring_produce(&tfile->tx_ring, skb); 1134 /* Do not touch the queue state of a device that is going down. */ 1135 if ((tun->flags & IFF_BACKPRESSURE) && netif_running(dev) && 1136 !qdisc_txq_has_no_queue(queue) && 1137 __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) { 1138 netif_tx_stop_queue(queue); 1139 /* Paired with smp_mb() in __tun_wake_queue() */ 1140 smp_mb__after_atomic(); 1141 if (!__ptr_ring_check_produce(&tfile->tx_ring)) 1142 netif_tx_wake_queue(queue); 1143 } 1144 spin_unlock(&tfile->tx_ring.producer_lock); 1145 1146 if (ret) { 1147 /* This should be a rare case if IFF_BACKPRESSURE is enabled and 1148 * a qdisc is present, but can happen due to lltx. 1149 * Since skb_tx_timestamp(), skb_orphan(), 1150 * run_ebpf_filter() and pskb_trim() could have tinkered 1151 * with the SKB, returning NETDEV_TX_BUSY is unsafe and 1152 * we must drop instead. 1153 */ 1154 drop_reason = SKB_DROP_REASON_FULL_RING; 1155 goto drop; 1156 } 1157 1158 /* dev->lltx requires to do our own update of trans_start */ 1159 txq_trans_cond_update(queue); 1160 1161 /* Notify and wake up reader process */ 1162 if (tfile->flags & TUN_FASYNC) 1163 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1164 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1165 1166 rcu_read_unlock(); 1167 return NETDEV_TX_OK; 1168 1169 drop: 1170 dev_core_stats_tx_dropped_inc(dev); 1171 skb_tx_error(skb); 1172 kfree_skb_reason(skb, drop_reason); 1173 rcu_read_unlock(); 1174 return NET_XMIT_DROP; 1175 } 1176 1177 static void tun_net_mclist(struct net_device *dev) 1178 { 1179 /* 1180 * This callback is supposed to deal with mc filter in 1181 * _rx_ path and has nothing to do with the _tx_ path. 1182 * In rx path we always accept everything userspace gives us. 1183 */ 1184 } 1185 1186 static netdev_features_t tun_net_fix_features(struct net_device *dev, 1187 netdev_features_t features) 1188 { 1189 struct tun_struct *tun = netdev_priv(dev); 1190 1191 return (features & tun->set_features) | (features & ~TUN_USER_FEATURES); 1192 } 1193 1194 static void tun_set_headroom(struct net_device *dev, int new_hr) 1195 { 1196 struct tun_struct *tun = netdev_priv(dev); 1197 size_t max_headroom; 1198 1199 max_headroom = min_t(size_t, SKB_MAX_HEAD(0), U16_MAX - 1); 1200 1201 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) 1202 max_headroom -= ETH_HLEN + NET_IP_ALIGN; 1203 else 1204 max_headroom -= 1; 1205 1206 tun->align = clamp_t(int, new_hr, NET_SKB_PAD, max_headroom); 1207 } 1208 1209 static void 1210 tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1211 { 1212 struct tun_struct *tun = netdev_priv(dev); 1213 1214 dev_get_tstats64(dev, stats); 1215 1216 stats->rx_frame_errors += 1217 (unsigned long)atomic_long_read(&tun->rx_frame_errors); 1218 } 1219 1220 static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog, 1221 struct netlink_ext_ack *extack) 1222 { 1223 struct tun_struct *tun = netdev_priv(dev); 1224 struct tun_file *tfile; 1225 struct bpf_prog *old_prog; 1226 int i; 1227 1228 old_prog = rtnl_dereference(tun->xdp_prog); 1229 rcu_assign_pointer(tun->xdp_prog, prog); 1230 if (old_prog) 1231 bpf_prog_put(old_prog); 1232 1233 for (i = 0; i < tun->numqueues; i++) { 1234 tfile = rtnl_dereference(tun->tfiles[i]); 1235 if (prog) 1236 sock_set_flag(&tfile->sk, SOCK_XDP); 1237 else 1238 sock_reset_flag(&tfile->sk, SOCK_XDP); 1239 } 1240 list_for_each_entry(tfile, &tun->disabled, next) { 1241 if (prog) 1242 sock_set_flag(&tfile->sk, SOCK_XDP); 1243 else 1244 sock_reset_flag(&tfile->sk, SOCK_XDP); 1245 } 1246 1247 return 0; 1248 } 1249 1250 static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp) 1251 { 1252 switch (xdp->command) { 1253 case XDP_SETUP_PROG: 1254 return tun_xdp_set(dev, xdp->prog, xdp->extack); 1255 default: 1256 return -EINVAL; 1257 } 1258 } 1259 1260 static int tun_net_change_carrier(struct net_device *dev, bool new_carrier) 1261 { 1262 if (new_carrier) { 1263 struct tun_struct *tun = netdev_priv(dev); 1264 1265 if (!tun->numqueues) 1266 return -EPERM; 1267 1268 netif_carrier_on(dev); 1269 } else { 1270 netif_carrier_off(dev); 1271 } 1272 return 0; 1273 } 1274 1275 static const struct net_device_ops tun_netdev_ops = { 1276 .ndo_init = tun_net_init, 1277 .ndo_uninit = tun_net_uninit, 1278 .ndo_open = tun_net_open, 1279 .ndo_stop = tun_net_close, 1280 .ndo_start_xmit = tun_net_xmit, 1281 .ndo_fix_features = tun_net_fix_features, 1282 .ndo_select_queue = tun_select_queue, 1283 .ndo_set_rx_headroom = tun_set_headroom, 1284 .ndo_get_stats64 = tun_net_get_stats64, 1285 .ndo_change_carrier = tun_net_change_carrier, 1286 }; 1287 1288 static void __tun_xdp_flush_tfile(struct tun_file *tfile) 1289 { 1290 /* Notify and wake up reader process */ 1291 if (tfile->flags & TUN_FASYNC) 1292 kill_fasync(&tfile->fasync, SIGIO, POLL_IN); 1293 tfile->socket.sk->sk_data_ready(tfile->socket.sk); 1294 } 1295 1296 static int tun_xdp_xmit(struct net_device *dev, int n, 1297 struct xdp_frame **frames, u32 flags) 1298 { 1299 struct tun_struct *tun = netdev_priv(dev); 1300 struct tun_file *tfile; 1301 u32 numqueues; 1302 int nxmit = 0; 1303 int i; 1304 1305 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1306 return -EINVAL; 1307 1308 rcu_read_lock(); 1309 1310 resample: 1311 numqueues = READ_ONCE(tun->numqueues); 1312 if (!numqueues) { 1313 rcu_read_unlock(); 1314 return -ENXIO; /* Caller will free/return all frames */ 1315 } 1316 1317 tfile = rcu_dereference(tun->tfiles[smp_processor_id() % 1318 numqueues]); 1319 if (unlikely(!tfile)) 1320 goto resample; 1321 1322 spin_lock(&tfile->tx_ring.producer_lock); 1323 for (i = 0; i < n; i++) { 1324 struct xdp_frame *xdp = frames[i]; 1325 /* Encode the XDP flag into lowest bit for consumer to differ 1326 * XDP buffer from sk_buff. 1327 */ 1328 void *frame = tun_xdp_to_ptr(xdp); 1329 1330 if (__ptr_ring_produce(&tfile->tx_ring, frame)) { 1331 dev_core_stats_tx_dropped_inc(dev); 1332 break; 1333 } 1334 nxmit++; 1335 } 1336 spin_unlock(&tfile->tx_ring.producer_lock); 1337 1338 if (flags & XDP_XMIT_FLUSH) 1339 __tun_xdp_flush_tfile(tfile); 1340 1341 rcu_read_unlock(); 1342 return nxmit; 1343 } 1344 1345 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp) 1346 { 1347 struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp); 1348 int nxmit; 1349 1350 if (unlikely(!frame)) 1351 return -EOVERFLOW; 1352 1353 nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH); 1354 if (!nxmit) 1355 xdp_return_frame_rx_napi(frame); 1356 return nxmit; 1357 } 1358 1359 static const struct net_device_ops tap_netdev_ops = { 1360 .ndo_init = tun_net_init, 1361 .ndo_uninit = tun_net_uninit, 1362 .ndo_open = tun_net_open, 1363 .ndo_stop = tun_net_close, 1364 .ndo_start_xmit = tun_net_xmit, 1365 .ndo_fix_features = tun_net_fix_features, 1366 .ndo_set_rx_mode = tun_net_mclist, 1367 .ndo_set_mac_address = eth_mac_addr, 1368 .ndo_validate_addr = eth_validate_addr, 1369 .ndo_select_queue = tun_select_queue, 1370 .ndo_features_check = passthru_features_check, 1371 .ndo_set_rx_headroom = tun_set_headroom, 1372 .ndo_bpf = tun_xdp, 1373 .ndo_xdp_xmit = tun_xdp_xmit, 1374 .ndo_change_carrier = tun_net_change_carrier, 1375 }; 1376 1377 static void tun_flow_init(struct tun_struct *tun) 1378 { 1379 int i; 1380 1381 for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) 1382 INIT_HLIST_HEAD(&tun->flows[i]); 1383 1384 tun->ageing_time = TUN_FLOW_EXPIRE; 1385 timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0); 1386 mod_timer(&tun->flow_gc_timer, 1387 round_jiffies_up(jiffies + tun->ageing_time)); 1388 } 1389 1390 static void tun_flow_uninit(struct tun_struct *tun) 1391 { 1392 timer_delete_sync(&tun->flow_gc_timer); 1393 tun_flow_flush(tun); 1394 } 1395 1396 #define MIN_MTU 68 1397 #define MAX_MTU 65535 1398 1399 /* Initialize net device. */ 1400 static void tun_net_initialize(struct net_device *dev) 1401 { 1402 struct tun_struct *tun = netdev_priv(dev); 1403 1404 switch (tun->flags & TUN_TYPE_MASK) { 1405 case IFF_TUN: 1406 dev->netdev_ops = &tun_netdev_ops; 1407 dev->header_ops = &ip_tunnel_header_ops; 1408 1409 /* Point-to-Point TUN Device */ 1410 dev->hard_header_len = 0; 1411 dev->addr_len = 0; 1412 dev->mtu = 1500; 1413 1414 /* Zero header length */ 1415 dev->type = ARPHRD_NONE; 1416 dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST; 1417 break; 1418 1419 case IFF_TAP: 1420 dev->netdev_ops = &tap_netdev_ops; 1421 /* Ethernet TAP Device */ 1422 ether_setup(dev); 1423 dev->priv_flags &= ~IFF_TX_SKB_SHARING; 1424 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 1425 1426 eth_hw_addr_random(dev); 1427 1428 /* Currently tun does not support XDP, only tap does. */ 1429 dev->xdp_features = NETDEV_XDP_ACT_BASIC | 1430 NETDEV_XDP_ACT_REDIRECT | 1431 NETDEV_XDP_ACT_NDO_XMIT; 1432 1433 break; 1434 } 1435 1436 dev->min_mtu = MIN_MTU; 1437 dev->max_mtu = MAX_MTU - dev->hard_header_len; 1438 } 1439 1440 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile) 1441 { 1442 struct sock *sk = tfile->socket.sk; 1443 1444 return (tun->dev->flags & IFF_UP) && sock_writeable(sk); 1445 } 1446 1447 /* Character device part */ 1448 1449 /* Poll */ 1450 static __poll_t tun_chr_poll(struct file *file, poll_table *wait) 1451 { 1452 struct tun_file *tfile = file->private_data; 1453 struct tun_struct *tun = tun_get(tfile); 1454 struct sock *sk; 1455 __poll_t mask = 0; 1456 1457 if (!tun) 1458 return EPOLLERR; 1459 1460 sk = tfile->socket.sk; 1461 1462 poll_wait(file, sk_sleep(sk), wait); 1463 1464 if (!ptr_ring_empty(&tfile->tx_ring)) 1465 mask |= EPOLLIN | EPOLLRDNORM; 1466 1467 /* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to 1468 * guarantee EPOLLOUT to be raised by either here or 1469 * tun_sock_write_space(). Then process could get notification 1470 * after it writes to a down device and meets -EIO. 1471 */ 1472 if (tun_sock_writeable(tun, tfile) || 1473 (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) && 1474 tun_sock_writeable(tun, tfile))) 1475 mask |= EPOLLOUT | EPOLLWRNORM; 1476 1477 if (tun->dev->reg_state != NETREG_REGISTERED) 1478 mask = EPOLLERR; 1479 1480 tun_put(tun); 1481 return mask; 1482 } 1483 1484 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile, 1485 size_t len, 1486 const struct iov_iter *it) 1487 { 1488 struct sk_buff *skb; 1489 size_t linear; 1490 int err; 1491 int i; 1492 1493 if (it->nr_segs > MAX_SKB_FRAGS + 1 || 1494 len > (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN)) 1495 return ERR_PTR(-EMSGSIZE); 1496 1497 local_bh_disable(); 1498 skb = napi_get_frags(&tfile->napi); 1499 local_bh_enable(); 1500 if (!skb) 1501 return ERR_PTR(-ENOMEM); 1502 1503 /* An skb recycled by napi_reuse_skb() keeps the protocol of a 1504 * previous frame. eth_type_trans() sets it after the virtio 1505 * header checks, which must not see a stale one. 1506 */ 1507 skb->protocol = 0; 1508 1509 linear = iov_iter_single_seg_count(it); 1510 err = __skb_grow(skb, linear); 1511 if (err) 1512 goto free; 1513 1514 skb->len = len; 1515 skb->data_len = len - linear; 1516 skb->truesize += skb->data_len; 1517 1518 for (i = 1; i < it->nr_segs; i++) { 1519 const struct iovec *iov = iter_iov(it) + i; 1520 size_t fragsz = iov->iov_len; 1521 struct page *page; 1522 void *frag; 1523 1524 if (fragsz == 0 || fragsz > PAGE_SIZE) { 1525 err = -EINVAL; 1526 goto free; 1527 } 1528 frag = netdev_alloc_frag(fragsz); 1529 if (!frag) { 1530 err = -ENOMEM; 1531 goto free; 1532 } 1533 page = virt_to_head_page(frag); 1534 skb_fill_page_desc(skb, i - 1, page, 1535 frag - page_address(page), fragsz); 1536 } 1537 1538 return skb; 1539 free: 1540 /* frees skb and all frags allocated with napi_alloc_frag() */ 1541 napi_free_frags(&tfile->napi); 1542 return ERR_PTR(err); 1543 } 1544 1545 /* prepad is the amount to reserve at front. len is length after that. 1546 * linear is a hint as to how much to copy (usually headers). */ 1547 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile, 1548 size_t prepad, size_t len, 1549 size_t linear, int noblock) 1550 { 1551 struct sock *sk = tfile->socket.sk; 1552 struct sk_buff *skb; 1553 int err; 1554 1555 /* Under a page? Don't bother with paged skb. */ 1556 if (prepad + len < PAGE_SIZE) 1557 linear = len; 1558 1559 if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER)) 1560 linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER); 1561 skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock, 1562 &err, PAGE_ALLOC_COSTLY_ORDER); 1563 if (!skb) 1564 return ERR_PTR(err); 1565 1566 skb_reserve(skb, prepad); 1567 skb_put(skb, linear); 1568 skb->data_len = len - linear; 1569 skb->len += len - linear; 1570 1571 return skb; 1572 } 1573 1574 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile, 1575 struct sk_buff *skb, int more) 1576 { 1577 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 1578 struct sk_buff_head process_queue; 1579 u32 rx_batched = tun->rx_batched; 1580 bool rcv = false; 1581 1582 if (!rx_batched || (!more && skb_queue_empty(queue))) { 1583 local_bh_disable(); 1584 skb_record_rx_queue(skb, tfile->queue_index); 1585 netif_receive_skb(skb); 1586 local_bh_enable(); 1587 return; 1588 } 1589 1590 spin_lock(&queue->lock); 1591 if (!more || skb_queue_len(queue) == rx_batched) { 1592 __skb_queue_head_init(&process_queue); 1593 skb_queue_splice_tail_init(queue, &process_queue); 1594 rcv = true; 1595 } else { 1596 __skb_queue_tail(queue, skb); 1597 } 1598 spin_unlock(&queue->lock); 1599 1600 if (rcv) { 1601 struct sk_buff *nskb; 1602 1603 local_bh_disable(); 1604 while ((nskb = __skb_dequeue(&process_queue))) { 1605 skb_record_rx_queue(nskb, tfile->queue_index); 1606 netif_receive_skb(nskb); 1607 } 1608 skb_record_rx_queue(skb, tfile->queue_index); 1609 netif_receive_skb(skb); 1610 local_bh_enable(); 1611 } 1612 } 1613 1614 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile, 1615 int len, int noblock, bool zerocopy) 1616 { 1617 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 1618 return false; 1619 1620 if (tfile->socket.sk->sk_sndbuf != INT_MAX) 1621 return false; 1622 1623 if (!noblock) 1624 return false; 1625 1626 if (zerocopy) 1627 return false; 1628 1629 if (SKB_DATA_ALIGN(len + TUN_RX_PAD + XDP_PACKET_HEADROOM) + 1630 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE) 1631 return false; 1632 1633 return true; 1634 } 1635 1636 static struct sk_buff *__tun_build_skb(struct tun_file *tfile, 1637 struct page_frag *alloc_frag, char *buf, 1638 int buflen, int len, int pad, 1639 int metasize) 1640 { 1641 struct sk_buff *skb = build_skb(buf, buflen); 1642 1643 if (!skb) 1644 return ERR_PTR(-ENOMEM); 1645 1646 skb_reserve(skb, pad); 1647 skb_put(skb, len); 1648 if (metasize) 1649 skb_metadata_set(skb, metasize); 1650 skb_set_owner_w(skb, tfile->socket.sk); 1651 1652 get_page(alloc_frag->page); 1653 alloc_frag->offset += buflen; 1654 1655 return skb; 1656 } 1657 1658 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog, 1659 struct xdp_buff *xdp, u32 act) 1660 { 1661 int err; 1662 1663 switch (act) { 1664 case XDP_REDIRECT: 1665 err = xdp_do_redirect(tun->dev, xdp, xdp_prog); 1666 if (err) { 1667 dev_core_stats_rx_dropped_inc(tun->dev); 1668 return err; 1669 } 1670 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data); 1671 break; 1672 case XDP_TX: 1673 err = tun_xdp_tx(tun->dev, xdp); 1674 if (err < 0) { 1675 dev_core_stats_rx_dropped_inc(tun->dev); 1676 return err; 1677 } 1678 dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data); 1679 break; 1680 case XDP_PASS: 1681 break; 1682 default: 1683 bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act); 1684 fallthrough; 1685 case XDP_ABORTED: 1686 trace_xdp_exception(tun->dev, xdp_prog, act); 1687 fallthrough; 1688 case XDP_DROP: 1689 dev_core_stats_rx_dropped_inc(tun->dev); 1690 break; 1691 } 1692 1693 return act; 1694 } 1695 1696 static struct sk_buff *tun_build_skb(struct tun_struct *tun, 1697 struct tun_file *tfile, 1698 struct iov_iter *from, 1699 struct virtio_net_hdr *hdr, 1700 int len, int *skb_xdp) 1701 { 1702 struct page_frag *alloc_frag = ¤t->task_frag; 1703 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 1704 struct bpf_prog *xdp_prog; 1705 int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 1706 char *buf; 1707 size_t copied; 1708 int pad = TUN_RX_PAD; 1709 int metasize = 0; 1710 int err = 0; 1711 1712 rcu_read_lock(); 1713 xdp_prog = rcu_dereference(tun->xdp_prog); 1714 if (xdp_prog) 1715 pad += XDP_PACKET_HEADROOM; 1716 buflen += SKB_DATA_ALIGN(len + pad); 1717 rcu_read_unlock(); 1718 1719 alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES); 1720 if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL))) 1721 return ERR_PTR(-ENOMEM); 1722 1723 buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset; 1724 copied = copy_page_from_iter(alloc_frag->page, 1725 alloc_frag->offset + pad, 1726 len, from); 1727 if (copied != len) 1728 return ERR_PTR(-EFAULT); 1729 1730 /* There's a small window that XDP may be set after the check 1731 * of xdp_prog above, this should be rare and for simplicity 1732 * we do XDP on skb in case the headroom is not enough. 1733 */ 1734 if (hdr->gso_type || !xdp_prog) { 1735 *skb_xdp = 1; 1736 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, 1737 pad, metasize); 1738 } 1739 1740 *skb_xdp = 0; 1741 1742 local_bh_disable(); 1743 rcu_read_lock(); 1744 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 1745 xdp_prog = rcu_dereference(tun->xdp_prog); 1746 if (xdp_prog) { 1747 struct xdp_buff xdp; 1748 u32 act; 1749 1750 xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq); 1751 xdp_prepare_buff(&xdp, buf, pad, len, true); 1752 1753 act = bpf_prog_run_xdp(xdp_prog, &xdp); 1754 if (act == XDP_REDIRECT || act == XDP_TX) { 1755 get_page(alloc_frag->page); 1756 alloc_frag->offset += buflen; 1757 } 1758 err = tun_xdp_act(tun, xdp_prog, &xdp, act); 1759 if (err < 0) { 1760 if (act == XDP_REDIRECT || act == XDP_TX) 1761 put_page(alloc_frag->page); 1762 goto out; 1763 } 1764 1765 if (err == XDP_REDIRECT) 1766 xdp_do_flush(); 1767 if (err != XDP_PASS) 1768 goto out; 1769 1770 pad = xdp.data - xdp.data_hard_start; 1771 len = xdp.data_end - xdp.data; 1772 1773 /* It is known that the xdp_buff was prepared with metadata 1774 * support, so the metasize will never be negative. 1775 */ 1776 metasize = xdp.data - xdp.data_meta; 1777 } 1778 bpf_net_ctx_clear(bpf_net_ctx); 1779 rcu_read_unlock(); 1780 local_bh_enable(); 1781 1782 return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad, 1783 metasize); 1784 1785 out: 1786 bpf_net_ctx_clear(bpf_net_ctx); 1787 rcu_read_unlock(); 1788 local_bh_enable(); 1789 return NULL; 1790 } 1791 1792 /* Get packet from user space buffer */ 1793 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile, 1794 void *msg_control, struct iov_iter *from, 1795 int noblock, bool more) 1796 { 1797 struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) }; 1798 struct sk_buff *skb; 1799 size_t total_len = iov_iter_count(from); 1800 size_t len = total_len, align = tun->align, linear; 1801 struct virtio_net_hdr_v1_hash_tunnel hdr; 1802 struct virtio_net_hdr *gso; 1803 int good_linear; 1804 int copylen; 1805 int hdr_len = 0; 1806 bool zerocopy = false; 1807 int err; 1808 u32 rxhash = 0; 1809 int skb_xdp = 1; 1810 bool frags = tun_napi_frags_enabled(tfile); 1811 enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED; 1812 netdev_features_t features = 0; 1813 1814 /* 1815 * Keep it easy and always zero the whole buffer, even if the 1816 * tunnel-related field will be touched only when the feature 1817 * is enabled and the hdr size id compatible. 1818 */ 1819 memset(&hdr, 0, sizeof(hdr)); 1820 gso = (struct virtio_net_hdr *)&hdr; 1821 1822 if (!(tun->flags & IFF_NO_PI)) { 1823 if (len < sizeof(pi)) 1824 return -EINVAL; 1825 len -= sizeof(pi); 1826 1827 if (!copy_from_iter_full(&pi, sizeof(pi), from)) 1828 return -EFAULT; 1829 } 1830 1831 if (tun->flags & IFF_VNET_HDR) { 1832 int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 1833 1834 features = tun_vnet_hdr_guest_features(vnet_hdr_sz); 1835 hdr_len = __tun_vnet_hdr_get(vnet_hdr_sz, tun->flags, 1836 features, from, gso); 1837 if (hdr_len < 0) 1838 return hdr_len; 1839 1840 len -= vnet_hdr_sz; 1841 } 1842 1843 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) { 1844 align += NET_IP_ALIGN; 1845 if (unlikely(len < ETH_HLEN || (hdr_len && hdr_len < ETH_HLEN))) 1846 return -EINVAL; 1847 } 1848 1849 good_linear = SKB_MAX_HEAD(align); 1850 1851 if (msg_control) { 1852 struct iov_iter i = *from; 1853 1854 /* There are 256 bytes to be copied in skb, so there is 1855 * enough room for skb expand head in case it is used. 1856 * The rest of the buffer is mapped from userspace. 1857 */ 1858 copylen = min(hdr_len ? hdr_len : GOODCOPY_LEN, good_linear); 1859 linear = copylen; 1860 iov_iter_advance(&i, copylen); 1861 if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS) 1862 zerocopy = true; 1863 } 1864 1865 if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) { 1866 /* For the packet that is not easy to be processed 1867 * (e.g gso or jumbo packet), we will do it at after 1868 * skb was created with generic XDP routine. 1869 */ 1870 skb = tun_build_skb(tun, tfile, from, gso, len, &skb_xdp); 1871 err = PTR_ERR_OR_ZERO(skb); 1872 if (err) 1873 goto drop; 1874 if (!skb) 1875 return total_len; 1876 } else { 1877 if (!zerocopy) { 1878 copylen = len; 1879 linear = min(hdr_len, good_linear); 1880 } 1881 1882 if (frags) { 1883 mutex_lock(&tfile->napi_mutex); 1884 skb = tun_napi_alloc_frags(tfile, copylen, from); 1885 /* tun_napi_alloc_frags() enforces a layout for the skb. 1886 * If zerocopy is enabled, then this layout will be 1887 * overwritten by zerocopy_sg_from_iter(). 1888 */ 1889 zerocopy = false; 1890 } else { 1891 if (!linear) 1892 linear = min_t(size_t, good_linear, copylen); 1893 1894 skb = tun_alloc_skb(tfile, align, copylen, linear, 1895 noblock); 1896 } 1897 1898 err = PTR_ERR_OR_ZERO(skb); 1899 if (err) 1900 goto drop; 1901 1902 if (zerocopy) 1903 err = zerocopy_sg_from_iter(skb, from); 1904 else 1905 err = skb_copy_datagram_from_iter(skb, 0, from, len); 1906 1907 if (err) { 1908 err = -EFAULT; 1909 drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT; 1910 goto drop; 1911 } 1912 } 1913 1914 skb->dev = tun->dev; 1915 switch (tun->flags & TUN_TYPE_MASK) { 1916 case IFF_TUN: 1917 if (tun->flags & IFF_NO_PI) { 1918 u8 ip_version; 1919 1920 drop_reason = pskb_may_pull_reason(skb, 1); 1921 if (drop_reason) { 1922 err = -EINVAL; 1923 goto drop; 1924 } 1925 ip_version = skb->data[0] >> 4; 1926 1927 switch (ip_version) { 1928 case 4: 1929 pi.proto = htons(ETH_P_IP); 1930 break; 1931 case 6: 1932 pi.proto = htons(ETH_P_IPV6); 1933 break; 1934 default: 1935 err = -EINVAL; 1936 goto drop; 1937 } 1938 } 1939 1940 skb_reset_network_header(skb); 1941 skb->protocol = pi.proto; 1942 break; 1943 case IFF_TAP: 1944 drop_reason = pskb_may_pull_reason(skb, ETH_HLEN); 1945 if (drop_reason) { 1946 err = -ENOMEM; 1947 goto drop; 1948 } 1949 skb_set_network_header(skb, ETH_HLEN); 1950 break; 1951 } 1952 1953 if (tun_vnet_hdr_tnl_to_skb(tun->flags, features, skb, &hdr, 1954 (tun->flags & TUN_TYPE_MASK) == IFF_TAP)) { 1955 atomic_long_inc(&tun->rx_frame_errors); 1956 err = -EINVAL; 1957 goto free_skb; 1958 } 1959 1960 if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) 1961 skb->protocol = eth_type_trans(skb, tun->dev); 1962 1963 /* copy skb_ubuf_info for callback when skb has no error */ 1964 if (zerocopy) { 1965 skb_zcopy_init(skb, msg_control); 1966 } else if (msg_control) { 1967 struct ubuf_info *uarg = msg_control; 1968 uarg->ops->complete(NULL, uarg, false); 1969 } 1970 1971 skb_reset_network_header(skb); 1972 skb_probe_transport_header(skb); 1973 skb_record_rx_queue(skb, tfile->queue_index); 1974 1975 if (skb_xdp) { 1976 struct bpf_prog *xdp_prog; 1977 int ret; 1978 1979 local_bh_disable(); 1980 rcu_read_lock(); 1981 xdp_prog = rcu_dereference(tun->xdp_prog); 1982 if (xdp_prog) { 1983 ret = do_xdp_generic(xdp_prog, &skb); 1984 if (ret != XDP_PASS) { 1985 rcu_read_unlock(); 1986 local_bh_enable(); 1987 goto unlock_frags; 1988 } 1989 1990 if (frags && skb != tfile->napi.skb) 1991 tfile->napi.skb = skb; 1992 } 1993 rcu_read_unlock(); 1994 local_bh_enable(); 1995 } 1996 1997 /* Compute the costly rx hash only if needed for flow updates. 1998 * We may get a very small possibility of OOO during switching, not 1999 * worth to optimize. 2000 */ 2001 if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 && 2002 !tfile->detached) 2003 rxhash = __skb_get_hash_symmetric(skb); 2004 2005 rcu_read_lock(); 2006 if (unlikely(!(tun->dev->flags & IFF_UP))) { 2007 err = -EIO; 2008 rcu_read_unlock(); 2009 drop_reason = SKB_DROP_REASON_DEV_READY; 2010 goto drop; 2011 } 2012 2013 if (frags) { 2014 u32 headlen; 2015 2016 /* Exercise flow dissector code path. */ 2017 skb_push(skb, ETH_HLEN); 2018 headlen = eth_get_headlen(tun->dev, skb->data, 2019 skb_headlen(skb)); 2020 2021 if (unlikely(headlen > skb_headlen(skb))) { 2022 WARN_ON_ONCE(1); 2023 err = -ENOMEM; 2024 dev_core_stats_rx_dropped_inc(tun->dev); 2025 napi_busy: 2026 napi_free_frags(&tfile->napi); 2027 rcu_read_unlock(); 2028 mutex_unlock(&tfile->napi_mutex); 2029 return err; 2030 } 2031 2032 if (likely(napi_schedule_prep(&tfile->napi))) { 2033 local_bh_disable(); 2034 napi_gro_frags(&tfile->napi); 2035 napi_complete(&tfile->napi); 2036 local_bh_enable(); 2037 } else { 2038 err = -EBUSY; 2039 goto napi_busy; 2040 } 2041 mutex_unlock(&tfile->napi_mutex); 2042 } else if (tfile->napi_enabled) { 2043 struct sk_buff_head *queue = &tfile->sk.sk_write_queue; 2044 int queue_len; 2045 2046 spin_lock_bh(&queue->lock); 2047 2048 if (unlikely(tfile->detached)) { 2049 spin_unlock_bh(&queue->lock); 2050 rcu_read_unlock(); 2051 err = -EBUSY; 2052 goto free_skb; 2053 } 2054 2055 __skb_queue_tail(queue, skb); 2056 queue_len = skb_queue_len(queue); 2057 spin_unlock(&queue->lock); 2058 2059 if (!more || queue_len > NAPI_POLL_WEIGHT) 2060 napi_schedule(&tfile->napi); 2061 2062 local_bh_enable(); 2063 } else if (!IS_ENABLED(CONFIG_4KSTACKS)) { 2064 tun_rx_batched(tun, tfile, skb, more); 2065 } else { 2066 netif_rx(skb); 2067 } 2068 rcu_read_unlock(); 2069 2070 preempt_disable(); 2071 dev_sw_netstats_rx_add(tun->dev, len); 2072 preempt_enable(); 2073 2074 if (rxhash) 2075 tun_flow_update(tun, rxhash, tfile); 2076 2077 return total_len; 2078 2079 drop: 2080 if (err != -EAGAIN) 2081 dev_core_stats_rx_dropped_inc(tun->dev); 2082 2083 free_skb: 2084 drop_reason = drop_reason ?: SKB_DROP_REASON_NOT_SPECIFIED; 2085 if (!IS_ERR_OR_NULL(skb)) 2086 kfree_skb_reason(skb, drop_reason); 2087 2088 unlock_frags: 2089 if (frags) { 2090 tfile->napi.skb = NULL; 2091 mutex_unlock(&tfile->napi_mutex); 2092 } 2093 2094 return err ?: total_len; 2095 } 2096 2097 static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from) 2098 { 2099 struct file *file = iocb->ki_filp; 2100 struct tun_file *tfile = file->private_data; 2101 struct tun_struct *tun = tun_get(tfile); 2102 ssize_t result; 2103 int noblock = 0; 2104 2105 if (!tun) 2106 return -EBADFD; 2107 2108 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT)) 2109 noblock = 1; 2110 2111 result = tun_get_user(tun, tfile, NULL, from, noblock, false); 2112 2113 tun_put(tun); 2114 return result; 2115 } 2116 2117 static ssize_t tun_put_user_xdp(struct tun_struct *tun, 2118 struct tun_file *tfile, 2119 struct xdp_frame *xdp_frame, 2120 struct iov_iter *iter) 2121 { 2122 int vnet_hdr_sz = 0; 2123 size_t size = xdp_frame->len; 2124 ssize_t ret; 2125 2126 if (tun->flags & IFF_VNET_HDR) { 2127 struct virtio_net_hdr gso = { 0 }; 2128 2129 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2130 ret = tun_vnet_hdr_put(vnet_hdr_sz, iter, &gso); 2131 if (ret) 2132 return ret; 2133 } 2134 2135 ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz; 2136 2137 preempt_disable(); 2138 dev_sw_netstats_tx_add(tun->dev, 1, ret); 2139 preempt_enable(); 2140 2141 return ret; 2142 } 2143 2144 /* Put packet to the user space buffer */ 2145 static ssize_t tun_put_user(struct tun_struct *tun, 2146 struct tun_file *tfile, 2147 struct sk_buff *skb, 2148 struct iov_iter *iter) 2149 { 2150 struct tun_pi pi = { 0, skb->protocol }; 2151 ssize_t total; 2152 int vlan_offset = 0; 2153 int vlan_hlen = 0; 2154 int vnet_hdr_sz = 0; 2155 int ret; 2156 2157 if (skb_vlan_tag_present(skb)) 2158 vlan_hlen = VLAN_HLEN; 2159 2160 if (tun->flags & IFF_VNET_HDR) 2161 vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz); 2162 2163 total = skb->len + vlan_hlen + vnet_hdr_sz; 2164 2165 if (!(tun->flags & IFF_NO_PI)) { 2166 if (iov_iter_count(iter) < sizeof(pi)) 2167 return -EINVAL; 2168 2169 total += sizeof(pi); 2170 if (iov_iter_count(iter) < total) { 2171 /* Packet will be striped */ 2172 pi.flags |= TUN_PKT_STRIP; 2173 } 2174 2175 if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi)) 2176 return -EFAULT; 2177 } 2178 2179 if (vnet_hdr_sz) { 2180 struct virtio_net_hdr_v1_hash_tunnel hdr; 2181 struct virtio_net_hdr *gso; 2182 2183 memset(&hdr, 0, sizeof(hdr)); 2184 ret = tun_vnet_hdr_tnl_from_skb(tun->flags, tun->dev, skb, 2185 &hdr); 2186 if (ret) 2187 return ret; 2188 2189 /* 2190 * Drop the packet if the configured header size is too small 2191 * WRT the enabled offloads. 2192 */ 2193 gso = (struct virtio_net_hdr *)&hdr; 2194 ret = __tun_vnet_hdr_put(vnet_hdr_sz, tun->dev->features, 2195 iter, gso); 2196 if (ret) 2197 return ret; 2198 } 2199 2200 if (vlan_hlen) { 2201 int ret; 2202 struct veth veth; 2203 2204 veth.h_vlan_proto = skb->vlan_proto; 2205 veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 2206 2207 vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto); 2208 2209 ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset); 2210 if (ret || !iov_iter_count(iter)) 2211 goto done; 2212 2213 ret = copy_to_iter(&veth, sizeof(veth), iter); 2214 if (ret != sizeof(veth) || !iov_iter_count(iter)) 2215 goto done; 2216 } 2217 2218 skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset); 2219 2220 done: 2221 /* caller is in process context, */ 2222 preempt_disable(); 2223 dev_sw_netstats_tx_add(tun->dev, 1, skb->len + vlan_hlen); 2224 preempt_enable(); 2225 2226 return total; 2227 } 2228 2229 /* Callers must hold ring.consumer_lock */ 2230 static void __tun_wake_queue(struct tun_struct *tun, 2231 struct tun_file *tfile, int consumed) 2232 { 2233 u16 queue_index = tfile->queue_index; 2234 struct netdev_queue *txq; 2235 2236 lockdep_assert_held(&tfile->tx_ring.consumer_lock); 2237 2238 if (!(tun->flags & IFF_BACKPRESSURE)) 2239 return; 2240 2241 /* A stop from tun_net_close() is not backpressure, leave it alone. */ 2242 if (unlikely(!netif_running(tun->dev))) 2243 return; 2244 2245 /* Only the current owner of the slot may wake its subqueue. */ 2246 if (unlikely(rcu_access_pointer(tun->tfiles[queue_index]) != tfile)) 2247 return; 2248 2249 txq = netdev_get_tx_queue(tun->dev, queue_index); 2250 2251 /* Paired with smp_mb__after_atomic() in tun_net_xmit() */ 2252 smp_mb(); 2253 if (netif_tx_queue_stopped(txq)) { 2254 tfile->cons_cnt += consumed; 2255 if (tfile->cons_cnt >= tfile->tx_ring.size / 2 || 2256 __ptr_ring_empty(&tfile->tx_ring)) { 2257 netif_tx_wake_queue(txq); 2258 tfile->cons_cnt = 0; 2259 } 2260 } 2261 } 2262 2263 static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile) 2264 { 2265 void *ptr; 2266 2267 spin_lock(&tfile->tx_ring.consumer_lock); 2268 ptr = __ptr_ring_consume(&tfile->tx_ring); 2269 if (ptr) 2270 __tun_wake_queue(tun, tfile, 1); 2271 2272 spin_unlock(&tfile->tx_ring.consumer_lock); 2273 return ptr; 2274 } 2275 2276 static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile, 2277 int noblock, int *err) 2278 { 2279 DECLARE_WAITQUEUE(wait, current); 2280 void *ptr = NULL; 2281 int error = 0; 2282 2283 ptr = tun_ring_consume(tun, tfile); 2284 if (ptr) 2285 goto out; 2286 if (noblock) { 2287 error = -EAGAIN; 2288 goto out; 2289 } 2290 2291 add_wait_queue(&tfile->socket.wq.wait, &wait); 2292 2293 while (1) { 2294 set_current_state(TASK_INTERRUPTIBLE); 2295 ptr = tun_ring_consume(tun, tfile); 2296 if (ptr) 2297 break; 2298 if (signal_pending(current)) { 2299 error = -ERESTARTSYS; 2300 break; 2301 } 2302 if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) { 2303 error = -EFAULT; 2304 break; 2305 } 2306 2307 schedule(); 2308 } 2309 2310 __set_current_state(TASK_RUNNING); 2311 remove_wait_queue(&tfile->socket.wq.wait, &wait); 2312 2313 out: 2314 *err = error; 2315 return ptr; 2316 } 2317 2318 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile, 2319 struct iov_iter *to, 2320 int noblock, void *ptr) 2321 { 2322 ssize_t ret; 2323 int err; 2324 2325 if (!iov_iter_count(to)) { 2326 tun_ptr_free(ptr); 2327 return 0; 2328 } 2329 2330 if (!ptr) { 2331 /* Read frames from ring */ 2332 ptr = tun_ring_recv(tun, tfile, noblock, &err); 2333 if (!ptr) 2334 return err; 2335 } 2336 2337 if (tun_is_xdp_frame(ptr)) { 2338 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2339 2340 ret = tun_put_user_xdp(tun, tfile, xdpf, to); 2341 xdp_return_frame(xdpf); 2342 } else { 2343 struct sk_buff *skb = ptr; 2344 2345 ret = tun_put_user(tun, tfile, skb, to); 2346 if (unlikely(ret < 0)) 2347 kfree_skb(skb); 2348 else 2349 consume_skb(skb); 2350 } 2351 2352 return ret; 2353 } 2354 2355 static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to) 2356 { 2357 struct file *file = iocb->ki_filp; 2358 struct tun_file *tfile = file->private_data; 2359 struct tun_struct *tun = tun_get(tfile); 2360 ssize_t len = iov_iter_count(to), ret; 2361 int noblock = 0; 2362 2363 if (!tun) 2364 return -EBADFD; 2365 2366 if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT)) 2367 noblock = 1; 2368 2369 ret = tun_do_read(tun, tfile, to, noblock, NULL); 2370 ret = min_t(ssize_t, ret, len); 2371 if (ret > 0) 2372 iocb->ki_pos = ret; 2373 tun_put(tun); 2374 return ret; 2375 } 2376 2377 static void tun_prog_free(struct rcu_head *rcu) 2378 { 2379 struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu); 2380 2381 bpf_prog_destroy(prog->prog); 2382 kfree(prog); 2383 } 2384 2385 static int __tun_set_ebpf(struct tun_struct *tun, 2386 struct tun_prog __rcu **prog_p, 2387 struct bpf_prog *prog) 2388 { 2389 struct tun_prog *old, *new = NULL; 2390 2391 if (prog) { 2392 new = kmalloc_obj(*new); 2393 if (!new) 2394 return -ENOMEM; 2395 new->prog = prog; 2396 } 2397 2398 spin_lock_bh(&tun->lock); 2399 old = rcu_dereference_protected(*prog_p, 2400 lockdep_is_held(&tun->lock)); 2401 rcu_assign_pointer(*prog_p, new); 2402 spin_unlock_bh(&tun->lock); 2403 2404 if (old) 2405 call_rcu(&old->rcu, tun_prog_free); 2406 2407 return 0; 2408 } 2409 2410 static void tun_free_netdev(struct net_device *dev) 2411 { 2412 struct tun_struct *tun = netdev_priv(dev); 2413 2414 BUG_ON(!(list_empty(&tun->disabled))); 2415 2416 tun_flow_uninit(tun); 2417 security_tun_dev_free_security(tun->security); 2418 __tun_set_ebpf(tun, &tun->steering_prog, NULL); 2419 __tun_set_ebpf(tun, &tun->filter_prog, NULL); 2420 } 2421 2422 static void tun_setup(struct net_device *dev) 2423 { 2424 struct tun_struct *tun = netdev_priv(dev); 2425 2426 tun->owner = INVALID_UID; 2427 tun->group = INVALID_GID; 2428 tun_default_link_ksettings(dev, &tun->link_ksettings); 2429 2430 dev->ethtool_ops = &tun_ethtool_ops; 2431 dev->needs_free_netdev = true; 2432 dev->priv_destructor = tun_free_netdev; 2433 /* We prefer our own queue length */ 2434 dev->tx_queue_len = TUN_READQ_SIZE; 2435 } 2436 2437 /* Trivial set of netlink ops to allow deleting tun or tap 2438 * device with netlink. 2439 */ 2440 static int tun_validate(struct nlattr *tb[], struct nlattr *data[], 2441 struct netlink_ext_ack *extack) 2442 { 2443 NL_SET_ERR_MSG(extack, 2444 "tun/tap creation via rtnetlink is not supported."); 2445 return -EOPNOTSUPP; 2446 } 2447 2448 static size_t tun_get_size(const struct net_device *dev) 2449 { 2450 BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t)); 2451 BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t)); 2452 2453 return nla_total_size(sizeof(uid_t)) + /* OWNER */ 2454 nla_total_size(sizeof(gid_t)) + /* GROUP */ 2455 nla_total_size(sizeof(u8)) + /* TYPE */ 2456 nla_total_size(sizeof(u8)) + /* PI */ 2457 nla_total_size(sizeof(u8)) + /* VNET_HDR */ 2458 nla_total_size(sizeof(u8)) + /* PERSIST */ 2459 nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */ 2460 nla_total_size(sizeof(u32)) + /* NUM_QUEUES */ 2461 nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */ 2462 0; 2463 } 2464 2465 static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev) 2466 { 2467 const struct tun_struct *tun = netdev_priv(dev); 2468 unsigned int flags = READ_ONCE(tun->flags); 2469 kuid_t owner = READ_ONCE(tun->owner); 2470 kgid_t group = READ_ONCE(tun->group); 2471 2472 if (nla_put_u8(skb, IFLA_TUN_TYPE, flags & TUN_TYPE_MASK)) 2473 goto nla_put_failure; 2474 if (uid_valid(owner) && 2475 nla_put_u32(skb, IFLA_TUN_OWNER, 2476 from_kuid_munged(current_user_ns(), owner))) 2477 goto nla_put_failure; 2478 if (gid_valid(group) && 2479 nla_put_u32(skb, IFLA_TUN_GROUP, 2480 from_kgid_munged(current_user_ns(), group))) 2481 goto nla_put_failure; 2482 if (nla_put_u8(skb, IFLA_TUN_PI, !(flags & IFF_NO_PI))) 2483 goto nla_put_failure; 2484 if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(flags & IFF_VNET_HDR))) 2485 goto nla_put_failure; 2486 if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(flags & IFF_PERSIST))) 2487 goto nla_put_failure; 2488 if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE, 2489 !!(flags & IFF_MULTI_QUEUE))) 2490 goto nla_put_failure; 2491 if (flags & IFF_MULTI_QUEUE) { 2492 if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES, 2493 READ_ONCE(tun->numqueues))) 2494 goto nla_put_failure; 2495 if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES, 2496 READ_ONCE(tun->numdisabled))) 2497 goto nla_put_failure; 2498 } 2499 2500 return 0; 2501 2502 nla_put_failure: 2503 return -EMSGSIZE; 2504 } 2505 2506 static struct rtnl_link_ops tun_link_ops __read_mostly = { 2507 .kind = DRV_NAME, 2508 .priv_size = sizeof(struct tun_struct), 2509 .setup = tun_setup, 2510 .validate = tun_validate, 2511 .get_size = tun_get_size, 2512 .fill_info = tun_fill_info, 2513 }; 2514 2515 static void tun_sock_write_space(struct sock *sk) 2516 { 2517 struct tun_file *tfile; 2518 wait_queue_head_t *wqueue; 2519 2520 if (!sock_writeable(sk)) 2521 return; 2522 2523 if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags)) 2524 return; 2525 2526 wqueue = sk_sleep(sk); 2527 if (wqueue && waitqueue_active(wqueue)) 2528 wake_up_interruptible_sync_poll(wqueue, EPOLLOUT | 2529 EPOLLWRNORM | EPOLLWRBAND); 2530 2531 tfile = container_of(sk, struct tun_file, sk); 2532 kill_fasync(&tfile->fasync, SIGIO, POLL_OUT); 2533 } 2534 2535 static void tun_put_page(struct tun_page *tpage) 2536 { 2537 if (tpage->page) 2538 __page_frag_cache_drain(tpage->page, tpage->count); 2539 } 2540 2541 static int tun_xdp_one(struct tun_struct *tun, 2542 struct tun_file *tfile, 2543 struct xdp_buff *xdp, int *flush, 2544 struct tun_page *tpage) 2545 { 2546 unsigned int datasize = xdp->data_end - xdp->data; 2547 struct virtio_net_hdr *gso = xdp->data_hard_start; 2548 struct virtio_net_hdr_v1_hash_tunnel *tnl_hdr; 2549 struct bpf_prog *xdp_prog; 2550 struct sk_buff *skb = NULL; 2551 struct sk_buff_head *queue; 2552 netdev_features_t features; 2553 u32 rxhash = 0, act; 2554 int buflen = xdp->frame_sz; 2555 int metasize = 0; 2556 int ret = 0; 2557 bool skb_xdp = false; 2558 struct page *page; 2559 2560 if (unlikely(datasize < ETH_HLEN)) { 2561 put_page(virt_to_head_page(xdp->data)); 2562 return -EINVAL; 2563 } 2564 2565 if (unlikely(!(READ_ONCE(tun->dev->flags) & IFF_UP))) { 2566 put_page(virt_to_head_page(xdp->data)); 2567 dev_core_stats_rx_dropped_inc(tun->dev); 2568 return -EIO; 2569 } 2570 2571 xdp_prog = rcu_dereference(tun->xdp_prog); 2572 if (xdp_prog) { 2573 if (gso->gso_type) { 2574 skb_xdp = true; 2575 goto build; 2576 } 2577 2578 xdp_init_buff(xdp, buflen, &tfile->xdp_rxq); 2579 2580 act = bpf_prog_run_xdp(xdp_prog, xdp); 2581 ret = tun_xdp_act(tun, xdp_prog, xdp, act); 2582 if (ret < 0) { 2583 put_page(virt_to_head_page(xdp->data)); 2584 return ret; 2585 } 2586 2587 switch (ret) { 2588 case XDP_REDIRECT: 2589 *flush = true; 2590 fallthrough; 2591 case XDP_TX: 2592 return 0; 2593 case XDP_PASS: 2594 break; 2595 default: 2596 page = virt_to_head_page(xdp->data); 2597 if (tpage->page == page) { 2598 ++tpage->count; 2599 } else { 2600 tun_put_page(tpage); 2601 tpage->page = page; 2602 tpage->count = 1; 2603 } 2604 return 0; 2605 } 2606 } 2607 2608 build: 2609 skb = build_skb(xdp->data_hard_start, buflen); 2610 if (!skb) { 2611 put_page(virt_to_head_page(xdp->data)); 2612 ret = -ENOMEM; 2613 goto out; 2614 } 2615 2616 skb_reserve(skb, xdp->data - xdp->data_hard_start); 2617 skb_put(skb, xdp->data_end - xdp->data); 2618 2619 /* The externally provided xdp_buff may have no metadata support, which 2620 * is marked by xdp->data_meta being xdp->data + 1. This will lead to a 2621 * metasize of -1 and is the reason why the condition checks for > 0. 2622 */ 2623 metasize = xdp->data - xdp->data_meta; 2624 if (metasize > 0) 2625 skb_metadata_set(skb, metasize); 2626 2627 features = tun_vnet_hdr_guest_features(READ_ONCE(tun->vnet_hdr_sz)); 2628 tnl_hdr = (struct virtio_net_hdr_v1_hash_tunnel *)gso; 2629 skb->dev = tun->dev; 2630 skb_set_network_header(skb, ETH_HLEN); 2631 /* Frames always start with an Ethernet header here, as the 2632 * eth_type_trans() call below expects. 2633 */ 2634 if (tun_vnet_hdr_tnl_to_skb(tun->flags, features, skb, tnl_hdr, 2635 true)) { 2636 atomic_long_inc(&tun->rx_frame_errors); 2637 kfree_skb(skb); 2638 ret = -EINVAL; 2639 goto out; 2640 } 2641 2642 skb->protocol = eth_type_trans(skb, tun->dev); 2643 skb_reset_network_header(skb); 2644 skb_probe_transport_header(skb); 2645 skb_record_rx_queue(skb, tfile->queue_index); 2646 2647 if (skb_xdp) { 2648 ret = do_xdp_generic(xdp_prog, &skb); 2649 if (ret != XDP_PASS) { 2650 ret = 0; 2651 goto out; 2652 } 2653 } 2654 2655 if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 && 2656 !tfile->detached) 2657 rxhash = __skb_get_hash_symmetric(skb); 2658 2659 if (tfile->napi_enabled) { 2660 queue = &tfile->sk.sk_write_queue; 2661 spin_lock(&queue->lock); 2662 2663 if (unlikely(tfile->detached)) { 2664 spin_unlock(&queue->lock); 2665 kfree_skb(skb); 2666 return -EBUSY; 2667 } 2668 2669 __skb_queue_tail(queue, skb); 2670 spin_unlock(&queue->lock); 2671 ret = 1; 2672 } else { 2673 netif_receive_skb(skb); 2674 ret = 0; 2675 } 2676 2677 /* No need to disable preemption here since this function is 2678 * always called with bh disabled 2679 */ 2680 dev_sw_netstats_rx_add(tun->dev, datasize); 2681 2682 if (rxhash) 2683 tun_flow_update(tun, rxhash, tfile); 2684 2685 out: 2686 return ret; 2687 } 2688 2689 static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len) 2690 { 2691 int ret, i; 2692 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2693 struct tun_struct *tun = tun_get(tfile); 2694 struct tun_msg_ctl *ctl = m->msg_control; 2695 struct xdp_buff *xdp; 2696 2697 if (!tun) 2698 return -EBADFD; 2699 2700 if (m->msg_controllen == sizeof(struct tun_msg_ctl) && 2701 ctl && ctl->type == TUN_MSG_PTR) { 2702 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 2703 struct tun_page tpage; 2704 int n = ctl->num; 2705 int flush = 0, queued = 0; 2706 2707 memset(&tpage, 0, sizeof(tpage)); 2708 2709 local_bh_disable(); 2710 rcu_read_lock(); 2711 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 2712 2713 for (i = 0; i < n; i++) { 2714 xdp = &((struct xdp_buff *)ctl->ptr)[i]; 2715 ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage); 2716 if (ret > 0) 2717 queued += ret; 2718 } 2719 2720 if (flush) 2721 xdp_do_flush(); 2722 2723 if (tfile->napi_enabled && queued > 0) 2724 napi_schedule(&tfile->napi); 2725 2726 bpf_net_ctx_clear(bpf_net_ctx); 2727 rcu_read_unlock(); 2728 local_bh_enable(); 2729 2730 tun_put_page(&tpage); 2731 2732 ret = total_len; 2733 goto out; 2734 } 2735 2736 ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter, 2737 m->msg_flags & MSG_DONTWAIT, 2738 m->msg_flags & MSG_MORE); 2739 out: 2740 tun_put(tun); 2741 return ret; 2742 } 2743 2744 static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len, 2745 int flags) 2746 { 2747 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2748 struct tun_struct *tun = tun_get(tfile); 2749 void *ptr = m->msg_control; 2750 int ret; 2751 2752 if (!tun) { 2753 ret = -EBADFD; 2754 goto out_free; 2755 } 2756 2757 if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) { 2758 ret = -EINVAL; 2759 goto out_put_tun; 2760 } 2761 if (flags & MSG_ERRQUEUE) { 2762 ret = sock_recv_errqueue(sock->sk, m, total_len, 2763 SOL_PACKET, TUN_TX_TIMESTAMP); 2764 goto out; 2765 } 2766 ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr); 2767 if (ret > (ssize_t)total_len) { 2768 m->msg_flags |= MSG_TRUNC; 2769 ret = flags & MSG_TRUNC ? ret : total_len; 2770 } 2771 out: 2772 tun_put(tun); 2773 return ret; 2774 2775 out_put_tun: 2776 tun_put(tun); 2777 out_free: 2778 tun_ptr_free(ptr); 2779 return ret; 2780 } 2781 2782 static int tun_ptr_peek_len(void *ptr) 2783 { 2784 if (likely(ptr)) { 2785 if (tun_is_xdp_frame(ptr)) { 2786 struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr); 2787 2788 return xdpf->len; 2789 } 2790 return __skb_array_len_with_tag(ptr); 2791 } else { 2792 return 0; 2793 } 2794 } 2795 2796 static int tun_peek_len(struct socket *sock) 2797 { 2798 struct tun_file *tfile = container_of(sock, struct tun_file, socket); 2799 struct tun_struct *tun; 2800 int ret = 0; 2801 2802 tun = tun_get(tfile); 2803 if (!tun) 2804 return 0; 2805 2806 ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len); 2807 tun_put(tun); 2808 2809 return ret; 2810 } 2811 2812 /* Ops structure to mimic raw sockets with tun */ 2813 static const struct proto_ops tun_socket_ops = { 2814 .peek_len = tun_peek_len, 2815 .sendmsg = tun_sendmsg, 2816 .recvmsg = tun_recvmsg, 2817 }; 2818 2819 static struct proto tun_proto = { 2820 .name = "tun", 2821 .owner = THIS_MODULE, 2822 .obj_size = sizeof(struct tun_file), 2823 }; 2824 2825 static int tun_flags(struct tun_struct *tun) 2826 { 2827 return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP); 2828 } 2829 2830 static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr, 2831 char *buf) 2832 { 2833 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2834 return sysfs_emit(buf, "0x%x\n", tun_flags(tun)); 2835 } 2836 2837 static ssize_t owner_show(struct device *dev, struct device_attribute *attr, 2838 char *buf) 2839 { 2840 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2841 return uid_valid(tun->owner)? 2842 sysfs_emit(buf, "%u\n", 2843 from_kuid_munged(current_user_ns(), tun->owner)) : 2844 sysfs_emit(buf, "-1\n"); 2845 } 2846 2847 static ssize_t group_show(struct device *dev, struct device_attribute *attr, 2848 char *buf) 2849 { 2850 struct tun_struct *tun = netdev_priv(to_net_dev(dev)); 2851 return gid_valid(tun->group) ? 2852 sysfs_emit(buf, "%u\n", 2853 from_kgid_munged(current_user_ns(), tun->group)) : 2854 sysfs_emit(buf, "-1\n"); 2855 } 2856 2857 static DEVICE_ATTR_RO(tun_flags); 2858 static DEVICE_ATTR_RO(owner); 2859 static DEVICE_ATTR_RO(group); 2860 2861 static struct attribute *tun_dev_attrs[] = { 2862 &dev_attr_tun_flags.attr, 2863 &dev_attr_owner.attr, 2864 &dev_attr_group.attr, 2865 NULL 2866 }; 2867 2868 static const struct attribute_group tun_attr_group = { 2869 .attrs = tun_dev_attrs 2870 }; 2871 2872 static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr) 2873 { 2874 struct tun_struct *tun; 2875 struct tun_file *tfile = file->private_data; 2876 struct tun_file *ntfile; 2877 struct net_device *dev; 2878 int err, i; 2879 2880 if (tfile->detached) 2881 return -EINVAL; 2882 2883 if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) { 2884 if (!capable(CAP_NET_ADMIN)) 2885 return -EPERM; 2886 2887 if (!(ifr->ifr_flags & IFF_NAPI) || 2888 (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP) 2889 return -EINVAL; 2890 } 2891 2892 dev = __dev_get_by_name(net, ifr->ifr_name); 2893 if (dev) { 2894 if (ifr->ifr_flags & IFF_TUN_EXCL) 2895 return -EBUSY; 2896 if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops) 2897 tun = netdev_priv(dev); 2898 else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops) 2899 tun = netdev_priv(dev); 2900 else 2901 return -EINVAL; 2902 2903 if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) != 2904 !!(tun->flags & IFF_MULTI_QUEUE)) 2905 return -EINVAL; 2906 2907 if (!tun_permitted(tun)) 2908 return -EPERM; 2909 err = security_tun_dev_open(tun->security); 2910 if (err < 0) 2911 return err; 2912 2913 err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER, 2914 ifr->ifr_flags & IFF_NAPI, 2915 ifr->ifr_flags & IFF_NAPI_FRAGS, true); 2916 if (err < 0) 2917 return err; 2918 2919 if (tun->flags & IFF_MULTI_QUEUE && 2920 (tun->numqueues + tun->numdisabled > 1)) { 2921 /* One or more queue has already been attached, no need 2922 * to initialize the device again. 2923 */ 2924 netdev_state_change(dev); 2925 return 0; 2926 } 2927 2928 WRITE_ONCE(tun->flags, (tun->flags & ~TUN_FEATURES) | 2929 (ifr->ifr_flags & TUN_FEATURES)); 2930 2931 netdev_state_change(dev); 2932 } else { 2933 char *name; 2934 unsigned long flags = 0; 2935 int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ? 2936 MAX_TAP_QUEUES : 1; 2937 2938 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 2939 return -EPERM; 2940 err = security_tun_dev_create(); 2941 if (err < 0) 2942 return err; 2943 2944 /* Set dev type */ 2945 if (ifr->ifr_flags & IFF_TUN) { 2946 /* TUN device */ 2947 flags |= IFF_TUN; 2948 name = "tun%d"; 2949 } else if (ifr->ifr_flags & IFF_TAP) { 2950 /* TAP device */ 2951 flags |= IFF_TAP; 2952 name = "tap%d"; 2953 } else 2954 return -EINVAL; 2955 2956 if (*ifr->ifr_name) 2957 name = ifr->ifr_name; 2958 2959 dev = alloc_netdev_mqs(sizeof(struct tun_struct), name, 2960 NET_NAME_UNKNOWN, tun_setup, queues, 2961 queues); 2962 2963 if (!dev) 2964 return -ENOMEM; 2965 2966 dev_net_set(dev, net); 2967 dev->rtnl_link_ops = &tun_link_ops; 2968 dev->ifindex = tfile->ifindex; 2969 dev->sysfs_groups[0] = &tun_attr_group; 2970 2971 tun = netdev_priv(dev); 2972 tun->dev = dev; 2973 tun->flags = flags; 2974 tun->txflt.count = 0; 2975 tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr); 2976 2977 tun->align = NET_SKB_PAD; 2978 tun->filter_attached = false; 2979 tun->sndbuf = tfile->socket.sk->sk_sndbuf; 2980 tun->rx_batched = 0; 2981 RCU_INIT_POINTER(tun->steering_prog, NULL); 2982 2983 tun->ifr = ifr; 2984 tun->file = file; 2985 2986 tun_net_initialize(dev); 2987 2988 err = register_netdevice(tun->dev); 2989 if (err < 0) { 2990 free_netdev(dev); 2991 return err; 2992 } 2993 /* free_netdev() won't check refcnt, to avoid race 2994 * with dev_put() we need publish tun after registration. 2995 */ 2996 rcu_assign_pointer(tfile->tun, tun); 2997 } 2998 2999 if (ifr->ifr_flags & IFF_NO_CARRIER) 3000 netif_carrier_off(tun->dev); 3001 else 3002 netif_carrier_on(tun->dev); 3003 3004 /* Make sure persistent devices do not get stuck in 3005 * xoff state. 3006 */ 3007 for (i = 0; i < tun->numqueues; i++) { 3008 ntfile = rtnl_dereference(tun->tfiles[i]); 3009 tun_force_wake_queue(tun, ntfile); 3010 } 3011 3012 strscpy(ifr->ifr_name, tun->dev->name); 3013 return 0; 3014 } 3015 3016 static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr) 3017 { 3018 strscpy(ifr->ifr_name, tun->dev->name); 3019 3020 ifr->ifr_flags = tun_flags(tun); 3021 3022 } 3023 3024 #define PLAIN_GSO (NETIF_F_GSO_UDP_L4 | NETIF_F_TSO | NETIF_F_TSO6) 3025 3026 /* This is like a cut-down ethtool ops, except done via tun fd so no 3027 * privs required. */ 3028 static int set_offload(struct tun_struct *tun, unsigned long arg) 3029 { 3030 netdev_features_t features = 0; 3031 3032 if (arg & TUN_F_CSUM) { 3033 features |= NETIF_F_HW_CSUM; 3034 arg &= ~TUN_F_CSUM; 3035 3036 if (arg & (TUN_F_TSO4|TUN_F_TSO6)) { 3037 if (arg & TUN_F_TSO_ECN) { 3038 features |= NETIF_F_TSO_ECN; 3039 arg &= ~TUN_F_TSO_ECN; 3040 } 3041 if (arg & TUN_F_TSO4) 3042 features |= NETIF_F_TSO; 3043 if (arg & TUN_F_TSO6) 3044 features |= NETIF_F_TSO6; 3045 arg &= ~(TUN_F_TSO4|TUN_F_TSO6); 3046 } 3047 3048 arg &= ~TUN_F_UFO; 3049 3050 /* TODO: for now USO4 and USO6 should work simultaneously */ 3051 if (arg & TUN_F_USO4 && arg & TUN_F_USO6) { 3052 features |= NETIF_F_GSO_UDP_L4; 3053 arg &= ~(TUN_F_USO4 | TUN_F_USO6); 3054 } 3055 3056 /* 3057 * Tunnel offload is allowed only if some plain offload is 3058 * available, too. 3059 */ 3060 if (features & PLAIN_GSO && arg & TUN_F_UDP_TUNNEL_GSO) { 3061 features |= NETIF_F_GSO_UDP_TUNNEL; 3062 if (arg & TUN_F_UDP_TUNNEL_GSO_CSUM) 3063 features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 3064 arg &= ~(TUN_F_UDP_TUNNEL_GSO | 3065 TUN_F_UDP_TUNNEL_GSO_CSUM); 3066 } 3067 } 3068 3069 /* This gives the user a way to test for new features in future by 3070 * trying to set them. */ 3071 if (arg) 3072 return -EINVAL; 3073 3074 tun->set_features = features; 3075 tun->dev->wanted_features &= ~TUN_USER_FEATURES; 3076 tun->dev->wanted_features |= features; 3077 netdev_update_features(tun->dev); 3078 3079 return 0; 3080 } 3081 3082 static void tun_detach_filter(struct tun_struct *tun, int n) 3083 { 3084 int i; 3085 struct tun_file *tfile; 3086 3087 for (i = 0; i < n; i++) { 3088 tfile = rtnl_dereference(tun->tfiles[i]); 3089 lock_sock(tfile->socket.sk); 3090 sk_detach_filter(tfile->socket.sk); 3091 release_sock(tfile->socket.sk); 3092 } 3093 3094 tun->filter_attached = false; 3095 } 3096 3097 static int tun_attach_filter(struct tun_struct *tun) 3098 { 3099 int i, ret = 0; 3100 struct tun_file *tfile; 3101 3102 for (i = 0; i < tun->numqueues; i++) { 3103 tfile = rtnl_dereference(tun->tfiles[i]); 3104 lock_sock(tfile->socket.sk); 3105 ret = sk_attach_filter(&tun->fprog, tfile->socket.sk); 3106 release_sock(tfile->socket.sk); 3107 if (ret) { 3108 tun_detach_filter(tun, i); 3109 return ret; 3110 } 3111 } 3112 3113 tun->filter_attached = true; 3114 return ret; 3115 } 3116 3117 static void tun_set_sndbuf(struct tun_struct *tun) 3118 { 3119 struct tun_file *tfile; 3120 int i; 3121 3122 for (i = 0; i < tun->numqueues; i++) { 3123 tfile = rtnl_dereference(tun->tfiles[i]); 3124 tfile->socket.sk->sk_sndbuf = tun->sndbuf; 3125 } 3126 } 3127 3128 static int tun_set_queue(struct file *file, struct ifreq *ifr) 3129 { 3130 struct tun_file *tfile = file->private_data; 3131 struct tun_struct *tun; 3132 int ret = 0; 3133 3134 rtnl_lock(); 3135 3136 if (ifr->ifr_flags & IFF_ATTACH_QUEUE) { 3137 tun = tfile->detached; 3138 if (!tun) { 3139 ret = -EINVAL; 3140 goto unlock; 3141 } 3142 ret = security_tun_dev_attach_queue(tun->security); 3143 if (ret < 0) 3144 goto unlock; 3145 ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI, 3146 tun->flags & IFF_NAPI_FRAGS, true); 3147 } else if (ifr->ifr_flags & IFF_DETACH_QUEUE) { 3148 tun = rtnl_dereference(tfile->tun); 3149 if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached) 3150 ret = -EINVAL; 3151 else 3152 __tun_detach(tfile, false); 3153 } else 3154 ret = -EINVAL; 3155 3156 if (ret >= 0) 3157 netdev_state_change(tun->dev); 3158 3159 unlock: 3160 rtnl_unlock(); 3161 return ret; 3162 } 3163 3164 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p, 3165 void __user *data) 3166 { 3167 struct bpf_prog *prog; 3168 int fd; 3169 3170 if (copy_from_user(&fd, data, sizeof(fd))) 3171 return -EFAULT; 3172 3173 if (fd == -1) { 3174 prog = NULL; 3175 } else { 3176 prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER); 3177 if (IS_ERR(prog)) 3178 return PTR_ERR(prog); 3179 } 3180 3181 return __tun_set_ebpf(tun, prog_p, prog); 3182 } 3183 3184 /* Return correct value for tun->dev->addr_len based on tun->dev->type. */ 3185 static unsigned char tun_get_addr_len(unsigned short type) 3186 { 3187 switch (type) { 3188 case ARPHRD_IP6GRE: 3189 case ARPHRD_TUNNEL6: 3190 return sizeof(struct in6_addr); 3191 case ARPHRD_IPGRE: 3192 case ARPHRD_TUNNEL: 3193 case ARPHRD_SIT: 3194 return 4; 3195 case ARPHRD_ETHER: 3196 return ETH_ALEN; 3197 case ARPHRD_IEEE802154: 3198 case ARPHRD_IEEE802154_MONITOR: 3199 return IEEE802154_EXTENDED_ADDR_LEN; 3200 case ARPHRD_PHONET_PIPE: 3201 case ARPHRD_PPP: 3202 case ARPHRD_NONE: 3203 return 0; 3204 case ARPHRD_6LOWPAN: 3205 return EUI64_ADDR_LEN; 3206 case ARPHRD_FDDI: 3207 return FDDI_K_ALEN; 3208 case ARPHRD_HIPPI: 3209 return HIPPI_ALEN; 3210 case ARPHRD_IEEE802: 3211 return FC_ALEN; 3212 case ARPHRD_ROSE: 3213 return ROSE_ADDR_LEN; 3214 case ARPHRD_NETROM: 3215 return AX25_ADDR_LEN; 3216 case ARPHRD_LOCALTLK: 3217 return LTALK_ALEN; 3218 default: 3219 return 0; 3220 } 3221 } 3222 3223 static long __tun_chr_ioctl(struct file *file, unsigned int cmd, 3224 unsigned long arg, int ifreq_len) 3225 { 3226 struct tun_file *tfile = file->private_data; 3227 struct net *net = sock_net(&tfile->sk); 3228 struct tun_struct *tun; 3229 void __user* argp = (void __user*)arg; 3230 unsigned int carrier; 3231 struct ifreq ifr; 3232 kuid_t owner; 3233 kgid_t group; 3234 int ifindex; 3235 int sndbuf; 3236 int ret; 3237 bool do_notify = false; 3238 3239 if (cmd == TUNSETIFF || cmd == TUNSETQUEUE || 3240 (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) { 3241 if (copy_from_user(&ifr, argp, ifreq_len)) 3242 return -EFAULT; 3243 } else { 3244 memset(&ifr, 0, sizeof(ifr)); 3245 } 3246 if (cmd == TUNGETFEATURES) { 3247 /* Currently this just means: "what IFF flags are valid?". 3248 * This is needed because we never checked for invalid flags on 3249 * TUNSETIFF. 3250 */ 3251 return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER | 3252 TUN_FEATURES, (unsigned int __user*)argp); 3253 } else if (cmd == TUNSETQUEUE) { 3254 return tun_set_queue(file, &ifr); 3255 } else if (cmd == SIOCGSKNS) { 3256 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 3257 return -EPERM; 3258 return open_related_ns(&net->ns, get_net_ns); 3259 } 3260 3261 rtnl_lock(); 3262 3263 tun = tun_get(tfile); 3264 if (cmd == TUNSETIFF) { 3265 ret = -EEXIST; 3266 if (tun) 3267 goto unlock; 3268 3269 ifr.ifr_name[IFNAMSIZ-1] = '\0'; 3270 3271 ret = tun_set_iff(net, file, &ifr); 3272 3273 if (ret) 3274 goto unlock; 3275 3276 if (copy_to_user(argp, &ifr, ifreq_len)) 3277 ret = -EFAULT; 3278 goto unlock; 3279 } 3280 if (cmd == TUNSETIFINDEX) { 3281 ret = -EPERM; 3282 if (tun) 3283 goto unlock; 3284 3285 ret = -EFAULT; 3286 if (copy_from_user(&ifindex, argp, sizeof(ifindex))) 3287 goto unlock; 3288 ret = -EINVAL; 3289 if (ifindex < 0) 3290 goto unlock; 3291 ret = 0; 3292 tfile->ifindex = ifindex; 3293 goto unlock; 3294 } 3295 3296 ret = -EBADFD; 3297 if (!tun) 3298 goto unlock; 3299 3300 netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd); 3301 3302 net = dev_net(tun->dev); 3303 ret = 0; 3304 switch (cmd) { 3305 case TUNGETIFF: 3306 tun_get_iff(tun, &ifr); 3307 3308 if (tfile->detached) 3309 ifr.ifr_flags |= IFF_DETACH_QUEUE; 3310 if (!tfile->socket.sk->sk_filter) 3311 ifr.ifr_flags |= IFF_NOFILTER; 3312 3313 if (copy_to_user(argp, &ifr, ifreq_len)) 3314 ret = -EFAULT; 3315 break; 3316 3317 case TUNSETNOCSUM: 3318 /* Disable/Enable checksum */ 3319 3320 /* [unimplemented] */ 3321 netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n", 3322 arg ? "disabled" : "enabled"); 3323 break; 3324 3325 case TUNSETPERSIST: 3326 /* Disable/Enable persist mode. Keep an extra reference to the 3327 * module to prevent the module being unprobed. 3328 */ 3329 if (arg && !(READ_ONCE(tun->flags) & IFF_PERSIST)) { 3330 WRITE_ONCE(tun->flags, READ_ONCE(tun->flags) | IFF_PERSIST); 3331 __module_get(THIS_MODULE); 3332 do_notify = true; 3333 } 3334 if (!arg && (READ_ONCE(tun->flags) & IFF_PERSIST)) { 3335 WRITE_ONCE(tun->flags, READ_ONCE(tun->flags) & ~IFF_PERSIST); 3336 module_put(THIS_MODULE); 3337 do_notify = true; 3338 } 3339 3340 netif_info(tun, drv, tun->dev, "persist %s\n", 3341 arg ? "enabled" : "disabled"); 3342 break; 3343 3344 case TUNSETOWNER: 3345 /* Set owner of the device */ 3346 owner = make_kuid(current_user_ns(), arg); 3347 if (!uid_valid(owner)) { 3348 ret = -EINVAL; 3349 break; 3350 } 3351 WRITE_ONCE(tun->owner, owner); 3352 do_notify = true; 3353 netif_info(tun, drv, tun->dev, "owner set to %u\n", 3354 from_kuid(&init_user_ns, owner)); 3355 break; 3356 3357 case TUNSETGROUP: 3358 /* Set group of the device */ 3359 group = make_kgid(current_user_ns(), arg); 3360 if (!gid_valid(group)) { 3361 ret = -EINVAL; 3362 break; 3363 } 3364 WRITE_ONCE(tun->group, group); 3365 do_notify = true; 3366 netif_info(tun, drv, tun->dev, "group set to %u\n", 3367 from_kgid(&init_user_ns, group)); 3368 break; 3369 3370 case TUNSETLINK: 3371 /* Only allow setting the type when the interface is down */ 3372 if (tun->dev->flags & IFF_UP) { 3373 netif_info(tun, drv, tun->dev, 3374 "Linktype set failed because interface is up\n"); 3375 ret = -EBUSY; 3376 } else if ((unsigned short)arg == ARPHRD_IEEE802154 || 3377 (unsigned short)arg == ARPHRD_IEEE802154_MONITOR || 3378 (unsigned short)arg == ARPHRD_6LOWPAN) { 3379 ret = -EINVAL; 3380 } else { 3381 ret = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE, 3382 tun->dev); 3383 ret = notifier_to_errno(ret); 3384 if (ret) { 3385 netif_info(tun, drv, tun->dev, 3386 "Refused to change device type\n"); 3387 break; 3388 } 3389 tun->dev->type = (int) arg; 3390 tun->dev->addr_len = tun_get_addr_len(tun->dev->type); 3391 netif_info(tun, drv, tun->dev, "linktype set to %d\n", 3392 tun->dev->type); 3393 call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE, 3394 tun->dev); 3395 } 3396 break; 3397 3398 case TUNSETDEBUG: 3399 tun->msg_enable = (u32)arg; 3400 break; 3401 3402 case TUNSETOFFLOAD: 3403 ret = set_offload(tun, arg); 3404 break; 3405 3406 case TUNSETTXFILTER: 3407 /* Can be set only for TAPs */ 3408 ret = -EINVAL; 3409 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3410 break; 3411 ret = update_filter(&tun->txflt, (void __user *)arg); 3412 break; 3413 3414 case SIOCGIFHWADDR: 3415 /* Get hw address */ 3416 netif_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name); 3417 if (copy_to_user(argp, &ifr, ifreq_len)) 3418 ret = -EFAULT; 3419 break; 3420 3421 case SIOCSIFHWADDR: 3422 /* Set hw address */ 3423 if (tun->dev->addr_len > sizeof(ifr.ifr_hwaddr)) { 3424 ret = -EINVAL; 3425 break; 3426 } 3427 ret = dev_set_mac_address_user(tun->dev, 3428 (struct sockaddr_storage *)&ifr.ifr_hwaddr, 3429 NULL); 3430 break; 3431 3432 case TUNGETSNDBUF: 3433 sndbuf = tfile->socket.sk->sk_sndbuf; 3434 if (copy_to_user(argp, &sndbuf, sizeof(sndbuf))) 3435 ret = -EFAULT; 3436 break; 3437 3438 case TUNSETSNDBUF: 3439 if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) { 3440 ret = -EFAULT; 3441 break; 3442 } 3443 if (sndbuf <= 0) { 3444 ret = -EINVAL; 3445 break; 3446 } 3447 3448 tun->sndbuf = sndbuf; 3449 tun_set_sndbuf(tun); 3450 break; 3451 3452 case TUNATTACHFILTER: 3453 /* Can be set only for TAPs */ 3454 ret = -EINVAL; 3455 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3456 break; 3457 ret = -EFAULT; 3458 if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog))) 3459 break; 3460 3461 ret = tun_attach_filter(tun); 3462 break; 3463 3464 case TUNDETACHFILTER: 3465 /* Can be set only for TAPs */ 3466 ret = -EINVAL; 3467 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3468 break; 3469 ret = 0; 3470 tun_detach_filter(tun, tun->numqueues); 3471 break; 3472 3473 case TUNGETFILTER: 3474 ret = -EINVAL; 3475 if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP) 3476 break; 3477 ret = -EFAULT; 3478 if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog))) 3479 break; 3480 ret = 0; 3481 break; 3482 3483 case TUNSETSTEERINGEBPF: 3484 ret = tun_set_ebpf(tun, &tun->steering_prog, argp); 3485 break; 3486 3487 case TUNSETFILTEREBPF: 3488 ret = tun_set_ebpf(tun, &tun->filter_prog, argp); 3489 break; 3490 3491 case TUNSETCARRIER: 3492 ret = -EFAULT; 3493 if (copy_from_user(&carrier, argp, sizeof(carrier))) 3494 goto unlock; 3495 3496 ret = tun_net_change_carrier(tun->dev, (bool)carrier); 3497 break; 3498 3499 case TUNGETDEVNETNS: 3500 ret = -EPERM; 3501 if (!ns_capable(net->user_ns, CAP_NET_ADMIN)) 3502 goto unlock; 3503 ret = open_related_ns(&net->ns, get_net_ns); 3504 break; 3505 3506 default: 3507 ret = tun_vnet_ioctl(&tun->vnet_hdr_sz, &tun->flags, cmd, argp); 3508 break; 3509 } 3510 3511 if (do_notify) 3512 netdev_state_change(tun->dev); 3513 3514 unlock: 3515 rtnl_unlock(); 3516 if (tun) 3517 tun_put(tun); 3518 return ret; 3519 } 3520 3521 static long tun_chr_ioctl(struct file *file, 3522 unsigned int cmd, unsigned long arg) 3523 { 3524 return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq)); 3525 } 3526 3527 #ifdef CONFIG_COMPAT 3528 static long tun_chr_compat_ioctl(struct file *file, 3529 unsigned int cmd, unsigned long arg) 3530 { 3531 switch (cmd) { 3532 case TUNSETIFF: 3533 case TUNGETIFF: 3534 case TUNSETTXFILTER: 3535 case TUNGETSNDBUF: 3536 case TUNSETSNDBUF: 3537 case SIOCGIFHWADDR: 3538 case SIOCSIFHWADDR: 3539 arg = (unsigned long)compat_ptr(arg); 3540 break; 3541 default: 3542 arg = (compat_ulong_t)arg; 3543 break; 3544 } 3545 3546 /* 3547 * compat_ifreq is shorter than ifreq, so we must not access beyond 3548 * the end of that structure. All fields that are used in this 3549 * driver are compatible though, we don't need to convert the 3550 * contents. 3551 */ 3552 return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq)); 3553 } 3554 #endif /* CONFIG_COMPAT */ 3555 3556 static int tun_chr_fasync(int fd, struct file *file, int on) 3557 { 3558 struct tun_file *tfile = file->private_data; 3559 int ret; 3560 3561 if (on) { 3562 ret = file_f_owner_allocate(file); 3563 if (ret) 3564 goto out; 3565 } 3566 3567 if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0) 3568 goto out; 3569 3570 if (on) { 3571 __f_setown(file, task_pid(current), PIDTYPE_TGID, 0); 3572 tfile->flags |= TUN_FASYNC; 3573 } else 3574 tfile->flags &= ~TUN_FASYNC; 3575 ret = 0; 3576 out: 3577 return ret; 3578 } 3579 3580 static int tun_chr_open(struct inode *inode, struct file * file) 3581 { 3582 struct net *net = current->nsproxy->net_ns; 3583 struct tun_file *tfile; 3584 3585 tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL, 3586 &tun_proto, 0); 3587 if (!tfile) 3588 return -ENOMEM; 3589 if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) { 3590 sk_free(&tfile->sk); 3591 return -ENOMEM; 3592 } 3593 3594 mutex_init(&tfile->napi_mutex); 3595 RCU_INIT_POINTER(tfile->tun, NULL); 3596 tfile->flags = 0; 3597 tfile->ifindex = 0; 3598 3599 init_waitqueue_head(&tfile->socket.wq.wait); 3600 3601 tfile->socket.file = file; 3602 tfile->socket.ops = &tun_socket_ops; 3603 3604 sock_init_data_uid(&tfile->socket, &tfile->sk, current_fsuid()); 3605 3606 tfile->sk.sk_write_space = tun_sock_write_space; 3607 tfile->sk.sk_sndbuf = INT_MAX; 3608 3609 file->private_data = tfile; 3610 INIT_LIST_HEAD(&tfile->next); 3611 3612 sock_set_flag(&tfile->sk, SOCK_ZEROCOPY); 3613 3614 /* tun groks IOCB_NOWAIT just fine, mark it as such */ 3615 file->f_mode |= FMODE_NOWAIT; 3616 return 0; 3617 } 3618 3619 static int tun_chr_close(struct inode *inode, struct file *file) 3620 { 3621 struct tun_file *tfile = file->private_data; 3622 3623 tun_detach(tfile, true); 3624 3625 return 0; 3626 } 3627 3628 #ifdef CONFIG_PROC_FS 3629 static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file) 3630 { 3631 struct tun_file *tfile = file->private_data; 3632 struct tun_struct *tun; 3633 struct ifreq ifr; 3634 3635 memset(&ifr, 0, sizeof(ifr)); 3636 3637 rtnl_lock(); 3638 tun = tun_get(tfile); 3639 if (tun) 3640 tun_get_iff(tun, &ifr); 3641 rtnl_unlock(); 3642 3643 if (tun) 3644 tun_put(tun); 3645 3646 seq_printf(m, "iff:\t%s\n", ifr.ifr_name); 3647 } 3648 #endif 3649 3650 static const struct file_operations tun_fops = { 3651 .owner = THIS_MODULE, 3652 .read_iter = tun_chr_read_iter, 3653 .write_iter = tun_chr_write_iter, 3654 .poll = tun_chr_poll, 3655 .unlocked_ioctl = tun_chr_ioctl, 3656 #ifdef CONFIG_COMPAT 3657 .compat_ioctl = tun_chr_compat_ioctl, 3658 #endif 3659 .open = tun_chr_open, 3660 .release = tun_chr_close, 3661 .fasync = tun_chr_fasync, 3662 #ifdef CONFIG_PROC_FS 3663 .show_fdinfo = tun_chr_show_fdinfo, 3664 #endif 3665 }; 3666 3667 static struct miscdevice tun_miscdev = { 3668 .minor = TUN_MINOR, 3669 .name = "tun", 3670 .nodename = "net/tun", 3671 .fops = &tun_fops, 3672 }; 3673 3674 /* ethtool interface */ 3675 3676 static void tun_default_link_ksettings(struct net_device *dev, 3677 struct ethtool_link_ksettings *cmd) 3678 { 3679 ethtool_link_ksettings_zero_link_mode(cmd, supported); 3680 ethtool_link_ksettings_zero_link_mode(cmd, advertising); 3681 cmd->base.speed = SPEED_10000; 3682 cmd->base.duplex = DUPLEX_FULL; 3683 cmd->base.port = PORT_TP; 3684 cmd->base.phy_address = 0; 3685 cmd->base.autoneg = AUTONEG_DISABLE; 3686 } 3687 3688 static int tun_get_link_ksettings(struct net_device *dev, 3689 struct ethtool_link_ksettings *cmd) 3690 { 3691 struct tun_struct *tun = netdev_priv(dev); 3692 3693 memcpy(cmd, &tun->link_ksettings, sizeof(*cmd)); 3694 return 0; 3695 } 3696 3697 static int tun_set_link_ksettings(struct net_device *dev, 3698 const struct ethtool_link_ksettings *cmd) 3699 { 3700 struct tun_struct *tun = netdev_priv(dev); 3701 3702 memcpy(&tun->link_ksettings, cmd, sizeof(*cmd)); 3703 return 0; 3704 } 3705 3706 static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 3707 { 3708 struct tun_struct *tun = netdev_priv(dev); 3709 3710 strscpy(info->driver, DRV_NAME, sizeof(info->driver)); 3711 strscpy(info->version, DRV_VERSION, sizeof(info->version)); 3712 3713 switch (tun->flags & TUN_TYPE_MASK) { 3714 case IFF_TUN: 3715 strscpy(info->bus_info, "tun", sizeof(info->bus_info)); 3716 break; 3717 case IFF_TAP: 3718 strscpy(info->bus_info, "tap", sizeof(info->bus_info)); 3719 break; 3720 } 3721 } 3722 3723 static u32 tun_get_msglevel(struct net_device *dev) 3724 { 3725 struct tun_struct *tun = netdev_priv(dev); 3726 3727 return tun->msg_enable; 3728 } 3729 3730 static void tun_set_msglevel(struct net_device *dev, u32 value) 3731 { 3732 struct tun_struct *tun = netdev_priv(dev); 3733 3734 tun->msg_enable = value; 3735 } 3736 3737 static int tun_get_coalesce(struct net_device *dev, 3738 struct ethtool_coalesce *ec, 3739 struct kernel_ethtool_coalesce *kernel_coal, 3740 struct netlink_ext_ack *extack) 3741 { 3742 struct tun_struct *tun = netdev_priv(dev); 3743 3744 ec->rx_max_coalesced_frames = tun->rx_batched; 3745 3746 return 0; 3747 } 3748 3749 static int tun_set_coalesce(struct net_device *dev, 3750 struct ethtool_coalesce *ec, 3751 struct kernel_ethtool_coalesce *kernel_coal, 3752 struct netlink_ext_ack *extack) 3753 { 3754 struct tun_struct *tun = netdev_priv(dev); 3755 3756 if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT) 3757 tun->rx_batched = NAPI_POLL_WEIGHT; 3758 else 3759 tun->rx_batched = ec->rx_max_coalesced_frames; 3760 3761 return 0; 3762 } 3763 3764 static void tun_get_channels(struct net_device *dev, 3765 struct ethtool_channels *channels) 3766 { 3767 struct tun_struct *tun = netdev_priv(dev); 3768 3769 channels->combined_count = tun->numqueues; 3770 channels->max_combined = tun->flags & IFF_MULTI_QUEUE ? MAX_TAP_QUEUES : 1; 3771 } 3772 3773 static const struct ethtool_ops tun_ethtool_ops = { 3774 .supported_coalesce_params = ETHTOOL_COALESCE_RX_MAX_FRAMES, 3775 .get_drvinfo = tun_get_drvinfo, 3776 .get_msglevel = tun_get_msglevel, 3777 .set_msglevel = tun_set_msglevel, 3778 .get_link = ethtool_op_get_link, 3779 .get_channels = tun_get_channels, 3780 .get_ts_info = ethtool_op_get_ts_info, 3781 .get_coalesce = tun_get_coalesce, 3782 .set_coalesce = tun_set_coalesce, 3783 .get_link_ksettings = tun_get_link_ksettings, 3784 .set_link_ksettings = tun_set_link_ksettings, 3785 }; 3786 3787 static int tun_queue_resize(struct tun_struct *tun) 3788 { 3789 struct net_device *dev = tun->dev; 3790 struct tun_file *tfile; 3791 struct ptr_ring **rings; 3792 int n = tun->numqueues + tun->numdisabled; 3793 int ret, i; 3794 3795 rings = kmalloc_objs(*rings, n); 3796 if (!rings) 3797 return -ENOMEM; 3798 3799 for (i = 0; i < tun->numqueues; i++) { 3800 tfile = rtnl_dereference(tun->tfiles[i]); 3801 rings[i] = &tfile->tx_ring; 3802 } 3803 list_for_each_entry(tfile, &tun->disabled, next) 3804 rings[i++] = &tfile->tx_ring; 3805 3806 ret = ptr_ring_resize_multiple_bh(rings, n, 3807 dev->tx_queue_len, GFP_KERNEL, 3808 tun_ptr_free); 3809 3810 if (!ret) { 3811 for (i = 0; i < tun->numqueues; i++) { 3812 tfile = rtnl_dereference(tun->tfiles[i]); 3813 tun_force_wake_queue(tun, tfile); 3814 } 3815 } 3816 3817 kfree(rings); 3818 return ret; 3819 } 3820 3821 static int tun_device_event(struct notifier_block *unused, 3822 unsigned long event, void *ptr) 3823 { 3824 struct net_device *dev = netdev_notifier_info_to_dev(ptr); 3825 struct tun_struct *tun = netdev_priv(dev); 3826 int i; 3827 3828 if (dev->rtnl_link_ops != &tun_link_ops) 3829 return NOTIFY_DONE; 3830 3831 switch (event) { 3832 case NETDEV_CHANGE_TX_QUEUE_LEN: 3833 if (tun_queue_resize(tun)) 3834 return NOTIFY_BAD; 3835 break; 3836 case NETDEV_UP: 3837 for (i = 0; i < tun->numqueues; i++) { 3838 struct tun_file *tfile; 3839 3840 tfile = rtnl_dereference(tun->tfiles[i]); 3841 tfile->socket.sk->sk_write_space(tfile->socket.sk); 3842 } 3843 break; 3844 default: 3845 break; 3846 } 3847 3848 return NOTIFY_DONE; 3849 } 3850 3851 static struct notifier_block tun_notifier_block __read_mostly = { 3852 .notifier_call = tun_device_event, 3853 }; 3854 3855 static int __init tun_init(void) 3856 { 3857 int ret = 0; 3858 3859 pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION); 3860 3861 ret = rtnl_link_register(&tun_link_ops); 3862 if (ret) { 3863 pr_err("Can't register link_ops\n"); 3864 goto err_linkops; 3865 } 3866 3867 ret = misc_register(&tun_miscdev); 3868 if (ret) { 3869 pr_err("Can't register misc device %d\n", TUN_MINOR); 3870 goto err_misc; 3871 } 3872 3873 ret = register_netdevice_notifier(&tun_notifier_block); 3874 if (ret) { 3875 pr_err("Can't register netdevice notifier\n"); 3876 goto err_notifier; 3877 } 3878 3879 return 0; 3880 3881 err_notifier: 3882 misc_deregister(&tun_miscdev); 3883 err_misc: 3884 rtnl_link_unregister(&tun_link_ops); 3885 err_linkops: 3886 return ret; 3887 } 3888 3889 static void __exit tun_cleanup(void) 3890 { 3891 misc_deregister(&tun_miscdev); 3892 rtnl_link_unregister(&tun_link_ops); 3893 unregister_netdevice_notifier(&tun_notifier_block); 3894 rcu_barrier(); 3895 } 3896 3897 /* Get an underlying socket object from tun file. Returns error unless file is 3898 * attached to a device. The returned object works like a packet socket, it 3899 * can be used for sock_sendmsg/sock_recvmsg. The caller is responsible for 3900 * holding a reference to the file for as long as the socket is in use. */ 3901 struct socket *tun_get_socket(struct file *file) 3902 { 3903 struct tun_file *tfile; 3904 if (file->f_op != &tun_fops) 3905 return ERR_PTR(-EINVAL); 3906 tfile = file->private_data; 3907 if (!tfile) 3908 return ERR_PTR(-EBADFD); 3909 return &tfile->socket; 3910 } 3911 EXPORT_SYMBOL_GPL(tun_get_socket); 3912 3913 struct ptr_ring *tun_get_tx_ring(struct file *file) 3914 { 3915 struct tun_file *tfile; 3916 3917 if (file->f_op != &tun_fops) 3918 return ERR_PTR(-EINVAL); 3919 tfile = file->private_data; 3920 if (!tfile) 3921 return ERR_PTR(-EBADFD); 3922 return &tfile->tx_ring; 3923 } 3924 EXPORT_SYMBOL_GPL(tun_get_tx_ring); 3925 3926 /* Callers must hold ring.consumer_lock */ 3927 void tun_wake_queue(struct file *file, int consumed) 3928 { 3929 struct tun_file *tfile; 3930 struct tun_struct *tun; 3931 3932 if (file->f_op != &tun_fops) 3933 return; 3934 3935 tfile = file->private_data; 3936 if (!tfile) 3937 return; 3938 3939 lockdep_assert_held(&tfile->tx_ring.consumer_lock); 3940 3941 rcu_read_lock(); 3942 3943 tun = rcu_dereference(tfile->tun); 3944 if (tun) 3945 __tun_wake_queue(tun, tfile, consumed); 3946 3947 rcu_read_unlock(); 3948 } 3949 EXPORT_SYMBOL_GPL(tun_wake_queue); 3950 3951 module_init(tun_init); 3952 module_exit(tun_cleanup); 3953 MODULE_DESCRIPTION(DRV_DESCRIPTION); 3954 MODULE_AUTHOR(DRV_COPYRIGHT); 3955 MODULE_LICENSE("GPL"); 3956 MODULE_ALIAS_MISCDEV(TUN_MINOR); 3957 MODULE_ALIAS("devname:net/tun"); 3958 MODULE_IMPORT_NS("NETDEV_INTERNAL"); 3959