xref: /linux/drivers/net/tun.c (revision 73e08d50eb139e7329032c89a4cb105cf43ef278)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  TUN - Universal TUN/TAP device driver.
4  *  Copyright (C) 1999-2002 Maxim Krasnyansky <maxk@qualcomm.com>
5  *
6  *  $Id: tun.c,v 1.15 2002/03/01 02:44:24 maxk Exp $
7  */
8 
9 /*
10  *  Changes:
11  *
12  *  Mike Kershaw <dragorn@kismetwireless.net> 2005/08/14
13  *    Add TUNSETLINK ioctl to set the link encapsulation
14  *
15  *  Mark Smith <markzzzsmith@yahoo.com.au>
16  *    Use eth_random_addr() for tap MAC address.
17  *
18  *  Harald Roelle <harald.roelle@ifi.lmu.de>  2004/04/20
19  *    Fixes in packet dropping, queue length setting and queue wakeup.
20  *    Increased default tx queue length.
21  *    Added ethtool API.
22  *    Minor cleanups
23  *
24  *  Daniel Podlejski <underley@underley.eu.org>
25  *    Modifications for 2.3.99-pre5 kernel.
26  */
27 
28 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29 
30 #define DRV_NAME	"tun"
31 #define DRV_VERSION	"1.6"
32 #define DRV_DESCRIPTION	"Universal TUN/TAP device driver"
33 #define DRV_COPYRIGHT	"(C) 1999-2004 Max Krasnyansky <maxk@qualcomm.com>"
34 
35 #include <linux/module.h>
36 #include <linux/errno.h>
37 #include <linux/kernel.h>
38 #include <linux/sched/signal.h>
39 #include <linux/major.h>
40 #include <linux/slab.h>
41 #include <linux/poll.h>
42 #include <linux/fcntl.h>
43 #include <linux/init.h>
44 #include <linux/skbuff.h>
45 #include <linux/netdevice.h>
46 #include <linux/etherdevice.h>
47 #include <linux/miscdevice.h>
48 #include <linux/ethtool.h>
49 #include <linux/rtnetlink.h>
50 #include <linux/compat.h>
51 #include <linux/if.h>
52 #include <linux/if_arp.h>
53 #include <linux/if_ether.h>
54 #include <linux/if_tun.h>
55 #include <linux/if_vlan.h>
56 #include <linux/crc32.h>
57 #include <linux/math.h>
58 #include <linux/nsproxy.h>
59 #include <linux/virtio_net.h>
60 #include <linux/rcupdate.h>
61 #include <net/net_namespace.h>
62 #include <net/netns/generic.h>
63 #include <net/rtnetlink.h>
64 #include <net/sock.h>
65 #include <net/xdp.h>
66 #include <net/ip_tunnels.h>
67 #include <linux/seq_file.h>
68 #include <linux/uio.h>
69 #include <linux/skb_array.h>
70 #include <linux/bpf.h>
71 #include <linux/bpf_trace.h>
72 #include <linux/mutex.h>
73 #include <linux/ieee802154.h>
74 #include <uapi/linux/if_ltalk.h>
75 #include <uapi/linux/if_fddi.h>
76 #include <uapi/linux/if_hippi.h>
77 #include <uapi/linux/if_fc.h>
78 #include <net/ax25.h>
79 #include <net/rose.h>
80 #include <net/6lowpan.h>
81 #include <net/rps.h>
82 
83 #include <linux/uaccess.h>
84 #include <linux/proc_fs.h>
85 
86 #include "tun_vnet.h"
87 
88 static void tun_default_link_ksettings(struct net_device *dev,
89 				       struct ethtool_link_ksettings *cmd);
90 
91 #define TUN_RX_PAD (NET_IP_ALIGN + NET_SKB_PAD)
92 
93 /* TUN device flags */
94 
95 /* IFF_ATTACH_QUEUE is never stored in device flags,
96  * overload it to mean fasync when stored there.
97  */
98 #define TUN_FASYNC	IFF_ATTACH_QUEUE
99 
100 #define TUN_FEATURES (IFF_NO_PI | IFF_ONE_QUEUE | IFF_VNET_HDR | \
101 		      IFF_MULTI_QUEUE | IFF_NAPI | IFF_NAPI_FRAGS | \
102 		      IFF_BACKPRESSURE)
103 
104 #define GOODCOPY_LEN 128
105 
106 #define FLT_EXACT_COUNT 8
107 struct tap_filter {
108 	unsigned int    count;    /* Number of addrs. Zero means disabled */
109 	u32             mask[2];  /* Mask of the hashed addrs */
110 	unsigned char	addr[FLT_EXACT_COUNT][ETH_ALEN];
111 };
112 
113 /* MAX_TAP_QUEUES 256 is chosen to allow rx/tx queues to be equal
114  * to max number of VCPUs in guest. */
115 #define MAX_TAP_QUEUES 256
116 #define MAX_TAP_FLOWS  4096
117 
118 #define TUN_FLOW_EXPIRE (3 * HZ)
119 
120 /* A tun_file connects an open character device to a tuntap netdevice. It
121  * also contains all socket related structures (except sock_fprog and tap_filter)
122  * to serve as one transmit queue for tuntap device. The sock_fprog and
123  * tap_filter were kept in tun_struct since they were used for filtering for the
124  * netdevice not for a specific queue (at least I didn't see the requirement for
125  * this).
126  *
127  * RCU usage:
128  * The tun_file and tun_struct are loosely coupled, the pointer from one to the
129  * other can only be read while rcu_read_lock or rtnl_lock is held.
130  */
131 struct tun_file {
132 	struct sock sk;
133 	struct socket socket;
134 	struct tun_struct __rcu *tun;
135 	struct fasync_struct *fasync;
136 	/* only used for fasnyc */
137 	unsigned int flags;
138 	union {
139 		u16 queue_index;
140 		unsigned int ifindex;
141 	};
142 	struct napi_struct napi;
143 	bool napi_enabled;
144 	bool napi_frags_enabled;
145 	struct mutex napi_mutex;	/* Protects access to the above napi */
146 	struct list_head next;
147 	struct tun_struct *detached;
148 	struct ptr_ring tx_ring;
149 	/* Protected by tx_ring.consumer_lock */
150 	int cons_cnt;
151 	struct xdp_rxq_info xdp_rxq;
152 };
153 
154 struct tun_page {
155 	struct page *page;
156 	int count;
157 };
158 
159 struct tun_flow_entry {
160 	struct hlist_node hash_link;
161 	struct rcu_head rcu;
162 	struct tun_struct *tun;
163 
164 	u32 rxhash;
165 	u32 rps_rxhash;
166 	int queue_index;
167 	unsigned long updated ____cacheline_aligned_in_smp;
168 };
169 
170 #define TUN_NUM_FLOW_ENTRIES 1024
171 #define TUN_MASK_FLOW_ENTRIES (TUN_NUM_FLOW_ENTRIES - 1)
172 
173 struct tun_prog {
174 	struct rcu_head rcu;
175 	struct bpf_prog *prog;
176 };
177 
178 /* Since the socket were moved to tun_file, to preserve the behavior of persist
179  * device, socket filter, sndbuf and vnet header size were restore when the
180  * file were attached to a persist device.
181  */
182 struct tun_struct {
183 	struct tun_file __rcu	*tfiles[MAX_TAP_QUEUES];
184 	unsigned int            numqueues;
185 	unsigned int 		flags;
186 	kuid_t			owner;
187 	kgid_t			group;
188 
189 	struct net_device	*dev;
190 	netdev_features_t	set_features;
191 #define TUN_USER_FEATURES (NETIF_F_HW_CSUM|NETIF_F_TSO_ECN|NETIF_F_TSO| \
192 			  NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4 | \
193 			  NETIF_F_GSO_UDP_TUNNEL | NETIF_F_GSO_UDP_TUNNEL_CSUM)
194 
195 	int			align;
196 	int			vnet_hdr_sz;
197 	int			sndbuf;
198 	struct tap_filter	txflt;
199 	struct sock_fprog	fprog;
200 	/* protected by rtnl lock */
201 	bool			filter_attached;
202 	u32			msg_enable;
203 	spinlock_t lock;
204 	struct hlist_head flows[TUN_NUM_FLOW_ENTRIES];
205 	struct timer_list flow_gc_timer;
206 	unsigned long ageing_time;
207 	unsigned int numdisabled;
208 	struct list_head disabled;
209 	void *security;
210 	u32 flow_count;
211 	u32 rx_batched;
212 	atomic_long_t rx_frame_errors;
213 	struct bpf_prog __rcu *xdp_prog;
214 	struct tun_prog __rcu *steering_prog;
215 	struct tun_prog __rcu *filter_prog;
216 	struct ethtool_link_ksettings link_ksettings;
217 	/* init args */
218 	struct file *file;
219 	struct ifreq *ifr;
220 };
221 
222 struct veth {
223 	__be16 h_vlan_proto;
224 	__be16 h_vlan_TCI;
225 };
226 
227 static void tun_flow_init(struct tun_struct *tun);
228 static void tun_flow_uninit(struct tun_struct *tun);
229 
230 static int tun_napi_receive(struct napi_struct *napi, int budget)
231 {
232 	struct tun_file *tfile = container_of(napi, struct tun_file, napi);
233 	struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
234 	struct sk_buff_head process_queue;
235 	struct sk_buff *skb;
236 	int received = 0;
237 
238 	__skb_queue_head_init(&process_queue);
239 
240 	spin_lock(&queue->lock);
241 	skb_queue_splice_tail_init(queue, &process_queue);
242 	spin_unlock(&queue->lock);
243 
244 	while (received < budget && (skb = __skb_dequeue(&process_queue))) {
245 		napi_gro_receive(napi, skb);
246 		++received;
247 	}
248 
249 	if (!skb_queue_empty(&process_queue)) {
250 		spin_lock(&queue->lock);
251 		skb_queue_splice(&process_queue, queue);
252 		spin_unlock(&queue->lock);
253 	}
254 
255 	return received;
256 }
257 
258 static int tun_napi_poll(struct napi_struct *napi, int budget)
259 {
260 	unsigned int received;
261 
262 	received = tun_napi_receive(napi, budget);
263 
264 	if (received < budget)
265 		napi_complete_done(napi, received);
266 
267 	return received;
268 }
269 
270 static void tun_napi_init(struct tun_struct *tun, struct tun_file *tfile,
271 			  bool napi_en, bool napi_frags)
272 {
273 	tfile->napi_enabled = napi_en;
274 	tfile->napi_frags_enabled = napi_en && napi_frags;
275 	if (napi_en) {
276 		netif_napi_add_tx(tun->dev, &tfile->napi, tun_napi_poll);
277 		napi_enable(&tfile->napi);
278 	}
279 }
280 
281 static void tun_napi_enable(struct tun_file *tfile)
282 {
283 	if (tfile->napi_enabled)
284 		napi_enable(&tfile->napi);
285 }
286 
287 static void tun_napi_disable(struct tun_file *tfile)
288 {
289 	if (tfile->napi_enabled)
290 		napi_disable(&tfile->napi);
291 }
292 
293 static void tun_napi_del(struct tun_file *tfile)
294 {
295 	if (tfile->napi_enabled)
296 		netif_napi_del(&tfile->napi);
297 }
298 
299 static bool tun_napi_frags_enabled(const struct tun_file *tfile)
300 {
301 	return tfile->napi_frags_enabled;
302 }
303 
304 static inline u32 tun_hashfn(u32 rxhash)
305 {
306 	return rxhash & TUN_MASK_FLOW_ENTRIES;
307 }
308 
309 static struct tun_flow_entry *tun_flow_find(struct hlist_head *head, u32 rxhash)
310 {
311 	struct tun_flow_entry *e;
312 
313 	hlist_for_each_entry_rcu(e, head, hash_link) {
314 		if (e->rxhash == rxhash)
315 			return e;
316 	}
317 	return NULL;
318 }
319 
320 static struct tun_flow_entry *tun_flow_create(struct tun_struct *tun,
321 					      struct hlist_head *head,
322 					      u32 rxhash, u16 queue_index)
323 {
324 	struct tun_flow_entry *e = kmalloc_obj(*e, GFP_ATOMIC);
325 
326 	if (e) {
327 		netif_info(tun, tx_queued, tun->dev,
328 			   "create flow: hash %u index %u\n",
329 			   rxhash, queue_index);
330 		e->updated = jiffies;
331 		e->rxhash = rxhash;
332 		e->rps_rxhash = 0;
333 		e->queue_index = queue_index;
334 		e->tun = tun;
335 		hlist_add_head_rcu(&e->hash_link, head);
336 		++tun->flow_count;
337 	}
338 	return e;
339 }
340 
341 static void tun_flow_delete(struct tun_struct *tun, struct tun_flow_entry *e)
342 {
343 	netif_info(tun, tx_queued, tun->dev, "delete flow: hash %u index %u\n",
344 		   e->rxhash, e->queue_index);
345 	hlist_del_rcu(&e->hash_link);
346 	kfree_rcu(e, rcu);
347 	--tun->flow_count;
348 }
349 
350 static void tun_flow_flush(struct tun_struct *tun)
351 {
352 	int i;
353 
354 	spin_lock_bh(&tun->lock);
355 	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
356 		struct tun_flow_entry *e;
357 		struct hlist_node *n;
358 
359 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link)
360 			tun_flow_delete(tun, e);
361 	}
362 	spin_unlock_bh(&tun->lock);
363 }
364 
365 static void tun_flow_delete_by_queue(struct tun_struct *tun, u16 queue_index)
366 {
367 	int i;
368 
369 	spin_lock_bh(&tun->lock);
370 	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
371 		struct tun_flow_entry *e;
372 		struct hlist_node *n;
373 
374 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
375 			if (e->queue_index == queue_index)
376 				tun_flow_delete(tun, e);
377 		}
378 	}
379 	spin_unlock_bh(&tun->lock);
380 }
381 
382 static void tun_flow_cleanup(struct timer_list *t)
383 {
384 	struct tun_struct *tun = timer_container_of(tun, t, flow_gc_timer);
385 	unsigned long delay = tun->ageing_time;
386 	unsigned long next_timer = jiffies + delay;
387 	unsigned long count = 0;
388 	int i;
389 
390 	spin_lock(&tun->lock);
391 	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++) {
392 		struct tun_flow_entry *e;
393 		struct hlist_node *n;
394 
395 		hlist_for_each_entry_safe(e, n, &tun->flows[i], hash_link) {
396 			unsigned long this_timer;
397 
398 			this_timer = e->updated + delay;
399 			if (time_before_eq(this_timer, jiffies)) {
400 				tun_flow_delete(tun, e);
401 				continue;
402 			}
403 			count++;
404 			if (time_before(this_timer, next_timer))
405 				next_timer = this_timer;
406 		}
407 	}
408 
409 	if (count)
410 		mod_timer(&tun->flow_gc_timer, round_jiffies_up(next_timer));
411 	spin_unlock(&tun->lock);
412 }
413 
414 static void tun_flow_update(struct tun_struct *tun, u32 rxhash,
415 			    struct tun_file *tfile)
416 {
417 	struct hlist_head *head;
418 	struct tun_flow_entry *e;
419 	unsigned long delay = tun->ageing_time;
420 	u16 queue_index = tfile->queue_index;
421 
422 	head = &tun->flows[tun_hashfn(rxhash)];
423 
424 	rcu_read_lock();
425 
426 	e = tun_flow_find(head, rxhash);
427 	if (likely(e)) {
428 		/* TODO: keep queueing to old queue until it's empty? */
429 		if (READ_ONCE(e->queue_index) != queue_index)
430 			WRITE_ONCE(e->queue_index, queue_index);
431 		if (e->updated != jiffies)
432 			e->updated = jiffies;
433 		sock_rps_record_flow_hash(e->rps_rxhash);
434 	} else {
435 		spin_lock_bh(&tun->lock);
436 		if (!tun_flow_find(head, rxhash) &&
437 		    tun->flow_count < MAX_TAP_FLOWS)
438 			tun_flow_create(tun, head, rxhash, queue_index);
439 
440 		if (!timer_pending(&tun->flow_gc_timer))
441 			mod_timer(&tun->flow_gc_timer,
442 				  round_jiffies_up(jiffies + delay));
443 		spin_unlock_bh(&tun->lock);
444 	}
445 
446 	rcu_read_unlock();
447 }
448 
449 /* Save the hash received in the stack receive path and update the
450  * flow_hash table accordingly.
451  */
452 static inline void tun_flow_save_rps_rxhash(struct tun_flow_entry *e, u32 hash)
453 {
454 	if (unlikely(e->rps_rxhash != hash))
455 		e->rps_rxhash = hash;
456 }
457 
458 /* We try to identify a flow through its rxhash. The reason that
459  * we do not check rxq no. is because some cards(e.g 82599), chooses
460  * the rxq based on the txq where the last packet of the flow comes. As
461  * the userspace application move between processors, we may get a
462  * different rxq no. here.
463  */
464 static u16 tun_automq_select_queue(struct tun_struct *tun, struct sk_buff *skb)
465 {
466 	struct tun_flow_entry *e;
467 	u32 txq, numqueues;
468 
469 	numqueues = READ_ONCE(tun->numqueues);
470 
471 	txq = __skb_get_hash_symmetric(skb);
472 	e = tun_flow_find(&tun->flows[tun_hashfn(txq)], txq);
473 	if (e) {
474 		tun_flow_save_rps_rxhash(e, txq);
475 		txq = e->queue_index;
476 	} else {
477 		txq = reciprocal_scale(txq, numqueues);
478 	}
479 
480 	return txq;
481 }
482 
483 static u16 tun_ebpf_select_queue(struct tun_struct *tun, struct sk_buff *skb)
484 {
485 	struct tun_prog *prog;
486 	u32 numqueues;
487 	u16 ret = 0;
488 
489 	numqueues = READ_ONCE(tun->numqueues);
490 	if (!numqueues)
491 		return 0;
492 
493 	prog = rcu_dereference(tun->steering_prog);
494 	if (prog)
495 		ret = bpf_prog_run_clear_cb(prog->prog, skb);
496 
497 	return ret % numqueues;
498 }
499 
500 static u16 tun_select_queue(struct net_device *dev, struct sk_buff *skb,
501 			    struct net_device *sb_dev)
502 {
503 	struct tun_struct *tun = netdev_priv(dev);
504 	u16 ret;
505 
506 	rcu_read_lock();
507 	if (rcu_dereference(tun->steering_prog))
508 		ret = tun_ebpf_select_queue(tun, skb);
509 	else
510 		ret = tun_automq_select_queue(tun, skb);
511 	rcu_read_unlock();
512 
513 	return ret;
514 }
515 
516 static inline bool tun_permitted(struct tun_struct *tun)
517 {
518 	const struct cred *cred = current_cred();
519 	struct net *net = dev_net(tun->dev);
520 
521 	/* If both uid and gid are set, *both* must match */
522 	if ((uid_valid(tun->owner) || gid_valid(tun->group)) &&
523 	    (!uid_valid(tun->owner) || uid_eq(cred->euid, tun->owner)) &&
524 	    (!gid_valid(tun->group) || in_egroup_p(tun->group)))
525 		return true;
526 
527 	return ns_capable(net->user_ns, CAP_NET_ADMIN);
528 }
529 
530 static void tun_set_real_num_queues(struct tun_struct *tun)
531 {
532 	netif_set_real_num_tx_queues(tun->dev, tun->numqueues);
533 	netif_set_real_num_rx_queues(tun->dev, tun->numqueues);
534 }
535 
536 static void tun_disable_queue(struct tun_struct *tun, struct tun_file *tfile)
537 {
538 	tfile->detached = tun;
539 	list_add_tail(&tfile->next, &tun->disabled);
540 	WRITE_ONCE(tun->numdisabled, tun->numdisabled + 1);
541 }
542 
543 static struct tun_struct *tun_enable_queue(struct tun_file *tfile)
544 {
545 	struct tun_struct *tun = tfile->detached;
546 
547 	tfile->detached = NULL;
548 	list_del_init(&tfile->next);
549 	WRITE_ONCE(tun->numdisabled, tun->numdisabled - 1);
550 	return tun;
551 }
552 
553 void tun_ptr_free(void *ptr)
554 {
555 	if (!ptr)
556 		return;
557 	if (tun_is_xdp_frame(ptr)) {
558 		struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
559 
560 		xdp_return_frame(xdpf);
561 	} else {
562 		__skb_array_destroy_skb(ptr);
563 	}
564 }
565 EXPORT_SYMBOL_GPL(tun_ptr_free);
566 
567 static void tun_queue_purge(struct tun_file *tfile)
568 {
569 	void *ptr;
570 
571 	while ((ptr = ptr_ring_consume(&tfile->tx_ring)) != NULL)
572 		tun_ptr_free(ptr);
573 
574 	skb_queue_purge(&tfile->sk.sk_write_queue);
575 	skb_queue_purge(&tfile->sk.sk_error_queue);
576 }
577 
578 static void __tun_detach(struct tun_file *tfile, bool clean)
579 {
580 	struct tun_file *ntfile;
581 	struct tun_struct *tun;
582 
583 	tun = rtnl_dereference(tfile->tun);
584 
585 	if (tun && clean) {
586 		if (!tfile->detached)
587 			tun_napi_disable(tfile);
588 		tun_napi_del(tfile);
589 	}
590 
591 	if (tun && !tfile->detached) {
592 		u16 index = tfile->queue_index;
593 		BUG_ON(index >= tun->numqueues);
594 
595 		spin_lock(&tfile->tx_ring.consumer_lock);
596 		rcu_assign_pointer(tun->tfiles[index],
597 				   tun->tfiles[tun->numqueues - 1]);
598 		spin_unlock(&tfile->tx_ring.consumer_lock);
599 		ntfile = rtnl_dereference(tun->tfiles[index]);
600 		spin_lock(&ntfile->tx_ring.consumer_lock);
601 		ntfile->queue_index = index;
602 		ntfile->xdp_rxq.queue_index = index;
603 		ntfile->cons_cnt = 0;
604 		spin_unlock(&ntfile->tx_ring.consumer_lock);
605 		rcu_assign_pointer(tun->tfiles[tun->numqueues - 1],
606 				   NULL);
607 
608 		WRITE_ONCE(tun->numqueues, tun->numqueues - 1);
609 		if (clean) {
610 			RCU_INIT_POINTER(tfile->tun, NULL);
611 			sock_put(&tfile->sk);
612 		} else {
613 			tun_disable_queue(tun, tfile);
614 			tun_napi_disable(tfile);
615 		}
616 
617 		synchronize_net();
618 		tun_flow_delete_by_queue(tun, tun->numqueues);
619 		/* Drop read queue */
620 		tun_queue_purge(tfile);
621 		spin_lock_bh(&ntfile->tx_ring.consumer_lock);
622 		spin_lock(&ntfile->tx_ring.producer_lock);
623 		ntfile->cons_cnt = 0;
624 		if (netif_running(tun->dev) &&
625 		    __ptr_ring_empty(&ntfile->tx_ring))
626 			netif_wake_subqueue(tun->dev, index);
627 		spin_unlock(&ntfile->tx_ring.producer_lock);
628 		spin_unlock_bh(&ntfile->tx_ring.consumer_lock);
629 		tun_set_real_num_queues(tun);
630 	} else if (tfile->detached && clean) {
631 		tun = tun_enable_queue(tfile);
632 		sock_put(&tfile->sk);
633 	}
634 
635 	if (clean) {
636 		if (tun && tun->numqueues == 0 && tun->numdisabled == 0) {
637 			netif_carrier_off(tun->dev);
638 
639 			if (!(tun->flags & IFF_PERSIST) &&
640 			    tun->dev->reg_state == NETREG_REGISTERED)
641 				unregister_netdevice(tun->dev);
642 		}
643 		if (tun)
644 			xdp_rxq_info_unreg(&tfile->xdp_rxq);
645 		ptr_ring_cleanup(&tfile->tx_ring, tun_ptr_free);
646 	}
647 }
648 
649 static void tun_detach(struct tun_file *tfile, bool clean)
650 {
651 	struct tun_struct *tun;
652 	struct net_device *dev;
653 
654 	rtnl_lock();
655 	tun = rtnl_dereference(tfile->tun);
656 	dev = tun ? tun->dev : NULL;
657 	__tun_detach(tfile, clean);
658 	if (dev)
659 		netdev_state_change(dev);
660 	rtnl_unlock();
661 
662 	if (clean)
663 		sock_put(&tfile->sk);
664 }
665 
666 static void tun_detach_all(struct net_device *dev)
667 {
668 	struct tun_struct *tun = netdev_priv(dev);
669 	struct tun_file *tfile, *tmp;
670 	int i, n = tun->numqueues;
671 
672 	for (i = 0; i < n; i++) {
673 		tfile = rtnl_dereference(tun->tfiles[i]);
674 		BUG_ON(!tfile);
675 		tun_napi_disable(tfile);
676 		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
677 		tfile->socket.sk->sk_data_ready(tfile->socket.sk);
678 		RCU_INIT_POINTER(tfile->tun, NULL);
679 		WRITE_ONCE(tun->numqueues, tun->numqueues - 1);
680 	}
681 	list_for_each_entry(tfile, &tun->disabled, next) {
682 		tfile->socket.sk->sk_shutdown = RCV_SHUTDOWN;
683 		tfile->socket.sk->sk_data_ready(tfile->socket.sk);
684 		RCU_INIT_POINTER(tfile->tun, NULL);
685 	}
686 	BUG_ON(tun->numqueues != 0);
687 
688 	synchronize_net();
689 	for (i = 0; i < n; i++) {
690 		tfile = rtnl_dereference(tun->tfiles[i]);
691 		mutex_lock(&tfile->napi_mutex);
692 		tun_napi_del(tfile);
693 		mutex_unlock(&tfile->napi_mutex);
694 		/* Drop read queue */
695 		tun_queue_purge(tfile);
696 		xdp_rxq_info_unreg(&tfile->xdp_rxq);
697 		sock_put(&tfile->sk);
698 	}
699 	list_for_each_entry_safe(tfile, tmp, &tun->disabled, next) {
700 		mutex_lock(&tfile->napi_mutex);
701 		tun_napi_del(tfile);
702 		mutex_unlock(&tfile->napi_mutex);
703 		tun_enable_queue(tfile);
704 		tun_queue_purge(tfile);
705 		xdp_rxq_info_unreg(&tfile->xdp_rxq);
706 		sock_put(&tfile->sk);
707 	}
708 	BUG_ON(tun->numdisabled != 0);
709 
710 	if (tun->flags & IFF_PERSIST)
711 		module_put(THIS_MODULE);
712 }
713 
714 static void tun_force_wake_queue(struct tun_struct *tun,
715 				 struct tun_file *tfile)
716 {
717 	/* Ensure that the producer can not stop the
718 	 * queue concurrently by taking locks.
719 	 */
720 	spin_lock_bh(&tfile->tx_ring.consumer_lock);
721 	spin_lock(&tfile->tx_ring.producer_lock);
722 	tfile->cons_cnt = 0;
723 	/* Tested under the locks that tun_net_close() takes, so this can not
724 	 * undo its stop. tun_net_open() wakes the queues of a device that
725 	 * comes back up.
726 	 */
727 	if (netif_running(tun->dev))
728 		netif_wake_subqueue(tun->dev, tfile->queue_index);
729 	spin_unlock(&tfile->tx_ring.producer_lock);
730 	spin_unlock_bh(&tfile->tx_ring.consumer_lock);
731 }
732 
733 static int tun_attach(struct tun_struct *tun, struct file *file,
734 		      bool skip_filter, bool napi, bool napi_frags,
735 		      bool publish_tun)
736 {
737 	struct tun_file *tfile = file->private_data;
738 	struct net_device *dev = tun->dev;
739 	int err;
740 
741 	err = security_tun_dev_attach(tfile->socket.sk, tun->security);
742 	if (err < 0)
743 		goto out;
744 
745 	err = -EINVAL;
746 	if (rtnl_dereference(tfile->tun) && !tfile->detached)
747 		goto out;
748 
749 	err = -EBUSY;
750 	if (!(tun->flags & IFF_MULTI_QUEUE) && tun->numqueues == 1)
751 		goto out;
752 
753 	err = -E2BIG;
754 	if (!tfile->detached &&
755 	    tun->numqueues + tun->numdisabled == MAX_TAP_QUEUES)
756 		goto out;
757 
758 	err = 0;
759 
760 	/* Re-attach the filter to persist device */
761 	if (!skip_filter && (tun->filter_attached == true)) {
762 		lock_sock(tfile->socket.sk);
763 		err = sk_attach_filter(&tun->fprog, tfile->socket.sk);
764 		release_sock(tfile->socket.sk);
765 		if (!err)
766 			goto out;
767 	}
768 
769 	if (!tfile->detached &&
770 	    ptr_ring_resize(&tfile->tx_ring, dev->tx_queue_len,
771 			    GFP_KERNEL, tun_ptr_free)) {
772 		err = -ENOMEM;
773 		goto out;
774 	}
775 
776 	spin_lock(&tfile->tx_ring.consumer_lock);
777 	tfile->queue_index = tun->numqueues;
778 	spin_unlock(&tfile->tx_ring.consumer_lock);
779 	tfile->socket.sk->sk_shutdown &= ~RCV_SHUTDOWN;
780 	tun_force_wake_queue(tun, tfile);
781 
782 	if (tfile->detached) {
783 		/* Re-attach detached tfile, updating XDP queue_index */
784 		WARN_ON(!xdp_rxq_info_is_reg(&tfile->xdp_rxq));
785 
786 		if (tfile->xdp_rxq.queue_index    != tfile->queue_index)
787 			tfile->xdp_rxq.queue_index = tfile->queue_index;
788 	} else {
789 		/* Setup XDP RX-queue info, for new tfile getting attached */
790 		err = xdp_rxq_info_reg(&tfile->xdp_rxq,
791 				       tun->dev, tfile->queue_index, 0);
792 		if (err < 0)
793 			goto out;
794 		err = xdp_rxq_info_reg_mem_model(&tfile->xdp_rxq,
795 						 MEM_TYPE_PAGE_SHARED, NULL);
796 		if (err < 0) {
797 			xdp_rxq_info_unreg(&tfile->xdp_rxq);
798 			goto out;
799 		}
800 		err = 0;
801 	}
802 
803 	if (tfile->detached) {
804 		tun_enable_queue(tfile);
805 		tun_napi_enable(tfile);
806 	} else {
807 		sock_hold(&tfile->sk);
808 		tun_napi_init(tun, tfile, napi, napi_frags);
809 	}
810 
811 	if (rtnl_dereference(tun->xdp_prog))
812 		sock_set_flag(&tfile->sk, SOCK_XDP);
813 
814 	/* device is allowed to go away first, so no need to hold extra
815 	 * refcnt.
816 	 */
817 
818 	/* Publish tfile->tun and tun->tfiles only after we've fully
819 	 * initialized tfile; otherwise we risk using half-initialized
820 	 * object.
821 	 */
822 	if (publish_tun)
823 		rcu_assign_pointer(tfile->tun, tun);
824 	rcu_assign_pointer(tun->tfiles[tun->numqueues], tfile);
825 	WRITE_ONCE(tun->numqueues, tun->numqueues + 1);
826 	tun_set_real_num_queues(tun);
827 out:
828 	return err;
829 }
830 
831 static struct tun_struct *tun_get(struct tun_file *tfile)
832 {
833 	struct tun_struct *tun;
834 
835 	rcu_read_lock();
836 	tun = rcu_dereference(tfile->tun);
837 	if (tun)
838 		dev_hold(tun->dev);
839 	rcu_read_unlock();
840 
841 	return tun;
842 }
843 
844 static void tun_put(struct tun_struct *tun)
845 {
846 	dev_put(tun->dev);
847 }
848 
849 /* TAP filtering */
850 static void addr_hash_set(u32 *mask, const u8 *addr)
851 {
852 	int n = ether_crc(ETH_ALEN, addr) >> 26;
853 	mask[n >> 5] |= (1 << (n & 31));
854 }
855 
856 static unsigned int addr_hash_test(const u32 *mask, const u8 *addr)
857 {
858 	int n = ether_crc(ETH_ALEN, addr) >> 26;
859 	return mask[n >> 5] & (1 << (n & 31));
860 }
861 
862 static int update_filter(struct tap_filter *filter, void __user *arg)
863 {
864 	struct { u8 u[ETH_ALEN]; } *addr;
865 	struct tun_filter uf;
866 	int err, alen, n, nexact;
867 
868 	if (copy_from_user(&uf, arg, sizeof(uf)))
869 		return -EFAULT;
870 
871 	if (!uf.count) {
872 		/* Disabled */
873 		filter->count = 0;
874 		return 0;
875 	}
876 
877 	alen = ETH_ALEN * uf.count;
878 	addr = memdup_user(arg + sizeof(uf), alen);
879 	if (IS_ERR(addr))
880 		return PTR_ERR(addr);
881 
882 	/* The filter is updated without holding any locks. Which is
883 	 * perfectly safe. We disable it first and in the worst
884 	 * case we'll accept a few undesired packets. */
885 	filter->count = 0;
886 	wmb();
887 
888 	/* Use first set of addresses as an exact filter */
889 	for (n = 0; n < uf.count && n < FLT_EXACT_COUNT; n++)
890 		memcpy(filter->addr[n], addr[n].u, ETH_ALEN);
891 
892 	nexact = n;
893 
894 	/* Remaining multicast addresses are hashed,
895 	 * unicast will leave the filter disabled. */
896 	memset(filter->mask, 0, sizeof(filter->mask));
897 	for (; n < uf.count; n++) {
898 		if (!is_multicast_ether_addr(addr[n].u)) {
899 			err = 0; /* no filter */
900 			goto free_addr;
901 		}
902 		addr_hash_set(filter->mask, addr[n].u);
903 	}
904 
905 	/* For ALLMULTI just set the mask to all ones.
906 	 * This overrides the mask populated above. */
907 	if ((uf.flags & TUN_FLT_ALLMULTI))
908 		memset(filter->mask, ~0, sizeof(filter->mask));
909 
910 	/* Now enable the filter */
911 	wmb();
912 	filter->count = nexact;
913 
914 	/* Return the number of exact filters */
915 	err = nexact;
916 free_addr:
917 	kfree(addr);
918 	return err;
919 }
920 
921 /* Returns: 0 - drop, !=0 - accept */
922 static int run_filter(struct tap_filter *filter, const struct sk_buff *skb)
923 {
924 	/* Cannot use eth_hdr(skb) here because skb_mac_hdr() is incorrect
925 	 * at this point. */
926 	struct ethhdr *eh = (struct ethhdr *) skb->data;
927 	int i;
928 
929 	/* Exact match */
930 	for (i = 0; i < filter->count; i++)
931 		if (ether_addr_equal(eh->h_dest, filter->addr[i]))
932 			return 1;
933 
934 	/* Inexact match (multicast only) */
935 	if (is_multicast_ether_addr(eh->h_dest))
936 		return addr_hash_test(filter->mask, eh->h_dest);
937 
938 	return 0;
939 }
940 
941 /*
942  * Checks whether the packet is accepted or not.
943  * Returns: 0 - drop, !=0 - accept
944  */
945 static int check_filter(struct tap_filter *filter, const struct sk_buff *skb)
946 {
947 	if (!filter->count)
948 		return 1;
949 
950 	return run_filter(filter, skb);
951 }
952 
953 /* Network device part of the driver */
954 
955 static const struct ethtool_ops tun_ethtool_ops;
956 
957 static int tun_net_init(struct net_device *dev)
958 {
959 	struct tun_struct *tun = netdev_priv(dev);
960 	struct ifreq *ifr = tun->ifr;
961 	int err;
962 
963 	spin_lock_init(&tun->lock);
964 
965 	err = security_tun_dev_alloc_security(&tun->security);
966 	if (err < 0)
967 		return err;
968 
969 	tun_flow_init(tun);
970 
971 	dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
972 	dev->hw_features = NETIF_F_SG | NETIF_F_FRAGLIST |
973 			   TUN_USER_FEATURES | NETIF_F_HW_VLAN_CTAG_TX |
974 			   NETIF_F_HW_VLAN_STAG_TX;
975 	dev->hw_enc_features = dev->hw_features;
976 	dev->features = dev->hw_features;
977 	dev->vlan_features = dev->features &
978 			     ~(NETIF_F_HW_VLAN_CTAG_TX |
979 			       NETIF_F_HW_VLAN_STAG_TX);
980 	dev->lltx = true;
981 
982 	tun->flags = (tun->flags & ~TUN_FEATURES) |
983 		      (ifr->ifr_flags & TUN_FEATURES);
984 
985 	INIT_LIST_HEAD(&tun->disabled);
986 	err = tun_attach(tun, tun->file, false, ifr->ifr_flags & IFF_NAPI,
987 			 ifr->ifr_flags & IFF_NAPI_FRAGS, false);
988 	if (err < 0) {
989 		tun_flow_uninit(tun);
990 		security_tun_dev_free_security(tun->security);
991 		return err;
992 	}
993 	return 0;
994 }
995 
996 /* Net device detach from fd. */
997 static void tun_net_uninit(struct net_device *dev)
998 {
999 	tun_detach_all(dev);
1000 }
1001 
1002 /* Net device open. */
1003 static int tun_net_open(struct net_device *dev)
1004 {
1005 	netif_tx_start_all_queues(dev);
1006 
1007 	return 0;
1008 }
1009 
1010 /* Net device close. */
1011 static int tun_net_close(struct net_device *dev)
1012 {
1013 	struct tun_struct *tun = netdev_priv(dev);
1014 	struct tun_file *tfile;
1015 	int i;
1016 
1017 	/* netif_running() is already false: take both ring locks to keep the
1018 	 * wake sites out, so the stop below is the last write to
1019 	 * __QUEUE_STATE_DRV_XOFF.
1020 	 */
1021 	for (i = 0; i < tun->numqueues; i++) {
1022 		tfile = rtnl_dereference(tun->tfiles[i]);
1023 
1024 		spin_lock_bh(&tfile->tx_ring.consumer_lock);
1025 		spin_lock(&tfile->tx_ring.producer_lock);
1026 		tfile->cons_cnt = 0;
1027 		spin_unlock(&tfile->tx_ring.producer_lock);
1028 		spin_unlock_bh(&tfile->tx_ring.consumer_lock);
1029 	}
1030 
1031 	netif_tx_stop_all_queues(dev);
1032 	return 0;
1033 }
1034 
1035 /* Net device start xmit */
1036 static void tun_automq_xmit(struct tun_struct *tun, struct sk_buff *skb)
1037 {
1038 #ifdef CONFIG_RPS
1039 	if (tun->numqueues == 1 && static_branch_unlikely(&rps_needed)) {
1040 		/* Select queue was not called for the skbuff, so we extract the
1041 		 * RPS hash and save it into the flow_table here.
1042 		 */
1043 		struct tun_flow_entry *e;
1044 		__u32 rxhash;
1045 
1046 		rxhash = __skb_get_hash_symmetric(skb);
1047 		e = tun_flow_find(&tun->flows[tun_hashfn(rxhash)], rxhash);
1048 		if (e)
1049 			tun_flow_save_rps_rxhash(e, rxhash);
1050 	}
1051 #endif
1052 }
1053 
1054 static unsigned int run_ebpf_filter(struct tun_struct *tun,
1055 				    struct sk_buff *skb,
1056 				    int len)
1057 {
1058 	struct tun_prog *prog = rcu_dereference(tun->filter_prog);
1059 
1060 	if (prog)
1061 		len = bpf_prog_run_clear_cb(prog->prog, skb);
1062 
1063 	return len;
1064 }
1065 
1066 /* Net device start xmit */
1067 static netdev_tx_t tun_net_xmit(struct sk_buff *skb, struct net_device *dev)
1068 {
1069 	enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1070 	struct tun_struct *tun = netdev_priv(dev);
1071 	int txq = skb->queue_mapping;
1072 	struct netdev_queue *queue;
1073 	struct tun_file *tfile;
1074 	int len = skb->len;
1075 	int ret;
1076 
1077 	rcu_read_lock();
1078 	tfile = rcu_dereference(tun->tfiles[txq]);
1079 
1080 	/* Drop packet if interface is not attached */
1081 	if (!tfile) {
1082 		drop_reason = SKB_DROP_REASON_DEV_READY;
1083 		goto drop;
1084 	}
1085 
1086 	if (!rcu_dereference(tun->steering_prog))
1087 		tun_automq_xmit(tun, skb);
1088 
1089 	netif_info(tun, tx_queued, tun->dev, "%s %d\n", __func__, skb->len);
1090 
1091 	/* Drop if the filter does not like it.
1092 	 * This is a noop if the filter is disabled.
1093 	 * Filter can be enabled only for the TAP devices. */
1094 	if (!check_filter(&tun->txflt, skb)) {
1095 		drop_reason = SKB_DROP_REASON_TAP_TXFILTER;
1096 		goto drop;
1097 	}
1098 
1099 	if (tfile->socket.sk->sk_filter) {
1100 		drop_reason = sk_filter_reason(tfile->socket.sk, skb);
1101 		if (drop_reason)
1102 			goto drop;
1103 	}
1104 
1105 	len = run_ebpf_filter(tun, skb, len);
1106 	if (len == 0) {
1107 		drop_reason = SKB_DROP_REASON_TAP_FILTER;
1108 		goto drop;
1109 	}
1110 
1111 	if (pskb_trim(skb, len)) {
1112 		drop_reason = SKB_DROP_REASON_NOMEM;
1113 		goto drop;
1114 	}
1115 
1116 	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) {
1117 		drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT;
1118 		goto drop;
1119 	}
1120 
1121 	skb_tx_timestamp(skb);
1122 
1123 	/* Orphan the skb - required as we might hang on to it
1124 	 * for indefinite time.
1125 	 */
1126 	skb_orphan(skb);
1127 
1128 	nf_reset_ct(skb);
1129 
1130 	queue = netdev_get_tx_queue(dev, txq);
1131 
1132 	spin_lock(&tfile->tx_ring.producer_lock);
1133 	ret = __ptr_ring_produce(&tfile->tx_ring, skb);
1134 	/* Do not touch the queue state of a device that is going down. */
1135 	if ((tun->flags & IFF_BACKPRESSURE) && netif_running(dev) &&
1136 	    !qdisc_txq_has_no_queue(queue) &&
1137 	    __ptr_ring_check_produce(&tfile->tx_ring) == -ENOSPC) {
1138 		netif_tx_stop_queue(queue);
1139 		/* Paired with smp_mb() in __tun_wake_queue() */
1140 		smp_mb__after_atomic();
1141 		if (!__ptr_ring_check_produce(&tfile->tx_ring))
1142 			netif_tx_wake_queue(queue);
1143 	}
1144 	spin_unlock(&tfile->tx_ring.producer_lock);
1145 
1146 	if (ret) {
1147 		/* This should be a rare case if IFF_BACKPRESSURE is enabled and
1148 		 * a qdisc is present, but can happen due to lltx.
1149 		 * Since skb_tx_timestamp(), skb_orphan(),
1150 		 * run_ebpf_filter() and pskb_trim() could have tinkered
1151 		 * with the SKB, returning NETDEV_TX_BUSY is unsafe and
1152 		 * we must drop instead.
1153 		 */
1154 		drop_reason = SKB_DROP_REASON_FULL_RING;
1155 		goto drop;
1156 	}
1157 
1158 	/* dev->lltx requires to do our own update of trans_start */
1159 	txq_trans_cond_update(queue);
1160 
1161 	/* Notify and wake up reader process */
1162 	if (tfile->flags & TUN_FASYNC)
1163 		kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1164 	tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1165 
1166 	rcu_read_unlock();
1167 	return NETDEV_TX_OK;
1168 
1169 drop:
1170 	dev_core_stats_tx_dropped_inc(dev);
1171 	skb_tx_error(skb);
1172 	kfree_skb_reason(skb, drop_reason);
1173 	rcu_read_unlock();
1174 	return NET_XMIT_DROP;
1175 }
1176 
1177 static void tun_net_mclist(struct net_device *dev)
1178 {
1179 	/*
1180 	 * This callback is supposed to deal with mc filter in
1181 	 * _rx_ path and has nothing to do with the _tx_ path.
1182 	 * In rx path we always accept everything userspace gives us.
1183 	 */
1184 }
1185 
1186 static netdev_features_t tun_net_fix_features(struct net_device *dev,
1187 	netdev_features_t features)
1188 {
1189 	struct tun_struct *tun = netdev_priv(dev);
1190 
1191 	return (features & tun->set_features) | (features & ~TUN_USER_FEATURES);
1192 }
1193 
1194 static void tun_set_headroom(struct net_device *dev, int new_hr)
1195 {
1196 	struct tun_struct *tun = netdev_priv(dev);
1197 	size_t max_headroom;
1198 
1199 	max_headroom = min_t(size_t, SKB_MAX_HEAD(0), U16_MAX - 1);
1200 
1201 	if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP)
1202 		max_headroom -= ETH_HLEN + NET_IP_ALIGN;
1203 	else
1204 		max_headroom -= 1;
1205 
1206 	tun->align = clamp_t(int, new_hr, NET_SKB_PAD, max_headroom);
1207 }
1208 
1209 static void
1210 tun_net_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
1211 {
1212 	struct tun_struct *tun = netdev_priv(dev);
1213 
1214 	dev_get_tstats64(dev, stats);
1215 
1216 	stats->rx_frame_errors +=
1217 		(unsigned long)atomic_long_read(&tun->rx_frame_errors);
1218 }
1219 
1220 static int tun_xdp_set(struct net_device *dev, struct bpf_prog *prog,
1221 		       struct netlink_ext_ack *extack)
1222 {
1223 	struct tun_struct *tun = netdev_priv(dev);
1224 	struct tun_file *tfile;
1225 	struct bpf_prog *old_prog;
1226 	int i;
1227 
1228 	old_prog = rtnl_dereference(tun->xdp_prog);
1229 	rcu_assign_pointer(tun->xdp_prog, prog);
1230 	if (old_prog)
1231 		bpf_prog_put(old_prog);
1232 
1233 	for (i = 0; i < tun->numqueues; i++) {
1234 		tfile = rtnl_dereference(tun->tfiles[i]);
1235 		if (prog)
1236 			sock_set_flag(&tfile->sk, SOCK_XDP);
1237 		else
1238 			sock_reset_flag(&tfile->sk, SOCK_XDP);
1239 	}
1240 	list_for_each_entry(tfile, &tun->disabled, next) {
1241 		if (prog)
1242 			sock_set_flag(&tfile->sk, SOCK_XDP);
1243 		else
1244 			sock_reset_flag(&tfile->sk, SOCK_XDP);
1245 	}
1246 
1247 	return 0;
1248 }
1249 
1250 static int tun_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1251 {
1252 	switch (xdp->command) {
1253 	case XDP_SETUP_PROG:
1254 		return tun_xdp_set(dev, xdp->prog, xdp->extack);
1255 	default:
1256 		return -EINVAL;
1257 	}
1258 }
1259 
1260 static int tun_net_change_carrier(struct net_device *dev, bool new_carrier)
1261 {
1262 	if (new_carrier) {
1263 		struct tun_struct *tun = netdev_priv(dev);
1264 
1265 		if (!tun->numqueues)
1266 			return -EPERM;
1267 
1268 		netif_carrier_on(dev);
1269 	} else {
1270 		netif_carrier_off(dev);
1271 	}
1272 	return 0;
1273 }
1274 
1275 static const struct net_device_ops tun_netdev_ops = {
1276 	.ndo_init		= tun_net_init,
1277 	.ndo_uninit		= tun_net_uninit,
1278 	.ndo_open		= tun_net_open,
1279 	.ndo_stop		= tun_net_close,
1280 	.ndo_start_xmit		= tun_net_xmit,
1281 	.ndo_fix_features	= tun_net_fix_features,
1282 	.ndo_select_queue	= tun_select_queue,
1283 	.ndo_set_rx_headroom	= tun_set_headroom,
1284 	.ndo_get_stats64	= tun_net_get_stats64,
1285 	.ndo_change_carrier	= tun_net_change_carrier,
1286 };
1287 
1288 static void __tun_xdp_flush_tfile(struct tun_file *tfile)
1289 {
1290 	/* Notify and wake up reader process */
1291 	if (tfile->flags & TUN_FASYNC)
1292 		kill_fasync(&tfile->fasync, SIGIO, POLL_IN);
1293 	tfile->socket.sk->sk_data_ready(tfile->socket.sk);
1294 }
1295 
1296 static int tun_xdp_xmit(struct net_device *dev, int n,
1297 			struct xdp_frame **frames, u32 flags)
1298 {
1299 	struct tun_struct *tun = netdev_priv(dev);
1300 	struct tun_file *tfile;
1301 	u32 numqueues;
1302 	int nxmit = 0;
1303 	int i;
1304 
1305 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
1306 		return -EINVAL;
1307 
1308 	rcu_read_lock();
1309 
1310 resample:
1311 	numqueues = READ_ONCE(tun->numqueues);
1312 	if (!numqueues) {
1313 		rcu_read_unlock();
1314 		return -ENXIO; /* Caller will free/return all frames */
1315 	}
1316 
1317 	tfile = rcu_dereference(tun->tfiles[smp_processor_id() %
1318 					    numqueues]);
1319 	if (unlikely(!tfile))
1320 		goto resample;
1321 
1322 	spin_lock(&tfile->tx_ring.producer_lock);
1323 	for (i = 0; i < n; i++) {
1324 		struct xdp_frame *xdp = frames[i];
1325 		/* Encode the XDP flag into lowest bit for consumer to differ
1326 		 * XDP buffer from sk_buff.
1327 		 */
1328 		void *frame = tun_xdp_to_ptr(xdp);
1329 
1330 		if (__ptr_ring_produce(&tfile->tx_ring, frame)) {
1331 			dev_core_stats_tx_dropped_inc(dev);
1332 			break;
1333 		}
1334 		nxmit++;
1335 	}
1336 	spin_unlock(&tfile->tx_ring.producer_lock);
1337 
1338 	if (flags & XDP_XMIT_FLUSH)
1339 		__tun_xdp_flush_tfile(tfile);
1340 
1341 	rcu_read_unlock();
1342 	return nxmit;
1343 }
1344 
1345 static int tun_xdp_tx(struct net_device *dev, struct xdp_buff *xdp)
1346 {
1347 	struct xdp_frame *frame = xdp_convert_buff_to_frame(xdp);
1348 	int nxmit;
1349 
1350 	if (unlikely(!frame))
1351 		return -EOVERFLOW;
1352 
1353 	nxmit = tun_xdp_xmit(dev, 1, &frame, XDP_XMIT_FLUSH);
1354 	if (!nxmit)
1355 		xdp_return_frame_rx_napi(frame);
1356 	return nxmit;
1357 }
1358 
1359 static const struct net_device_ops tap_netdev_ops = {
1360 	.ndo_init		= tun_net_init,
1361 	.ndo_uninit		= tun_net_uninit,
1362 	.ndo_open		= tun_net_open,
1363 	.ndo_stop		= tun_net_close,
1364 	.ndo_start_xmit		= tun_net_xmit,
1365 	.ndo_fix_features	= tun_net_fix_features,
1366 	.ndo_set_rx_mode	= tun_net_mclist,
1367 	.ndo_set_mac_address	= eth_mac_addr,
1368 	.ndo_validate_addr	= eth_validate_addr,
1369 	.ndo_select_queue	= tun_select_queue,
1370 	.ndo_features_check	= passthru_features_check,
1371 	.ndo_set_rx_headroom	= tun_set_headroom,
1372 	.ndo_bpf		= tun_xdp,
1373 	.ndo_xdp_xmit		= tun_xdp_xmit,
1374 	.ndo_change_carrier	= tun_net_change_carrier,
1375 };
1376 
1377 static void tun_flow_init(struct tun_struct *tun)
1378 {
1379 	int i;
1380 
1381 	for (i = 0; i < TUN_NUM_FLOW_ENTRIES; i++)
1382 		INIT_HLIST_HEAD(&tun->flows[i]);
1383 
1384 	tun->ageing_time = TUN_FLOW_EXPIRE;
1385 	timer_setup(&tun->flow_gc_timer, tun_flow_cleanup, 0);
1386 	mod_timer(&tun->flow_gc_timer,
1387 		  round_jiffies_up(jiffies + tun->ageing_time));
1388 }
1389 
1390 static void tun_flow_uninit(struct tun_struct *tun)
1391 {
1392 	timer_delete_sync(&tun->flow_gc_timer);
1393 	tun_flow_flush(tun);
1394 }
1395 
1396 #define MIN_MTU 68
1397 #define MAX_MTU 65535
1398 
1399 /* Initialize net device. */
1400 static void tun_net_initialize(struct net_device *dev)
1401 {
1402 	struct tun_struct *tun = netdev_priv(dev);
1403 
1404 	switch (tun->flags & TUN_TYPE_MASK) {
1405 	case IFF_TUN:
1406 		dev->netdev_ops = &tun_netdev_ops;
1407 		dev->header_ops = &ip_tunnel_header_ops;
1408 
1409 		/* Point-to-Point TUN Device */
1410 		dev->hard_header_len = 0;
1411 		dev->addr_len = 0;
1412 		dev->mtu = 1500;
1413 
1414 		/* Zero header length */
1415 		dev->type = ARPHRD_NONE;
1416 		dev->flags = IFF_POINTOPOINT | IFF_NOARP | IFF_MULTICAST;
1417 		break;
1418 
1419 	case IFF_TAP:
1420 		dev->netdev_ops = &tap_netdev_ops;
1421 		/* Ethernet TAP Device */
1422 		ether_setup(dev);
1423 		dev->priv_flags &= ~IFF_TX_SKB_SHARING;
1424 		dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;
1425 
1426 		eth_hw_addr_random(dev);
1427 
1428 		/* Currently tun does not support XDP, only tap does. */
1429 		dev->xdp_features = NETDEV_XDP_ACT_BASIC |
1430 				    NETDEV_XDP_ACT_REDIRECT |
1431 				    NETDEV_XDP_ACT_NDO_XMIT;
1432 
1433 		break;
1434 	}
1435 
1436 	dev->min_mtu = MIN_MTU;
1437 	dev->max_mtu = MAX_MTU - dev->hard_header_len;
1438 }
1439 
1440 static bool tun_sock_writeable(struct tun_struct *tun, struct tun_file *tfile)
1441 {
1442 	struct sock *sk = tfile->socket.sk;
1443 
1444 	return (tun->dev->flags & IFF_UP) && sock_writeable(sk);
1445 }
1446 
1447 /* Character device part */
1448 
1449 /* Poll */
1450 static __poll_t tun_chr_poll(struct file *file, poll_table *wait)
1451 {
1452 	struct tun_file *tfile = file->private_data;
1453 	struct tun_struct *tun = tun_get(tfile);
1454 	struct sock *sk;
1455 	__poll_t mask = 0;
1456 
1457 	if (!tun)
1458 		return EPOLLERR;
1459 
1460 	sk = tfile->socket.sk;
1461 
1462 	poll_wait(file, sk_sleep(sk), wait);
1463 
1464 	if (!ptr_ring_empty(&tfile->tx_ring))
1465 		mask |= EPOLLIN | EPOLLRDNORM;
1466 
1467 	/* Make sure SOCKWQ_ASYNC_NOSPACE is set if not writable to
1468 	 * guarantee EPOLLOUT to be raised by either here or
1469 	 * tun_sock_write_space(). Then process could get notification
1470 	 * after it writes to a down device and meets -EIO.
1471 	 */
1472 	if (tun_sock_writeable(tun, tfile) ||
1473 	    (!test_and_set_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags) &&
1474 	     tun_sock_writeable(tun, tfile)))
1475 		mask |= EPOLLOUT | EPOLLWRNORM;
1476 
1477 	if (tun->dev->reg_state != NETREG_REGISTERED)
1478 		mask = EPOLLERR;
1479 
1480 	tun_put(tun);
1481 	return mask;
1482 }
1483 
1484 static struct sk_buff *tun_napi_alloc_frags(struct tun_file *tfile,
1485 					    size_t len,
1486 					    const struct iov_iter *it)
1487 {
1488 	struct sk_buff *skb;
1489 	size_t linear;
1490 	int err;
1491 	int i;
1492 
1493 	if (it->nr_segs > MAX_SKB_FRAGS + 1 ||
1494 	    len > (ETH_MAX_MTU - NET_SKB_PAD - NET_IP_ALIGN))
1495 		return ERR_PTR(-EMSGSIZE);
1496 
1497 	local_bh_disable();
1498 	skb = napi_get_frags(&tfile->napi);
1499 	local_bh_enable();
1500 	if (!skb)
1501 		return ERR_PTR(-ENOMEM);
1502 
1503 	/* An skb recycled by napi_reuse_skb() keeps the protocol of a
1504 	 * previous frame. eth_type_trans() sets it after the virtio
1505 	 * header checks, which must not see a stale one.
1506 	 */
1507 	skb->protocol = 0;
1508 
1509 	linear = iov_iter_single_seg_count(it);
1510 	err = __skb_grow(skb, linear);
1511 	if (err)
1512 		goto free;
1513 
1514 	skb->len = len;
1515 	skb->data_len = len - linear;
1516 	skb->truesize += skb->data_len;
1517 
1518 	for (i = 1; i < it->nr_segs; i++) {
1519 		const struct iovec *iov = iter_iov(it) + i;
1520 		size_t fragsz = iov->iov_len;
1521 		struct page *page;
1522 		void *frag;
1523 
1524 		if (fragsz == 0 || fragsz > PAGE_SIZE) {
1525 			err = -EINVAL;
1526 			goto free;
1527 		}
1528 		frag = netdev_alloc_frag(fragsz);
1529 		if (!frag) {
1530 			err = -ENOMEM;
1531 			goto free;
1532 		}
1533 		page = virt_to_head_page(frag);
1534 		skb_fill_page_desc(skb, i - 1, page,
1535 				   frag - page_address(page), fragsz);
1536 	}
1537 
1538 	return skb;
1539 free:
1540 	/* frees skb and all frags allocated with napi_alloc_frag() */
1541 	napi_free_frags(&tfile->napi);
1542 	return ERR_PTR(err);
1543 }
1544 
1545 /* prepad is the amount to reserve at front.  len is length after that.
1546  * linear is a hint as to how much to copy (usually headers). */
1547 static struct sk_buff *tun_alloc_skb(struct tun_file *tfile,
1548 				     size_t prepad, size_t len,
1549 				     size_t linear, int noblock)
1550 {
1551 	struct sock *sk = tfile->socket.sk;
1552 	struct sk_buff *skb;
1553 	int err;
1554 
1555 	/* Under a page?  Don't bother with paged skb. */
1556 	if (prepad + len < PAGE_SIZE)
1557 		linear = len;
1558 
1559 	if (len - linear > MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER))
1560 		linear = len - MAX_SKB_FRAGS * (PAGE_SIZE << PAGE_ALLOC_COSTLY_ORDER);
1561 	skb = sock_alloc_send_pskb(sk, prepad + linear, len - linear, noblock,
1562 				   &err, PAGE_ALLOC_COSTLY_ORDER);
1563 	if (!skb)
1564 		return ERR_PTR(err);
1565 
1566 	skb_reserve(skb, prepad);
1567 	skb_put(skb, linear);
1568 	skb->data_len = len - linear;
1569 	skb->len += len - linear;
1570 
1571 	return skb;
1572 }
1573 
1574 static void tun_rx_batched(struct tun_struct *tun, struct tun_file *tfile,
1575 			   struct sk_buff *skb, int more)
1576 {
1577 	struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
1578 	struct sk_buff_head process_queue;
1579 	u32 rx_batched = tun->rx_batched;
1580 	bool rcv = false;
1581 
1582 	if (!rx_batched || (!more && skb_queue_empty(queue))) {
1583 		local_bh_disable();
1584 		skb_record_rx_queue(skb, tfile->queue_index);
1585 		netif_receive_skb(skb);
1586 		local_bh_enable();
1587 		return;
1588 	}
1589 
1590 	spin_lock(&queue->lock);
1591 	if (!more || skb_queue_len(queue) == rx_batched) {
1592 		__skb_queue_head_init(&process_queue);
1593 		skb_queue_splice_tail_init(queue, &process_queue);
1594 		rcv = true;
1595 	} else {
1596 		__skb_queue_tail(queue, skb);
1597 	}
1598 	spin_unlock(&queue->lock);
1599 
1600 	if (rcv) {
1601 		struct sk_buff *nskb;
1602 
1603 		local_bh_disable();
1604 		while ((nskb = __skb_dequeue(&process_queue))) {
1605 			skb_record_rx_queue(nskb, tfile->queue_index);
1606 			netif_receive_skb(nskb);
1607 		}
1608 		skb_record_rx_queue(skb, tfile->queue_index);
1609 		netif_receive_skb(skb);
1610 		local_bh_enable();
1611 	}
1612 }
1613 
1614 static bool tun_can_build_skb(struct tun_struct *tun, struct tun_file *tfile,
1615 			      int len, int noblock, bool zerocopy)
1616 {
1617 	if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
1618 		return false;
1619 
1620 	if (tfile->socket.sk->sk_sndbuf != INT_MAX)
1621 		return false;
1622 
1623 	if (!noblock)
1624 		return false;
1625 
1626 	if (zerocopy)
1627 		return false;
1628 
1629 	if (SKB_DATA_ALIGN(len + TUN_RX_PAD + XDP_PACKET_HEADROOM) +
1630 	    SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) > PAGE_SIZE)
1631 		return false;
1632 
1633 	return true;
1634 }
1635 
1636 static struct sk_buff *__tun_build_skb(struct tun_file *tfile,
1637 				       struct page_frag *alloc_frag, char *buf,
1638 				       int buflen, int len, int pad,
1639 				       int metasize)
1640 {
1641 	struct sk_buff *skb = build_skb(buf, buflen);
1642 
1643 	if (!skb)
1644 		return ERR_PTR(-ENOMEM);
1645 
1646 	skb_reserve(skb, pad);
1647 	skb_put(skb, len);
1648 	if (metasize)
1649 		skb_metadata_set(skb, metasize);
1650 	skb_set_owner_w(skb, tfile->socket.sk);
1651 
1652 	get_page(alloc_frag->page);
1653 	alloc_frag->offset += buflen;
1654 
1655 	return skb;
1656 }
1657 
1658 static int tun_xdp_act(struct tun_struct *tun, struct bpf_prog *xdp_prog,
1659 		       struct xdp_buff *xdp, u32 act)
1660 {
1661 	int err;
1662 
1663 	switch (act) {
1664 	case XDP_REDIRECT:
1665 		err = xdp_do_redirect(tun->dev, xdp, xdp_prog);
1666 		if (err) {
1667 			dev_core_stats_rx_dropped_inc(tun->dev);
1668 			return err;
1669 		}
1670 		dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data);
1671 		break;
1672 	case XDP_TX:
1673 		err = tun_xdp_tx(tun->dev, xdp);
1674 		if (err < 0) {
1675 			dev_core_stats_rx_dropped_inc(tun->dev);
1676 			return err;
1677 		}
1678 		dev_sw_netstats_rx_add(tun->dev, xdp->data_end - xdp->data);
1679 		break;
1680 	case XDP_PASS:
1681 		break;
1682 	default:
1683 		bpf_warn_invalid_xdp_action(tun->dev, xdp_prog, act);
1684 		fallthrough;
1685 	case XDP_ABORTED:
1686 		trace_xdp_exception(tun->dev, xdp_prog, act);
1687 		fallthrough;
1688 	case XDP_DROP:
1689 		dev_core_stats_rx_dropped_inc(tun->dev);
1690 		break;
1691 	}
1692 
1693 	return act;
1694 }
1695 
1696 static struct sk_buff *tun_build_skb(struct tun_struct *tun,
1697 				     struct tun_file *tfile,
1698 				     struct iov_iter *from,
1699 				     struct virtio_net_hdr *hdr,
1700 				     int len, int *skb_xdp)
1701 {
1702 	struct page_frag *alloc_frag = &current->task_frag;
1703 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
1704 	struct bpf_prog *xdp_prog;
1705 	int buflen = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1706 	char *buf;
1707 	size_t copied;
1708 	int pad = TUN_RX_PAD;
1709 	int metasize = 0;
1710 	int err = 0;
1711 
1712 	rcu_read_lock();
1713 	xdp_prog = rcu_dereference(tun->xdp_prog);
1714 	if (xdp_prog)
1715 		pad += XDP_PACKET_HEADROOM;
1716 	buflen += SKB_DATA_ALIGN(len + pad);
1717 	rcu_read_unlock();
1718 
1719 	alloc_frag->offset = ALIGN((u64)alloc_frag->offset, SMP_CACHE_BYTES);
1720 	if (unlikely(!skb_page_frag_refill(buflen, alloc_frag, GFP_KERNEL)))
1721 		return ERR_PTR(-ENOMEM);
1722 
1723 	buf = (char *)page_address(alloc_frag->page) + alloc_frag->offset;
1724 	copied = copy_page_from_iter(alloc_frag->page,
1725 				     alloc_frag->offset + pad,
1726 				     len, from);
1727 	if (copied != len)
1728 		return ERR_PTR(-EFAULT);
1729 
1730 	/* There's a small window that XDP may be set after the check
1731 	 * of xdp_prog above, this should be rare and for simplicity
1732 	 * we do XDP on skb in case the headroom is not enough.
1733 	 */
1734 	if (hdr->gso_type || !xdp_prog) {
1735 		*skb_xdp = 1;
1736 		return __tun_build_skb(tfile, alloc_frag, buf, buflen, len,
1737 				       pad, metasize);
1738 	}
1739 
1740 	*skb_xdp = 0;
1741 
1742 	local_bh_disable();
1743 	rcu_read_lock();
1744 	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
1745 	xdp_prog = rcu_dereference(tun->xdp_prog);
1746 	if (xdp_prog) {
1747 		struct xdp_buff xdp;
1748 		u32 act;
1749 
1750 		xdp_init_buff(&xdp, buflen, &tfile->xdp_rxq);
1751 		xdp_prepare_buff(&xdp, buf, pad, len, true);
1752 
1753 		act = bpf_prog_run_xdp(xdp_prog, &xdp);
1754 		if (act == XDP_REDIRECT || act == XDP_TX) {
1755 			get_page(alloc_frag->page);
1756 			alloc_frag->offset += buflen;
1757 		}
1758 		err = tun_xdp_act(tun, xdp_prog, &xdp, act);
1759 		if (err < 0) {
1760 			if (act == XDP_REDIRECT || act == XDP_TX)
1761 				put_page(alloc_frag->page);
1762 			goto out;
1763 		}
1764 
1765 		if (err == XDP_REDIRECT)
1766 			xdp_do_flush();
1767 		if (err != XDP_PASS)
1768 			goto out;
1769 
1770 		pad = xdp.data - xdp.data_hard_start;
1771 		len = xdp.data_end - xdp.data;
1772 
1773 		/* It is known that the xdp_buff was prepared with metadata
1774 		 * support, so the metasize will never be negative.
1775 		 */
1776 		metasize = xdp.data - xdp.data_meta;
1777 	}
1778 	bpf_net_ctx_clear(bpf_net_ctx);
1779 	rcu_read_unlock();
1780 	local_bh_enable();
1781 
1782 	return __tun_build_skb(tfile, alloc_frag, buf, buflen, len, pad,
1783 			       metasize);
1784 
1785 out:
1786 	bpf_net_ctx_clear(bpf_net_ctx);
1787 	rcu_read_unlock();
1788 	local_bh_enable();
1789 	return NULL;
1790 }
1791 
1792 /* Get packet from user space buffer */
1793 static ssize_t tun_get_user(struct tun_struct *tun, struct tun_file *tfile,
1794 			    void *msg_control, struct iov_iter *from,
1795 			    int noblock, bool more)
1796 {
1797 	struct tun_pi pi = { 0, cpu_to_be16(ETH_P_IP) };
1798 	struct sk_buff *skb;
1799 	size_t total_len = iov_iter_count(from);
1800 	size_t len = total_len, align = tun->align, linear;
1801 	struct virtio_net_hdr_v1_hash_tunnel hdr;
1802 	struct virtio_net_hdr *gso;
1803 	int good_linear;
1804 	int copylen;
1805 	int hdr_len = 0;
1806 	bool zerocopy = false;
1807 	int err;
1808 	u32 rxhash = 0;
1809 	int skb_xdp = 1;
1810 	bool frags = tun_napi_frags_enabled(tfile);
1811 	enum skb_drop_reason drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1812 	netdev_features_t features = 0;
1813 
1814 	/*
1815 	 * Keep it easy and always zero the whole buffer, even if the
1816 	 * tunnel-related field will be touched only when the feature
1817 	 * is enabled and the hdr size id compatible.
1818 	 */
1819 	memset(&hdr, 0, sizeof(hdr));
1820 	gso = (struct virtio_net_hdr *)&hdr;
1821 
1822 	if (!(tun->flags & IFF_NO_PI)) {
1823 		if (len < sizeof(pi))
1824 			return -EINVAL;
1825 		len -= sizeof(pi);
1826 
1827 		if (!copy_from_iter_full(&pi, sizeof(pi), from))
1828 			return -EFAULT;
1829 	}
1830 
1831 	if (tun->flags & IFF_VNET_HDR) {
1832 		int vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
1833 
1834 		features = tun_vnet_hdr_guest_features(vnet_hdr_sz);
1835 		hdr_len = __tun_vnet_hdr_get(vnet_hdr_sz, tun->flags,
1836 					     features, from, gso);
1837 		if (hdr_len < 0)
1838 			return hdr_len;
1839 
1840 		len -= vnet_hdr_sz;
1841 	}
1842 
1843 	if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP) {
1844 		align += NET_IP_ALIGN;
1845 		if (unlikely(len < ETH_HLEN || (hdr_len && hdr_len < ETH_HLEN)))
1846 			return -EINVAL;
1847 	}
1848 
1849 	good_linear = SKB_MAX_HEAD(align);
1850 
1851 	if (msg_control) {
1852 		struct iov_iter i = *from;
1853 
1854 		/* There are 256 bytes to be copied in skb, so there is
1855 		 * enough room for skb expand head in case it is used.
1856 		 * The rest of the buffer is mapped from userspace.
1857 		 */
1858 		copylen = min(hdr_len ? hdr_len : GOODCOPY_LEN, good_linear);
1859 		linear = copylen;
1860 		iov_iter_advance(&i, copylen);
1861 		if (iov_iter_npages(&i, INT_MAX) <= MAX_SKB_FRAGS)
1862 			zerocopy = true;
1863 	}
1864 
1865 	if (!frags && tun_can_build_skb(tun, tfile, len, noblock, zerocopy)) {
1866 		/* For the packet that is not easy to be processed
1867 		 * (e.g gso or jumbo packet), we will do it at after
1868 		 * skb was created with generic XDP routine.
1869 		 */
1870 		skb = tun_build_skb(tun, tfile, from, gso, len, &skb_xdp);
1871 		err = PTR_ERR_OR_ZERO(skb);
1872 		if (err)
1873 			goto drop;
1874 		if (!skb)
1875 			return total_len;
1876 	} else {
1877 		if (!zerocopy) {
1878 			copylen = len;
1879 			linear = min(hdr_len, good_linear);
1880 		}
1881 
1882 		if (frags) {
1883 			mutex_lock(&tfile->napi_mutex);
1884 			skb = tun_napi_alloc_frags(tfile, copylen, from);
1885 			/* tun_napi_alloc_frags() enforces a layout for the skb.
1886 			 * If zerocopy is enabled, then this layout will be
1887 			 * overwritten by zerocopy_sg_from_iter().
1888 			 */
1889 			zerocopy = false;
1890 		} else {
1891 			if (!linear)
1892 				linear = min_t(size_t, good_linear, copylen);
1893 
1894 			skb = tun_alloc_skb(tfile, align, copylen, linear,
1895 					    noblock);
1896 		}
1897 
1898 		err = PTR_ERR_OR_ZERO(skb);
1899 		if (err)
1900 			goto drop;
1901 
1902 		if (zerocopy)
1903 			err = zerocopy_sg_from_iter(skb, from);
1904 		else
1905 			err = skb_copy_datagram_from_iter(skb, 0, from, len);
1906 
1907 		if (err) {
1908 			err = -EFAULT;
1909 			drop_reason = SKB_DROP_REASON_SKB_UCOPY_FAULT;
1910 			goto drop;
1911 		}
1912 	}
1913 
1914 	skb->dev = tun->dev;
1915 	switch (tun->flags & TUN_TYPE_MASK) {
1916 	case IFF_TUN:
1917 		if (tun->flags & IFF_NO_PI) {
1918 			u8 ip_version;
1919 
1920 			drop_reason = pskb_may_pull_reason(skb, 1);
1921 			if (drop_reason) {
1922 				err = -EINVAL;
1923 				goto drop;
1924 			}
1925 			ip_version = skb->data[0] >> 4;
1926 
1927 			switch (ip_version) {
1928 			case 4:
1929 				pi.proto = htons(ETH_P_IP);
1930 				break;
1931 			case 6:
1932 				pi.proto = htons(ETH_P_IPV6);
1933 				break;
1934 			default:
1935 				err = -EINVAL;
1936 				goto drop;
1937 			}
1938 		}
1939 
1940 		skb_reset_network_header(skb);
1941 		skb->protocol = pi.proto;
1942 		break;
1943 	case IFF_TAP:
1944 		drop_reason = pskb_may_pull_reason(skb, ETH_HLEN);
1945 		if (drop_reason) {
1946 			err = -ENOMEM;
1947 			goto drop;
1948 		}
1949 		skb_set_network_header(skb, ETH_HLEN);
1950 		break;
1951 	}
1952 
1953 	if (tun_vnet_hdr_tnl_to_skb(tun->flags, features, skb, &hdr,
1954 				    (tun->flags & TUN_TYPE_MASK) == IFF_TAP)) {
1955 		atomic_long_inc(&tun->rx_frame_errors);
1956 		err = -EINVAL;
1957 		goto free_skb;
1958 	}
1959 
1960 	if ((tun->flags & TUN_TYPE_MASK) == IFF_TAP)
1961 		skb->protocol = eth_type_trans(skb, tun->dev);
1962 
1963 	/* copy skb_ubuf_info for callback when skb has no error */
1964 	if (zerocopy) {
1965 		skb_zcopy_init(skb, msg_control);
1966 	} else if (msg_control) {
1967 		struct ubuf_info *uarg = msg_control;
1968 		uarg->ops->complete(NULL, uarg, false);
1969 	}
1970 
1971 	skb_reset_network_header(skb);
1972 	skb_probe_transport_header(skb);
1973 	skb_record_rx_queue(skb, tfile->queue_index);
1974 
1975 	if (skb_xdp) {
1976 		struct bpf_prog *xdp_prog;
1977 		int ret;
1978 
1979 		local_bh_disable();
1980 		rcu_read_lock();
1981 		xdp_prog = rcu_dereference(tun->xdp_prog);
1982 		if (xdp_prog) {
1983 			ret = do_xdp_generic(xdp_prog, &skb);
1984 			if (ret != XDP_PASS) {
1985 				rcu_read_unlock();
1986 				local_bh_enable();
1987 				goto unlock_frags;
1988 			}
1989 
1990 			if (frags && skb != tfile->napi.skb)
1991 				tfile->napi.skb = skb;
1992 		}
1993 		rcu_read_unlock();
1994 		local_bh_enable();
1995 	}
1996 
1997 	/* Compute the costly rx hash only if needed for flow updates.
1998 	 * We may get a very small possibility of OOO during switching, not
1999 	 * worth to optimize.
2000 	 */
2001 	if (!rcu_access_pointer(tun->steering_prog) && tun->numqueues > 1 &&
2002 	    !tfile->detached)
2003 		rxhash = __skb_get_hash_symmetric(skb);
2004 
2005 	rcu_read_lock();
2006 	if (unlikely(!(tun->dev->flags & IFF_UP))) {
2007 		err = -EIO;
2008 		rcu_read_unlock();
2009 		drop_reason = SKB_DROP_REASON_DEV_READY;
2010 		goto drop;
2011 	}
2012 
2013 	if (frags) {
2014 		u32 headlen;
2015 
2016 		/* Exercise flow dissector code path. */
2017 		skb_push(skb, ETH_HLEN);
2018 		headlen = eth_get_headlen(tun->dev, skb->data,
2019 					  skb_headlen(skb));
2020 
2021 		if (unlikely(headlen > skb_headlen(skb))) {
2022 			WARN_ON_ONCE(1);
2023 			err = -ENOMEM;
2024 			dev_core_stats_rx_dropped_inc(tun->dev);
2025 napi_busy:
2026 			napi_free_frags(&tfile->napi);
2027 			rcu_read_unlock();
2028 			mutex_unlock(&tfile->napi_mutex);
2029 			return err;
2030 		}
2031 
2032 		if (likely(napi_schedule_prep(&tfile->napi))) {
2033 			local_bh_disable();
2034 			napi_gro_frags(&tfile->napi);
2035 			napi_complete(&tfile->napi);
2036 			local_bh_enable();
2037 		} else {
2038 			err = -EBUSY;
2039 			goto napi_busy;
2040 		}
2041 		mutex_unlock(&tfile->napi_mutex);
2042 	} else if (tfile->napi_enabled) {
2043 		struct sk_buff_head *queue = &tfile->sk.sk_write_queue;
2044 		int queue_len;
2045 
2046 		spin_lock_bh(&queue->lock);
2047 
2048 		if (unlikely(tfile->detached)) {
2049 			spin_unlock_bh(&queue->lock);
2050 			rcu_read_unlock();
2051 			err = -EBUSY;
2052 			goto free_skb;
2053 		}
2054 
2055 		__skb_queue_tail(queue, skb);
2056 		queue_len = skb_queue_len(queue);
2057 		spin_unlock(&queue->lock);
2058 
2059 		if (!more || queue_len > NAPI_POLL_WEIGHT)
2060 			napi_schedule(&tfile->napi);
2061 
2062 		local_bh_enable();
2063 	} else if (!IS_ENABLED(CONFIG_4KSTACKS)) {
2064 		tun_rx_batched(tun, tfile, skb, more);
2065 	} else {
2066 		netif_rx(skb);
2067 	}
2068 	rcu_read_unlock();
2069 
2070 	preempt_disable();
2071 	dev_sw_netstats_rx_add(tun->dev, len);
2072 	preempt_enable();
2073 
2074 	if (rxhash)
2075 		tun_flow_update(tun, rxhash, tfile);
2076 
2077 	return total_len;
2078 
2079 drop:
2080 	if (err != -EAGAIN)
2081 		dev_core_stats_rx_dropped_inc(tun->dev);
2082 
2083 free_skb:
2084 	drop_reason = drop_reason ?: SKB_DROP_REASON_NOT_SPECIFIED;
2085 	if (!IS_ERR_OR_NULL(skb))
2086 		kfree_skb_reason(skb, drop_reason);
2087 
2088 unlock_frags:
2089 	if (frags) {
2090 		tfile->napi.skb = NULL;
2091 		mutex_unlock(&tfile->napi_mutex);
2092 	}
2093 
2094 	return err ?: total_len;
2095 }
2096 
2097 static ssize_t tun_chr_write_iter(struct kiocb *iocb, struct iov_iter *from)
2098 {
2099 	struct file *file = iocb->ki_filp;
2100 	struct tun_file *tfile = file->private_data;
2101 	struct tun_struct *tun = tun_get(tfile);
2102 	ssize_t result;
2103 	int noblock = 0;
2104 
2105 	if (!tun)
2106 		return -EBADFD;
2107 
2108 	if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
2109 		noblock = 1;
2110 
2111 	result = tun_get_user(tun, tfile, NULL, from, noblock, false);
2112 
2113 	tun_put(tun);
2114 	return result;
2115 }
2116 
2117 static ssize_t tun_put_user_xdp(struct tun_struct *tun,
2118 				struct tun_file *tfile,
2119 				struct xdp_frame *xdp_frame,
2120 				struct iov_iter *iter)
2121 {
2122 	int vnet_hdr_sz = 0;
2123 	size_t size = xdp_frame->len;
2124 	ssize_t ret;
2125 
2126 	if (tun->flags & IFF_VNET_HDR) {
2127 		struct virtio_net_hdr gso = { 0 };
2128 
2129 		vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2130 		ret = tun_vnet_hdr_put(vnet_hdr_sz, iter, &gso);
2131 		if (ret)
2132 			return ret;
2133 	}
2134 
2135 	ret = copy_to_iter(xdp_frame->data, size, iter) + vnet_hdr_sz;
2136 
2137 	preempt_disable();
2138 	dev_sw_netstats_tx_add(tun->dev, 1, ret);
2139 	preempt_enable();
2140 
2141 	return ret;
2142 }
2143 
2144 /* Put packet to the user space buffer */
2145 static ssize_t tun_put_user(struct tun_struct *tun,
2146 			    struct tun_file *tfile,
2147 			    struct sk_buff *skb,
2148 			    struct iov_iter *iter)
2149 {
2150 	struct tun_pi pi = { 0, skb->protocol };
2151 	ssize_t total;
2152 	int vlan_offset = 0;
2153 	int vlan_hlen = 0;
2154 	int vnet_hdr_sz = 0;
2155 	int ret;
2156 
2157 	if (skb_vlan_tag_present(skb))
2158 		vlan_hlen = VLAN_HLEN;
2159 
2160 	if (tun->flags & IFF_VNET_HDR)
2161 		vnet_hdr_sz = READ_ONCE(tun->vnet_hdr_sz);
2162 
2163 	total = skb->len + vlan_hlen + vnet_hdr_sz;
2164 
2165 	if (!(tun->flags & IFF_NO_PI)) {
2166 		if (iov_iter_count(iter) < sizeof(pi))
2167 			return -EINVAL;
2168 
2169 		total += sizeof(pi);
2170 		if (iov_iter_count(iter) < total) {
2171 			/* Packet will be striped */
2172 			pi.flags |= TUN_PKT_STRIP;
2173 		}
2174 
2175 		if (copy_to_iter(&pi, sizeof(pi), iter) != sizeof(pi))
2176 			return -EFAULT;
2177 	}
2178 
2179 	if (vnet_hdr_sz) {
2180 		struct virtio_net_hdr_v1_hash_tunnel hdr;
2181 		struct virtio_net_hdr *gso;
2182 
2183 		memset(&hdr, 0, sizeof(hdr));
2184 		ret = tun_vnet_hdr_tnl_from_skb(tun->flags, tun->dev, skb,
2185 						&hdr);
2186 		if (ret)
2187 			return ret;
2188 
2189 		/*
2190 		 * Drop the packet if the configured header size is too small
2191 		 * WRT the enabled offloads.
2192 		 */
2193 		gso = (struct virtio_net_hdr *)&hdr;
2194 		ret = __tun_vnet_hdr_put(vnet_hdr_sz, tun->dev->features,
2195 					 iter, gso);
2196 		if (ret)
2197 			return ret;
2198 	}
2199 
2200 	if (vlan_hlen) {
2201 		int ret;
2202 		struct veth veth;
2203 
2204 		veth.h_vlan_proto = skb->vlan_proto;
2205 		veth.h_vlan_TCI = htons(skb_vlan_tag_get(skb));
2206 
2207 		vlan_offset = offsetof(struct vlan_ethhdr, h_vlan_proto);
2208 
2209 		ret = skb_copy_datagram_iter(skb, 0, iter, vlan_offset);
2210 		if (ret || !iov_iter_count(iter))
2211 			goto done;
2212 
2213 		ret = copy_to_iter(&veth, sizeof(veth), iter);
2214 		if (ret != sizeof(veth) || !iov_iter_count(iter))
2215 			goto done;
2216 	}
2217 
2218 	skb_copy_datagram_iter(skb, vlan_offset, iter, skb->len - vlan_offset);
2219 
2220 done:
2221 	/* caller is in process context, */
2222 	preempt_disable();
2223 	dev_sw_netstats_tx_add(tun->dev, 1, skb->len + vlan_hlen);
2224 	preempt_enable();
2225 
2226 	return total;
2227 }
2228 
2229 /* Callers must hold ring.consumer_lock */
2230 static void __tun_wake_queue(struct tun_struct *tun,
2231 			     struct tun_file *tfile, int consumed)
2232 {
2233 	u16 queue_index = tfile->queue_index;
2234 	struct netdev_queue *txq;
2235 
2236 	lockdep_assert_held(&tfile->tx_ring.consumer_lock);
2237 
2238 	if (!(tun->flags & IFF_BACKPRESSURE))
2239 		return;
2240 
2241 	/* A stop from tun_net_close() is not backpressure, leave it alone. */
2242 	if (unlikely(!netif_running(tun->dev)))
2243 		return;
2244 
2245 	/* Only the current owner of the slot may wake its subqueue. */
2246 	if (unlikely(rcu_access_pointer(tun->tfiles[queue_index]) != tfile))
2247 		return;
2248 
2249 	txq = netdev_get_tx_queue(tun->dev, queue_index);
2250 
2251 	/* Paired with smp_mb__after_atomic() in tun_net_xmit() */
2252 	smp_mb();
2253 	if (netif_tx_queue_stopped(txq)) {
2254 		tfile->cons_cnt += consumed;
2255 		if (tfile->cons_cnt >= tfile->tx_ring.size / 2 ||
2256 		    __ptr_ring_empty(&tfile->tx_ring)) {
2257 			netif_tx_wake_queue(txq);
2258 			tfile->cons_cnt = 0;
2259 		}
2260 	}
2261 }
2262 
2263 static void *tun_ring_consume(struct tun_struct *tun, struct tun_file *tfile)
2264 {
2265 	void *ptr;
2266 
2267 	spin_lock(&tfile->tx_ring.consumer_lock);
2268 	ptr = __ptr_ring_consume(&tfile->tx_ring);
2269 	if (ptr)
2270 		__tun_wake_queue(tun, tfile, 1);
2271 
2272 	spin_unlock(&tfile->tx_ring.consumer_lock);
2273 	return ptr;
2274 }
2275 
2276 static void *tun_ring_recv(struct tun_struct *tun, struct tun_file *tfile,
2277 			   int noblock, int *err)
2278 {
2279 	DECLARE_WAITQUEUE(wait, current);
2280 	void *ptr = NULL;
2281 	int error = 0;
2282 
2283 	ptr = tun_ring_consume(tun, tfile);
2284 	if (ptr)
2285 		goto out;
2286 	if (noblock) {
2287 		error = -EAGAIN;
2288 		goto out;
2289 	}
2290 
2291 	add_wait_queue(&tfile->socket.wq.wait, &wait);
2292 
2293 	while (1) {
2294 		set_current_state(TASK_INTERRUPTIBLE);
2295 		ptr = tun_ring_consume(tun, tfile);
2296 		if (ptr)
2297 			break;
2298 		if (signal_pending(current)) {
2299 			error = -ERESTARTSYS;
2300 			break;
2301 		}
2302 		if (tfile->socket.sk->sk_shutdown & RCV_SHUTDOWN) {
2303 			error = -EFAULT;
2304 			break;
2305 		}
2306 
2307 		schedule();
2308 	}
2309 
2310 	__set_current_state(TASK_RUNNING);
2311 	remove_wait_queue(&tfile->socket.wq.wait, &wait);
2312 
2313 out:
2314 	*err = error;
2315 	return ptr;
2316 }
2317 
2318 static ssize_t tun_do_read(struct tun_struct *tun, struct tun_file *tfile,
2319 			   struct iov_iter *to,
2320 			   int noblock, void *ptr)
2321 {
2322 	ssize_t ret;
2323 	int err;
2324 
2325 	if (!iov_iter_count(to)) {
2326 		tun_ptr_free(ptr);
2327 		return 0;
2328 	}
2329 
2330 	if (!ptr) {
2331 		/* Read frames from ring */
2332 		ptr = tun_ring_recv(tun, tfile, noblock, &err);
2333 		if (!ptr)
2334 			return err;
2335 	}
2336 
2337 	if (tun_is_xdp_frame(ptr)) {
2338 		struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
2339 
2340 		ret = tun_put_user_xdp(tun, tfile, xdpf, to);
2341 		xdp_return_frame(xdpf);
2342 	} else {
2343 		struct sk_buff *skb = ptr;
2344 
2345 		ret = tun_put_user(tun, tfile, skb, to);
2346 		if (unlikely(ret < 0))
2347 			kfree_skb(skb);
2348 		else
2349 			consume_skb(skb);
2350 	}
2351 
2352 	return ret;
2353 }
2354 
2355 static ssize_t tun_chr_read_iter(struct kiocb *iocb, struct iov_iter *to)
2356 {
2357 	struct file *file = iocb->ki_filp;
2358 	struct tun_file *tfile = file->private_data;
2359 	struct tun_struct *tun = tun_get(tfile);
2360 	ssize_t len = iov_iter_count(to), ret;
2361 	int noblock = 0;
2362 
2363 	if (!tun)
2364 		return -EBADFD;
2365 
2366 	if ((file->f_flags & O_NONBLOCK) || (iocb->ki_flags & IOCB_NOWAIT))
2367 		noblock = 1;
2368 
2369 	ret = tun_do_read(tun, tfile, to, noblock, NULL);
2370 	ret = min_t(ssize_t, ret, len);
2371 	if (ret > 0)
2372 		iocb->ki_pos = ret;
2373 	tun_put(tun);
2374 	return ret;
2375 }
2376 
2377 static void tun_prog_free(struct rcu_head *rcu)
2378 {
2379 	struct tun_prog *prog = container_of(rcu, struct tun_prog, rcu);
2380 
2381 	bpf_prog_destroy(prog->prog);
2382 	kfree(prog);
2383 }
2384 
2385 static int __tun_set_ebpf(struct tun_struct *tun,
2386 			  struct tun_prog __rcu **prog_p,
2387 			  struct bpf_prog *prog)
2388 {
2389 	struct tun_prog *old, *new = NULL;
2390 
2391 	if (prog) {
2392 		new = kmalloc_obj(*new);
2393 		if (!new)
2394 			return -ENOMEM;
2395 		new->prog = prog;
2396 	}
2397 
2398 	spin_lock_bh(&tun->lock);
2399 	old = rcu_dereference_protected(*prog_p,
2400 					lockdep_is_held(&tun->lock));
2401 	rcu_assign_pointer(*prog_p, new);
2402 	spin_unlock_bh(&tun->lock);
2403 
2404 	if (old)
2405 		call_rcu(&old->rcu, tun_prog_free);
2406 
2407 	return 0;
2408 }
2409 
2410 static void tun_free_netdev(struct net_device *dev)
2411 {
2412 	struct tun_struct *tun = netdev_priv(dev);
2413 
2414 	BUG_ON(!(list_empty(&tun->disabled)));
2415 
2416 	tun_flow_uninit(tun);
2417 	security_tun_dev_free_security(tun->security);
2418 	__tun_set_ebpf(tun, &tun->steering_prog, NULL);
2419 	__tun_set_ebpf(tun, &tun->filter_prog, NULL);
2420 }
2421 
2422 static void tun_setup(struct net_device *dev)
2423 {
2424 	struct tun_struct *tun = netdev_priv(dev);
2425 
2426 	tun->owner = INVALID_UID;
2427 	tun->group = INVALID_GID;
2428 	tun_default_link_ksettings(dev, &tun->link_ksettings);
2429 
2430 	dev->ethtool_ops = &tun_ethtool_ops;
2431 	dev->needs_free_netdev = true;
2432 	dev->priv_destructor = tun_free_netdev;
2433 	/* We prefer our own queue length */
2434 	dev->tx_queue_len = TUN_READQ_SIZE;
2435 }
2436 
2437 /* Trivial set of netlink ops to allow deleting tun or tap
2438  * device with netlink.
2439  */
2440 static int tun_validate(struct nlattr *tb[], struct nlattr *data[],
2441 			struct netlink_ext_ack *extack)
2442 {
2443 	NL_SET_ERR_MSG(extack,
2444 		       "tun/tap creation via rtnetlink is not supported.");
2445 	return -EOPNOTSUPP;
2446 }
2447 
2448 static size_t tun_get_size(const struct net_device *dev)
2449 {
2450 	BUILD_BUG_ON(sizeof(u32) != sizeof(uid_t));
2451 	BUILD_BUG_ON(sizeof(u32) != sizeof(gid_t));
2452 
2453 	return nla_total_size(sizeof(uid_t)) + /* OWNER */
2454 	       nla_total_size(sizeof(gid_t)) + /* GROUP */
2455 	       nla_total_size(sizeof(u8)) + /* TYPE */
2456 	       nla_total_size(sizeof(u8)) + /* PI */
2457 	       nla_total_size(sizeof(u8)) + /* VNET_HDR */
2458 	       nla_total_size(sizeof(u8)) + /* PERSIST */
2459 	       nla_total_size(sizeof(u8)) + /* MULTI_QUEUE */
2460 	       nla_total_size(sizeof(u32)) + /* NUM_QUEUES */
2461 	       nla_total_size(sizeof(u32)) + /* NUM_DISABLED_QUEUES */
2462 	       0;
2463 }
2464 
2465 static int tun_fill_info(struct sk_buff *skb, const struct net_device *dev)
2466 {
2467 	const struct tun_struct *tun = netdev_priv(dev);
2468 	unsigned int flags = READ_ONCE(tun->flags);
2469 	kuid_t owner = READ_ONCE(tun->owner);
2470 	kgid_t group = READ_ONCE(tun->group);
2471 
2472 	if (nla_put_u8(skb, IFLA_TUN_TYPE, flags & TUN_TYPE_MASK))
2473 		goto nla_put_failure;
2474 	if (uid_valid(owner) &&
2475 	    nla_put_u32(skb, IFLA_TUN_OWNER,
2476 			from_kuid_munged(current_user_ns(), owner)))
2477 		goto nla_put_failure;
2478 	if (gid_valid(group) &&
2479 	    nla_put_u32(skb, IFLA_TUN_GROUP,
2480 			from_kgid_munged(current_user_ns(), group)))
2481 		goto nla_put_failure;
2482 	if (nla_put_u8(skb, IFLA_TUN_PI, !(flags & IFF_NO_PI)))
2483 		goto nla_put_failure;
2484 	if (nla_put_u8(skb, IFLA_TUN_VNET_HDR, !!(flags & IFF_VNET_HDR)))
2485 		goto nla_put_failure;
2486 	if (nla_put_u8(skb, IFLA_TUN_PERSIST, !!(flags & IFF_PERSIST)))
2487 		goto nla_put_failure;
2488 	if (nla_put_u8(skb, IFLA_TUN_MULTI_QUEUE,
2489 		       !!(flags & IFF_MULTI_QUEUE)))
2490 		goto nla_put_failure;
2491 	if (flags & IFF_MULTI_QUEUE) {
2492 		if (nla_put_u32(skb, IFLA_TUN_NUM_QUEUES,
2493 				READ_ONCE(tun->numqueues)))
2494 			goto nla_put_failure;
2495 		if (nla_put_u32(skb, IFLA_TUN_NUM_DISABLED_QUEUES,
2496 				READ_ONCE(tun->numdisabled)))
2497 			goto nla_put_failure;
2498 	}
2499 
2500 	return 0;
2501 
2502 nla_put_failure:
2503 	return -EMSGSIZE;
2504 }
2505 
2506 static struct rtnl_link_ops tun_link_ops __read_mostly = {
2507 	.kind		= DRV_NAME,
2508 	.priv_size	= sizeof(struct tun_struct),
2509 	.setup		= tun_setup,
2510 	.validate	= tun_validate,
2511 	.get_size       = tun_get_size,
2512 	.fill_info      = tun_fill_info,
2513 };
2514 
2515 static void tun_sock_write_space(struct sock *sk)
2516 {
2517 	struct tun_file *tfile;
2518 	wait_queue_head_t *wqueue;
2519 
2520 	if (!sock_writeable(sk))
2521 		return;
2522 
2523 	if (!test_and_clear_bit(SOCKWQ_ASYNC_NOSPACE, &sk->sk_socket->flags))
2524 		return;
2525 
2526 	wqueue = sk_sleep(sk);
2527 	if (wqueue && waitqueue_active(wqueue))
2528 		wake_up_interruptible_sync_poll(wqueue, EPOLLOUT |
2529 						EPOLLWRNORM | EPOLLWRBAND);
2530 
2531 	tfile = container_of(sk, struct tun_file, sk);
2532 	kill_fasync(&tfile->fasync, SIGIO, POLL_OUT);
2533 }
2534 
2535 static void tun_put_page(struct tun_page *tpage)
2536 {
2537 	if (tpage->page)
2538 		__page_frag_cache_drain(tpage->page, tpage->count);
2539 }
2540 
2541 static int tun_xdp_one(struct tun_struct *tun,
2542 		       struct tun_file *tfile,
2543 		       struct xdp_buff *xdp, int *flush,
2544 		       struct tun_page *tpage)
2545 {
2546 	unsigned int datasize = xdp->data_end - xdp->data;
2547 	struct virtio_net_hdr *gso = xdp->data_hard_start;
2548 	struct virtio_net_hdr_v1_hash_tunnel *tnl_hdr;
2549 	struct bpf_prog *xdp_prog;
2550 	struct sk_buff *skb = NULL;
2551 	struct sk_buff_head *queue;
2552 	netdev_features_t features;
2553 	u32 rxhash = 0, act;
2554 	int buflen = xdp->frame_sz;
2555 	int metasize = 0;
2556 	int ret = 0;
2557 	bool skb_xdp = false;
2558 	struct page *page;
2559 
2560 	if (unlikely(datasize < ETH_HLEN)) {
2561 		put_page(virt_to_head_page(xdp->data));
2562 		return -EINVAL;
2563 	}
2564 
2565 	if (unlikely(!(READ_ONCE(tun->dev->flags) & IFF_UP))) {
2566 		put_page(virt_to_head_page(xdp->data));
2567 		dev_core_stats_rx_dropped_inc(tun->dev);
2568 		return -EIO;
2569 	}
2570 
2571 	xdp_prog = rcu_dereference(tun->xdp_prog);
2572 	if (xdp_prog) {
2573 		if (gso->gso_type) {
2574 			skb_xdp = true;
2575 			goto build;
2576 		}
2577 
2578 		xdp_init_buff(xdp, buflen, &tfile->xdp_rxq);
2579 
2580 		act = bpf_prog_run_xdp(xdp_prog, xdp);
2581 		ret = tun_xdp_act(tun, xdp_prog, xdp, act);
2582 		if (ret < 0) {
2583 			put_page(virt_to_head_page(xdp->data));
2584 			return ret;
2585 		}
2586 
2587 		switch (ret) {
2588 		case XDP_REDIRECT:
2589 			*flush = true;
2590 			fallthrough;
2591 		case XDP_TX:
2592 			return 0;
2593 		case XDP_PASS:
2594 			break;
2595 		default:
2596 			page = virt_to_head_page(xdp->data);
2597 			if (tpage->page == page) {
2598 				++tpage->count;
2599 			} else {
2600 				tun_put_page(tpage);
2601 				tpage->page = page;
2602 				tpage->count = 1;
2603 			}
2604 			return 0;
2605 		}
2606 	}
2607 
2608 build:
2609 	skb = build_skb(xdp->data_hard_start, buflen);
2610 	if (!skb) {
2611 		put_page(virt_to_head_page(xdp->data));
2612 		ret = -ENOMEM;
2613 		goto out;
2614 	}
2615 
2616 	skb_reserve(skb, xdp->data - xdp->data_hard_start);
2617 	skb_put(skb, xdp->data_end - xdp->data);
2618 
2619 	/* The externally provided xdp_buff may have no metadata support, which
2620 	 * is marked by xdp->data_meta being xdp->data + 1. This will lead to a
2621 	 * metasize of -1 and is the reason why the condition checks for > 0.
2622 	 */
2623 	metasize = xdp->data - xdp->data_meta;
2624 	if (metasize > 0)
2625 		skb_metadata_set(skb, metasize);
2626 
2627 	features = tun_vnet_hdr_guest_features(READ_ONCE(tun->vnet_hdr_sz));
2628 	tnl_hdr = (struct virtio_net_hdr_v1_hash_tunnel *)gso;
2629 	skb->dev = tun->dev;
2630 	skb_set_network_header(skb, ETH_HLEN);
2631 	/* Frames always start with an Ethernet header here, as the
2632 	 * eth_type_trans() call below expects.
2633 	 */
2634 	if (tun_vnet_hdr_tnl_to_skb(tun->flags, features, skb, tnl_hdr,
2635 				    true)) {
2636 		atomic_long_inc(&tun->rx_frame_errors);
2637 		kfree_skb(skb);
2638 		ret = -EINVAL;
2639 		goto out;
2640 	}
2641 
2642 	skb->protocol = eth_type_trans(skb, tun->dev);
2643 	skb_reset_network_header(skb);
2644 	skb_probe_transport_header(skb);
2645 	skb_record_rx_queue(skb, tfile->queue_index);
2646 
2647 	if (skb_xdp) {
2648 		ret = do_xdp_generic(xdp_prog, &skb);
2649 		if (ret != XDP_PASS) {
2650 			ret = 0;
2651 			goto out;
2652 		}
2653 	}
2654 
2655 	if (!rcu_dereference(tun->steering_prog) && tun->numqueues > 1 &&
2656 	    !tfile->detached)
2657 		rxhash = __skb_get_hash_symmetric(skb);
2658 
2659 	if (tfile->napi_enabled) {
2660 		queue = &tfile->sk.sk_write_queue;
2661 		spin_lock(&queue->lock);
2662 
2663 		if (unlikely(tfile->detached)) {
2664 			spin_unlock(&queue->lock);
2665 			kfree_skb(skb);
2666 			return -EBUSY;
2667 		}
2668 
2669 		__skb_queue_tail(queue, skb);
2670 		spin_unlock(&queue->lock);
2671 		ret = 1;
2672 	} else {
2673 		netif_receive_skb(skb);
2674 		ret = 0;
2675 	}
2676 
2677 	/* No need to disable preemption here since this function is
2678 	 * always called with bh disabled
2679 	 */
2680 	dev_sw_netstats_rx_add(tun->dev, datasize);
2681 
2682 	if (rxhash)
2683 		tun_flow_update(tun, rxhash, tfile);
2684 
2685 out:
2686 	return ret;
2687 }
2688 
2689 static int tun_sendmsg(struct socket *sock, struct msghdr *m, size_t total_len)
2690 {
2691 	int ret, i;
2692 	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2693 	struct tun_struct *tun = tun_get(tfile);
2694 	struct tun_msg_ctl *ctl = m->msg_control;
2695 	struct xdp_buff *xdp;
2696 
2697 	if (!tun)
2698 		return -EBADFD;
2699 
2700 	if (m->msg_controllen == sizeof(struct tun_msg_ctl) &&
2701 	    ctl && ctl->type == TUN_MSG_PTR) {
2702 		struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
2703 		struct tun_page tpage;
2704 		int n = ctl->num;
2705 		int flush = 0, queued = 0;
2706 
2707 		memset(&tpage, 0, sizeof(tpage));
2708 
2709 		local_bh_disable();
2710 		rcu_read_lock();
2711 		bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
2712 
2713 		for (i = 0; i < n; i++) {
2714 			xdp = &((struct xdp_buff *)ctl->ptr)[i];
2715 			ret = tun_xdp_one(tun, tfile, xdp, &flush, &tpage);
2716 			if (ret > 0)
2717 				queued += ret;
2718 		}
2719 
2720 		if (flush)
2721 			xdp_do_flush();
2722 
2723 		if (tfile->napi_enabled && queued > 0)
2724 			napi_schedule(&tfile->napi);
2725 
2726 		bpf_net_ctx_clear(bpf_net_ctx);
2727 		rcu_read_unlock();
2728 		local_bh_enable();
2729 
2730 		tun_put_page(&tpage);
2731 
2732 		ret = total_len;
2733 		goto out;
2734 	}
2735 
2736 	ret = tun_get_user(tun, tfile, ctl ? ctl->ptr : NULL, &m->msg_iter,
2737 			   m->msg_flags & MSG_DONTWAIT,
2738 			   m->msg_flags & MSG_MORE);
2739 out:
2740 	tun_put(tun);
2741 	return ret;
2742 }
2743 
2744 static int tun_recvmsg(struct socket *sock, struct msghdr *m, size_t total_len,
2745 		       int flags)
2746 {
2747 	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2748 	struct tun_struct *tun = tun_get(tfile);
2749 	void *ptr = m->msg_control;
2750 	int ret;
2751 
2752 	if (!tun) {
2753 		ret = -EBADFD;
2754 		goto out_free;
2755 	}
2756 
2757 	if (flags & ~(MSG_DONTWAIT|MSG_TRUNC|MSG_ERRQUEUE)) {
2758 		ret = -EINVAL;
2759 		goto out_put_tun;
2760 	}
2761 	if (flags & MSG_ERRQUEUE) {
2762 		ret = sock_recv_errqueue(sock->sk, m, total_len,
2763 					 SOL_PACKET, TUN_TX_TIMESTAMP);
2764 		goto out;
2765 	}
2766 	ret = tun_do_read(tun, tfile, &m->msg_iter, flags & MSG_DONTWAIT, ptr);
2767 	if (ret > (ssize_t)total_len) {
2768 		m->msg_flags |= MSG_TRUNC;
2769 		ret = flags & MSG_TRUNC ? ret : total_len;
2770 	}
2771 out:
2772 	tun_put(tun);
2773 	return ret;
2774 
2775 out_put_tun:
2776 	tun_put(tun);
2777 out_free:
2778 	tun_ptr_free(ptr);
2779 	return ret;
2780 }
2781 
2782 static int tun_ptr_peek_len(void *ptr)
2783 {
2784 	if (likely(ptr)) {
2785 		if (tun_is_xdp_frame(ptr)) {
2786 			struct xdp_frame *xdpf = tun_ptr_to_xdp(ptr);
2787 
2788 			return xdpf->len;
2789 		}
2790 		return __skb_array_len_with_tag(ptr);
2791 	} else {
2792 		return 0;
2793 	}
2794 }
2795 
2796 static int tun_peek_len(struct socket *sock)
2797 {
2798 	struct tun_file *tfile = container_of(sock, struct tun_file, socket);
2799 	struct tun_struct *tun;
2800 	int ret = 0;
2801 
2802 	tun = tun_get(tfile);
2803 	if (!tun)
2804 		return 0;
2805 
2806 	ret = PTR_RING_PEEK_CALL(&tfile->tx_ring, tun_ptr_peek_len);
2807 	tun_put(tun);
2808 
2809 	return ret;
2810 }
2811 
2812 /* Ops structure to mimic raw sockets with tun */
2813 static const struct proto_ops tun_socket_ops = {
2814 	.peek_len = tun_peek_len,
2815 	.sendmsg = tun_sendmsg,
2816 	.recvmsg = tun_recvmsg,
2817 };
2818 
2819 static struct proto tun_proto = {
2820 	.name		= "tun",
2821 	.owner		= THIS_MODULE,
2822 	.obj_size	= sizeof(struct tun_file),
2823 };
2824 
2825 static int tun_flags(struct tun_struct *tun)
2826 {
2827 	return tun->flags & (TUN_FEATURES | IFF_PERSIST | IFF_TUN | IFF_TAP);
2828 }
2829 
2830 static ssize_t tun_flags_show(struct device *dev, struct device_attribute *attr,
2831 			      char *buf)
2832 {
2833 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2834 	return sysfs_emit(buf, "0x%x\n", tun_flags(tun));
2835 }
2836 
2837 static ssize_t owner_show(struct device *dev, struct device_attribute *attr,
2838 			  char *buf)
2839 {
2840 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2841 	return uid_valid(tun->owner)?
2842 		sysfs_emit(buf, "%u\n",
2843 			   from_kuid_munged(current_user_ns(), tun->owner)) :
2844 		sysfs_emit(buf, "-1\n");
2845 }
2846 
2847 static ssize_t group_show(struct device *dev, struct device_attribute *attr,
2848 			  char *buf)
2849 {
2850 	struct tun_struct *tun = netdev_priv(to_net_dev(dev));
2851 	return gid_valid(tun->group) ?
2852 		sysfs_emit(buf, "%u\n",
2853 			   from_kgid_munged(current_user_ns(), tun->group)) :
2854 		sysfs_emit(buf, "-1\n");
2855 }
2856 
2857 static DEVICE_ATTR_RO(tun_flags);
2858 static DEVICE_ATTR_RO(owner);
2859 static DEVICE_ATTR_RO(group);
2860 
2861 static struct attribute *tun_dev_attrs[] = {
2862 	&dev_attr_tun_flags.attr,
2863 	&dev_attr_owner.attr,
2864 	&dev_attr_group.attr,
2865 	NULL
2866 };
2867 
2868 static const struct attribute_group tun_attr_group = {
2869 	.attrs = tun_dev_attrs
2870 };
2871 
2872 static int tun_set_iff(struct net *net, struct file *file, struct ifreq *ifr)
2873 {
2874 	struct tun_struct *tun;
2875 	struct tun_file *tfile = file->private_data;
2876 	struct tun_file *ntfile;
2877 	struct net_device *dev;
2878 	int err, i;
2879 
2880 	if (tfile->detached)
2881 		return -EINVAL;
2882 
2883 	if ((ifr->ifr_flags & IFF_NAPI_FRAGS)) {
2884 		if (!capable(CAP_NET_ADMIN))
2885 			return -EPERM;
2886 
2887 		if (!(ifr->ifr_flags & IFF_NAPI) ||
2888 		    (ifr->ifr_flags & TUN_TYPE_MASK) != IFF_TAP)
2889 			return -EINVAL;
2890 	}
2891 
2892 	dev = __dev_get_by_name(net, ifr->ifr_name);
2893 	if (dev) {
2894 		if (ifr->ifr_flags & IFF_TUN_EXCL)
2895 			return -EBUSY;
2896 		if ((ifr->ifr_flags & IFF_TUN) && dev->netdev_ops == &tun_netdev_ops)
2897 			tun = netdev_priv(dev);
2898 		else if ((ifr->ifr_flags & IFF_TAP) && dev->netdev_ops == &tap_netdev_ops)
2899 			tun = netdev_priv(dev);
2900 		else
2901 			return -EINVAL;
2902 
2903 		if (!!(ifr->ifr_flags & IFF_MULTI_QUEUE) !=
2904 		    !!(tun->flags & IFF_MULTI_QUEUE))
2905 			return -EINVAL;
2906 
2907 		if (!tun_permitted(tun))
2908 			return -EPERM;
2909 		err = security_tun_dev_open(tun->security);
2910 		if (err < 0)
2911 			return err;
2912 
2913 		err = tun_attach(tun, file, ifr->ifr_flags & IFF_NOFILTER,
2914 				 ifr->ifr_flags & IFF_NAPI,
2915 				 ifr->ifr_flags & IFF_NAPI_FRAGS, true);
2916 		if (err < 0)
2917 			return err;
2918 
2919 		if (tun->flags & IFF_MULTI_QUEUE &&
2920 		    (tun->numqueues + tun->numdisabled > 1)) {
2921 			/* One or more queue has already been attached, no need
2922 			 * to initialize the device again.
2923 			 */
2924 			netdev_state_change(dev);
2925 			return 0;
2926 		}
2927 
2928 		WRITE_ONCE(tun->flags, (tun->flags & ~TUN_FEATURES) |
2929 				       (ifr->ifr_flags & TUN_FEATURES));
2930 
2931 		netdev_state_change(dev);
2932 	} else {
2933 		char *name;
2934 		unsigned long flags = 0;
2935 		int queues = ifr->ifr_flags & IFF_MULTI_QUEUE ?
2936 			     MAX_TAP_QUEUES : 1;
2937 
2938 		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
2939 			return -EPERM;
2940 		err = security_tun_dev_create();
2941 		if (err < 0)
2942 			return err;
2943 
2944 		/* Set dev type */
2945 		if (ifr->ifr_flags & IFF_TUN) {
2946 			/* TUN device */
2947 			flags |= IFF_TUN;
2948 			name = "tun%d";
2949 		} else if (ifr->ifr_flags & IFF_TAP) {
2950 			/* TAP device */
2951 			flags |= IFF_TAP;
2952 			name = "tap%d";
2953 		} else
2954 			return -EINVAL;
2955 
2956 		if (*ifr->ifr_name)
2957 			name = ifr->ifr_name;
2958 
2959 		dev = alloc_netdev_mqs(sizeof(struct tun_struct), name,
2960 				       NET_NAME_UNKNOWN, tun_setup, queues,
2961 				       queues);
2962 
2963 		if (!dev)
2964 			return -ENOMEM;
2965 
2966 		dev_net_set(dev, net);
2967 		dev->rtnl_link_ops = &tun_link_ops;
2968 		dev->ifindex = tfile->ifindex;
2969 		dev->sysfs_groups[0] = &tun_attr_group;
2970 
2971 		tun = netdev_priv(dev);
2972 		tun->dev = dev;
2973 		tun->flags = flags;
2974 		tun->txflt.count = 0;
2975 		tun->vnet_hdr_sz = sizeof(struct virtio_net_hdr);
2976 
2977 		tun->align = NET_SKB_PAD;
2978 		tun->filter_attached = false;
2979 		tun->sndbuf = tfile->socket.sk->sk_sndbuf;
2980 		tun->rx_batched = 0;
2981 		RCU_INIT_POINTER(tun->steering_prog, NULL);
2982 
2983 		tun->ifr = ifr;
2984 		tun->file = file;
2985 
2986 		tun_net_initialize(dev);
2987 
2988 		err = register_netdevice(tun->dev);
2989 		if (err < 0) {
2990 			free_netdev(dev);
2991 			return err;
2992 		}
2993 		/* free_netdev() won't check refcnt, to avoid race
2994 		 * with dev_put() we need publish tun after registration.
2995 		 */
2996 		rcu_assign_pointer(tfile->tun, tun);
2997 	}
2998 
2999 	if (ifr->ifr_flags & IFF_NO_CARRIER)
3000 		netif_carrier_off(tun->dev);
3001 	else
3002 		netif_carrier_on(tun->dev);
3003 
3004 	/* Make sure persistent devices do not get stuck in
3005 	 * xoff state.
3006 	 */
3007 	for (i = 0; i < tun->numqueues; i++) {
3008 		ntfile = rtnl_dereference(tun->tfiles[i]);
3009 		tun_force_wake_queue(tun, ntfile);
3010 	}
3011 
3012 	strscpy(ifr->ifr_name, tun->dev->name);
3013 	return 0;
3014 }
3015 
3016 static void tun_get_iff(struct tun_struct *tun, struct ifreq *ifr)
3017 {
3018 	strscpy(ifr->ifr_name, tun->dev->name);
3019 
3020 	ifr->ifr_flags = tun_flags(tun);
3021 
3022 }
3023 
3024 #define PLAIN_GSO (NETIF_F_GSO_UDP_L4 | NETIF_F_TSO | NETIF_F_TSO6)
3025 
3026 /* This is like a cut-down ethtool ops, except done via tun fd so no
3027  * privs required. */
3028 static int set_offload(struct tun_struct *tun, unsigned long arg)
3029 {
3030 	netdev_features_t features = 0;
3031 
3032 	if (arg & TUN_F_CSUM) {
3033 		features |= NETIF_F_HW_CSUM;
3034 		arg &= ~TUN_F_CSUM;
3035 
3036 		if (arg & (TUN_F_TSO4|TUN_F_TSO6)) {
3037 			if (arg & TUN_F_TSO_ECN) {
3038 				features |= NETIF_F_TSO_ECN;
3039 				arg &= ~TUN_F_TSO_ECN;
3040 			}
3041 			if (arg & TUN_F_TSO4)
3042 				features |= NETIF_F_TSO;
3043 			if (arg & TUN_F_TSO6)
3044 				features |= NETIF_F_TSO6;
3045 			arg &= ~(TUN_F_TSO4|TUN_F_TSO6);
3046 		}
3047 
3048 		arg &= ~TUN_F_UFO;
3049 
3050 		/* TODO: for now USO4 and USO6 should work simultaneously */
3051 		if (arg & TUN_F_USO4 && arg & TUN_F_USO6) {
3052 			features |= NETIF_F_GSO_UDP_L4;
3053 			arg &= ~(TUN_F_USO4 | TUN_F_USO6);
3054 		}
3055 
3056 		/*
3057 		 * Tunnel offload is allowed only if some plain offload is
3058 		 * available, too.
3059 		 */
3060 		if (features & PLAIN_GSO && arg & TUN_F_UDP_TUNNEL_GSO) {
3061 			features |= NETIF_F_GSO_UDP_TUNNEL;
3062 			if (arg & TUN_F_UDP_TUNNEL_GSO_CSUM)
3063 				features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
3064 			arg &= ~(TUN_F_UDP_TUNNEL_GSO |
3065 				 TUN_F_UDP_TUNNEL_GSO_CSUM);
3066 		}
3067 	}
3068 
3069 	/* This gives the user a way to test for new features in future by
3070 	 * trying to set them. */
3071 	if (arg)
3072 		return -EINVAL;
3073 
3074 	tun->set_features = features;
3075 	tun->dev->wanted_features &= ~TUN_USER_FEATURES;
3076 	tun->dev->wanted_features |= features;
3077 	netdev_update_features(tun->dev);
3078 
3079 	return 0;
3080 }
3081 
3082 static void tun_detach_filter(struct tun_struct *tun, int n)
3083 {
3084 	int i;
3085 	struct tun_file *tfile;
3086 
3087 	for (i = 0; i < n; i++) {
3088 		tfile = rtnl_dereference(tun->tfiles[i]);
3089 		lock_sock(tfile->socket.sk);
3090 		sk_detach_filter(tfile->socket.sk);
3091 		release_sock(tfile->socket.sk);
3092 	}
3093 
3094 	tun->filter_attached = false;
3095 }
3096 
3097 static int tun_attach_filter(struct tun_struct *tun)
3098 {
3099 	int i, ret = 0;
3100 	struct tun_file *tfile;
3101 
3102 	for (i = 0; i < tun->numqueues; i++) {
3103 		tfile = rtnl_dereference(tun->tfiles[i]);
3104 		lock_sock(tfile->socket.sk);
3105 		ret = sk_attach_filter(&tun->fprog, tfile->socket.sk);
3106 		release_sock(tfile->socket.sk);
3107 		if (ret) {
3108 			tun_detach_filter(tun, i);
3109 			return ret;
3110 		}
3111 	}
3112 
3113 	tun->filter_attached = true;
3114 	return ret;
3115 }
3116 
3117 static void tun_set_sndbuf(struct tun_struct *tun)
3118 {
3119 	struct tun_file *tfile;
3120 	int i;
3121 
3122 	for (i = 0; i < tun->numqueues; i++) {
3123 		tfile = rtnl_dereference(tun->tfiles[i]);
3124 		tfile->socket.sk->sk_sndbuf = tun->sndbuf;
3125 	}
3126 }
3127 
3128 static int tun_set_queue(struct file *file, struct ifreq *ifr)
3129 {
3130 	struct tun_file *tfile = file->private_data;
3131 	struct tun_struct *tun;
3132 	int ret = 0;
3133 
3134 	rtnl_lock();
3135 
3136 	if (ifr->ifr_flags & IFF_ATTACH_QUEUE) {
3137 		tun = tfile->detached;
3138 		if (!tun) {
3139 			ret = -EINVAL;
3140 			goto unlock;
3141 		}
3142 		ret = security_tun_dev_attach_queue(tun->security);
3143 		if (ret < 0)
3144 			goto unlock;
3145 		ret = tun_attach(tun, file, false, tun->flags & IFF_NAPI,
3146 				 tun->flags & IFF_NAPI_FRAGS, true);
3147 	} else if (ifr->ifr_flags & IFF_DETACH_QUEUE) {
3148 		tun = rtnl_dereference(tfile->tun);
3149 		if (!tun || !(tun->flags & IFF_MULTI_QUEUE) || tfile->detached)
3150 			ret = -EINVAL;
3151 		else
3152 			__tun_detach(tfile, false);
3153 	} else
3154 		ret = -EINVAL;
3155 
3156 	if (ret >= 0)
3157 		netdev_state_change(tun->dev);
3158 
3159 unlock:
3160 	rtnl_unlock();
3161 	return ret;
3162 }
3163 
3164 static int tun_set_ebpf(struct tun_struct *tun, struct tun_prog __rcu **prog_p,
3165 			void __user *data)
3166 {
3167 	struct bpf_prog *prog;
3168 	int fd;
3169 
3170 	if (copy_from_user(&fd, data, sizeof(fd)))
3171 		return -EFAULT;
3172 
3173 	if (fd == -1) {
3174 		prog = NULL;
3175 	} else {
3176 		prog = bpf_prog_get_type(fd, BPF_PROG_TYPE_SOCKET_FILTER);
3177 		if (IS_ERR(prog))
3178 			return PTR_ERR(prog);
3179 	}
3180 
3181 	return __tun_set_ebpf(tun, prog_p, prog);
3182 }
3183 
3184 /* Return correct value for tun->dev->addr_len based on tun->dev->type. */
3185 static unsigned char tun_get_addr_len(unsigned short type)
3186 {
3187 	switch (type) {
3188 	case ARPHRD_IP6GRE:
3189 	case ARPHRD_TUNNEL6:
3190 		return sizeof(struct in6_addr);
3191 	case ARPHRD_IPGRE:
3192 	case ARPHRD_TUNNEL:
3193 	case ARPHRD_SIT:
3194 		return 4;
3195 	case ARPHRD_ETHER:
3196 		return ETH_ALEN;
3197 	case ARPHRD_IEEE802154:
3198 	case ARPHRD_IEEE802154_MONITOR:
3199 		return IEEE802154_EXTENDED_ADDR_LEN;
3200 	case ARPHRD_PHONET_PIPE:
3201 	case ARPHRD_PPP:
3202 	case ARPHRD_NONE:
3203 		return 0;
3204 	case ARPHRD_6LOWPAN:
3205 		return EUI64_ADDR_LEN;
3206 	case ARPHRD_FDDI:
3207 		return FDDI_K_ALEN;
3208 	case ARPHRD_HIPPI:
3209 		return HIPPI_ALEN;
3210 	case ARPHRD_IEEE802:
3211 		return FC_ALEN;
3212 	case ARPHRD_ROSE:
3213 		return ROSE_ADDR_LEN;
3214 	case ARPHRD_NETROM:
3215 		return AX25_ADDR_LEN;
3216 	case ARPHRD_LOCALTLK:
3217 		return LTALK_ALEN;
3218 	default:
3219 		return 0;
3220 	}
3221 }
3222 
3223 static long __tun_chr_ioctl(struct file *file, unsigned int cmd,
3224 			    unsigned long arg, int ifreq_len)
3225 {
3226 	struct tun_file *tfile = file->private_data;
3227 	struct net *net = sock_net(&tfile->sk);
3228 	struct tun_struct *tun;
3229 	void __user* argp = (void __user*)arg;
3230 	unsigned int carrier;
3231 	struct ifreq ifr;
3232 	kuid_t owner;
3233 	kgid_t group;
3234 	int ifindex;
3235 	int sndbuf;
3236 	int ret;
3237 	bool do_notify = false;
3238 
3239 	if (cmd == TUNSETIFF || cmd == TUNSETQUEUE ||
3240 	    (_IOC_TYPE(cmd) == SOCK_IOC_TYPE && cmd != SIOCGSKNS)) {
3241 		if (copy_from_user(&ifr, argp, ifreq_len))
3242 			return -EFAULT;
3243 	} else {
3244 		memset(&ifr, 0, sizeof(ifr));
3245 	}
3246 	if (cmd == TUNGETFEATURES) {
3247 		/* Currently this just means: "what IFF flags are valid?".
3248 		 * This is needed because we never checked for invalid flags on
3249 		 * TUNSETIFF.
3250 		 */
3251 		return put_user(IFF_TUN | IFF_TAP | IFF_NO_CARRIER |
3252 				TUN_FEATURES, (unsigned int __user*)argp);
3253 	} else if (cmd == TUNSETQUEUE) {
3254 		return tun_set_queue(file, &ifr);
3255 	} else if (cmd == SIOCGSKNS) {
3256 		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
3257 			return -EPERM;
3258 		return open_related_ns(&net->ns, get_net_ns);
3259 	}
3260 
3261 	rtnl_lock();
3262 
3263 	tun = tun_get(tfile);
3264 	if (cmd == TUNSETIFF) {
3265 		ret = -EEXIST;
3266 		if (tun)
3267 			goto unlock;
3268 
3269 		ifr.ifr_name[IFNAMSIZ-1] = '\0';
3270 
3271 		ret = tun_set_iff(net, file, &ifr);
3272 
3273 		if (ret)
3274 			goto unlock;
3275 
3276 		if (copy_to_user(argp, &ifr, ifreq_len))
3277 			ret = -EFAULT;
3278 		goto unlock;
3279 	}
3280 	if (cmd == TUNSETIFINDEX) {
3281 		ret = -EPERM;
3282 		if (tun)
3283 			goto unlock;
3284 
3285 		ret = -EFAULT;
3286 		if (copy_from_user(&ifindex, argp, sizeof(ifindex)))
3287 			goto unlock;
3288 		ret = -EINVAL;
3289 		if (ifindex < 0)
3290 			goto unlock;
3291 		ret = 0;
3292 		tfile->ifindex = ifindex;
3293 		goto unlock;
3294 	}
3295 
3296 	ret = -EBADFD;
3297 	if (!tun)
3298 		goto unlock;
3299 
3300 	netif_info(tun, drv, tun->dev, "tun_chr_ioctl cmd %u\n", cmd);
3301 
3302 	net = dev_net(tun->dev);
3303 	ret = 0;
3304 	switch (cmd) {
3305 	case TUNGETIFF:
3306 		tun_get_iff(tun, &ifr);
3307 
3308 		if (tfile->detached)
3309 			ifr.ifr_flags |= IFF_DETACH_QUEUE;
3310 		if (!tfile->socket.sk->sk_filter)
3311 			ifr.ifr_flags |= IFF_NOFILTER;
3312 
3313 		if (copy_to_user(argp, &ifr, ifreq_len))
3314 			ret = -EFAULT;
3315 		break;
3316 
3317 	case TUNSETNOCSUM:
3318 		/* Disable/Enable checksum */
3319 
3320 		/* [unimplemented] */
3321 		netif_info(tun, drv, tun->dev, "ignored: set checksum %s\n",
3322 			   arg ? "disabled" : "enabled");
3323 		break;
3324 
3325 	case TUNSETPERSIST:
3326 		/* Disable/Enable persist mode. Keep an extra reference to the
3327 		 * module to prevent the module being unprobed.
3328 		 */
3329 		if (arg && !(READ_ONCE(tun->flags) & IFF_PERSIST)) {
3330 			WRITE_ONCE(tun->flags, READ_ONCE(tun->flags) | IFF_PERSIST);
3331 			__module_get(THIS_MODULE);
3332 			do_notify = true;
3333 		}
3334 		if (!arg && (READ_ONCE(tun->flags) & IFF_PERSIST)) {
3335 			WRITE_ONCE(tun->flags, READ_ONCE(tun->flags) & ~IFF_PERSIST);
3336 			module_put(THIS_MODULE);
3337 			do_notify = true;
3338 		}
3339 
3340 		netif_info(tun, drv, tun->dev, "persist %s\n",
3341 			   arg ? "enabled" : "disabled");
3342 		break;
3343 
3344 	case TUNSETOWNER:
3345 		/* Set owner of the device */
3346 		owner = make_kuid(current_user_ns(), arg);
3347 		if (!uid_valid(owner)) {
3348 			ret = -EINVAL;
3349 			break;
3350 		}
3351 		WRITE_ONCE(tun->owner, owner);
3352 		do_notify = true;
3353 		netif_info(tun, drv, tun->dev, "owner set to %u\n",
3354 			   from_kuid(&init_user_ns, owner));
3355 		break;
3356 
3357 	case TUNSETGROUP:
3358 		/* Set group of the device */
3359 		group = make_kgid(current_user_ns(), arg);
3360 		if (!gid_valid(group)) {
3361 			ret = -EINVAL;
3362 			break;
3363 		}
3364 		WRITE_ONCE(tun->group, group);
3365 		do_notify = true;
3366 		netif_info(tun, drv, tun->dev, "group set to %u\n",
3367 			   from_kgid(&init_user_ns, group));
3368 		break;
3369 
3370 	case TUNSETLINK:
3371 		/* Only allow setting the type when the interface is down */
3372 		if (tun->dev->flags & IFF_UP) {
3373 			netif_info(tun, drv, tun->dev,
3374 				   "Linktype set failed because interface is up\n");
3375 			ret = -EBUSY;
3376 		} else if ((unsigned short)arg == ARPHRD_IEEE802154 ||
3377 			   (unsigned short)arg == ARPHRD_IEEE802154_MONITOR ||
3378 			   (unsigned short)arg == ARPHRD_6LOWPAN) {
3379 			ret = -EINVAL;
3380 		} else {
3381 			ret = call_netdevice_notifiers(NETDEV_PRE_TYPE_CHANGE,
3382 						       tun->dev);
3383 			ret = notifier_to_errno(ret);
3384 			if (ret) {
3385 				netif_info(tun, drv, tun->dev,
3386 					   "Refused to change device type\n");
3387 				break;
3388 			}
3389 			tun->dev->type = (int) arg;
3390 			tun->dev->addr_len = tun_get_addr_len(tun->dev->type);
3391 			netif_info(tun, drv, tun->dev, "linktype set to %d\n",
3392 				   tun->dev->type);
3393 			call_netdevice_notifiers(NETDEV_POST_TYPE_CHANGE,
3394 						 tun->dev);
3395 		}
3396 		break;
3397 
3398 	case TUNSETDEBUG:
3399 		tun->msg_enable = (u32)arg;
3400 		break;
3401 
3402 	case TUNSETOFFLOAD:
3403 		ret = set_offload(tun, arg);
3404 		break;
3405 
3406 	case TUNSETTXFILTER:
3407 		/* Can be set only for TAPs */
3408 		ret = -EINVAL;
3409 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3410 			break;
3411 		ret = update_filter(&tun->txflt, (void __user *)arg);
3412 		break;
3413 
3414 	case SIOCGIFHWADDR:
3415 		/* Get hw address */
3416 		netif_get_mac_address(&ifr.ifr_hwaddr, net, tun->dev->name);
3417 		if (copy_to_user(argp, &ifr, ifreq_len))
3418 			ret = -EFAULT;
3419 		break;
3420 
3421 	case SIOCSIFHWADDR:
3422 		/* Set hw address */
3423 		if (tun->dev->addr_len > sizeof(ifr.ifr_hwaddr)) {
3424 			ret = -EINVAL;
3425 			break;
3426 		}
3427 		ret = dev_set_mac_address_user(tun->dev,
3428 					       (struct sockaddr_storage *)&ifr.ifr_hwaddr,
3429 					       NULL);
3430 		break;
3431 
3432 	case TUNGETSNDBUF:
3433 		sndbuf = tfile->socket.sk->sk_sndbuf;
3434 		if (copy_to_user(argp, &sndbuf, sizeof(sndbuf)))
3435 			ret = -EFAULT;
3436 		break;
3437 
3438 	case TUNSETSNDBUF:
3439 		if (copy_from_user(&sndbuf, argp, sizeof(sndbuf))) {
3440 			ret = -EFAULT;
3441 			break;
3442 		}
3443 		if (sndbuf <= 0) {
3444 			ret = -EINVAL;
3445 			break;
3446 		}
3447 
3448 		tun->sndbuf = sndbuf;
3449 		tun_set_sndbuf(tun);
3450 		break;
3451 
3452 	case TUNATTACHFILTER:
3453 		/* Can be set only for TAPs */
3454 		ret = -EINVAL;
3455 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3456 			break;
3457 		ret = -EFAULT;
3458 		if (copy_from_user(&tun->fprog, argp, sizeof(tun->fprog)))
3459 			break;
3460 
3461 		ret = tun_attach_filter(tun);
3462 		break;
3463 
3464 	case TUNDETACHFILTER:
3465 		/* Can be set only for TAPs */
3466 		ret = -EINVAL;
3467 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3468 			break;
3469 		ret = 0;
3470 		tun_detach_filter(tun, tun->numqueues);
3471 		break;
3472 
3473 	case TUNGETFILTER:
3474 		ret = -EINVAL;
3475 		if ((tun->flags & TUN_TYPE_MASK) != IFF_TAP)
3476 			break;
3477 		ret = -EFAULT;
3478 		if (copy_to_user(argp, &tun->fprog, sizeof(tun->fprog)))
3479 			break;
3480 		ret = 0;
3481 		break;
3482 
3483 	case TUNSETSTEERINGEBPF:
3484 		ret = tun_set_ebpf(tun, &tun->steering_prog, argp);
3485 		break;
3486 
3487 	case TUNSETFILTEREBPF:
3488 		ret = tun_set_ebpf(tun, &tun->filter_prog, argp);
3489 		break;
3490 
3491 	case TUNSETCARRIER:
3492 		ret = -EFAULT;
3493 		if (copy_from_user(&carrier, argp, sizeof(carrier)))
3494 			goto unlock;
3495 
3496 		ret = tun_net_change_carrier(tun->dev, (bool)carrier);
3497 		break;
3498 
3499 	case TUNGETDEVNETNS:
3500 		ret = -EPERM;
3501 		if (!ns_capable(net->user_ns, CAP_NET_ADMIN))
3502 			goto unlock;
3503 		ret = open_related_ns(&net->ns, get_net_ns);
3504 		break;
3505 
3506 	default:
3507 		ret = tun_vnet_ioctl(&tun->vnet_hdr_sz, &tun->flags, cmd, argp);
3508 		break;
3509 	}
3510 
3511 	if (do_notify)
3512 		netdev_state_change(tun->dev);
3513 
3514 unlock:
3515 	rtnl_unlock();
3516 	if (tun)
3517 		tun_put(tun);
3518 	return ret;
3519 }
3520 
3521 static long tun_chr_ioctl(struct file *file,
3522 			  unsigned int cmd, unsigned long arg)
3523 {
3524 	return __tun_chr_ioctl(file, cmd, arg, sizeof (struct ifreq));
3525 }
3526 
3527 #ifdef CONFIG_COMPAT
3528 static long tun_chr_compat_ioctl(struct file *file,
3529 			 unsigned int cmd, unsigned long arg)
3530 {
3531 	switch (cmd) {
3532 	case TUNSETIFF:
3533 	case TUNGETIFF:
3534 	case TUNSETTXFILTER:
3535 	case TUNGETSNDBUF:
3536 	case TUNSETSNDBUF:
3537 	case SIOCGIFHWADDR:
3538 	case SIOCSIFHWADDR:
3539 		arg = (unsigned long)compat_ptr(arg);
3540 		break;
3541 	default:
3542 		arg = (compat_ulong_t)arg;
3543 		break;
3544 	}
3545 
3546 	/*
3547 	 * compat_ifreq is shorter than ifreq, so we must not access beyond
3548 	 * the end of that structure. All fields that are used in this
3549 	 * driver are compatible though, we don't need to convert the
3550 	 * contents.
3551 	 */
3552 	return __tun_chr_ioctl(file, cmd, arg, sizeof(struct compat_ifreq));
3553 }
3554 #endif /* CONFIG_COMPAT */
3555 
3556 static int tun_chr_fasync(int fd, struct file *file, int on)
3557 {
3558 	struct tun_file *tfile = file->private_data;
3559 	int ret;
3560 
3561 	if (on) {
3562 		ret = file_f_owner_allocate(file);
3563 		if (ret)
3564 			goto out;
3565 	}
3566 
3567 	if ((ret = fasync_helper(fd, file, on, &tfile->fasync)) < 0)
3568 		goto out;
3569 
3570 	if (on) {
3571 		__f_setown(file, task_pid(current), PIDTYPE_TGID, 0);
3572 		tfile->flags |= TUN_FASYNC;
3573 	} else
3574 		tfile->flags &= ~TUN_FASYNC;
3575 	ret = 0;
3576 out:
3577 	return ret;
3578 }
3579 
3580 static int tun_chr_open(struct inode *inode, struct file * file)
3581 {
3582 	struct net *net = current->nsproxy->net_ns;
3583 	struct tun_file *tfile;
3584 
3585 	tfile = (struct tun_file *)sk_alloc(net, AF_UNSPEC, GFP_KERNEL,
3586 					    &tun_proto, 0);
3587 	if (!tfile)
3588 		return -ENOMEM;
3589 	if (ptr_ring_init(&tfile->tx_ring, 0, GFP_KERNEL)) {
3590 		sk_free(&tfile->sk);
3591 		return -ENOMEM;
3592 	}
3593 
3594 	mutex_init(&tfile->napi_mutex);
3595 	RCU_INIT_POINTER(tfile->tun, NULL);
3596 	tfile->flags = 0;
3597 	tfile->ifindex = 0;
3598 
3599 	init_waitqueue_head(&tfile->socket.wq.wait);
3600 
3601 	tfile->socket.file = file;
3602 	tfile->socket.ops = &tun_socket_ops;
3603 
3604 	sock_init_data_uid(&tfile->socket, &tfile->sk, current_fsuid());
3605 
3606 	tfile->sk.sk_write_space = tun_sock_write_space;
3607 	tfile->sk.sk_sndbuf = INT_MAX;
3608 
3609 	file->private_data = tfile;
3610 	INIT_LIST_HEAD(&tfile->next);
3611 
3612 	sock_set_flag(&tfile->sk, SOCK_ZEROCOPY);
3613 
3614 	/* tun groks IOCB_NOWAIT just fine, mark it as such */
3615 	file->f_mode |= FMODE_NOWAIT;
3616 	return 0;
3617 }
3618 
3619 static int tun_chr_close(struct inode *inode, struct file *file)
3620 {
3621 	struct tun_file *tfile = file->private_data;
3622 
3623 	tun_detach(tfile, true);
3624 
3625 	return 0;
3626 }
3627 
3628 #ifdef CONFIG_PROC_FS
3629 static void tun_chr_show_fdinfo(struct seq_file *m, struct file *file)
3630 {
3631 	struct tun_file *tfile = file->private_data;
3632 	struct tun_struct *tun;
3633 	struct ifreq ifr;
3634 
3635 	memset(&ifr, 0, sizeof(ifr));
3636 
3637 	rtnl_lock();
3638 	tun = tun_get(tfile);
3639 	if (tun)
3640 		tun_get_iff(tun, &ifr);
3641 	rtnl_unlock();
3642 
3643 	if (tun)
3644 		tun_put(tun);
3645 
3646 	seq_printf(m, "iff:\t%s\n", ifr.ifr_name);
3647 }
3648 #endif
3649 
3650 static const struct file_operations tun_fops = {
3651 	.owner	= THIS_MODULE,
3652 	.read_iter  = tun_chr_read_iter,
3653 	.write_iter = tun_chr_write_iter,
3654 	.poll	= tun_chr_poll,
3655 	.unlocked_ioctl	= tun_chr_ioctl,
3656 #ifdef CONFIG_COMPAT
3657 	.compat_ioctl = tun_chr_compat_ioctl,
3658 #endif
3659 	.open	= tun_chr_open,
3660 	.release = tun_chr_close,
3661 	.fasync = tun_chr_fasync,
3662 #ifdef CONFIG_PROC_FS
3663 	.show_fdinfo = tun_chr_show_fdinfo,
3664 #endif
3665 };
3666 
3667 static struct miscdevice tun_miscdev = {
3668 	.minor = TUN_MINOR,
3669 	.name = "tun",
3670 	.nodename = "net/tun",
3671 	.fops = &tun_fops,
3672 };
3673 
3674 /* ethtool interface */
3675 
3676 static void tun_default_link_ksettings(struct net_device *dev,
3677 				       struct ethtool_link_ksettings *cmd)
3678 {
3679 	ethtool_link_ksettings_zero_link_mode(cmd, supported);
3680 	ethtool_link_ksettings_zero_link_mode(cmd, advertising);
3681 	cmd->base.speed		= SPEED_10000;
3682 	cmd->base.duplex	= DUPLEX_FULL;
3683 	cmd->base.port		= PORT_TP;
3684 	cmd->base.phy_address	= 0;
3685 	cmd->base.autoneg	= AUTONEG_DISABLE;
3686 }
3687 
3688 static int tun_get_link_ksettings(struct net_device *dev,
3689 				  struct ethtool_link_ksettings *cmd)
3690 {
3691 	struct tun_struct *tun = netdev_priv(dev);
3692 
3693 	memcpy(cmd, &tun->link_ksettings, sizeof(*cmd));
3694 	return 0;
3695 }
3696 
3697 static int tun_set_link_ksettings(struct net_device *dev,
3698 				  const struct ethtool_link_ksettings *cmd)
3699 {
3700 	struct tun_struct *tun = netdev_priv(dev);
3701 
3702 	memcpy(&tun->link_ksettings, cmd, sizeof(*cmd));
3703 	return 0;
3704 }
3705 
3706 static void tun_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
3707 {
3708 	struct tun_struct *tun = netdev_priv(dev);
3709 
3710 	strscpy(info->driver, DRV_NAME, sizeof(info->driver));
3711 	strscpy(info->version, DRV_VERSION, sizeof(info->version));
3712 
3713 	switch (tun->flags & TUN_TYPE_MASK) {
3714 	case IFF_TUN:
3715 		strscpy(info->bus_info, "tun", sizeof(info->bus_info));
3716 		break;
3717 	case IFF_TAP:
3718 		strscpy(info->bus_info, "tap", sizeof(info->bus_info));
3719 		break;
3720 	}
3721 }
3722 
3723 static u32 tun_get_msglevel(struct net_device *dev)
3724 {
3725 	struct tun_struct *tun = netdev_priv(dev);
3726 
3727 	return tun->msg_enable;
3728 }
3729 
3730 static void tun_set_msglevel(struct net_device *dev, u32 value)
3731 {
3732 	struct tun_struct *tun = netdev_priv(dev);
3733 
3734 	tun->msg_enable = value;
3735 }
3736 
3737 static int tun_get_coalesce(struct net_device *dev,
3738 			    struct ethtool_coalesce *ec,
3739 			    struct kernel_ethtool_coalesce *kernel_coal,
3740 			    struct netlink_ext_ack *extack)
3741 {
3742 	struct tun_struct *tun = netdev_priv(dev);
3743 
3744 	ec->rx_max_coalesced_frames = tun->rx_batched;
3745 
3746 	return 0;
3747 }
3748 
3749 static int tun_set_coalesce(struct net_device *dev,
3750 			    struct ethtool_coalesce *ec,
3751 			    struct kernel_ethtool_coalesce *kernel_coal,
3752 			    struct netlink_ext_ack *extack)
3753 {
3754 	struct tun_struct *tun = netdev_priv(dev);
3755 
3756 	if (ec->rx_max_coalesced_frames > NAPI_POLL_WEIGHT)
3757 		tun->rx_batched = NAPI_POLL_WEIGHT;
3758 	else
3759 		tun->rx_batched = ec->rx_max_coalesced_frames;
3760 
3761 	return 0;
3762 }
3763 
3764 static void tun_get_channels(struct net_device *dev,
3765 			     struct ethtool_channels *channels)
3766 {
3767 	struct tun_struct *tun = netdev_priv(dev);
3768 
3769 	channels->combined_count = tun->numqueues;
3770 	channels->max_combined = tun->flags & IFF_MULTI_QUEUE ? MAX_TAP_QUEUES : 1;
3771 }
3772 
3773 static const struct ethtool_ops tun_ethtool_ops = {
3774 	.supported_coalesce_params = ETHTOOL_COALESCE_RX_MAX_FRAMES,
3775 	.get_drvinfo	= tun_get_drvinfo,
3776 	.get_msglevel	= tun_get_msglevel,
3777 	.set_msglevel	= tun_set_msglevel,
3778 	.get_link	= ethtool_op_get_link,
3779 	.get_channels   = tun_get_channels,
3780 	.get_ts_info	= ethtool_op_get_ts_info,
3781 	.get_coalesce   = tun_get_coalesce,
3782 	.set_coalesce   = tun_set_coalesce,
3783 	.get_link_ksettings = tun_get_link_ksettings,
3784 	.set_link_ksettings = tun_set_link_ksettings,
3785 };
3786 
3787 static int tun_queue_resize(struct tun_struct *tun)
3788 {
3789 	struct net_device *dev = tun->dev;
3790 	struct tun_file *tfile;
3791 	struct ptr_ring **rings;
3792 	int n = tun->numqueues + tun->numdisabled;
3793 	int ret, i;
3794 
3795 	rings = kmalloc_objs(*rings, n);
3796 	if (!rings)
3797 		return -ENOMEM;
3798 
3799 	for (i = 0; i < tun->numqueues; i++) {
3800 		tfile = rtnl_dereference(tun->tfiles[i]);
3801 		rings[i] = &tfile->tx_ring;
3802 	}
3803 	list_for_each_entry(tfile, &tun->disabled, next)
3804 		rings[i++] = &tfile->tx_ring;
3805 
3806 	ret = ptr_ring_resize_multiple_bh(rings, n,
3807 					  dev->tx_queue_len, GFP_KERNEL,
3808 					  tun_ptr_free);
3809 
3810 	if (!ret) {
3811 		for (i = 0; i < tun->numqueues; i++) {
3812 			tfile = rtnl_dereference(tun->tfiles[i]);
3813 			tun_force_wake_queue(tun, tfile);
3814 		}
3815 	}
3816 
3817 	kfree(rings);
3818 	return ret;
3819 }
3820 
3821 static int tun_device_event(struct notifier_block *unused,
3822 			    unsigned long event, void *ptr)
3823 {
3824 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
3825 	struct tun_struct *tun = netdev_priv(dev);
3826 	int i;
3827 
3828 	if (dev->rtnl_link_ops != &tun_link_ops)
3829 		return NOTIFY_DONE;
3830 
3831 	switch (event) {
3832 	case NETDEV_CHANGE_TX_QUEUE_LEN:
3833 		if (tun_queue_resize(tun))
3834 			return NOTIFY_BAD;
3835 		break;
3836 	case NETDEV_UP:
3837 		for (i = 0; i < tun->numqueues; i++) {
3838 			struct tun_file *tfile;
3839 
3840 			tfile = rtnl_dereference(tun->tfiles[i]);
3841 			tfile->socket.sk->sk_write_space(tfile->socket.sk);
3842 		}
3843 		break;
3844 	default:
3845 		break;
3846 	}
3847 
3848 	return NOTIFY_DONE;
3849 }
3850 
3851 static struct notifier_block tun_notifier_block __read_mostly = {
3852 	.notifier_call	= tun_device_event,
3853 };
3854 
3855 static int __init tun_init(void)
3856 {
3857 	int ret = 0;
3858 
3859 	pr_info("%s, %s\n", DRV_DESCRIPTION, DRV_VERSION);
3860 
3861 	ret = rtnl_link_register(&tun_link_ops);
3862 	if (ret) {
3863 		pr_err("Can't register link_ops\n");
3864 		goto err_linkops;
3865 	}
3866 
3867 	ret = misc_register(&tun_miscdev);
3868 	if (ret) {
3869 		pr_err("Can't register misc device %d\n", TUN_MINOR);
3870 		goto err_misc;
3871 	}
3872 
3873 	ret = register_netdevice_notifier(&tun_notifier_block);
3874 	if (ret) {
3875 		pr_err("Can't register netdevice notifier\n");
3876 		goto err_notifier;
3877 	}
3878 
3879 	return  0;
3880 
3881 err_notifier:
3882 	misc_deregister(&tun_miscdev);
3883 err_misc:
3884 	rtnl_link_unregister(&tun_link_ops);
3885 err_linkops:
3886 	return ret;
3887 }
3888 
3889 static void __exit tun_cleanup(void)
3890 {
3891 	misc_deregister(&tun_miscdev);
3892 	rtnl_link_unregister(&tun_link_ops);
3893 	unregister_netdevice_notifier(&tun_notifier_block);
3894 	rcu_barrier();
3895 }
3896 
3897 /* Get an underlying socket object from tun file.  Returns error unless file is
3898  * attached to a device.  The returned object works like a packet socket, it
3899  * can be used for sock_sendmsg/sock_recvmsg.  The caller is responsible for
3900  * holding a reference to the file for as long as the socket is in use. */
3901 struct socket *tun_get_socket(struct file *file)
3902 {
3903 	struct tun_file *tfile;
3904 	if (file->f_op != &tun_fops)
3905 		return ERR_PTR(-EINVAL);
3906 	tfile = file->private_data;
3907 	if (!tfile)
3908 		return ERR_PTR(-EBADFD);
3909 	return &tfile->socket;
3910 }
3911 EXPORT_SYMBOL_GPL(tun_get_socket);
3912 
3913 struct ptr_ring *tun_get_tx_ring(struct file *file)
3914 {
3915 	struct tun_file *tfile;
3916 
3917 	if (file->f_op != &tun_fops)
3918 		return ERR_PTR(-EINVAL);
3919 	tfile = file->private_data;
3920 	if (!tfile)
3921 		return ERR_PTR(-EBADFD);
3922 	return &tfile->tx_ring;
3923 }
3924 EXPORT_SYMBOL_GPL(tun_get_tx_ring);
3925 
3926 /* Callers must hold ring.consumer_lock */
3927 void tun_wake_queue(struct file *file, int consumed)
3928 {
3929 	struct tun_file *tfile;
3930 	struct tun_struct *tun;
3931 
3932 	if (file->f_op != &tun_fops)
3933 		return;
3934 
3935 	tfile = file->private_data;
3936 	if (!tfile)
3937 		return;
3938 
3939 	lockdep_assert_held(&tfile->tx_ring.consumer_lock);
3940 
3941 	rcu_read_lock();
3942 
3943 	tun = rcu_dereference(tfile->tun);
3944 	if (tun)
3945 		__tun_wake_queue(tun, tfile, consumed);
3946 
3947 	rcu_read_unlock();
3948 }
3949 EXPORT_SYMBOL_GPL(tun_wake_queue);
3950 
3951 module_init(tun_init);
3952 module_exit(tun_cleanup);
3953 MODULE_DESCRIPTION(DRV_DESCRIPTION);
3954 MODULE_AUTHOR(DRV_COPYRIGHT);
3955 MODULE_LICENSE("GPL");
3956 MODULE_ALIAS_MISCDEV(TUN_MINOR);
3957 MODULE_ALIAS("devname:net/tun");
3958 MODULE_IMPORT_NS("NETDEV_INTERNAL");
3959