xref: /linux/net/xfrm/xfrm_policy.c (revision c2933b2befe25309f4c5cfbea0ca80909735fd76)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * xfrm_policy.c
4  *
5  * Changes:
6  *	Mitsuru KANDA @USAGI
7  * 	Kazunori MIYAZAWA @USAGI
8  * 	Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9  * 		IPv6 support
10  * 	Kazunori MIYAZAWA @USAGI
11  * 	YOSHIFUJI Hideaki
12  * 		Split up af-specific portion
13  *	Derek Atkins <derek@ihtfp.com>		Add the post_input processor
14  *
15  */
16 
17 #include <linux/err.h>
18 #include <linux/slab.h>
19 #include <linux/kmod.h>
20 #include <linux/list.h>
21 #include <linux/spinlock.h>
22 #include <linux/workqueue.h>
23 #include <linux/notifier.h>
24 #include <linux/netdevice.h>
25 #include <linux/netfilter.h>
26 #include <linux/module.h>
27 #include <linux/cache.h>
28 #include <linux/cpu.h>
29 #include <linux/audit.h>
30 #include <linux/rhashtable.h>
31 #include <linux/if_tunnel.h>
32 #include <linux/icmp.h>
33 #include <net/dst.h>
34 #include <net/flow.h>
35 #include <net/inet_ecn.h>
36 #include <net/xfrm.h>
37 #include <net/ip.h>
38 #include <net/gre.h>
39 #if IS_ENABLED(CONFIG_IPV6_MIP6)
40 #include <net/mip6.h>
41 #endif
42 #ifdef CONFIG_XFRM_STATISTICS
43 #include <net/snmp.h>
44 #endif
45 #ifdef CONFIG_XFRM_ESPINTCP
46 #include <net/espintcp.h>
47 #endif
48 #include <net/inet_dscp.h>
49 
50 #include "xfrm_hash.h"
51 
52 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
53 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
54 #define XFRM_MAX_QUEUE_LEN	100
55 
56 struct xfrm_flo {
57 	struct dst_entry *dst_orig;
58 	u8 flags;
59 };
60 
61 /* prefixes smaller than this are stored in lists, not trees. */
62 #define INEXACT_PREFIXLEN_IPV4	16
63 #define INEXACT_PREFIXLEN_IPV6	48
64 
65 struct xfrm_pol_inexact_node {
66 	struct rb_node node;
67 	union {
68 		xfrm_address_t addr;
69 		struct rcu_head rcu;
70 	};
71 	u8 prefixlen;
72 
73 	struct rb_root root;
74 
75 	/* the policies matching this node, can be empty list */
76 	struct hlist_head hhead;
77 };
78 
79 /* xfrm inexact policy search tree:
80  * xfrm_pol_inexact_bin = hash(dir,type,family,if_id);
81  *  |
82  * +---- root_d: sorted by daddr:prefix
83  * |                 |
84  * |        xfrm_pol_inexact_node
85  * |                 |
86  * |                 +- root: sorted by saddr/prefix
87  * |                 |              |
88  * |                 |         xfrm_pol_inexact_node
89  * |                 |              |
90  * |                 |              + root: unused
91  * |                 |              |
92  * |                 |              + hhead: saddr:daddr policies
93  * |                 |
94  * |                 +- coarse policies and all any:daddr policies
95  * |
96  * +---- root_s: sorted by saddr:prefix
97  * |                 |
98  * |        xfrm_pol_inexact_node
99  * |                 |
100  * |                 + root: unused
101  * |                 |
102  * |                 + hhead: saddr:any policies
103  * |
104  * +---- coarse policies and all any:any policies
105  *
106  * Lookups return four candidate lists:
107  * 1. any:any list from top-level xfrm_pol_inexact_bin
108  * 2. any:daddr list from daddr tree
109  * 3. saddr:daddr list from 2nd level daddr tree
110  * 4. saddr:any list from saddr tree
111  *
112  * This result set then needs to be searched for the policy with
113  * the lowest priority.  If two candidates have the same priority, the
114  * struct xfrm_policy pos member with the lower number is used.
115  *
116  * This replicates previous single-list-search algorithm which would
117  * return first matching policy in the (ordered-by-priority) list.
118  */
119 
120 struct xfrm_pol_inexact_key {
121 	possible_net_t net;
122 	u32 if_id;
123 	u16 family;
124 	u8 dir, type;
125 };
126 
127 struct xfrm_pol_inexact_bin {
128 	struct xfrm_pol_inexact_key k;
129 	struct rhash_head head;
130 	/* list containing '*:*' policies */
131 	struct hlist_head hhead;
132 
133 	seqcount_spinlock_t count;
134 	/* tree sorted by daddr/prefix */
135 	struct rb_root root_d;
136 
137 	/* tree sorted by saddr/prefix */
138 	struct rb_root root_s;
139 
140 	/* slow path below */
141 	struct list_head inexact_bins;
142 	struct rcu_head rcu;
143 };
144 
145 enum xfrm_pol_inexact_candidate_type {
146 	XFRM_POL_CAND_BOTH,
147 	XFRM_POL_CAND_SADDR,
148 	XFRM_POL_CAND_DADDR,
149 	XFRM_POL_CAND_ANY,
150 
151 	XFRM_POL_CAND_MAX,
152 };
153 
154 struct xfrm_pol_inexact_candidates {
155 	struct hlist_head *res[XFRM_POL_CAND_MAX];
156 };
157 
158 struct xfrm_flow_keys {
159 	struct flow_dissector_key_basic basic;
160 	struct flow_dissector_key_control control;
161 	union {
162 		struct flow_dissector_key_ipv4_addrs ipv4;
163 		struct flow_dissector_key_ipv6_addrs ipv6;
164 	} addrs;
165 	struct flow_dissector_key_ip ip;
166 	struct flow_dissector_key_icmp icmp;
167 	struct flow_dissector_key_ports ports;
168 	struct flow_dissector_key_keyid gre;
169 };
170 
171 static struct flow_dissector xfrm_session_dissector __ro_after_init;
172 
173 static DEFINE_SPINLOCK(xfrm_if_cb_lock);
174 static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
175 
176 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
177 static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
178 						__read_mostly;
179 
180 static struct kmem_cache *xfrm_dst_cache __ro_after_init;
181 
182 static struct rhashtable xfrm_policy_inexact_table;
183 static const struct rhashtable_params xfrm_pol_inexact_params;
184 
185 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr);
186 static int stale_bundle(struct dst_entry *dst);
187 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
188 static void xfrm_policy_queue_process(struct timer_list *t);
189 
190 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
191 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
192 						int dir);
193 
194 static struct xfrm_pol_inexact_bin *
195 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir,
196 			   u32 if_id);
197 
198 static struct xfrm_pol_inexact_bin *
199 xfrm_policy_inexact_lookup_rcu(struct net *net,
200 			       u8 type, u16 family, u8 dir, u32 if_id);
201 static struct xfrm_policy *
202 xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy,
203 			bool excl);
204 
205 static bool
206 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
207 				    struct xfrm_pol_inexact_bin *b,
208 				    const xfrm_address_t *saddr,
209 				    const xfrm_address_t *daddr);
210 
xfrm_pol_hold_rcu(struct xfrm_policy * policy)211 static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
212 {
213 	return refcount_inc_not_zero(&policy->refcnt);
214 }
215 
216 static inline bool
__xfrm4_selector_match(const struct xfrm_selector * sel,const struct flowi * fl)217 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
218 {
219 	const struct flowi4 *fl4 = &fl->u.ip4;
220 
221 	return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
222 		addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
223 		!((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
224 		!((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
225 		(fl4->flowi4_proto == sel->proto || !sel->proto) &&
226 		(fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
227 }
228 
229 static inline bool
__xfrm6_selector_match(const struct xfrm_selector * sel,const struct flowi * fl)230 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
231 {
232 	const struct flowi6 *fl6 = &fl->u.ip6;
233 
234 	return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
235 		addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
236 		!((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
237 		!((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
238 		(fl6->flowi6_proto == sel->proto || !sel->proto) &&
239 		(fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
240 }
241 
xfrm_selector_match(const struct xfrm_selector * sel,const struct flowi * fl,unsigned short family)242 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
243 			 unsigned short family)
244 {
245 	switch (family) {
246 	case AF_INET:
247 		return __xfrm4_selector_match(sel, fl);
248 	case AF_INET6:
249 		return __xfrm6_selector_match(sel, fl);
250 	}
251 	return false;
252 }
253 
xfrm_policy_get_afinfo(unsigned short family)254 static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
255 {
256 	const struct xfrm_policy_afinfo *afinfo;
257 
258 	if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
259 		return NULL;
260 	rcu_read_lock();
261 	afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
262 	if (unlikely(!afinfo))
263 		rcu_read_unlock();
264 	return afinfo;
265 }
266 
267 /* Called with rcu_read_lock(). */
xfrm_if_get_cb(void)268 static const struct xfrm_if_cb *xfrm_if_get_cb(void)
269 {
270 	return rcu_dereference(xfrm_if_cb);
271 }
272 
__xfrm_dst_lookup(int family,const struct xfrm_dst_lookup_params * params)273 struct dst_entry *__xfrm_dst_lookup(int family,
274 				    const struct xfrm_dst_lookup_params *params)
275 {
276 	const struct xfrm_policy_afinfo *afinfo;
277 	struct dst_entry *dst;
278 
279 	afinfo = xfrm_policy_get_afinfo(family);
280 	if (unlikely(afinfo == NULL))
281 		return ERR_PTR(-EAFNOSUPPORT);
282 
283 	dst = afinfo->dst_lookup(params);
284 
285 	rcu_read_unlock();
286 
287 	return dst;
288 }
289 EXPORT_SYMBOL(__xfrm_dst_lookup);
290 
xfrm_dst_lookup(struct xfrm_state * x,dscp_t dscp,int oif,xfrm_address_t * prev_saddr,xfrm_address_t * prev_daddr,int family,u32 mark)291 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
292 						dscp_t dscp, int oif,
293 						xfrm_address_t *prev_saddr,
294 						xfrm_address_t *prev_daddr,
295 						int family, u32 mark)
296 {
297 	struct xfrm_dst_lookup_params params;
298 	struct net *net = xs_net(x);
299 	xfrm_address_t *saddr = &x->props.saddr;
300 	xfrm_address_t *daddr = &x->id.daddr;
301 	struct dst_entry *dst;
302 
303 	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
304 		saddr = x->coaddr;
305 		daddr = prev_daddr;
306 	}
307 	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
308 		saddr = prev_saddr;
309 		daddr = x->coaddr;
310 	}
311 
312 	params.net = net;
313 	params.saddr = saddr;
314 	params.daddr = daddr;
315 	params.dscp = dscp;
316 	params.oif = oif;
317 	params.mark = mark;
318 	params.ipproto = x->id.proto;
319 	if (x->encap) {
320 		switch (x->encap->encap_type) {
321 		case UDP_ENCAP_ESPINUDP:
322 			params.ipproto = IPPROTO_UDP;
323 			params.uli.ports.sport = x->encap->encap_sport;
324 			params.uli.ports.dport = x->encap->encap_dport;
325 			break;
326 		case TCP_ENCAP_ESPINTCP:
327 			params.ipproto = IPPROTO_TCP;
328 			params.uli.ports.sport = x->encap->encap_sport;
329 			params.uli.ports.dport = x->encap->encap_dport;
330 			break;
331 		}
332 	}
333 
334 	dst = __xfrm_dst_lookup(family, &params);
335 
336 	if (!IS_ERR(dst)) {
337 		if (prev_saddr != saddr)
338 			memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
339 		if (prev_daddr != daddr)
340 			memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
341 	}
342 
343 	return dst;
344 }
345 
make_jiffies(long secs)346 static inline unsigned long make_jiffies(long secs)
347 {
348 	if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
349 		return MAX_SCHEDULE_TIMEOUT-1;
350 	else
351 		return secs*HZ;
352 }
353 
xfrm_policy_timer(struct timer_list * t)354 static void xfrm_policy_timer(struct timer_list *t)
355 {
356 	struct xfrm_policy *xp = from_timer(xp, t, timer);
357 	time64_t now = ktime_get_real_seconds();
358 	time64_t next = TIME64_MAX;
359 	int warn = 0;
360 	int dir;
361 
362 	read_lock(&xp->lock);
363 
364 	if (unlikely(xp->walk.dead))
365 		goto out;
366 
367 	dir = xfrm_policy_id2dir(xp->index);
368 
369 	if (xp->lft.hard_add_expires_seconds) {
370 		time64_t tmo = xp->lft.hard_add_expires_seconds +
371 			xp->curlft.add_time - now;
372 		if (tmo <= 0)
373 			goto expired;
374 		if (tmo < next)
375 			next = tmo;
376 	}
377 	if (xp->lft.hard_use_expires_seconds) {
378 		time64_t tmo = xp->lft.hard_use_expires_seconds +
379 			(READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
380 		if (tmo <= 0)
381 			goto expired;
382 		if (tmo < next)
383 			next = tmo;
384 	}
385 	if (xp->lft.soft_add_expires_seconds) {
386 		time64_t tmo = xp->lft.soft_add_expires_seconds +
387 			xp->curlft.add_time - now;
388 		if (tmo <= 0) {
389 			warn = 1;
390 			tmo = XFRM_KM_TIMEOUT;
391 		}
392 		if (tmo < next)
393 			next = tmo;
394 	}
395 	if (xp->lft.soft_use_expires_seconds) {
396 		time64_t tmo = xp->lft.soft_use_expires_seconds +
397 			(READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
398 		if (tmo <= 0) {
399 			warn = 1;
400 			tmo = XFRM_KM_TIMEOUT;
401 		}
402 		if (tmo < next)
403 			next = tmo;
404 	}
405 
406 	if (warn)
407 		km_policy_expired(xp, dir, 0, 0);
408 	if (next != TIME64_MAX &&
409 	    !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
410 		xfrm_pol_hold(xp);
411 
412 out:
413 	read_unlock(&xp->lock);
414 	xfrm_pol_put(xp);
415 	return;
416 
417 expired:
418 	read_unlock(&xp->lock);
419 	if (!xfrm_policy_delete(xp, dir))
420 		km_policy_expired(xp, dir, 1, 0);
421 	xfrm_pol_put(xp);
422 }
423 
424 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
425  * SPD calls.
426  */
427 
xfrm_policy_alloc(struct net * net,gfp_t gfp)428 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
429 {
430 	struct xfrm_policy *policy;
431 
432 	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
433 
434 	if (policy) {
435 		write_pnet(&policy->xp_net, net);
436 		INIT_LIST_HEAD(&policy->walk.all);
437 		INIT_HLIST_HEAD(&policy->state_cache_list);
438 		INIT_HLIST_NODE(&policy->bydst);
439 		INIT_HLIST_NODE(&policy->byidx);
440 		rwlock_init(&policy->lock);
441 		refcount_set(&policy->refcnt, 1);
442 		skb_queue_head_init(&policy->polq.hold_queue);
443 		timer_setup(&policy->timer, xfrm_policy_timer, 0);
444 		timer_setup(&policy->polq.hold_timer,
445 			    xfrm_policy_queue_process, 0);
446 	}
447 	return policy;
448 }
449 EXPORT_SYMBOL(xfrm_policy_alloc);
450 
xfrm_policy_destroy_rcu(struct rcu_head * head)451 static void xfrm_policy_destroy_rcu(struct rcu_head *head)
452 {
453 	struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);
454 
455 	security_xfrm_policy_free(policy->security);
456 	kfree(policy);
457 }
458 
459 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
460 
xfrm_policy_destroy(struct xfrm_policy * policy)461 void xfrm_policy_destroy(struct xfrm_policy *policy)
462 {
463 	BUG_ON(!policy->walk.dead);
464 
465 	if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
466 		BUG();
467 
468 	xfrm_dev_policy_free(policy);
469 	call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
470 }
471 EXPORT_SYMBOL(xfrm_policy_destroy);
472 
473 /* Rule must be locked. Release descendant resources, announce
474  * entry dead. The rule must be unlinked from lists to the moment.
475  */
476 
xfrm_policy_kill(struct xfrm_policy * policy)477 static void xfrm_policy_kill(struct xfrm_policy *policy)
478 {
479 	struct net *net = xp_net(policy);
480 	struct xfrm_state *x;
481 
482 	xfrm_dev_policy_delete(policy);
483 
484 	write_lock_bh(&policy->lock);
485 	policy->walk.dead = 1;
486 	write_unlock_bh(&policy->lock);
487 
488 	atomic_inc(&policy->genid);
489 
490 	if (del_timer(&policy->polq.hold_timer))
491 		xfrm_pol_put(policy);
492 	skb_queue_purge(&policy->polq.hold_queue);
493 
494 	if (del_timer(&policy->timer))
495 		xfrm_pol_put(policy);
496 
497 	/* XXX: Flush state cache */
498 	spin_lock_bh(&net->xfrm.xfrm_state_lock);
499 	hlist_for_each_entry_rcu(x, &policy->state_cache_list, state_cache) {
500 		hlist_del_init_rcu(&x->state_cache);
501 	}
502 	spin_unlock_bh(&net->xfrm.xfrm_state_lock);
503 
504 	xfrm_pol_put(policy);
505 }
506 
507 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
508 
idx_hash(struct net * net,u32 index)509 static inline unsigned int idx_hash(struct net *net, u32 index)
510 {
511 	return __idx_hash(index, net->xfrm.policy_idx_hmask);
512 }
513 
514 /* calculate policy hash thresholds */
__get_hash_thresh(struct net * net,unsigned short family,int dir,u8 * dbits,u8 * sbits)515 static void __get_hash_thresh(struct net *net,
516 			      unsigned short family, int dir,
517 			      u8 *dbits, u8 *sbits)
518 {
519 	switch (family) {
520 	case AF_INET:
521 		*dbits = net->xfrm.policy_bydst[dir].dbits4;
522 		*sbits = net->xfrm.policy_bydst[dir].sbits4;
523 		break;
524 
525 	case AF_INET6:
526 		*dbits = net->xfrm.policy_bydst[dir].dbits6;
527 		*sbits = net->xfrm.policy_bydst[dir].sbits6;
528 		break;
529 
530 	default:
531 		*dbits = 0;
532 		*sbits = 0;
533 	}
534 }
535 
policy_hash_bysel(struct net * net,const struct xfrm_selector * sel,unsigned short family,int dir)536 static struct hlist_head *policy_hash_bysel(struct net *net,
537 					    const struct xfrm_selector *sel,
538 					    unsigned short family, int dir)
539 {
540 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
541 	unsigned int hash;
542 	u8 dbits;
543 	u8 sbits;
544 
545 	__get_hash_thresh(net, family, dir, &dbits, &sbits);
546 	hash = __sel_hash(sel, family, hmask, dbits, sbits);
547 
548 	if (hash == hmask + 1)
549 		return NULL;
550 
551 	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
552 		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
553 }
554 
policy_hash_direct(struct net * net,const xfrm_address_t * daddr,const xfrm_address_t * saddr,unsigned short family,int dir)555 static struct hlist_head *policy_hash_direct(struct net *net,
556 					     const xfrm_address_t *daddr,
557 					     const xfrm_address_t *saddr,
558 					     unsigned short family, int dir)
559 {
560 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
561 	unsigned int hash;
562 	u8 dbits;
563 	u8 sbits;
564 
565 	__get_hash_thresh(net, family, dir, &dbits, &sbits);
566 	hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
567 
568 	return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
569 		     lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
570 }
571 
xfrm_dst_hash_transfer(struct net * net,struct hlist_head * list,struct hlist_head * ndsttable,unsigned int nhashmask,int dir)572 static void xfrm_dst_hash_transfer(struct net *net,
573 				   struct hlist_head *list,
574 				   struct hlist_head *ndsttable,
575 				   unsigned int nhashmask,
576 				   int dir)
577 {
578 	struct hlist_node *tmp, *entry0 = NULL;
579 	struct xfrm_policy *pol;
580 	unsigned int h0 = 0;
581 	u8 dbits;
582 	u8 sbits;
583 
584 redo:
585 	hlist_for_each_entry_safe(pol, tmp, list, bydst) {
586 		unsigned int h;
587 
588 		__get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
589 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
590 				pol->family, nhashmask, dbits, sbits);
591 		if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
592 			hlist_del_rcu(&pol->bydst);
593 			hlist_add_head_rcu(&pol->bydst, ndsttable + h);
594 			h0 = h;
595 		} else {
596 			if (h != h0)
597 				continue;
598 			hlist_del_rcu(&pol->bydst);
599 			hlist_add_behind_rcu(&pol->bydst, entry0);
600 		}
601 		entry0 = &pol->bydst;
602 	}
603 	if (!hlist_empty(list)) {
604 		entry0 = NULL;
605 		goto redo;
606 	}
607 }
608 
xfrm_idx_hash_transfer(struct hlist_head * list,struct hlist_head * nidxtable,unsigned int nhashmask)609 static void xfrm_idx_hash_transfer(struct hlist_head *list,
610 				   struct hlist_head *nidxtable,
611 				   unsigned int nhashmask)
612 {
613 	struct hlist_node *tmp;
614 	struct xfrm_policy *pol;
615 
616 	hlist_for_each_entry_safe(pol, tmp, list, byidx) {
617 		unsigned int h;
618 
619 		h = __idx_hash(pol->index, nhashmask);
620 		hlist_add_head(&pol->byidx, nidxtable+h);
621 	}
622 }
623 
xfrm_new_hash_mask(unsigned int old_hmask)624 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
625 {
626 	return ((old_hmask + 1) << 1) - 1;
627 }
628 
xfrm_bydst_resize(struct net * net,int dir)629 static void xfrm_bydst_resize(struct net *net, int dir)
630 {
631 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
632 	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
633 	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
634 	struct hlist_head *ndst = xfrm_hash_alloc(nsize);
635 	struct hlist_head *odst;
636 	int i;
637 
638 	if (!ndst)
639 		return;
640 
641 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
642 	write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
643 
644 	odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
645 				lockdep_is_held(&net->xfrm.xfrm_policy_lock));
646 
647 	for (i = hmask; i >= 0; i--)
648 		xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
649 
650 	rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
651 	net->xfrm.policy_bydst[dir].hmask = nhashmask;
652 
653 	write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
654 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
655 
656 	synchronize_rcu();
657 
658 	xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
659 }
660 
xfrm_byidx_resize(struct net * net)661 static void xfrm_byidx_resize(struct net *net)
662 {
663 	unsigned int hmask = net->xfrm.policy_idx_hmask;
664 	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
665 	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
666 	struct hlist_head *oidx = net->xfrm.policy_byidx;
667 	struct hlist_head *nidx = xfrm_hash_alloc(nsize);
668 	int i;
669 
670 	if (!nidx)
671 		return;
672 
673 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
674 
675 	for (i = hmask; i >= 0; i--)
676 		xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
677 
678 	net->xfrm.policy_byidx = nidx;
679 	net->xfrm.policy_idx_hmask = nhashmask;
680 
681 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
682 
683 	xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
684 }
685 
xfrm_bydst_should_resize(struct net * net,int dir,int * total)686 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
687 {
688 	unsigned int cnt = net->xfrm.policy_count[dir];
689 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
690 
691 	if (total)
692 		*total += cnt;
693 
694 	if ((hmask + 1) < xfrm_policy_hashmax &&
695 	    cnt > hmask)
696 		return 1;
697 
698 	return 0;
699 }
700 
xfrm_byidx_should_resize(struct net * net,int total)701 static inline int xfrm_byidx_should_resize(struct net *net, int total)
702 {
703 	unsigned int hmask = net->xfrm.policy_idx_hmask;
704 
705 	if ((hmask + 1) < xfrm_policy_hashmax &&
706 	    total > hmask)
707 		return 1;
708 
709 	return 0;
710 }
711 
xfrm_spd_getinfo(struct net * net,struct xfrmk_spdinfo * si)712 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
713 {
714 	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
715 	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
716 	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
717 	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
718 	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
719 	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
720 	si->spdhcnt = net->xfrm.policy_idx_hmask;
721 	si->spdhmcnt = xfrm_policy_hashmax;
722 }
723 EXPORT_SYMBOL(xfrm_spd_getinfo);
724 
725 static DEFINE_MUTEX(hash_resize_mutex);
xfrm_hash_resize(struct work_struct * work)726 static void xfrm_hash_resize(struct work_struct *work)
727 {
728 	struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
729 	int dir, total;
730 
731 	mutex_lock(&hash_resize_mutex);
732 
733 	total = 0;
734 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
735 		if (xfrm_bydst_should_resize(net, dir, &total))
736 			xfrm_bydst_resize(net, dir);
737 	}
738 	if (xfrm_byidx_should_resize(net, total))
739 		xfrm_byidx_resize(net);
740 
741 	mutex_unlock(&hash_resize_mutex);
742 }
743 
744 /* Make sure *pol can be inserted into fastbin.
745  * Useful to check that later insert requests will be successful
746  * (provided xfrm_policy_lock is held throughout).
747  */
748 static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_alloc_bin(const struct xfrm_policy * pol,u8 dir)749 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
750 {
751 	struct xfrm_pol_inexact_bin *bin, *prev;
752 	struct xfrm_pol_inexact_key k = {
753 		.family = pol->family,
754 		.type = pol->type,
755 		.dir = dir,
756 		.if_id = pol->if_id,
757 	};
758 	struct net *net = xp_net(pol);
759 
760 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
761 
762 	write_pnet(&k.net, net);
763 	bin = rhashtable_lookup_fast(&xfrm_policy_inexact_table, &k,
764 				     xfrm_pol_inexact_params);
765 	if (bin)
766 		return bin;
767 
768 	bin = kzalloc(sizeof(*bin), GFP_ATOMIC);
769 	if (!bin)
770 		return NULL;
771 
772 	bin->k = k;
773 	INIT_HLIST_HEAD(&bin->hhead);
774 	bin->root_d = RB_ROOT;
775 	bin->root_s = RB_ROOT;
776 	seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock);
777 
778 	prev = rhashtable_lookup_get_insert_key(&xfrm_policy_inexact_table,
779 						&bin->k, &bin->head,
780 						xfrm_pol_inexact_params);
781 	if (!prev) {
782 		list_add(&bin->inexact_bins, &net->xfrm.inexact_bins);
783 		return bin;
784 	}
785 
786 	kfree(bin);
787 
788 	return IS_ERR(prev) ? NULL : prev;
789 }
790 
xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t * addr,int family,u8 prefixlen)791 static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr,
792 					       int family, u8 prefixlen)
793 {
794 	if (xfrm_addr_any(addr, family))
795 		return true;
796 
797 	if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6)
798 		return true;
799 
800 	if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4)
801 		return true;
802 
803 	return false;
804 }
805 
806 static bool
xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy * policy)807 xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy)
808 {
809 	const xfrm_address_t *addr;
810 	bool saddr_any, daddr_any;
811 	u8 prefixlen;
812 
813 	addr = &policy->selector.saddr;
814 	prefixlen = policy->selector.prefixlen_s;
815 
816 	saddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
817 						       policy->family,
818 						       prefixlen);
819 	addr = &policy->selector.daddr;
820 	prefixlen = policy->selector.prefixlen_d;
821 	daddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
822 						       policy->family,
823 						       prefixlen);
824 	return saddr_any && daddr_any;
825 }
826 
xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node * node,const xfrm_address_t * addr,u8 prefixlen)827 static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node,
828 				       const xfrm_address_t *addr, u8 prefixlen)
829 {
830 	node->addr = *addr;
831 	node->prefixlen = prefixlen;
832 }
833 
834 static struct xfrm_pol_inexact_node *
xfrm_pol_inexact_node_alloc(const xfrm_address_t * addr,u8 prefixlen)835 xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen)
836 {
837 	struct xfrm_pol_inexact_node *node;
838 
839 	node = kzalloc(sizeof(*node), GFP_ATOMIC);
840 	if (node)
841 		xfrm_pol_inexact_node_init(node, addr, prefixlen);
842 
843 	return node;
844 }
845 
xfrm_policy_addr_delta(const xfrm_address_t * a,const xfrm_address_t * b,u8 prefixlen,u16 family)846 static int xfrm_policy_addr_delta(const xfrm_address_t *a,
847 				  const xfrm_address_t *b,
848 				  u8 prefixlen, u16 family)
849 {
850 	u32 ma, mb, mask;
851 	unsigned int pdw, pbi;
852 	int delta = 0;
853 
854 	switch (family) {
855 	case AF_INET:
856 		if (prefixlen == 0)
857 			return 0;
858 		mask = ~0U << (32 - prefixlen);
859 		ma = ntohl(a->a4) & mask;
860 		mb = ntohl(b->a4) & mask;
861 		if (ma < mb)
862 			delta = -1;
863 		else if (ma > mb)
864 			delta = 1;
865 		break;
866 	case AF_INET6:
867 		pdw = prefixlen >> 5;
868 		pbi = prefixlen & 0x1f;
869 
870 		if (pdw) {
871 			delta = memcmp(a->a6, b->a6, pdw << 2);
872 			if (delta)
873 				return delta;
874 		}
875 		if (pbi) {
876 			mask = ~0U << (32 - pbi);
877 			ma = ntohl(a->a6[pdw]) & mask;
878 			mb = ntohl(b->a6[pdw]) & mask;
879 			if (ma < mb)
880 				delta = -1;
881 			else if (ma > mb)
882 				delta = 1;
883 		}
884 		break;
885 	default:
886 		break;
887 	}
888 
889 	return delta;
890 }
891 
xfrm_policy_inexact_list_reinsert(struct net * net,struct xfrm_pol_inexact_node * n,u16 family)892 static void xfrm_policy_inexact_list_reinsert(struct net *net,
893 					      struct xfrm_pol_inexact_node *n,
894 					      u16 family)
895 {
896 	unsigned int matched_s, matched_d;
897 	struct xfrm_policy *policy, *p;
898 
899 	matched_s = 0;
900 	matched_d = 0;
901 
902 	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
903 		struct hlist_node *newpos = NULL;
904 		bool matches_s, matches_d;
905 
906 		if (policy->walk.dead || !policy->bydst_reinsert)
907 			continue;
908 
909 		WARN_ON_ONCE(policy->family != family);
910 
911 		policy->bydst_reinsert = false;
912 		hlist_for_each_entry(p, &n->hhead, bydst) {
913 			if (policy->priority > p->priority)
914 				newpos = &p->bydst;
915 			else if (policy->priority == p->priority &&
916 				 policy->pos > p->pos)
917 				newpos = &p->bydst;
918 			else
919 				break;
920 		}
921 
922 		if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
923 			hlist_add_behind_rcu(&policy->bydst, newpos);
924 		else
925 			hlist_add_head_rcu(&policy->bydst, &n->hhead);
926 
927 		/* paranoia checks follow.
928 		 * Check that the reinserted policy matches at least
929 		 * saddr or daddr for current node prefix.
930 		 *
931 		 * Matching both is fine, matching saddr in one policy
932 		 * (but not daddr) and then matching only daddr in another
933 		 * is a bug.
934 		 */
935 		matches_s = xfrm_policy_addr_delta(&policy->selector.saddr,
936 						   &n->addr,
937 						   n->prefixlen,
938 						   family) == 0;
939 		matches_d = xfrm_policy_addr_delta(&policy->selector.daddr,
940 						   &n->addr,
941 						   n->prefixlen,
942 						   family) == 0;
943 		if (matches_s && matches_d)
944 			continue;
945 
946 		WARN_ON_ONCE(!matches_s && !matches_d);
947 		if (matches_s)
948 			matched_s++;
949 		if (matches_d)
950 			matched_d++;
951 		WARN_ON_ONCE(matched_s && matched_d);
952 	}
953 }
954 
xfrm_policy_inexact_node_reinsert(struct net * net,struct xfrm_pol_inexact_node * n,struct rb_root * new,u16 family)955 static void xfrm_policy_inexact_node_reinsert(struct net *net,
956 					      struct xfrm_pol_inexact_node *n,
957 					      struct rb_root *new,
958 					      u16 family)
959 {
960 	struct xfrm_pol_inexact_node *node;
961 	struct rb_node **p, *parent;
962 
963 	/* we should not have another subtree here */
964 	WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root));
965 restart:
966 	parent = NULL;
967 	p = &new->rb_node;
968 	while (*p) {
969 		u8 prefixlen;
970 		int delta;
971 
972 		parent = *p;
973 		node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
974 
975 		prefixlen = min(node->prefixlen, n->prefixlen);
976 
977 		delta = xfrm_policy_addr_delta(&n->addr, &node->addr,
978 					       prefixlen, family);
979 		if (delta < 0) {
980 			p = &parent->rb_left;
981 		} else if (delta > 0) {
982 			p = &parent->rb_right;
983 		} else {
984 			bool same_prefixlen = node->prefixlen == n->prefixlen;
985 			struct xfrm_policy *tmp;
986 
987 			hlist_for_each_entry(tmp, &n->hhead, bydst) {
988 				tmp->bydst_reinsert = true;
989 				hlist_del_rcu(&tmp->bydst);
990 			}
991 
992 			node->prefixlen = prefixlen;
993 
994 			xfrm_policy_inexact_list_reinsert(net, node, family);
995 
996 			if (same_prefixlen) {
997 				kfree_rcu(n, rcu);
998 				return;
999 			}
1000 
1001 			rb_erase(*p, new);
1002 			kfree_rcu(n, rcu);
1003 			n = node;
1004 			goto restart;
1005 		}
1006 	}
1007 
1008 	rb_link_node_rcu(&n->node, parent, p);
1009 	rb_insert_color(&n->node, new);
1010 }
1011 
1012 /* merge nodes v and n */
xfrm_policy_inexact_node_merge(struct net * net,struct xfrm_pol_inexact_node * v,struct xfrm_pol_inexact_node * n,u16 family)1013 static void xfrm_policy_inexact_node_merge(struct net *net,
1014 					   struct xfrm_pol_inexact_node *v,
1015 					   struct xfrm_pol_inexact_node *n,
1016 					   u16 family)
1017 {
1018 	struct xfrm_pol_inexact_node *node;
1019 	struct xfrm_policy *tmp;
1020 	struct rb_node *rnode;
1021 
1022 	/* To-be-merged node v has a subtree.
1023 	 *
1024 	 * Dismantle it and insert its nodes to n->root.
1025 	 */
1026 	while ((rnode = rb_first(&v->root)) != NULL) {
1027 		node = rb_entry(rnode, struct xfrm_pol_inexact_node, node);
1028 		rb_erase(&node->node, &v->root);
1029 		xfrm_policy_inexact_node_reinsert(net, node, &n->root,
1030 						  family);
1031 	}
1032 
1033 	hlist_for_each_entry(tmp, &v->hhead, bydst) {
1034 		tmp->bydst_reinsert = true;
1035 		hlist_del_rcu(&tmp->bydst);
1036 	}
1037 
1038 	xfrm_policy_inexact_list_reinsert(net, n, family);
1039 }
1040 
1041 static struct xfrm_pol_inexact_node *
xfrm_policy_inexact_insert_node(struct net * net,struct rb_root * root,xfrm_address_t * addr,u16 family,u8 prefixlen,u8 dir)1042 xfrm_policy_inexact_insert_node(struct net *net,
1043 				struct rb_root *root,
1044 				xfrm_address_t *addr,
1045 				u16 family, u8 prefixlen, u8 dir)
1046 {
1047 	struct xfrm_pol_inexact_node *cached = NULL;
1048 	struct rb_node **p, *parent = NULL;
1049 	struct xfrm_pol_inexact_node *node;
1050 
1051 	p = &root->rb_node;
1052 	while (*p) {
1053 		int delta;
1054 
1055 		parent = *p;
1056 		node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
1057 
1058 		delta = xfrm_policy_addr_delta(addr, &node->addr,
1059 					       node->prefixlen,
1060 					       family);
1061 		if (delta == 0 && prefixlen >= node->prefixlen) {
1062 			WARN_ON_ONCE(cached); /* ipsec policies got lost */
1063 			return node;
1064 		}
1065 
1066 		if (delta < 0)
1067 			p = &parent->rb_left;
1068 		else
1069 			p = &parent->rb_right;
1070 
1071 		if (prefixlen < node->prefixlen) {
1072 			delta = xfrm_policy_addr_delta(addr, &node->addr,
1073 						       prefixlen,
1074 						       family);
1075 			if (delta)
1076 				continue;
1077 
1078 			/* This node is a subnet of the new prefix. It needs
1079 			 * to be removed and re-inserted with the smaller
1080 			 * prefix and all nodes that are now also covered
1081 			 * by the reduced prefixlen.
1082 			 */
1083 			rb_erase(&node->node, root);
1084 
1085 			if (!cached) {
1086 				xfrm_pol_inexact_node_init(node, addr,
1087 							   prefixlen);
1088 				cached = node;
1089 			} else {
1090 				/* This node also falls within the new
1091 				 * prefixlen. Merge the to-be-reinserted
1092 				 * node and this one.
1093 				 */
1094 				xfrm_policy_inexact_node_merge(net, node,
1095 							       cached, family);
1096 				kfree_rcu(node, rcu);
1097 			}
1098 
1099 			/* restart */
1100 			p = &root->rb_node;
1101 			parent = NULL;
1102 		}
1103 	}
1104 
1105 	node = cached;
1106 	if (!node) {
1107 		node = xfrm_pol_inexact_node_alloc(addr, prefixlen);
1108 		if (!node)
1109 			return NULL;
1110 	}
1111 
1112 	rb_link_node_rcu(&node->node, parent, p);
1113 	rb_insert_color(&node->node, root);
1114 
1115 	return node;
1116 }
1117 
xfrm_policy_inexact_gc_tree(struct rb_root * r,bool rm)1118 static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm)
1119 {
1120 	struct xfrm_pol_inexact_node *node;
1121 	struct rb_node *rn = rb_first(r);
1122 
1123 	while (rn) {
1124 		node = rb_entry(rn, struct xfrm_pol_inexact_node, node);
1125 
1126 		xfrm_policy_inexact_gc_tree(&node->root, rm);
1127 		rn = rb_next(rn);
1128 
1129 		if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) {
1130 			WARN_ON_ONCE(rm);
1131 			continue;
1132 		}
1133 
1134 		rb_erase(&node->node, r);
1135 		kfree_rcu(node, rcu);
1136 	}
1137 }
1138 
__xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin * b,bool net_exit)1139 static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit)
1140 {
1141 	write_seqcount_begin(&b->count);
1142 	xfrm_policy_inexact_gc_tree(&b->root_d, net_exit);
1143 	xfrm_policy_inexact_gc_tree(&b->root_s, net_exit);
1144 	write_seqcount_end(&b->count);
1145 
1146 	if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) ||
1147 	    !hlist_empty(&b->hhead)) {
1148 		WARN_ON_ONCE(net_exit);
1149 		return;
1150 	}
1151 
1152 	if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head,
1153 				   xfrm_pol_inexact_params) == 0) {
1154 		list_del(&b->inexact_bins);
1155 		kfree_rcu(b, rcu);
1156 	}
1157 }
1158 
xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin * b)1159 static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b)
1160 {
1161 	struct net *net = read_pnet(&b->k.net);
1162 
1163 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1164 	__xfrm_policy_inexact_prune_bin(b, false);
1165 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1166 }
1167 
__xfrm_policy_inexact_flush(struct net * net)1168 static void __xfrm_policy_inexact_flush(struct net *net)
1169 {
1170 	struct xfrm_pol_inexact_bin *bin, *t;
1171 
1172 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1173 
1174 	list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins)
1175 		__xfrm_policy_inexact_prune_bin(bin, false);
1176 }
1177 
1178 static struct hlist_head *
xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin * bin,struct xfrm_policy * policy,u8 dir)1179 xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin,
1180 				struct xfrm_policy *policy, u8 dir)
1181 {
1182 	struct xfrm_pol_inexact_node *n;
1183 	struct net *net;
1184 
1185 	net = xp_net(policy);
1186 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1187 
1188 	if (xfrm_policy_inexact_insert_use_any_list(policy))
1189 		return &bin->hhead;
1190 
1191 	if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr,
1192 					       policy->family,
1193 					       policy->selector.prefixlen_d)) {
1194 		write_seqcount_begin(&bin->count);
1195 		n = xfrm_policy_inexact_insert_node(net,
1196 						    &bin->root_s,
1197 						    &policy->selector.saddr,
1198 						    policy->family,
1199 						    policy->selector.prefixlen_s,
1200 						    dir);
1201 		write_seqcount_end(&bin->count);
1202 		if (!n)
1203 			return NULL;
1204 
1205 		return &n->hhead;
1206 	}
1207 
1208 	/* daddr is fixed */
1209 	write_seqcount_begin(&bin->count);
1210 	n = xfrm_policy_inexact_insert_node(net,
1211 					    &bin->root_d,
1212 					    &policy->selector.daddr,
1213 					    policy->family,
1214 					    policy->selector.prefixlen_d, dir);
1215 	write_seqcount_end(&bin->count);
1216 	if (!n)
1217 		return NULL;
1218 
1219 	/* saddr is wildcard */
1220 	if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr,
1221 					       policy->family,
1222 					       policy->selector.prefixlen_s))
1223 		return &n->hhead;
1224 
1225 	write_seqcount_begin(&bin->count);
1226 	n = xfrm_policy_inexact_insert_node(net,
1227 					    &n->root,
1228 					    &policy->selector.saddr,
1229 					    policy->family,
1230 					    policy->selector.prefixlen_s, dir);
1231 	write_seqcount_end(&bin->count);
1232 	if (!n)
1233 		return NULL;
1234 
1235 	return &n->hhead;
1236 }
1237 
1238 static struct xfrm_policy *
xfrm_policy_inexact_insert(struct xfrm_policy * policy,u8 dir,int excl)1239 xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
1240 {
1241 	struct xfrm_pol_inexact_bin *bin;
1242 	struct xfrm_policy *delpol;
1243 	struct hlist_head *chain;
1244 	struct net *net;
1245 
1246 	bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1247 	if (!bin)
1248 		return ERR_PTR(-ENOMEM);
1249 
1250 	net = xp_net(policy);
1251 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1252 
1253 	chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir);
1254 	if (!chain) {
1255 		__xfrm_policy_inexact_prune_bin(bin, false);
1256 		return ERR_PTR(-ENOMEM);
1257 	}
1258 
1259 	delpol = xfrm_policy_insert_list(chain, policy, excl);
1260 	if (delpol && excl) {
1261 		__xfrm_policy_inexact_prune_bin(bin, false);
1262 		return ERR_PTR(-EEXIST);
1263 	}
1264 
1265 	if (delpol)
1266 		__xfrm_policy_inexact_prune_bin(bin, false);
1267 
1268 	return delpol;
1269 }
1270 
xfrm_policy_is_dead_or_sk(const struct xfrm_policy * policy)1271 static bool xfrm_policy_is_dead_or_sk(const struct xfrm_policy *policy)
1272 {
1273 	int dir;
1274 
1275 	if (policy->walk.dead)
1276 		return true;
1277 
1278 	dir = xfrm_policy_id2dir(policy->index);
1279 	return dir >= XFRM_POLICY_MAX;
1280 }
1281 
xfrm_hash_rebuild(struct work_struct * work)1282 static void xfrm_hash_rebuild(struct work_struct *work)
1283 {
1284 	struct net *net = container_of(work, struct net,
1285 				       xfrm.policy_hthresh.work);
1286 	struct xfrm_policy *pol;
1287 	struct xfrm_policy *policy;
1288 	struct hlist_head *chain;
1289 	struct hlist_node *newpos;
1290 	int dir;
1291 	unsigned seq;
1292 	u8 lbits4, rbits4, lbits6, rbits6;
1293 
1294 	mutex_lock(&hash_resize_mutex);
1295 
1296 	/* read selector prefixlen thresholds */
1297 	do {
1298 		seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
1299 
1300 		lbits4 = net->xfrm.policy_hthresh.lbits4;
1301 		rbits4 = net->xfrm.policy_hthresh.rbits4;
1302 		lbits6 = net->xfrm.policy_hthresh.lbits6;
1303 		rbits6 = net->xfrm.policy_hthresh.rbits6;
1304 	} while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
1305 
1306 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1307 	write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
1308 
1309 	/* make sure that we can insert the indirect policies again before
1310 	 * we start with destructive action.
1311 	 */
1312 	list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
1313 		struct xfrm_pol_inexact_bin *bin;
1314 		u8 dbits, sbits;
1315 
1316 		if (xfrm_policy_is_dead_or_sk(policy))
1317 			continue;
1318 
1319 		dir = xfrm_policy_id2dir(policy->index);
1320 		if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1321 			if (policy->family == AF_INET) {
1322 				dbits = rbits4;
1323 				sbits = lbits4;
1324 			} else {
1325 				dbits = rbits6;
1326 				sbits = lbits6;
1327 			}
1328 		} else {
1329 			if (policy->family == AF_INET) {
1330 				dbits = lbits4;
1331 				sbits = rbits4;
1332 			} else {
1333 				dbits = lbits6;
1334 				sbits = rbits6;
1335 			}
1336 		}
1337 
1338 		if (policy->selector.prefixlen_d < dbits ||
1339 		    policy->selector.prefixlen_s < sbits)
1340 			continue;
1341 
1342 		bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1343 		if (!bin)
1344 			goto out_unlock;
1345 
1346 		if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir))
1347 			goto out_unlock;
1348 	}
1349 
1350 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
1351 		if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1352 			/* dir out => dst = remote, src = local */
1353 			net->xfrm.policy_bydst[dir].dbits4 = rbits4;
1354 			net->xfrm.policy_bydst[dir].sbits4 = lbits4;
1355 			net->xfrm.policy_bydst[dir].dbits6 = rbits6;
1356 			net->xfrm.policy_bydst[dir].sbits6 = lbits6;
1357 		} else {
1358 			/* dir in/fwd => dst = local, src = remote */
1359 			net->xfrm.policy_bydst[dir].dbits4 = lbits4;
1360 			net->xfrm.policy_bydst[dir].sbits4 = rbits4;
1361 			net->xfrm.policy_bydst[dir].dbits6 = lbits6;
1362 			net->xfrm.policy_bydst[dir].sbits6 = rbits6;
1363 		}
1364 	}
1365 
1366 	/* re-insert all policies by order of creation */
1367 	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
1368 		if (xfrm_policy_is_dead_or_sk(policy))
1369 			continue;
1370 
1371 		hlist_del_rcu(&policy->bydst);
1372 
1373 		newpos = NULL;
1374 		dir = xfrm_policy_id2dir(policy->index);
1375 		chain = policy_hash_bysel(net, &policy->selector,
1376 					  policy->family, dir);
1377 
1378 		if (!chain) {
1379 			void *p = xfrm_policy_inexact_insert(policy, dir, 0);
1380 
1381 			WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p));
1382 			continue;
1383 		}
1384 
1385 		hlist_for_each_entry(pol, chain, bydst) {
1386 			if (policy->priority >= pol->priority)
1387 				newpos = &pol->bydst;
1388 			else
1389 				break;
1390 		}
1391 		if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1392 			hlist_add_behind_rcu(&policy->bydst, newpos);
1393 		else
1394 			hlist_add_head_rcu(&policy->bydst, chain);
1395 	}
1396 
1397 out_unlock:
1398 	__xfrm_policy_inexact_flush(net);
1399 	write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
1400 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1401 
1402 	mutex_unlock(&hash_resize_mutex);
1403 }
1404 
xfrm_policy_hash_rebuild(struct net * net)1405 void xfrm_policy_hash_rebuild(struct net *net)
1406 {
1407 	schedule_work(&net->xfrm.policy_hthresh.work);
1408 }
1409 EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
1410 
1411 /* Generate new index... KAME seems to generate them ordered by cost
1412  * of an absolute inpredictability of ordering of rules. This will not pass. */
xfrm_gen_index(struct net * net,int dir,u32 index)1413 static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
1414 {
1415 	for (;;) {
1416 		struct hlist_head *list;
1417 		struct xfrm_policy *p;
1418 		u32 idx;
1419 		int found;
1420 
1421 		if (!index) {
1422 			idx = (net->xfrm.idx_generator | dir);
1423 			net->xfrm.idx_generator += 8;
1424 		} else {
1425 			idx = index;
1426 			index = 0;
1427 		}
1428 
1429 		if (idx == 0)
1430 			idx = 8;
1431 		list = net->xfrm.policy_byidx + idx_hash(net, idx);
1432 		found = 0;
1433 		hlist_for_each_entry(p, list, byidx) {
1434 			if (p->index == idx) {
1435 				found = 1;
1436 				break;
1437 			}
1438 		}
1439 		if (!found)
1440 			return idx;
1441 	}
1442 }
1443 
selector_cmp(struct xfrm_selector * s1,struct xfrm_selector * s2)1444 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
1445 {
1446 	u32 *p1 = (u32 *) s1;
1447 	u32 *p2 = (u32 *) s2;
1448 	int len = sizeof(struct xfrm_selector) / sizeof(u32);
1449 	int i;
1450 
1451 	for (i = 0; i < len; i++) {
1452 		if (p1[i] != p2[i])
1453 			return 1;
1454 	}
1455 
1456 	return 0;
1457 }
1458 
xfrm_policy_requeue(struct xfrm_policy * old,struct xfrm_policy * new)1459 static void xfrm_policy_requeue(struct xfrm_policy *old,
1460 				struct xfrm_policy *new)
1461 {
1462 	struct xfrm_policy_queue *pq = &old->polq;
1463 	struct sk_buff_head list;
1464 
1465 	if (skb_queue_empty(&pq->hold_queue))
1466 		return;
1467 
1468 	__skb_queue_head_init(&list);
1469 
1470 	spin_lock_bh(&pq->hold_queue.lock);
1471 	skb_queue_splice_init(&pq->hold_queue, &list);
1472 	if (del_timer(&pq->hold_timer))
1473 		xfrm_pol_put(old);
1474 	spin_unlock_bh(&pq->hold_queue.lock);
1475 
1476 	pq = &new->polq;
1477 
1478 	spin_lock_bh(&pq->hold_queue.lock);
1479 	skb_queue_splice(&list, &pq->hold_queue);
1480 	pq->timeout = XFRM_QUEUE_TMO_MIN;
1481 	if (!mod_timer(&pq->hold_timer, jiffies))
1482 		xfrm_pol_hold(new);
1483 	spin_unlock_bh(&pq->hold_queue.lock);
1484 }
1485 
xfrm_policy_mark_match(const struct xfrm_mark * mark,struct xfrm_policy * pol)1486 static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark,
1487 					  struct xfrm_policy *pol)
1488 {
1489 	return mark->v == pol->mark.v && mark->m == pol->mark.m;
1490 }
1491 
xfrm_pol_bin_key(const void * data,u32 len,u32 seed)1492 static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
1493 {
1494 	const struct xfrm_pol_inexact_key *k = data;
1495 	u32 a = k->type << 24 | k->dir << 16 | k->family;
1496 
1497 	return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)),
1498 			    seed);
1499 }
1500 
xfrm_pol_bin_obj(const void * data,u32 len,u32 seed)1501 static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
1502 {
1503 	const struct xfrm_pol_inexact_bin *b = data;
1504 
1505 	return xfrm_pol_bin_key(&b->k, 0, seed);
1506 }
1507 
xfrm_pol_bin_cmp(struct rhashtable_compare_arg * arg,const void * ptr)1508 static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
1509 			    const void *ptr)
1510 {
1511 	const struct xfrm_pol_inexact_key *key = arg->key;
1512 	const struct xfrm_pol_inexact_bin *b = ptr;
1513 	int ret;
1514 
1515 	if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net)))
1516 		return -1;
1517 
1518 	ret = b->k.dir ^ key->dir;
1519 	if (ret)
1520 		return ret;
1521 
1522 	ret = b->k.type ^ key->type;
1523 	if (ret)
1524 		return ret;
1525 
1526 	ret = b->k.family ^ key->family;
1527 	if (ret)
1528 		return ret;
1529 
1530 	return b->k.if_id ^ key->if_id;
1531 }
1532 
1533 static const struct rhashtable_params xfrm_pol_inexact_params = {
1534 	.head_offset		= offsetof(struct xfrm_pol_inexact_bin, head),
1535 	.hashfn			= xfrm_pol_bin_key,
1536 	.obj_hashfn		= xfrm_pol_bin_obj,
1537 	.obj_cmpfn		= xfrm_pol_bin_cmp,
1538 	.automatic_shrinking	= true,
1539 };
1540 
xfrm_policy_insert_list(struct hlist_head * chain,struct xfrm_policy * policy,bool excl)1541 static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
1542 						   struct xfrm_policy *policy,
1543 						   bool excl)
1544 {
1545 	struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;
1546 
1547 	hlist_for_each_entry(pol, chain, bydst) {
1548 		if (pol->type == policy->type &&
1549 		    pol->if_id == policy->if_id &&
1550 		    !selector_cmp(&pol->selector, &policy->selector) &&
1551 		    xfrm_policy_mark_match(&policy->mark, pol) &&
1552 		    xfrm_sec_ctx_match(pol->security, policy->security) &&
1553 		    !WARN_ON(delpol)) {
1554 			if (excl)
1555 				return ERR_PTR(-EEXIST);
1556 			delpol = pol;
1557 			if (policy->priority > pol->priority)
1558 				continue;
1559 		} else if (policy->priority >= pol->priority) {
1560 			newpos = pol;
1561 			continue;
1562 		}
1563 		if (delpol)
1564 			break;
1565 	}
1566 
1567 	if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1568 		hlist_add_behind_rcu(&policy->bydst, &newpos->bydst);
1569 	else
1570 		/* Packet offload policies enter to the head
1571 		 * to speed-up lookups.
1572 		 */
1573 		hlist_add_head_rcu(&policy->bydst, chain);
1574 
1575 	return delpol;
1576 }
1577 
xfrm_policy_insert(int dir,struct xfrm_policy * policy,int excl)1578 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
1579 {
1580 	struct net *net = xp_net(policy);
1581 	struct xfrm_policy *delpol;
1582 	struct hlist_head *chain;
1583 
1584 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1585 	chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
1586 	if (chain)
1587 		delpol = xfrm_policy_insert_list(chain, policy, excl);
1588 	else
1589 		delpol = xfrm_policy_inexact_insert(policy, dir, excl);
1590 
1591 	if (IS_ERR(delpol)) {
1592 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1593 		return PTR_ERR(delpol);
1594 	}
1595 
1596 	__xfrm_policy_link(policy, dir);
1597 
1598 	/* After previous checking, family can either be AF_INET or AF_INET6 */
1599 	if (policy->family == AF_INET)
1600 		rt_genid_bump_ipv4(net);
1601 	else
1602 		rt_genid_bump_ipv6(net);
1603 
1604 	if (delpol) {
1605 		xfrm_policy_requeue(delpol, policy);
1606 		__xfrm_policy_unlink(delpol, dir);
1607 	}
1608 	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
1609 	hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
1610 	policy->curlft.add_time = ktime_get_real_seconds();
1611 	policy->curlft.use_time = 0;
1612 	if (!mod_timer(&policy->timer, jiffies + HZ))
1613 		xfrm_pol_hold(policy);
1614 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1615 
1616 	if (delpol)
1617 		xfrm_policy_kill(delpol);
1618 	else if (xfrm_bydst_should_resize(net, dir, NULL))
1619 		schedule_work(&net->xfrm.policy_hash_work);
1620 
1621 	return 0;
1622 }
1623 EXPORT_SYMBOL(xfrm_policy_insert);
1624 
1625 static struct xfrm_policy *
__xfrm_policy_bysel_ctx(struct hlist_head * chain,const struct xfrm_mark * mark,u32 if_id,u8 type,int dir,struct xfrm_selector * sel,struct xfrm_sec_ctx * ctx)1626 __xfrm_policy_bysel_ctx(struct hlist_head *chain, const struct xfrm_mark *mark,
1627 			u32 if_id, u8 type, int dir, struct xfrm_selector *sel,
1628 			struct xfrm_sec_ctx *ctx)
1629 {
1630 	struct xfrm_policy *pol;
1631 
1632 	if (!chain)
1633 		return NULL;
1634 
1635 	hlist_for_each_entry(pol, chain, bydst) {
1636 		if (pol->type == type &&
1637 		    pol->if_id == if_id &&
1638 		    xfrm_policy_mark_match(mark, pol) &&
1639 		    !selector_cmp(sel, &pol->selector) &&
1640 		    xfrm_sec_ctx_match(ctx, pol->security))
1641 			return pol;
1642 	}
1643 
1644 	return NULL;
1645 }
1646 
1647 struct xfrm_policy *
xfrm_policy_bysel_ctx(struct net * net,const struct xfrm_mark * mark,u32 if_id,u8 type,int dir,struct xfrm_selector * sel,struct xfrm_sec_ctx * ctx,int delete,int * err)1648 xfrm_policy_bysel_ctx(struct net *net, const struct xfrm_mark *mark, u32 if_id,
1649 		      u8 type, int dir, struct xfrm_selector *sel,
1650 		      struct xfrm_sec_ctx *ctx, int delete, int *err)
1651 {
1652 	struct xfrm_pol_inexact_bin *bin = NULL;
1653 	struct xfrm_policy *pol, *ret = NULL;
1654 	struct hlist_head *chain;
1655 
1656 	*err = 0;
1657 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1658 	chain = policy_hash_bysel(net, sel, sel->family, dir);
1659 	if (!chain) {
1660 		struct xfrm_pol_inexact_candidates cand;
1661 		int i;
1662 
1663 		bin = xfrm_policy_inexact_lookup(net, type,
1664 						 sel->family, dir, if_id);
1665 		if (!bin) {
1666 			spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1667 			return NULL;
1668 		}
1669 
1670 		if (!xfrm_policy_find_inexact_candidates(&cand, bin,
1671 							 &sel->saddr,
1672 							 &sel->daddr)) {
1673 			spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1674 			return NULL;
1675 		}
1676 
1677 		pol = NULL;
1678 		for (i = 0; i < ARRAY_SIZE(cand.res); i++) {
1679 			struct xfrm_policy *tmp;
1680 
1681 			tmp = __xfrm_policy_bysel_ctx(cand.res[i], mark,
1682 						      if_id, type, dir,
1683 						      sel, ctx);
1684 			if (!tmp)
1685 				continue;
1686 
1687 			if (!pol || tmp->pos < pol->pos)
1688 				pol = tmp;
1689 		}
1690 	} else {
1691 		pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,
1692 					      sel, ctx);
1693 	}
1694 
1695 	if (pol) {
1696 		xfrm_pol_hold(pol);
1697 		if (delete) {
1698 			*err = security_xfrm_policy_delete(pol->security);
1699 			if (*err) {
1700 				spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1701 				return pol;
1702 			}
1703 			__xfrm_policy_unlink(pol, dir);
1704 		}
1705 		ret = pol;
1706 	}
1707 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1708 
1709 	if (ret && delete)
1710 		xfrm_policy_kill(ret);
1711 	if (bin && delete)
1712 		xfrm_policy_inexact_prune_bin(bin);
1713 	return ret;
1714 }
1715 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
1716 
1717 struct xfrm_policy *
xfrm_policy_byid(struct net * net,const struct xfrm_mark * mark,u32 if_id,u8 type,int dir,u32 id,int delete,int * err)1718 xfrm_policy_byid(struct net *net, const struct xfrm_mark *mark, u32 if_id,
1719 		 u8 type, int dir, u32 id, int delete, int *err)
1720 {
1721 	struct xfrm_policy *pol, *ret;
1722 	struct hlist_head *chain;
1723 
1724 	*err = -ENOENT;
1725 	if (xfrm_policy_id2dir(id) != dir)
1726 		return NULL;
1727 
1728 	*err = 0;
1729 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1730 	chain = net->xfrm.policy_byidx + idx_hash(net, id);
1731 	ret = NULL;
1732 	hlist_for_each_entry(pol, chain, byidx) {
1733 		if (pol->type == type && pol->index == id &&
1734 		    pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {
1735 			xfrm_pol_hold(pol);
1736 			if (delete) {
1737 				*err = security_xfrm_policy_delete(
1738 								pol->security);
1739 				if (*err) {
1740 					spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1741 					return pol;
1742 				}
1743 				__xfrm_policy_unlink(pol, dir);
1744 			}
1745 			ret = pol;
1746 			break;
1747 		}
1748 	}
1749 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1750 
1751 	if (ret && delete)
1752 		xfrm_policy_kill(ret);
1753 	return ret;
1754 }
1755 EXPORT_SYMBOL(xfrm_policy_byid);
1756 
1757 #ifdef CONFIG_SECURITY_NETWORK_XFRM
1758 static inline int
xfrm_policy_flush_secctx_check(struct net * net,u8 type,bool task_valid)1759 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1760 {
1761 	struct xfrm_policy *pol;
1762 	int err = 0;
1763 
1764 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1765 		if (pol->walk.dead ||
1766 		    xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1767 		    pol->type != type)
1768 			continue;
1769 
1770 		err = security_xfrm_policy_delete(pol->security);
1771 		if (err) {
1772 			xfrm_audit_policy_delete(pol, 0, task_valid);
1773 			return err;
1774 		}
1775 	}
1776 	return err;
1777 }
1778 
xfrm_dev_policy_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)1779 static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1780 						     struct net_device *dev,
1781 						     bool task_valid)
1782 {
1783 	struct xfrm_policy *pol;
1784 	int err = 0;
1785 
1786 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1787 		if (pol->walk.dead ||
1788 		    xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1789 		    pol->xdo.dev != dev)
1790 			continue;
1791 
1792 		err = security_xfrm_policy_delete(pol->security);
1793 		if (err) {
1794 			xfrm_audit_policy_delete(pol, 0, task_valid);
1795 			return err;
1796 		}
1797 	}
1798 	return err;
1799 }
1800 #else
1801 static inline int
xfrm_policy_flush_secctx_check(struct net * net,u8 type,bool task_valid)1802 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1803 {
1804 	return 0;
1805 }
1806 
xfrm_dev_policy_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)1807 static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1808 						     struct net_device *dev,
1809 						     bool task_valid)
1810 {
1811 	return 0;
1812 }
1813 #endif
1814 
xfrm_policy_flush(struct net * net,u8 type,bool task_valid)1815 int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
1816 {
1817 	int dir, err = 0, cnt = 0;
1818 	struct xfrm_policy *pol;
1819 
1820 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1821 
1822 	err = xfrm_policy_flush_secctx_check(net, type, task_valid);
1823 	if (err)
1824 		goto out;
1825 
1826 again:
1827 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1828 		if (pol->walk.dead)
1829 			continue;
1830 
1831 		dir = xfrm_policy_id2dir(pol->index);
1832 		if (dir >= XFRM_POLICY_MAX ||
1833 		    pol->type != type)
1834 			continue;
1835 
1836 		__xfrm_policy_unlink(pol, dir);
1837 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1838 		cnt++;
1839 		xfrm_audit_policy_delete(pol, 1, task_valid);
1840 		xfrm_policy_kill(pol);
1841 		spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1842 		goto again;
1843 	}
1844 	if (cnt)
1845 		__xfrm_policy_inexact_flush(net);
1846 	else
1847 		err = -ESRCH;
1848 out:
1849 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1850 	return err;
1851 }
1852 EXPORT_SYMBOL(xfrm_policy_flush);
1853 
xfrm_dev_policy_flush(struct net * net,struct net_device * dev,bool task_valid)1854 int xfrm_dev_policy_flush(struct net *net, struct net_device *dev,
1855 			  bool task_valid)
1856 {
1857 	int dir, err = 0, cnt = 0;
1858 	struct xfrm_policy *pol;
1859 
1860 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1861 
1862 	err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid);
1863 	if (err)
1864 		goto out;
1865 
1866 again:
1867 	list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1868 		if (pol->walk.dead)
1869 			continue;
1870 
1871 		dir = xfrm_policy_id2dir(pol->index);
1872 		if (dir >= XFRM_POLICY_MAX ||
1873 		    pol->xdo.dev != dev)
1874 			continue;
1875 
1876 		__xfrm_policy_unlink(pol, dir);
1877 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1878 		cnt++;
1879 		xfrm_audit_policy_delete(pol, 1, task_valid);
1880 		xfrm_policy_kill(pol);
1881 		spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1882 		goto again;
1883 	}
1884 	if (cnt)
1885 		__xfrm_policy_inexact_flush(net);
1886 	else
1887 		err = -ESRCH;
1888 out:
1889 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1890 	return err;
1891 }
1892 EXPORT_SYMBOL(xfrm_dev_policy_flush);
1893 
xfrm_policy_walk(struct net * net,struct xfrm_policy_walk * walk,int (* func)(struct xfrm_policy *,int,int,void *),void * data)1894 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
1895 		     int (*func)(struct xfrm_policy *, int, int, void*),
1896 		     void *data)
1897 {
1898 	struct xfrm_policy *pol;
1899 	struct xfrm_policy_walk_entry *x;
1900 	int error = 0;
1901 
1902 	if (walk->type >= XFRM_POLICY_TYPE_MAX &&
1903 	    walk->type != XFRM_POLICY_TYPE_ANY)
1904 		return -EINVAL;
1905 
1906 	if (list_empty(&walk->walk.all) && walk->seq != 0)
1907 		return 0;
1908 
1909 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1910 	if (list_empty(&walk->walk.all))
1911 		x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
1912 	else
1913 		x = list_first_entry(&walk->walk.all,
1914 				     struct xfrm_policy_walk_entry, all);
1915 
1916 	list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
1917 		if (x->dead)
1918 			continue;
1919 		pol = container_of(x, struct xfrm_policy, walk);
1920 		if (walk->type != XFRM_POLICY_TYPE_ANY &&
1921 		    walk->type != pol->type)
1922 			continue;
1923 		error = func(pol, xfrm_policy_id2dir(pol->index),
1924 			     walk->seq, data);
1925 		if (error) {
1926 			list_move_tail(&walk->walk.all, &x->all);
1927 			goto out;
1928 		}
1929 		walk->seq++;
1930 	}
1931 	if (walk->seq == 0) {
1932 		error = -ENOENT;
1933 		goto out;
1934 	}
1935 	list_del_init(&walk->walk.all);
1936 out:
1937 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1938 	return error;
1939 }
1940 EXPORT_SYMBOL(xfrm_policy_walk);
1941 
xfrm_policy_walk_init(struct xfrm_policy_walk * walk,u8 type)1942 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1943 {
1944 	INIT_LIST_HEAD(&walk->walk.all);
1945 	walk->walk.dead = 1;
1946 	walk->type = type;
1947 	walk->seq = 0;
1948 }
1949 EXPORT_SYMBOL(xfrm_policy_walk_init);
1950 
xfrm_policy_walk_done(struct xfrm_policy_walk * walk,struct net * net)1951 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
1952 {
1953 	if (list_empty(&walk->walk.all))
1954 		return;
1955 
1956 	spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
1957 	list_del(&walk->walk.all);
1958 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1959 }
1960 EXPORT_SYMBOL(xfrm_policy_walk_done);
1961 
1962 /*
1963  * Find policy to apply to this flow.
1964  *
1965  * Returns 0 if policy found, else an -errno.
1966  */
xfrm_policy_match(const struct xfrm_policy * pol,const struct flowi * fl,u8 type,u16 family,u32 if_id)1967 static int xfrm_policy_match(const struct xfrm_policy *pol,
1968 			     const struct flowi *fl,
1969 			     u8 type, u16 family, u32 if_id)
1970 {
1971 	const struct xfrm_selector *sel = &pol->selector;
1972 	int ret = -ESRCH;
1973 	bool match;
1974 
1975 	if (pol->family != family ||
1976 	    pol->if_id != if_id ||
1977 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
1978 	    pol->type != type)
1979 		return ret;
1980 
1981 	match = xfrm_selector_match(sel, fl, family);
1982 	if (match)
1983 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid);
1984 	return ret;
1985 }
1986 
1987 static struct xfrm_pol_inexact_node *
xfrm_policy_lookup_inexact_addr(const struct rb_root * r,seqcount_spinlock_t * count,const xfrm_address_t * addr,u16 family)1988 xfrm_policy_lookup_inexact_addr(const struct rb_root *r,
1989 				seqcount_spinlock_t *count,
1990 				const xfrm_address_t *addr, u16 family)
1991 {
1992 	const struct rb_node *parent;
1993 	int seq;
1994 
1995 again:
1996 	seq = read_seqcount_begin(count);
1997 
1998 	parent = rcu_dereference_raw(r->rb_node);
1999 	while (parent) {
2000 		struct xfrm_pol_inexact_node *node;
2001 		int delta;
2002 
2003 		node = rb_entry(parent, struct xfrm_pol_inexact_node, node);
2004 
2005 		delta = xfrm_policy_addr_delta(addr, &node->addr,
2006 					       node->prefixlen, family);
2007 		if (delta < 0) {
2008 			parent = rcu_dereference_raw(parent->rb_left);
2009 			continue;
2010 		} else if (delta > 0) {
2011 			parent = rcu_dereference_raw(parent->rb_right);
2012 			continue;
2013 		}
2014 
2015 		return node;
2016 	}
2017 
2018 	if (read_seqcount_retry(count, seq))
2019 		goto again;
2020 
2021 	return NULL;
2022 }
2023 
2024 static bool
xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates * cand,struct xfrm_pol_inexact_bin * b,const xfrm_address_t * saddr,const xfrm_address_t * daddr)2025 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
2026 				    struct xfrm_pol_inexact_bin *b,
2027 				    const xfrm_address_t *saddr,
2028 				    const xfrm_address_t *daddr)
2029 {
2030 	struct xfrm_pol_inexact_node *n;
2031 	u16 family;
2032 
2033 	if (!b)
2034 		return false;
2035 
2036 	family = b->k.family;
2037 	memset(cand, 0, sizeof(*cand));
2038 	cand->res[XFRM_POL_CAND_ANY] = &b->hhead;
2039 
2040 	n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr,
2041 					    family);
2042 	if (n) {
2043 		cand->res[XFRM_POL_CAND_DADDR] = &n->hhead;
2044 		n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr,
2045 						    family);
2046 		if (n)
2047 			cand->res[XFRM_POL_CAND_BOTH] = &n->hhead;
2048 	}
2049 
2050 	n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr,
2051 					    family);
2052 	if (n)
2053 		cand->res[XFRM_POL_CAND_SADDR] = &n->hhead;
2054 
2055 	return true;
2056 }
2057 
2058 static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup_rcu(struct net * net,u8 type,u16 family,u8 dir,u32 if_id)2059 xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family,
2060 			       u8 dir, u32 if_id)
2061 {
2062 	struct xfrm_pol_inexact_key k = {
2063 		.family = family,
2064 		.type = type,
2065 		.dir = dir,
2066 		.if_id = if_id,
2067 	};
2068 
2069 	write_pnet(&k.net, net);
2070 
2071 	return rhashtable_lookup(&xfrm_policy_inexact_table, &k,
2072 				 xfrm_pol_inexact_params);
2073 }
2074 
2075 static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup(struct net * net,u8 type,u16 family,u8 dir,u32 if_id)2076 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family,
2077 			   u8 dir, u32 if_id)
2078 {
2079 	struct xfrm_pol_inexact_bin *bin;
2080 
2081 	lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
2082 
2083 	rcu_read_lock();
2084 	bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2085 	rcu_read_unlock();
2086 
2087 	return bin;
2088 }
2089 
2090 static struct xfrm_policy *
__xfrm_policy_eval_candidates(struct hlist_head * chain,struct xfrm_policy * prefer,const struct flowi * fl,u8 type,u16 family,u32 if_id)2091 __xfrm_policy_eval_candidates(struct hlist_head *chain,
2092 			      struct xfrm_policy *prefer,
2093 			      const struct flowi *fl,
2094 			      u8 type, u16 family, u32 if_id)
2095 {
2096 	u32 priority = prefer ? prefer->priority : ~0u;
2097 	struct xfrm_policy *pol;
2098 
2099 	if (!chain)
2100 		return NULL;
2101 
2102 	hlist_for_each_entry_rcu(pol, chain, bydst) {
2103 		int err;
2104 
2105 		if (pol->priority > priority)
2106 			break;
2107 
2108 		err = xfrm_policy_match(pol, fl, type, family, if_id);
2109 		if (err) {
2110 			if (err != -ESRCH)
2111 				return ERR_PTR(err);
2112 
2113 			continue;
2114 		}
2115 
2116 		if (prefer) {
2117 			/* matches.  Is it older than *prefer? */
2118 			if (pol->priority == priority &&
2119 			    prefer->pos < pol->pos)
2120 				return prefer;
2121 		}
2122 
2123 		return pol;
2124 	}
2125 
2126 	return NULL;
2127 }
2128 
2129 static struct xfrm_policy *
xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates * cand,struct xfrm_policy * prefer,const struct flowi * fl,u8 type,u16 family,u32 if_id)2130 xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
2131 			    struct xfrm_policy *prefer,
2132 			    const struct flowi *fl,
2133 			    u8 type, u16 family, u32 if_id)
2134 {
2135 	struct xfrm_policy *tmp;
2136 	int i;
2137 
2138 	for (i = 0; i < ARRAY_SIZE(cand->res); i++) {
2139 		tmp = __xfrm_policy_eval_candidates(cand->res[i],
2140 						    prefer,
2141 						    fl, type, family, if_id);
2142 		if (!tmp)
2143 			continue;
2144 
2145 		if (IS_ERR(tmp))
2146 			return tmp;
2147 		prefer = tmp;
2148 	}
2149 
2150 	return prefer;
2151 }
2152 
xfrm_policy_lookup_bytype(struct net * net,u8 type,const struct flowi * fl,u16 family,u8 dir,u32 if_id)2153 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
2154 						     const struct flowi *fl,
2155 						     u16 family, u8 dir,
2156 						     u32 if_id)
2157 {
2158 	struct xfrm_pol_inexact_candidates cand;
2159 	const xfrm_address_t *daddr, *saddr;
2160 	struct xfrm_pol_inexact_bin *bin;
2161 	struct xfrm_policy *pol, *ret;
2162 	struct hlist_head *chain;
2163 	unsigned int sequence;
2164 	int err;
2165 
2166 	daddr = xfrm_flowi_daddr(fl, family);
2167 	saddr = xfrm_flowi_saddr(fl, family);
2168 	if (unlikely(!daddr || !saddr))
2169 		return NULL;
2170 
2171 	rcu_read_lock();
2172  retry:
2173 	do {
2174 		sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
2175 		chain = policy_hash_direct(net, daddr, saddr, family, dir);
2176 	} while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence));
2177 
2178 	ret = NULL;
2179 	hlist_for_each_entry_rcu(pol, chain, bydst) {
2180 		err = xfrm_policy_match(pol, fl, type, family, if_id);
2181 		if (err) {
2182 			if (err == -ESRCH)
2183 				continue;
2184 			else {
2185 				ret = ERR_PTR(err);
2186 				goto fail;
2187 			}
2188 		} else {
2189 			ret = pol;
2190 			break;
2191 		}
2192 	}
2193 	if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET)
2194 		goto skip_inexact;
2195 
2196 	bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2197 	if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr,
2198 							 daddr))
2199 		goto skip_inexact;
2200 
2201 	pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,
2202 					  family, if_id);
2203 	if (pol) {
2204 		ret = pol;
2205 		if (IS_ERR(pol))
2206 			goto fail;
2207 	}
2208 
2209 skip_inexact:
2210 	if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence))
2211 		goto retry;
2212 
2213 	if (ret && !xfrm_pol_hold_rcu(ret))
2214 		goto retry;
2215 fail:
2216 	rcu_read_unlock();
2217 
2218 	return ret;
2219 }
2220 
xfrm_policy_lookup(struct net * net,const struct flowi * fl,u16 family,u8 dir,u32 if_id)2221 static struct xfrm_policy *xfrm_policy_lookup(struct net *net,
2222 					      const struct flowi *fl,
2223 					      u16 family, u8 dir, u32 if_id)
2224 {
2225 #ifdef CONFIG_XFRM_SUB_POLICY
2226 	struct xfrm_policy *pol;
2227 
2228 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,
2229 					dir, if_id);
2230 	if (pol != NULL)
2231 		return pol;
2232 #endif
2233 	return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family,
2234 					 dir, if_id);
2235 }
2236 
xfrm_sk_policy_lookup(const struct sock * sk,int dir,const struct flowi * fl,u16 family,u32 if_id)2237 static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
2238 						 const struct flowi *fl,
2239 						 u16 family, u32 if_id)
2240 {
2241 	struct xfrm_policy *pol;
2242 
2243 	rcu_read_lock();
2244  again:
2245 	pol = rcu_dereference(sk->sk_policy[dir]);
2246 	if (pol != NULL) {
2247 		bool match;
2248 		int err = 0;
2249 
2250 		if (pol->family != family) {
2251 			pol = NULL;
2252 			goto out;
2253 		}
2254 
2255 		match = xfrm_selector_match(&pol->selector, fl, family);
2256 		if (match) {
2257 			if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v ||
2258 			    pol->if_id != if_id) {
2259 				pol = NULL;
2260 				goto out;
2261 			}
2262 			err = security_xfrm_policy_lookup(pol->security,
2263 						      fl->flowi_secid);
2264 			if (!err) {
2265 				if (!xfrm_pol_hold_rcu(pol))
2266 					goto again;
2267 			} else if (err == -ESRCH) {
2268 				pol = NULL;
2269 			} else {
2270 				pol = ERR_PTR(err);
2271 			}
2272 		} else
2273 			pol = NULL;
2274 	}
2275 out:
2276 	rcu_read_unlock();
2277 	return pol;
2278 }
2279 
xfrm_gen_pos_slow(struct net * net)2280 static u32 xfrm_gen_pos_slow(struct net *net)
2281 {
2282 	struct xfrm_policy *policy;
2283 	u32 i = 0;
2284 
2285 	/* oldest entry is last in list */
2286 	list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
2287 		if (!xfrm_policy_is_dead_or_sk(policy))
2288 			policy->pos = ++i;
2289 	}
2290 
2291 	return i;
2292 }
2293 
xfrm_gen_pos(struct net * net)2294 static u32 xfrm_gen_pos(struct net *net)
2295 {
2296 	const struct xfrm_policy *policy;
2297 	u32 i = 0;
2298 
2299 	/* most recently added policy is at the head of the list */
2300 	list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
2301 		if (xfrm_policy_is_dead_or_sk(policy))
2302 			continue;
2303 
2304 		if (policy->pos == UINT_MAX)
2305 			return xfrm_gen_pos_slow(net);
2306 
2307 		i = policy->pos + 1;
2308 		break;
2309 	}
2310 
2311 	return i;
2312 }
2313 
__xfrm_policy_link(struct xfrm_policy * pol,int dir)2314 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
2315 {
2316 	struct net *net = xp_net(pol);
2317 
2318 	switch (dir) {
2319 	case XFRM_POLICY_IN:
2320 	case XFRM_POLICY_FWD:
2321 	case XFRM_POLICY_OUT:
2322 		pol->pos = xfrm_gen_pos(net);
2323 		break;
2324 	}
2325 
2326 	list_add(&pol->walk.all, &net->xfrm.policy_all);
2327 	net->xfrm.policy_count[dir]++;
2328 	xfrm_pol_hold(pol);
2329 }
2330 
__xfrm_policy_unlink(struct xfrm_policy * pol,int dir)2331 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
2332 						int dir)
2333 {
2334 	struct net *net = xp_net(pol);
2335 
2336 	if (list_empty(&pol->walk.all))
2337 		return NULL;
2338 
2339 	/* Socket policies are not hashed. */
2340 	if (!hlist_unhashed(&pol->bydst)) {
2341 		hlist_del_rcu(&pol->bydst);
2342 		hlist_del(&pol->byidx);
2343 	}
2344 
2345 	list_del_init(&pol->walk.all);
2346 	net->xfrm.policy_count[dir]--;
2347 
2348 	return pol;
2349 }
2350 
xfrm_sk_policy_link(struct xfrm_policy * pol,int dir)2351 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
2352 {
2353 	__xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
2354 }
2355 
xfrm_sk_policy_unlink(struct xfrm_policy * pol,int dir)2356 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
2357 {
2358 	__xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
2359 }
2360 
xfrm_policy_delete(struct xfrm_policy * pol,int dir)2361 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
2362 {
2363 	struct net *net = xp_net(pol);
2364 
2365 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2366 	pol = __xfrm_policy_unlink(pol, dir);
2367 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2368 	if (pol) {
2369 		xfrm_policy_kill(pol);
2370 		return 0;
2371 	}
2372 	return -ENOENT;
2373 }
2374 EXPORT_SYMBOL(xfrm_policy_delete);
2375 
xfrm_sk_policy_insert(struct sock * sk,int dir,struct xfrm_policy * pol)2376 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
2377 {
2378 	struct net *net = sock_net(sk);
2379 	struct xfrm_policy *old_pol;
2380 
2381 #ifdef CONFIG_XFRM_SUB_POLICY
2382 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
2383 		return -EINVAL;
2384 #endif
2385 
2386 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2387 	old_pol = rcu_dereference_protected(sk->sk_policy[dir],
2388 				lockdep_is_held(&net->xfrm.xfrm_policy_lock));
2389 	if (pol) {
2390 		pol->curlft.add_time = ktime_get_real_seconds();
2391 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
2392 		xfrm_sk_policy_link(pol, dir);
2393 	}
2394 	rcu_assign_pointer(sk->sk_policy[dir], pol);
2395 	if (old_pol) {
2396 		if (pol)
2397 			xfrm_policy_requeue(old_pol, pol);
2398 
2399 		/* Unlinking succeeds always. This is the only function
2400 		 * allowed to delete or replace socket policy.
2401 		 */
2402 		xfrm_sk_policy_unlink(old_pol, dir);
2403 	}
2404 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2405 
2406 	if (old_pol) {
2407 		xfrm_policy_kill(old_pol);
2408 	}
2409 	return 0;
2410 }
2411 
clone_policy(const struct xfrm_policy * old,int dir)2412 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
2413 {
2414 	struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
2415 	struct net *net = xp_net(old);
2416 
2417 	if (newp) {
2418 		newp->selector = old->selector;
2419 		if (security_xfrm_policy_clone(old->security,
2420 					       &newp->security)) {
2421 			kfree(newp);
2422 			return NULL;  /* ENOMEM */
2423 		}
2424 		newp->lft = old->lft;
2425 		newp->curlft = old->curlft;
2426 		newp->mark = old->mark;
2427 		newp->if_id = old->if_id;
2428 		newp->action = old->action;
2429 		newp->flags = old->flags;
2430 		newp->xfrm_nr = old->xfrm_nr;
2431 		newp->index = old->index;
2432 		newp->type = old->type;
2433 		newp->family = old->family;
2434 		memcpy(newp->xfrm_vec, old->xfrm_vec,
2435 		       newp->xfrm_nr*sizeof(struct xfrm_tmpl));
2436 		spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2437 		xfrm_sk_policy_link(newp, dir);
2438 		spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2439 		xfrm_pol_put(newp);
2440 	}
2441 	return newp;
2442 }
2443 
__xfrm_sk_clone_policy(struct sock * sk,const struct sock * osk)2444 int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
2445 {
2446 	const struct xfrm_policy *p;
2447 	struct xfrm_policy *np;
2448 	int i, ret = 0;
2449 
2450 	rcu_read_lock();
2451 	for (i = 0; i < 2; i++) {
2452 		p = rcu_dereference(osk->sk_policy[i]);
2453 		if (p) {
2454 			np = clone_policy(p, i);
2455 			if (unlikely(!np)) {
2456 				ret = -ENOMEM;
2457 				break;
2458 			}
2459 			rcu_assign_pointer(sk->sk_policy[i], np);
2460 		}
2461 	}
2462 	rcu_read_unlock();
2463 	return ret;
2464 }
2465 
2466 static int
xfrm_get_saddr(unsigned short family,xfrm_address_t * saddr,const struct xfrm_dst_lookup_params * params)2467 xfrm_get_saddr(unsigned short family, xfrm_address_t *saddr,
2468 	       const struct xfrm_dst_lookup_params *params)
2469 {
2470 	int err;
2471 	const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2472 
2473 	if (unlikely(afinfo == NULL))
2474 		return -EINVAL;
2475 	err = afinfo->get_saddr(saddr, params);
2476 	rcu_read_unlock();
2477 	return err;
2478 }
2479 
2480 /* Resolve list of templates for the flow, given policy. */
2481 
2482 static int
xfrm_tmpl_resolve_one(struct xfrm_policy * policy,const struct flowi * fl,struct xfrm_state ** xfrm,unsigned short family)2483 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
2484 		      struct xfrm_state **xfrm, unsigned short family)
2485 {
2486 	struct net *net = xp_net(policy);
2487 	int nx;
2488 	int i, error;
2489 	xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
2490 	xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
2491 	xfrm_address_t tmp;
2492 
2493 	for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
2494 		struct xfrm_state *x;
2495 		xfrm_address_t *remote = daddr;
2496 		xfrm_address_t *local  = saddr;
2497 		struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
2498 
2499 		if (tmpl->mode == XFRM_MODE_TUNNEL ||
2500 		    tmpl->mode == XFRM_MODE_IPTFS ||
2501 		    tmpl->mode == XFRM_MODE_BEET) {
2502 			remote = &tmpl->id.daddr;
2503 			local = &tmpl->saddr;
2504 			if (xfrm_addr_any(local, tmpl->encap_family)) {
2505 				struct xfrm_dst_lookup_params params;
2506 
2507 				memset(&params, 0, sizeof(params));
2508 				params.net = net;
2509 				params.oif = fl->flowi_oif;
2510 				params.daddr = remote;
2511 				error = xfrm_get_saddr(tmpl->encap_family, &tmp,
2512 						       &params);
2513 				if (error)
2514 					goto fail;
2515 				local = &tmp;
2516 			}
2517 		}
2518 
2519 		x = xfrm_state_find(remote, local, fl, tmpl, policy, &error,
2520 				    family, policy->if_id);
2521 		if (x && x->dir && x->dir != XFRM_SA_DIR_OUT) {
2522 			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEDIRERROR);
2523 			xfrm_state_put(x);
2524 			error = -EINVAL;
2525 			goto fail;
2526 		}
2527 
2528 		if (x && x->km.state == XFRM_STATE_VALID) {
2529 			xfrm[nx++] = x;
2530 			daddr = remote;
2531 			saddr = local;
2532 			continue;
2533 		}
2534 		if (x) {
2535 			error = (x->km.state == XFRM_STATE_ERROR ?
2536 				 -EINVAL : -EAGAIN);
2537 			xfrm_state_put(x);
2538 		} else if (error == -ESRCH) {
2539 			error = -EAGAIN;
2540 		}
2541 
2542 		if (!tmpl->optional)
2543 			goto fail;
2544 	}
2545 	return nx;
2546 
2547 fail:
2548 	for (nx--; nx >= 0; nx--)
2549 		xfrm_state_put(xfrm[nx]);
2550 	return error;
2551 }
2552 
2553 static int
xfrm_tmpl_resolve(struct xfrm_policy ** pols,int npols,const struct flowi * fl,struct xfrm_state ** xfrm,unsigned short family)2554 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
2555 		  struct xfrm_state **xfrm, unsigned short family)
2556 {
2557 	struct xfrm_state *tp[XFRM_MAX_DEPTH];
2558 	struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
2559 	int cnx = 0;
2560 	int error;
2561 	int ret;
2562 	int i;
2563 
2564 	for (i = 0; i < npols; i++) {
2565 		if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
2566 			error = -ENOBUFS;
2567 			goto fail;
2568 		}
2569 
2570 		ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
2571 		if (ret < 0) {
2572 			error = ret;
2573 			goto fail;
2574 		} else
2575 			cnx += ret;
2576 	}
2577 
2578 	/* found states are sorted for outbound processing */
2579 	if (npols > 1)
2580 		xfrm_state_sort(xfrm, tpp, cnx, family);
2581 
2582 	return cnx;
2583 
2584  fail:
2585 	for (cnx--; cnx >= 0; cnx--)
2586 		xfrm_state_put(tpp[cnx]);
2587 	return error;
2588 
2589 }
2590 
xfrm_get_dscp(const struct flowi * fl,int family)2591 static dscp_t xfrm_get_dscp(const struct flowi *fl, int family)
2592 {
2593 	if (family == AF_INET)
2594 		return inet_dsfield_to_dscp(fl->u.ip4.flowi4_tos);
2595 
2596 	return 0;
2597 }
2598 
xfrm_alloc_dst(struct net * net,int family)2599 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
2600 {
2601 	const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2602 	struct dst_ops *dst_ops;
2603 	struct xfrm_dst *xdst;
2604 
2605 	if (!afinfo)
2606 		return ERR_PTR(-EINVAL);
2607 
2608 	switch (family) {
2609 	case AF_INET:
2610 		dst_ops = &net->xfrm.xfrm4_dst_ops;
2611 		break;
2612 #if IS_ENABLED(CONFIG_IPV6)
2613 	case AF_INET6:
2614 		dst_ops = &net->xfrm.xfrm6_dst_ops;
2615 		break;
2616 #endif
2617 	default:
2618 		BUG();
2619 	}
2620 	xdst = dst_alloc(dst_ops, NULL, DST_OBSOLETE_NONE, 0);
2621 
2622 	if (likely(xdst)) {
2623 		memset_after(xdst, 0, u.dst);
2624 	} else
2625 		xdst = ERR_PTR(-ENOBUFS);
2626 
2627 	rcu_read_unlock();
2628 
2629 	return xdst;
2630 }
2631 
xfrm_init_path(struct xfrm_dst * path,struct dst_entry * dst,int nfheader_len)2632 static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
2633 			   int nfheader_len)
2634 {
2635 	if (dst->ops->family == AF_INET6) {
2636 		path->path_cookie = rt6_get_cookie(dst_rt6_info(dst));
2637 		path->u.rt6.rt6i_nfheader_len = nfheader_len;
2638 	}
2639 }
2640 
xfrm_fill_dst(struct xfrm_dst * xdst,struct net_device * dev,const struct flowi * fl)2641 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
2642 				const struct flowi *fl)
2643 {
2644 	const struct xfrm_policy_afinfo *afinfo =
2645 		xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
2646 	int err;
2647 
2648 	if (!afinfo)
2649 		return -EINVAL;
2650 
2651 	err = afinfo->fill_dst(xdst, dev, fl);
2652 
2653 	rcu_read_unlock();
2654 
2655 	return err;
2656 }
2657 
2658 
2659 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
2660  * all the metrics... Shortly, bundle a bundle.
2661  */
2662 
xfrm_bundle_create(struct xfrm_policy * policy,struct xfrm_state ** xfrm,struct xfrm_dst ** bundle,int nx,const struct flowi * fl,struct dst_entry * dst)2663 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
2664 					    struct xfrm_state **xfrm,
2665 					    struct xfrm_dst **bundle,
2666 					    int nx,
2667 					    const struct flowi *fl,
2668 					    struct dst_entry *dst)
2669 {
2670 	const struct xfrm_state_afinfo *afinfo;
2671 	const struct xfrm_mode *inner_mode;
2672 	struct net *net = xp_net(policy);
2673 	unsigned long now = jiffies;
2674 	struct net_device *dev;
2675 	struct xfrm_dst *xdst_prev = NULL;
2676 	struct xfrm_dst *xdst0 = NULL;
2677 	int i = 0;
2678 	int err;
2679 	int header_len = 0;
2680 	int nfheader_len = 0;
2681 	int trailer_len = 0;
2682 	int family = policy->selector.family;
2683 	xfrm_address_t saddr, daddr;
2684 	dscp_t dscp;
2685 
2686 	xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
2687 
2688 	dscp = xfrm_get_dscp(fl, family);
2689 
2690 	dst_hold(dst);
2691 
2692 	for (; i < nx; i++) {
2693 		struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
2694 		struct dst_entry *dst1 = &xdst->u.dst;
2695 
2696 		err = PTR_ERR(xdst);
2697 		if (IS_ERR(xdst)) {
2698 			dst_release(dst);
2699 			goto put_states;
2700 		}
2701 
2702 		bundle[i] = xdst;
2703 		if (!xdst_prev)
2704 			xdst0 = xdst;
2705 		else
2706 			/* Ref count is taken during xfrm_alloc_dst()
2707 			 * No need to do dst_clone() on dst1
2708 			 */
2709 			xfrm_dst_set_child(xdst_prev, &xdst->u.dst);
2710 
2711 		if (xfrm[i]->sel.family == AF_UNSPEC) {
2712 			inner_mode = xfrm_ip2inner_mode(xfrm[i],
2713 							xfrm_af2proto(family));
2714 			if (!inner_mode) {
2715 				err = -EAFNOSUPPORT;
2716 				dst_release(dst);
2717 				goto put_states;
2718 			}
2719 		} else
2720 			inner_mode = &xfrm[i]->inner_mode;
2721 
2722 		xdst->route = dst;
2723 		dst_copy_metrics(dst1, dst);
2724 
2725 		if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
2726 			__u32 mark = 0;
2727 			int oif;
2728 
2729 			if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m)
2730 				mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]);
2731 
2732 			if (xfrm[i]->xso.type != XFRM_DEV_OFFLOAD_PACKET)
2733 				family = xfrm[i]->props.family;
2734 
2735 			oif = fl->flowi_oif ? : fl->flowi_l3mdev;
2736 			dst = xfrm_dst_lookup(xfrm[i], dscp, oif, &saddr,
2737 					      &daddr, family, mark);
2738 			err = PTR_ERR(dst);
2739 			if (IS_ERR(dst))
2740 				goto put_states;
2741 		} else
2742 			dst_hold(dst);
2743 
2744 		dst1->xfrm = xfrm[i];
2745 		xdst->xfrm_genid = xfrm[i]->genid;
2746 
2747 		dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2748 		dst1->lastuse = now;
2749 
2750 		dst1->input = dst_discard;
2751 
2752 		if (xfrm[i]->mode_cbs && xfrm[i]->mode_cbs->output) {
2753 			dst1->output = xfrm[i]->mode_cbs->output;
2754 		} else {
2755 			rcu_read_lock();
2756 			afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family);
2757 			if (likely(afinfo))
2758 				dst1->output = afinfo->output;
2759 			else
2760 				dst1->output = dst_discard_out;
2761 			rcu_read_unlock();
2762 		}
2763 
2764 		xdst_prev = xdst;
2765 
2766 		header_len += xfrm[i]->props.header_len;
2767 		if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
2768 			nfheader_len += xfrm[i]->props.header_len;
2769 		trailer_len += xfrm[i]->props.trailer_len;
2770 	}
2771 
2772 	xfrm_dst_set_child(xdst_prev, dst);
2773 	xdst0->path = dst;
2774 
2775 	err = -ENODEV;
2776 	dev = dst->dev;
2777 	if (!dev)
2778 		goto free_dst;
2779 
2780 	xfrm_init_path(xdst0, dst, nfheader_len);
2781 	xfrm_init_pmtu(bundle, nx);
2782 
2783 	for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst;
2784 	     xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) {
2785 		err = xfrm_fill_dst(xdst_prev, dev, fl);
2786 		if (err)
2787 			goto free_dst;
2788 
2789 		xdst_prev->u.dst.header_len = header_len;
2790 		xdst_prev->u.dst.trailer_len = trailer_len;
2791 		header_len -= xdst_prev->u.dst.xfrm->props.header_len;
2792 		trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;
2793 	}
2794 
2795 	return &xdst0->u.dst;
2796 
2797 put_states:
2798 	for (; i < nx; i++)
2799 		xfrm_state_put(xfrm[i]);
2800 free_dst:
2801 	if (xdst0)
2802 		dst_release_immediate(&xdst0->u.dst);
2803 
2804 	return ERR_PTR(err);
2805 }
2806 
xfrm_expand_policies(const struct flowi * fl,u16 family,struct xfrm_policy ** pols,int * num_pols,int * num_xfrms)2807 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
2808 				struct xfrm_policy **pols,
2809 				int *num_pols, int *num_xfrms)
2810 {
2811 	int i;
2812 
2813 	if (*num_pols == 0 || !pols[0]) {
2814 		*num_pols = 0;
2815 		*num_xfrms = 0;
2816 		return 0;
2817 	}
2818 	if (IS_ERR(pols[0])) {
2819 		*num_pols = 0;
2820 		return PTR_ERR(pols[0]);
2821 	}
2822 
2823 	*num_xfrms = pols[0]->xfrm_nr;
2824 
2825 #ifdef CONFIG_XFRM_SUB_POLICY
2826 	if (pols[0]->action == XFRM_POLICY_ALLOW &&
2827 	    pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2828 		pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
2829 						    XFRM_POLICY_TYPE_MAIN,
2830 						    fl, family,
2831 						    XFRM_POLICY_OUT,
2832 						    pols[0]->if_id);
2833 		if (pols[1]) {
2834 			if (IS_ERR(pols[1])) {
2835 				xfrm_pols_put(pols, *num_pols);
2836 				*num_pols = 0;
2837 				return PTR_ERR(pols[1]);
2838 			}
2839 			(*num_pols)++;
2840 			(*num_xfrms) += pols[1]->xfrm_nr;
2841 		}
2842 	}
2843 #endif
2844 	for (i = 0; i < *num_pols; i++) {
2845 		if (pols[i]->action != XFRM_POLICY_ALLOW) {
2846 			*num_xfrms = -1;
2847 			break;
2848 		}
2849 	}
2850 
2851 	return 0;
2852 
2853 }
2854 
2855 static struct xfrm_dst *
xfrm_resolve_and_create_bundle(struct xfrm_policy ** pols,int num_pols,const struct flowi * fl,u16 family,struct dst_entry * dst_orig)2856 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
2857 			       const struct flowi *fl, u16 family,
2858 			       struct dst_entry *dst_orig)
2859 {
2860 	struct net *net = xp_net(pols[0]);
2861 	struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
2862 	struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
2863 	struct xfrm_dst *xdst;
2864 	struct dst_entry *dst;
2865 	int err;
2866 
2867 	/* Try to instantiate a bundle */
2868 	err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
2869 	if (err <= 0) {
2870 		if (err == 0)
2871 			return NULL;
2872 
2873 		if (err != -EAGAIN)
2874 			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2875 		return ERR_PTR(err);
2876 	}
2877 
2878 	dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig);
2879 	if (IS_ERR(dst)) {
2880 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
2881 		return ERR_CAST(dst);
2882 	}
2883 
2884 	xdst = (struct xfrm_dst *)dst;
2885 	xdst->num_xfrms = err;
2886 	xdst->num_pols = num_pols;
2887 	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2888 	xdst->policy_genid = atomic_read(&pols[0]->genid);
2889 
2890 	return xdst;
2891 }
2892 
xfrm_policy_queue_process(struct timer_list * t)2893 static void xfrm_policy_queue_process(struct timer_list *t)
2894 {
2895 	struct sk_buff *skb;
2896 	struct sock *sk;
2897 	struct dst_entry *dst;
2898 	struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);
2899 	struct net *net = xp_net(pol);
2900 	struct xfrm_policy_queue *pq = &pol->polq;
2901 	struct flowi fl;
2902 	struct sk_buff_head list;
2903 	__u32 skb_mark;
2904 
2905 	spin_lock(&pq->hold_queue.lock);
2906 	skb = skb_peek(&pq->hold_queue);
2907 	if (!skb) {
2908 		spin_unlock(&pq->hold_queue.lock);
2909 		goto out;
2910 	}
2911 	dst = skb_dst(skb);
2912 	sk = skb->sk;
2913 
2914 	/* Fixup the mark to support VTI. */
2915 	skb_mark = skb->mark;
2916 	skb->mark = pol->mark.v;
2917 	xfrm_decode_session(net, skb, &fl, dst->ops->family);
2918 	skb->mark = skb_mark;
2919 	spin_unlock(&pq->hold_queue.lock);
2920 
2921 	dst_hold(xfrm_dst_path(dst));
2922 	dst = xfrm_lookup(net, xfrm_dst_path(dst), &fl, sk, XFRM_LOOKUP_QUEUE);
2923 	if (IS_ERR(dst))
2924 		goto purge_queue;
2925 
2926 	if (dst->flags & DST_XFRM_QUEUE) {
2927 		dst_release(dst);
2928 
2929 		if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
2930 			goto purge_queue;
2931 
2932 		pq->timeout = pq->timeout << 1;
2933 		if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
2934 			xfrm_pol_hold(pol);
2935 		goto out;
2936 	}
2937 
2938 	dst_release(dst);
2939 
2940 	__skb_queue_head_init(&list);
2941 
2942 	spin_lock(&pq->hold_queue.lock);
2943 	pq->timeout = 0;
2944 	skb_queue_splice_init(&pq->hold_queue, &list);
2945 	spin_unlock(&pq->hold_queue.lock);
2946 
2947 	while (!skb_queue_empty(&list)) {
2948 		skb = __skb_dequeue(&list);
2949 
2950 		/* Fixup the mark to support VTI. */
2951 		skb_mark = skb->mark;
2952 		skb->mark = pol->mark.v;
2953 		xfrm_decode_session(net, skb, &fl, skb_dst(skb)->ops->family);
2954 		skb->mark = skb_mark;
2955 
2956 		dst_hold(xfrm_dst_path(skb_dst(skb)));
2957 		dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0);
2958 		if (IS_ERR(dst)) {
2959 			kfree_skb(skb);
2960 			continue;
2961 		}
2962 
2963 		nf_reset_ct(skb);
2964 		skb_dst_drop(skb);
2965 		skb_dst_set(skb, dst);
2966 
2967 		dst_output(net, skb_to_full_sk(skb), skb);
2968 	}
2969 
2970 out:
2971 	xfrm_pol_put(pol);
2972 	return;
2973 
2974 purge_queue:
2975 	pq->timeout = 0;
2976 	skb_queue_purge(&pq->hold_queue);
2977 	xfrm_pol_put(pol);
2978 }
2979 
xdst_queue_output(struct net * net,struct sock * sk,struct sk_buff * skb)2980 static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
2981 {
2982 	unsigned long sched_next;
2983 	struct dst_entry *dst = skb_dst(skb);
2984 	struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
2985 	struct xfrm_policy *pol = xdst->pols[0];
2986 	struct xfrm_policy_queue *pq = &pol->polq;
2987 
2988 	if (unlikely(skb_fclone_busy(sk, skb))) {
2989 		kfree_skb(skb);
2990 		return 0;
2991 	}
2992 
2993 	if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
2994 		kfree_skb(skb);
2995 		return -EAGAIN;
2996 	}
2997 
2998 	skb_dst_force(skb);
2999 
3000 	spin_lock_bh(&pq->hold_queue.lock);
3001 
3002 	if (!pq->timeout)
3003 		pq->timeout = XFRM_QUEUE_TMO_MIN;
3004 
3005 	sched_next = jiffies + pq->timeout;
3006 
3007 	if (del_timer(&pq->hold_timer)) {
3008 		if (time_before(pq->hold_timer.expires, sched_next))
3009 			sched_next = pq->hold_timer.expires;
3010 		xfrm_pol_put(pol);
3011 	}
3012 
3013 	__skb_queue_tail(&pq->hold_queue, skb);
3014 	if (!mod_timer(&pq->hold_timer, sched_next))
3015 		xfrm_pol_hold(pol);
3016 
3017 	spin_unlock_bh(&pq->hold_queue.lock);
3018 
3019 	return 0;
3020 }
3021 
xfrm_create_dummy_bundle(struct net * net,struct xfrm_flo * xflo,const struct flowi * fl,int num_xfrms,u16 family)3022 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
3023 						 struct xfrm_flo *xflo,
3024 						 const struct flowi *fl,
3025 						 int num_xfrms,
3026 						 u16 family)
3027 {
3028 	int err;
3029 	struct net_device *dev;
3030 	struct dst_entry *dst;
3031 	struct dst_entry *dst1;
3032 	struct xfrm_dst *xdst;
3033 
3034 	xdst = xfrm_alloc_dst(net, family);
3035 	if (IS_ERR(xdst))
3036 		return xdst;
3037 
3038 	if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
3039 	    net->xfrm.sysctl_larval_drop ||
3040 	    num_xfrms <= 0)
3041 		return xdst;
3042 
3043 	dst = xflo->dst_orig;
3044 	dst1 = &xdst->u.dst;
3045 	dst_hold(dst);
3046 	xdst->route = dst;
3047 
3048 	dst_copy_metrics(dst1, dst);
3049 
3050 	dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
3051 	dst1->flags |= DST_XFRM_QUEUE;
3052 	dst1->lastuse = jiffies;
3053 
3054 	dst1->input = dst_discard;
3055 	dst1->output = xdst_queue_output;
3056 
3057 	dst_hold(dst);
3058 	xfrm_dst_set_child(xdst, dst);
3059 	xdst->path = dst;
3060 
3061 	xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
3062 
3063 	err = -ENODEV;
3064 	dev = dst->dev;
3065 	if (!dev)
3066 		goto free_dst;
3067 
3068 	err = xfrm_fill_dst(xdst, dev, fl);
3069 	if (err)
3070 		goto free_dst;
3071 
3072 out:
3073 	return xdst;
3074 
3075 free_dst:
3076 	dst_release(dst1);
3077 	xdst = ERR_PTR(err);
3078 	goto out;
3079 }
3080 
xfrm_bundle_lookup(struct net * net,const struct flowi * fl,u16 family,u8 dir,struct xfrm_flo * xflo,u32 if_id)3081 static struct xfrm_dst *xfrm_bundle_lookup(struct net *net,
3082 					   const struct flowi *fl,
3083 					   u16 family, u8 dir,
3084 					   struct xfrm_flo *xflo, u32 if_id)
3085 {
3086 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3087 	int num_pols = 0, num_xfrms = 0, err;
3088 	struct xfrm_dst *xdst;
3089 
3090 	/* Resolve policies to use if we couldn't get them from
3091 	 * previous cache entry */
3092 	num_pols = 1;
3093 	pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id);
3094 	err = xfrm_expand_policies(fl, family, pols,
3095 					   &num_pols, &num_xfrms);
3096 	if (err < 0)
3097 		goto inc_error;
3098 	if (num_pols == 0)
3099 		return NULL;
3100 	if (num_xfrms <= 0)
3101 		goto make_dummy_bundle;
3102 
3103 	xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
3104 					      xflo->dst_orig);
3105 	if (IS_ERR(xdst)) {
3106 		err = PTR_ERR(xdst);
3107 		if (err == -EREMOTE) {
3108 			xfrm_pols_put(pols, num_pols);
3109 			return NULL;
3110 		}
3111 
3112 		if (err != -EAGAIN)
3113 			goto error;
3114 		goto make_dummy_bundle;
3115 	} else if (xdst == NULL) {
3116 		num_xfrms = 0;
3117 		goto make_dummy_bundle;
3118 	}
3119 
3120 	return xdst;
3121 
3122 make_dummy_bundle:
3123 	/* We found policies, but there's no bundles to instantiate:
3124 	 * either because the policy blocks, has no transformations or
3125 	 * we could not build template (no xfrm_states).*/
3126 	xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
3127 	if (IS_ERR(xdst)) {
3128 		xfrm_pols_put(pols, num_pols);
3129 		return ERR_CAST(xdst);
3130 	}
3131 	xdst->num_pols = num_pols;
3132 	xdst->num_xfrms = num_xfrms;
3133 	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
3134 
3135 	return xdst;
3136 
3137 inc_error:
3138 	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
3139 error:
3140 	xfrm_pols_put(pols, num_pols);
3141 	return ERR_PTR(err);
3142 }
3143 
make_blackhole(struct net * net,u16 family,struct dst_entry * dst_orig)3144 static struct dst_entry *make_blackhole(struct net *net, u16 family,
3145 					struct dst_entry *dst_orig)
3146 {
3147 	const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
3148 	struct dst_entry *ret;
3149 
3150 	if (!afinfo) {
3151 		dst_release(dst_orig);
3152 		return ERR_PTR(-EINVAL);
3153 	} else {
3154 		ret = afinfo->blackhole_route(net, dst_orig);
3155 	}
3156 	rcu_read_unlock();
3157 
3158 	return ret;
3159 }
3160 
3161 /* Finds/creates a bundle for given flow and if_id
3162  *
3163  * At the moment we eat a raw IP route. Mostly to speed up lookups
3164  * on interfaces with disabled IPsec.
3165  *
3166  * xfrm_lookup uses an if_id of 0 by default, and is provided for
3167  * compatibility
3168  */
xfrm_lookup_with_ifid(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags,u32 if_id)3169 struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
3170 					struct dst_entry *dst_orig,
3171 					const struct flowi *fl,
3172 					const struct sock *sk,
3173 					int flags, u32 if_id)
3174 {
3175 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3176 	struct xfrm_dst *xdst;
3177 	struct dst_entry *dst, *route;
3178 	u16 family = dst_orig->ops->family;
3179 	u8 dir = XFRM_POLICY_OUT;
3180 	int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
3181 
3182 	dst = NULL;
3183 	xdst = NULL;
3184 	route = NULL;
3185 
3186 	sk = sk_const_to_full_sk(sk);
3187 	if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
3188 		num_pols = 1;
3189 		pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl, family,
3190 						if_id);
3191 		err = xfrm_expand_policies(fl, family, pols,
3192 					   &num_pols, &num_xfrms);
3193 		if (err < 0)
3194 			goto dropdst;
3195 
3196 		if (num_pols) {
3197 			if (num_xfrms <= 0) {
3198 				drop_pols = num_pols;
3199 				goto no_transform;
3200 			}
3201 
3202 			xdst = xfrm_resolve_and_create_bundle(
3203 					pols, num_pols, fl,
3204 					family, dst_orig);
3205 
3206 			if (IS_ERR(xdst)) {
3207 				xfrm_pols_put(pols, num_pols);
3208 				err = PTR_ERR(xdst);
3209 				if (err == -EREMOTE)
3210 					goto nopol;
3211 
3212 				goto dropdst;
3213 			} else if (xdst == NULL) {
3214 				num_xfrms = 0;
3215 				drop_pols = num_pols;
3216 				goto no_transform;
3217 			}
3218 
3219 			route = xdst->route;
3220 		}
3221 	}
3222 
3223 	if (xdst == NULL) {
3224 		struct xfrm_flo xflo;
3225 
3226 		xflo.dst_orig = dst_orig;
3227 		xflo.flags = flags;
3228 
3229 		/* To accelerate a bit...  */
3230 		if (!if_id && ((dst_orig->flags & DST_NOXFRM) ||
3231 			       !net->xfrm.policy_count[XFRM_POLICY_OUT]))
3232 			goto nopol;
3233 
3234 		xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id);
3235 		if (xdst == NULL)
3236 			goto nopol;
3237 		if (IS_ERR(xdst)) {
3238 			err = PTR_ERR(xdst);
3239 			goto dropdst;
3240 		}
3241 
3242 		num_pols = xdst->num_pols;
3243 		num_xfrms = xdst->num_xfrms;
3244 		memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
3245 		route = xdst->route;
3246 	}
3247 
3248 	dst = &xdst->u.dst;
3249 	if (route == NULL && num_xfrms > 0) {
3250 		/* The only case when xfrm_bundle_lookup() returns a
3251 		 * bundle with null route, is when the template could
3252 		 * not be resolved. It means policies are there, but
3253 		 * bundle could not be created, since we don't yet
3254 		 * have the xfrm_state's. We need to wait for KM to
3255 		 * negotiate new SA's or bail out with error.*/
3256 		if (net->xfrm.sysctl_larval_drop) {
3257 			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3258 			err = -EREMOTE;
3259 			goto error;
3260 		}
3261 
3262 		err = -EAGAIN;
3263 
3264 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3265 		goto error;
3266 	}
3267 
3268 no_transform:
3269 	if (num_pols == 0)
3270 		goto nopol;
3271 
3272 	if ((flags & XFRM_LOOKUP_ICMP) &&
3273 	    !(pols[0]->flags & XFRM_POLICY_ICMP)) {
3274 		err = -ENOENT;
3275 		goto error;
3276 	}
3277 
3278 	for (i = 0; i < num_pols; i++)
3279 		WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds());
3280 
3281 	if (num_xfrms < 0) {
3282 		/* Prohibit the flow */
3283 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
3284 		err = -EPERM;
3285 		goto error;
3286 	} else if (num_xfrms > 0) {
3287 		/* Flow transformed */
3288 		dst_release(dst_orig);
3289 	} else {
3290 		/* Flow passes untransformed */
3291 		dst_release(dst);
3292 		dst = dst_orig;
3293 	}
3294 
3295 ok:
3296 	xfrm_pols_put(pols, drop_pols);
3297 	if (dst && dst->xfrm &&
3298 	    (dst->xfrm->props.mode == XFRM_MODE_TUNNEL ||
3299 	     dst->xfrm->props.mode == XFRM_MODE_IPTFS))
3300 		dst->flags |= DST_XFRM_TUNNEL;
3301 	return dst;
3302 
3303 nopol:
3304 	if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) &&
3305 	    net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3306 		err = -EPERM;
3307 		goto error;
3308 	}
3309 	if (!(flags & XFRM_LOOKUP_ICMP)) {
3310 		dst = dst_orig;
3311 		goto ok;
3312 	}
3313 	err = -ENOENT;
3314 error:
3315 	dst_release(dst);
3316 dropdst:
3317 	if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
3318 		dst_release(dst_orig);
3319 	xfrm_pols_put(pols, drop_pols);
3320 	return ERR_PTR(err);
3321 }
3322 EXPORT_SYMBOL(xfrm_lookup_with_ifid);
3323 
3324 /* Main function: finds/creates a bundle for given flow.
3325  *
3326  * At the moment we eat a raw IP route. Mostly to speed up lookups
3327  * on interfaces with disabled IPsec.
3328  */
xfrm_lookup(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)3329 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
3330 			      const struct flowi *fl, const struct sock *sk,
3331 			      int flags)
3332 {
3333 	return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0);
3334 }
3335 EXPORT_SYMBOL(xfrm_lookup);
3336 
3337 /* Callers of xfrm_lookup_route() must ensure a call to dst_output().
3338  * Otherwise we may send out blackholed packets.
3339  */
xfrm_lookup_route(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)3340 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
3341 				    const struct flowi *fl,
3342 				    const struct sock *sk, int flags)
3343 {
3344 	struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
3345 					    flags | XFRM_LOOKUP_QUEUE |
3346 					    XFRM_LOOKUP_KEEP_DST_REF);
3347 
3348 	if (PTR_ERR(dst) == -EREMOTE)
3349 		return make_blackhole(net, dst_orig->ops->family, dst_orig);
3350 
3351 	if (IS_ERR(dst))
3352 		dst_release(dst_orig);
3353 
3354 	return dst;
3355 }
3356 EXPORT_SYMBOL(xfrm_lookup_route);
3357 
3358 static inline int
xfrm_secpath_reject(int idx,struct sk_buff * skb,const struct flowi * fl)3359 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
3360 {
3361 	struct sec_path *sp = skb_sec_path(skb);
3362 	struct xfrm_state *x;
3363 
3364 	if (!sp || idx < 0 || idx >= sp->len)
3365 		return 0;
3366 	x = sp->xvec[idx];
3367 	if (!x->type->reject)
3368 		return 0;
3369 	return x->type->reject(x, skb, fl);
3370 }
3371 
3372 /* When skb is transformed back to its "native" form, we have to
3373  * check policy restrictions. At the moment we make this in maximally
3374  * stupid way. Shame on me. :-) Of course, connected sockets must
3375  * have policy cached at them.
3376  */
3377 
3378 static inline int
xfrm_state_ok(const struct xfrm_tmpl * tmpl,const struct xfrm_state * x,unsigned short family,u32 if_id)3379 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
3380 	      unsigned short family, u32 if_id)
3381 {
3382 	if (xfrm_state_kern(x))
3383 		return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
3384 	return	x->id.proto == tmpl->id.proto &&
3385 		(x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
3386 		(x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
3387 		x->props.mode == tmpl->mode &&
3388 		(tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
3389 		 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
3390 		!(x->props.mode != XFRM_MODE_TRANSPORT &&
3391 		  xfrm_state_addr_cmp(tmpl, x, family)) &&
3392 		(if_id == 0 || if_id == x->if_id);
3393 }
3394 
3395 /*
3396  * 0 or more than 0 is returned when validation is succeeded (either bypass
3397  * because of optional transport mode, or next index of the matched secpath
3398  * state with the template.
3399  * -1 is returned when no matching template is found.
3400  * Otherwise "-2 - errored_index" is returned.
3401  */
3402 static inline int
xfrm_policy_ok(const struct xfrm_tmpl * tmpl,const struct sec_path * sp,int start,unsigned short family,u32 if_id)3403 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
3404 	       unsigned short family, u32 if_id)
3405 {
3406 	int idx = start;
3407 
3408 	if (tmpl->optional) {
3409 		if (tmpl->mode == XFRM_MODE_TRANSPORT)
3410 			return start;
3411 	} else
3412 		start = -1;
3413 	for (; idx < sp->len; idx++) {
3414 		if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id))
3415 			return ++idx;
3416 		if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
3417 			if (idx < sp->verified_cnt) {
3418 				/* Secpath entry previously verified, consider optional and
3419 				 * continue searching
3420 				 */
3421 				continue;
3422 			}
3423 
3424 			if (start == -1)
3425 				start = -2-idx;
3426 			break;
3427 		}
3428 	}
3429 	return start;
3430 }
3431 
3432 static void
decode_session4(const struct xfrm_flow_keys * flkeys,struct flowi * fl,bool reverse)3433 decode_session4(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
3434 {
3435 	struct flowi4 *fl4 = &fl->u.ip4;
3436 
3437 	memset(fl4, 0, sizeof(struct flowi4));
3438 
3439 	if (reverse) {
3440 		fl4->saddr = flkeys->addrs.ipv4.dst;
3441 		fl4->daddr = flkeys->addrs.ipv4.src;
3442 		fl4->fl4_sport = flkeys->ports.dst;
3443 		fl4->fl4_dport = flkeys->ports.src;
3444 	} else {
3445 		fl4->saddr = flkeys->addrs.ipv4.src;
3446 		fl4->daddr = flkeys->addrs.ipv4.dst;
3447 		fl4->fl4_sport = flkeys->ports.src;
3448 		fl4->fl4_dport = flkeys->ports.dst;
3449 	}
3450 
3451 	switch (flkeys->basic.ip_proto) {
3452 	case IPPROTO_GRE:
3453 		fl4->fl4_gre_key = flkeys->gre.keyid;
3454 		break;
3455 	case IPPROTO_ICMP:
3456 		fl4->fl4_icmp_type = flkeys->icmp.type;
3457 		fl4->fl4_icmp_code = flkeys->icmp.code;
3458 		break;
3459 	}
3460 
3461 	fl4->flowi4_proto = flkeys->basic.ip_proto;
3462 	fl4->flowi4_tos = flkeys->ip.tos & ~INET_ECN_MASK;
3463 }
3464 
3465 #if IS_ENABLED(CONFIG_IPV6)
3466 static void
decode_session6(const struct xfrm_flow_keys * flkeys,struct flowi * fl,bool reverse)3467 decode_session6(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
3468 {
3469 	struct flowi6 *fl6 = &fl->u.ip6;
3470 
3471 	memset(fl6, 0, sizeof(struct flowi6));
3472 
3473 	if (reverse) {
3474 		fl6->saddr = flkeys->addrs.ipv6.dst;
3475 		fl6->daddr = flkeys->addrs.ipv6.src;
3476 		fl6->fl6_sport = flkeys->ports.dst;
3477 		fl6->fl6_dport = flkeys->ports.src;
3478 	} else {
3479 		fl6->saddr = flkeys->addrs.ipv6.src;
3480 		fl6->daddr = flkeys->addrs.ipv6.dst;
3481 		fl6->fl6_sport = flkeys->ports.src;
3482 		fl6->fl6_dport = flkeys->ports.dst;
3483 	}
3484 
3485 	switch (flkeys->basic.ip_proto) {
3486 	case IPPROTO_GRE:
3487 		fl6->fl6_gre_key = flkeys->gre.keyid;
3488 		break;
3489 	case IPPROTO_ICMPV6:
3490 		fl6->fl6_icmp_type = flkeys->icmp.type;
3491 		fl6->fl6_icmp_code = flkeys->icmp.code;
3492 		break;
3493 	}
3494 
3495 	fl6->flowi6_proto = flkeys->basic.ip_proto;
3496 }
3497 #endif
3498 
__xfrm_decode_session(struct net * net,struct sk_buff * skb,struct flowi * fl,unsigned int family,int reverse)3499 int __xfrm_decode_session(struct net *net, struct sk_buff *skb, struct flowi *fl,
3500 			  unsigned int family, int reverse)
3501 {
3502 	struct xfrm_flow_keys flkeys;
3503 
3504 	memset(&flkeys, 0, sizeof(flkeys));
3505 	__skb_flow_dissect(net, skb, &xfrm_session_dissector, &flkeys,
3506 			   NULL, 0, 0, 0, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
3507 
3508 	switch (family) {
3509 	case AF_INET:
3510 		decode_session4(&flkeys, fl, reverse);
3511 		break;
3512 #if IS_ENABLED(CONFIG_IPV6)
3513 	case AF_INET6:
3514 		decode_session6(&flkeys, fl, reverse);
3515 		break;
3516 #endif
3517 	default:
3518 		return -EAFNOSUPPORT;
3519 	}
3520 
3521 	fl->flowi_mark = skb->mark;
3522 	if (reverse) {
3523 		fl->flowi_oif = skb->skb_iif;
3524 	} else {
3525 		int oif = 0;
3526 
3527 		if (skb_dst(skb) && skb_dst(skb)->dev)
3528 			oif = skb_dst(skb)->dev->ifindex;
3529 
3530 		fl->flowi_oif = oif;
3531 	}
3532 
3533 	return security_xfrm_decode_session(skb, &fl->flowi_secid);
3534 }
3535 EXPORT_SYMBOL(__xfrm_decode_session);
3536 
secpath_has_nontransport(const struct sec_path * sp,int k,int * idxp)3537 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
3538 {
3539 	for (; k < sp->len; k++) {
3540 		if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
3541 			*idxp = k;
3542 			return 1;
3543 		}
3544 	}
3545 
3546 	return 0;
3547 }
3548 
icmp_err_packet(const struct flowi * fl,unsigned short family)3549 static bool icmp_err_packet(const struct flowi *fl, unsigned short family)
3550 {
3551 	const struct flowi4 *fl4 = &fl->u.ip4;
3552 
3553 	if (family == AF_INET &&
3554 	    fl4->flowi4_proto == IPPROTO_ICMP &&
3555 	    (fl4->fl4_icmp_type == ICMP_DEST_UNREACH ||
3556 	     fl4->fl4_icmp_type == ICMP_TIME_EXCEEDED))
3557 		return true;
3558 
3559 #if IS_ENABLED(CONFIG_IPV6)
3560 	if (family == AF_INET6) {
3561 		const struct flowi6 *fl6 = &fl->u.ip6;
3562 
3563 		if (fl6->flowi6_proto == IPPROTO_ICMPV6 &&
3564 		    (fl6->fl6_icmp_type == ICMPV6_DEST_UNREACH ||
3565 		    fl6->fl6_icmp_type == ICMPV6_PKT_TOOBIG ||
3566 		    fl6->fl6_icmp_type == ICMPV6_TIME_EXCEED))
3567 			return true;
3568 	}
3569 #endif
3570 	return false;
3571 }
3572 
xfrm_icmp_flow_decode(struct sk_buff * skb,unsigned short family,const struct flowi * fl,struct flowi * fl1)3573 static bool xfrm_icmp_flow_decode(struct sk_buff *skb, unsigned short family,
3574 				  const struct flowi *fl, struct flowi *fl1)
3575 {
3576 	bool ret = true;
3577 	struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
3578 	int hl = family == AF_INET ? (sizeof(struct iphdr) +  sizeof(struct icmphdr)) :
3579 		 (sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr));
3580 
3581 	if (!newskb)
3582 		return true;
3583 
3584 	if (!pskb_pull(newskb, hl))
3585 		goto out;
3586 
3587 	skb_reset_network_header(newskb);
3588 
3589 	if (xfrm_decode_session_reverse(dev_net(skb->dev), newskb, fl1, family) < 0)
3590 		goto out;
3591 
3592 	fl1->flowi_oif = fl->flowi_oif;
3593 	fl1->flowi_mark = fl->flowi_mark;
3594 	fl1->flowi_tos = fl->flowi_tos;
3595 	nf_nat_decode_session(newskb, fl1, family);
3596 	ret = false;
3597 
3598 out:
3599 	consume_skb(newskb);
3600 	return ret;
3601 }
3602 
xfrm_selector_inner_icmp_match(struct sk_buff * skb,unsigned short family,const struct xfrm_selector * sel,const struct flowi * fl)3603 static bool xfrm_selector_inner_icmp_match(struct sk_buff *skb, unsigned short family,
3604 					   const struct xfrm_selector *sel,
3605 					   const struct flowi *fl)
3606 {
3607 	bool ret = false;
3608 
3609 	if (icmp_err_packet(fl, family)) {
3610 		struct flowi fl1;
3611 
3612 		if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3613 			return ret;
3614 
3615 		ret = xfrm_selector_match(sel, &fl1, family);
3616 	}
3617 
3618 	return ret;
3619 }
3620 
3621 static inline struct
xfrm_in_fwd_icmp(struct sk_buff * skb,const struct flowi * fl,unsigned short family,u32 if_id)3622 xfrm_policy *xfrm_in_fwd_icmp(struct sk_buff *skb,
3623 			      const struct flowi *fl, unsigned short family,
3624 			      u32 if_id)
3625 {
3626 	struct xfrm_policy *pol = NULL;
3627 
3628 	if (icmp_err_packet(fl, family)) {
3629 		struct flowi fl1;
3630 		struct net *net = dev_net(skb->dev);
3631 
3632 		if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3633 			return pol;
3634 
3635 		pol = xfrm_policy_lookup(net, &fl1, family, XFRM_POLICY_FWD, if_id);
3636 		if (IS_ERR(pol))
3637 			pol = NULL;
3638 	}
3639 
3640 	return pol;
3641 }
3642 
3643 static inline struct
xfrm_out_fwd_icmp(struct sk_buff * skb,struct flowi * fl,unsigned short family,struct dst_entry * dst)3644 dst_entry *xfrm_out_fwd_icmp(struct sk_buff *skb, struct flowi *fl,
3645 			     unsigned short family, struct dst_entry *dst)
3646 {
3647 	if (icmp_err_packet(fl, family)) {
3648 		struct net *net = dev_net(skb->dev);
3649 		struct dst_entry *dst2;
3650 		struct flowi fl1;
3651 
3652 		if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3653 			return dst;
3654 
3655 		dst_hold(dst);
3656 
3657 		dst2 = xfrm_lookup(net, dst, &fl1, NULL, (XFRM_LOOKUP_QUEUE | XFRM_LOOKUP_ICMP));
3658 
3659 		if (IS_ERR(dst2))
3660 			return dst;
3661 
3662 		if (dst2->xfrm) {
3663 			dst_release(dst);
3664 			dst = dst2;
3665 		} else {
3666 			dst_release(dst2);
3667 		}
3668 	}
3669 
3670 	return dst;
3671 }
3672 
__xfrm_policy_check(struct sock * sk,int dir,struct sk_buff * skb,unsigned short family)3673 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
3674 			unsigned short family)
3675 {
3676 	struct net *net = dev_net(skb->dev);
3677 	struct xfrm_policy *pol;
3678 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3679 	int npols = 0;
3680 	int xfrm_nr;
3681 	int pi;
3682 	int reverse;
3683 	struct flowi fl;
3684 	int xerr_idx = -1;
3685 	const struct xfrm_if_cb *ifcb;
3686 	struct sec_path *sp;
3687 	u32 if_id = 0;
3688 
3689 	rcu_read_lock();
3690 	ifcb = xfrm_if_get_cb();
3691 
3692 	if (ifcb) {
3693 		struct xfrm_if_decode_session_result r;
3694 
3695 		if (ifcb->decode_session(skb, family, &r)) {
3696 			if_id = r.if_id;
3697 			net = r.net;
3698 		}
3699 	}
3700 	rcu_read_unlock();
3701 
3702 	reverse = dir & ~XFRM_POLICY_MASK;
3703 	dir &= XFRM_POLICY_MASK;
3704 
3705 	if (__xfrm_decode_session(net, skb, &fl, family, reverse) < 0) {
3706 		XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
3707 		return 0;
3708 	}
3709 
3710 	nf_nat_decode_session(skb, &fl, family);
3711 
3712 	/* First, check used SA against their selectors. */
3713 	sp = skb_sec_path(skb);
3714 	if (sp) {
3715 		int i;
3716 
3717 		for (i = sp->len - 1; i >= 0; i--) {
3718 			struct xfrm_state *x = sp->xvec[i];
3719 			int ret = 0;
3720 
3721 			if (!xfrm_selector_match(&x->sel, &fl, family)) {
3722 				ret = 1;
3723 				if (x->props.flags & XFRM_STATE_ICMP &&
3724 				    xfrm_selector_inner_icmp_match(skb, family, &x->sel, &fl))
3725 					ret = 0;
3726 				if (ret) {
3727 					XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
3728 					return 0;
3729 				}
3730 			}
3731 		}
3732 	}
3733 
3734 	pol = NULL;
3735 	sk = sk_to_full_sk(sk);
3736 	if (sk && sk->sk_policy[dir]) {
3737 		pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);
3738 		if (IS_ERR(pol)) {
3739 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3740 			return 0;
3741 		}
3742 	}
3743 
3744 	if (!pol)
3745 		pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);
3746 
3747 	if (IS_ERR(pol)) {
3748 		XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3749 		return 0;
3750 	}
3751 
3752 	if (!pol && dir == XFRM_POLICY_FWD)
3753 		pol = xfrm_in_fwd_icmp(skb, &fl, family, if_id);
3754 
3755 	if (!pol) {
3756 		const bool is_crypto_offload = sp &&
3757 			(xfrm_input_state(skb)->xso.type == XFRM_DEV_OFFLOAD_CRYPTO);
3758 
3759 		if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3760 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3761 			return 0;
3762 		}
3763 
3764 		if (sp && secpath_has_nontransport(sp, 0, &xerr_idx) && !is_crypto_offload) {
3765 			xfrm_secpath_reject(xerr_idx, skb, &fl);
3766 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3767 			return 0;
3768 		}
3769 		return 1;
3770 	}
3771 
3772 	/* This lockless write can happen from different cpus. */
3773 	WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds());
3774 
3775 	pols[0] = pol;
3776 	npols++;
3777 #ifdef CONFIG_XFRM_SUB_POLICY
3778 	if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
3779 		pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
3780 						    &fl, family,
3781 						    XFRM_POLICY_IN, if_id);
3782 		if (pols[1]) {
3783 			if (IS_ERR(pols[1])) {
3784 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3785 				xfrm_pol_put(pols[0]);
3786 				return 0;
3787 			}
3788 			/* This write can happen from different cpus. */
3789 			WRITE_ONCE(pols[1]->curlft.use_time,
3790 				   ktime_get_real_seconds());
3791 			npols++;
3792 		}
3793 	}
3794 #endif
3795 
3796 	if (pol->action == XFRM_POLICY_ALLOW) {
3797 		static struct sec_path dummy;
3798 		struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
3799 		struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
3800 		struct xfrm_tmpl **tpp = tp;
3801 		int ti = 0;
3802 		int i, k;
3803 
3804 		sp = skb_sec_path(skb);
3805 		if (!sp)
3806 			sp = &dummy;
3807 
3808 		for (pi = 0; pi < npols; pi++) {
3809 			if (pols[pi] != pol &&
3810 			    pols[pi]->action != XFRM_POLICY_ALLOW) {
3811 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3812 				goto reject;
3813 			}
3814 			if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
3815 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
3816 				goto reject_error;
3817 			}
3818 			for (i = 0; i < pols[pi]->xfrm_nr; i++)
3819 				tpp[ti++] = &pols[pi]->xfrm_vec[i];
3820 		}
3821 		xfrm_nr = ti;
3822 
3823 		if (npols > 1) {
3824 			xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
3825 			tpp = stp;
3826 		}
3827 
3828 		/* For each tunnel xfrm, find the first matching tmpl.
3829 		 * For each tmpl before that, find corresponding xfrm.
3830 		 * Order is _important_. Later we will implement
3831 		 * some barriers, but at the moment barriers
3832 		 * are implied between each two transformations.
3833 		 * Upon success, marks secpath entries as having been
3834 		 * verified to allow them to be skipped in future policy
3835 		 * checks (e.g. nested tunnels).
3836 		 */
3837 		for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
3838 			k = xfrm_policy_ok(tpp[i], sp, k, family, if_id);
3839 			if (k < 0) {
3840 				if (k < -1)
3841 					/* "-2 - errored_index" returned */
3842 					xerr_idx = -(2+k);
3843 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3844 				goto reject;
3845 			}
3846 		}
3847 
3848 		if (secpath_has_nontransport(sp, k, &xerr_idx)) {
3849 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3850 			goto reject;
3851 		}
3852 
3853 		xfrm_pols_put(pols, npols);
3854 		sp->verified_cnt = k;
3855 
3856 		return 1;
3857 	}
3858 	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3859 
3860 reject:
3861 	xfrm_secpath_reject(xerr_idx, skb, &fl);
3862 reject_error:
3863 	xfrm_pols_put(pols, npols);
3864 	return 0;
3865 }
3866 EXPORT_SYMBOL(__xfrm_policy_check);
3867 
__xfrm_route_forward(struct sk_buff * skb,unsigned short family)3868 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
3869 {
3870 	struct net *net = dev_net(skb->dev);
3871 	struct flowi fl;
3872 	struct dst_entry *dst;
3873 	int res = 1;
3874 
3875 	if (xfrm_decode_session(net, skb, &fl, family) < 0) {
3876 		XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3877 		return 0;
3878 	}
3879 
3880 	skb_dst_force(skb);
3881 	if (!skb_dst(skb)) {
3882 		XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3883 		return 0;
3884 	}
3885 
3886 	dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
3887 	if (IS_ERR(dst)) {
3888 		res = 0;
3889 		dst = NULL;
3890 	}
3891 
3892 	if (dst && !dst->xfrm)
3893 		dst = xfrm_out_fwd_icmp(skb, &fl, family, dst);
3894 
3895 	skb_dst_set(skb, dst);
3896 	return res;
3897 }
3898 EXPORT_SYMBOL(__xfrm_route_forward);
3899 
3900 /* Optimize later using cookies and generation ids. */
3901 
xfrm_dst_check(struct dst_entry * dst,u32 cookie)3902 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
3903 {
3904 	/* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
3905 	 * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
3906 	 * get validated by dst_ops->check on every use.  We do this
3907 	 * because when a normal route referenced by an XFRM dst is
3908 	 * obsoleted we do not go looking around for all parent
3909 	 * referencing XFRM dsts so that we can invalidate them.  It
3910 	 * is just too much work.  Instead we make the checks here on
3911 	 * every use.  For example:
3912 	 *
3913 	 *	XFRM dst A --> IPv4 dst X
3914 	 *
3915 	 * X is the "xdst->route" of A (X is also the "dst->path" of A
3916 	 * in this example).  If X is marked obsolete, "A" will not
3917 	 * notice.  That's what we are validating here via the
3918 	 * stale_bundle() check.
3919 	 *
3920 	 * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
3921 	 * be marked on it.
3922 	 * This will force stale_bundle() to fail on any xdst bundle with
3923 	 * this dst linked in it.
3924 	 */
3925 	if (dst->obsolete < 0 && !stale_bundle(dst))
3926 		return dst;
3927 
3928 	return NULL;
3929 }
3930 
stale_bundle(struct dst_entry * dst)3931 static int stale_bundle(struct dst_entry *dst)
3932 {
3933 	return !xfrm_bundle_ok((struct xfrm_dst *)dst);
3934 }
3935 
xfrm_dst_ifdown(struct dst_entry * dst,struct net_device * dev)3936 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
3937 {
3938 	while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {
3939 		dst->dev = blackhole_netdev;
3940 		dev_hold(dst->dev);
3941 		dev_put(dev);
3942 	}
3943 }
3944 EXPORT_SYMBOL(xfrm_dst_ifdown);
3945 
xfrm_link_failure(struct sk_buff * skb)3946 static void xfrm_link_failure(struct sk_buff *skb)
3947 {
3948 	/* Impossible. Such dst must be popped before reaches point of failure. */
3949 }
3950 
xfrm_negative_advice(struct sock * sk,struct dst_entry * dst)3951 static void xfrm_negative_advice(struct sock *sk, struct dst_entry *dst)
3952 {
3953 	if (dst->obsolete)
3954 		sk_dst_reset(sk);
3955 }
3956 
xfrm_init_pmtu(struct xfrm_dst ** bundle,int nr)3957 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr)
3958 {
3959 	while (nr--) {
3960 		struct xfrm_dst *xdst = bundle[nr];
3961 		u32 pmtu, route_mtu_cached;
3962 		struct dst_entry *dst;
3963 
3964 		dst = &xdst->u.dst;
3965 		pmtu = dst_mtu(xfrm_dst_child(dst));
3966 		xdst->child_mtu_cached = pmtu;
3967 
3968 		pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
3969 
3970 		route_mtu_cached = dst_mtu(xdst->route);
3971 		xdst->route_mtu_cached = route_mtu_cached;
3972 
3973 		if (pmtu > route_mtu_cached)
3974 			pmtu = route_mtu_cached;
3975 
3976 		dst_metric_set(dst, RTAX_MTU, pmtu);
3977 	}
3978 }
3979 
3980 /* Check that the bundle accepts the flow and its components are
3981  * still valid.
3982  */
3983 
xfrm_bundle_ok(struct xfrm_dst * first)3984 static int xfrm_bundle_ok(struct xfrm_dst *first)
3985 {
3986 	struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
3987 	struct dst_entry *dst = &first->u.dst;
3988 	struct xfrm_dst *xdst;
3989 	int start_from, nr;
3990 	u32 mtu;
3991 
3992 	if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) ||
3993 	    (dst->dev && !netif_running(dst->dev)))
3994 		return 0;
3995 
3996 	if (dst->flags & DST_XFRM_QUEUE)
3997 		return 1;
3998 
3999 	start_from = nr = 0;
4000 	do {
4001 		struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
4002 
4003 		if (dst->xfrm->km.state != XFRM_STATE_VALID)
4004 			return 0;
4005 		if (xdst->xfrm_genid != dst->xfrm->genid)
4006 			return 0;
4007 		if (xdst->num_pols > 0 &&
4008 		    xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
4009 			return 0;
4010 
4011 		bundle[nr++] = xdst;
4012 
4013 		mtu = dst_mtu(xfrm_dst_child(dst));
4014 		if (xdst->child_mtu_cached != mtu) {
4015 			start_from = nr;
4016 			xdst->child_mtu_cached = mtu;
4017 		}
4018 
4019 		if (!dst_check(xdst->route, xdst->route_cookie))
4020 			return 0;
4021 		mtu = dst_mtu(xdst->route);
4022 		if (xdst->route_mtu_cached != mtu) {
4023 			start_from = nr;
4024 			xdst->route_mtu_cached = mtu;
4025 		}
4026 
4027 		dst = xfrm_dst_child(dst);
4028 	} while (dst->xfrm);
4029 
4030 	if (likely(!start_from))
4031 		return 1;
4032 
4033 	xdst = bundle[start_from - 1];
4034 	mtu = xdst->child_mtu_cached;
4035 	while (start_from--) {
4036 		dst = &xdst->u.dst;
4037 
4038 		mtu = xfrm_state_mtu(dst->xfrm, mtu);
4039 		if (mtu > xdst->route_mtu_cached)
4040 			mtu = xdst->route_mtu_cached;
4041 		dst_metric_set(dst, RTAX_MTU, mtu);
4042 		if (!start_from)
4043 			break;
4044 
4045 		xdst = bundle[start_from - 1];
4046 		xdst->child_mtu_cached = mtu;
4047 	}
4048 
4049 	return 1;
4050 }
4051 
xfrm_default_advmss(const struct dst_entry * dst)4052 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
4053 {
4054 	return dst_metric_advmss(xfrm_dst_path(dst));
4055 }
4056 
xfrm_mtu(const struct dst_entry * dst)4057 static unsigned int xfrm_mtu(const struct dst_entry *dst)
4058 {
4059 	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
4060 
4061 	return mtu ? : dst_mtu(xfrm_dst_path(dst));
4062 }
4063 
xfrm_get_dst_nexthop(const struct dst_entry * dst,const void * daddr)4064 static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst,
4065 					const void *daddr)
4066 {
4067 	while (dst->xfrm) {
4068 		const struct xfrm_state *xfrm = dst->xfrm;
4069 
4070 		dst = xfrm_dst_child(dst);
4071 
4072 		if (xfrm->props.mode == XFRM_MODE_TRANSPORT)
4073 			continue;
4074 		if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)
4075 			daddr = xfrm->coaddr;
4076 		else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))
4077 			daddr = &xfrm->id.daddr;
4078 	}
4079 	return daddr;
4080 }
4081 
xfrm_neigh_lookup(const struct dst_entry * dst,struct sk_buff * skb,const void * daddr)4082 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
4083 					   struct sk_buff *skb,
4084 					   const void *daddr)
4085 {
4086 	const struct dst_entry *path = xfrm_dst_path(dst);
4087 
4088 	if (!skb)
4089 		daddr = xfrm_get_dst_nexthop(dst, daddr);
4090 	return path->ops->neigh_lookup(path, skb, daddr);
4091 }
4092 
xfrm_confirm_neigh(const struct dst_entry * dst,const void * daddr)4093 static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr)
4094 {
4095 	const struct dst_entry *path = xfrm_dst_path(dst);
4096 
4097 	daddr = xfrm_get_dst_nexthop(dst, daddr);
4098 	path->ops->confirm_neigh(path, daddr);
4099 }
4100 
xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo * afinfo,int family)4101 int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
4102 {
4103 	int err = 0;
4104 
4105 	if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
4106 		return -EAFNOSUPPORT;
4107 
4108 	spin_lock(&xfrm_policy_afinfo_lock);
4109 	if (unlikely(xfrm_policy_afinfo[family] != NULL))
4110 		err = -EEXIST;
4111 	else {
4112 		struct dst_ops *dst_ops = afinfo->dst_ops;
4113 		if (likely(dst_ops->kmem_cachep == NULL))
4114 			dst_ops->kmem_cachep = xfrm_dst_cache;
4115 		if (likely(dst_ops->check == NULL))
4116 			dst_ops->check = xfrm_dst_check;
4117 		if (likely(dst_ops->default_advmss == NULL))
4118 			dst_ops->default_advmss = xfrm_default_advmss;
4119 		if (likely(dst_ops->mtu == NULL))
4120 			dst_ops->mtu = xfrm_mtu;
4121 		if (likely(dst_ops->negative_advice == NULL))
4122 			dst_ops->negative_advice = xfrm_negative_advice;
4123 		if (likely(dst_ops->link_failure == NULL))
4124 			dst_ops->link_failure = xfrm_link_failure;
4125 		if (likely(dst_ops->neigh_lookup == NULL))
4126 			dst_ops->neigh_lookup = xfrm_neigh_lookup;
4127 		if (likely(!dst_ops->confirm_neigh))
4128 			dst_ops->confirm_neigh = xfrm_confirm_neigh;
4129 		rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
4130 	}
4131 	spin_unlock(&xfrm_policy_afinfo_lock);
4132 
4133 	return err;
4134 }
4135 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
4136 
xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo * afinfo)4137 void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
4138 {
4139 	struct dst_ops *dst_ops = afinfo->dst_ops;
4140 	int i;
4141 
4142 	for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
4143 		if (xfrm_policy_afinfo[i] != afinfo)
4144 			continue;
4145 		RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
4146 		break;
4147 	}
4148 
4149 	synchronize_rcu();
4150 
4151 	dst_ops->kmem_cachep = NULL;
4152 	dst_ops->check = NULL;
4153 	dst_ops->negative_advice = NULL;
4154 	dst_ops->link_failure = NULL;
4155 }
4156 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
4157 
xfrm_if_register_cb(const struct xfrm_if_cb * ifcb)4158 void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
4159 {
4160 	spin_lock(&xfrm_if_cb_lock);
4161 	rcu_assign_pointer(xfrm_if_cb, ifcb);
4162 	spin_unlock(&xfrm_if_cb_lock);
4163 }
4164 EXPORT_SYMBOL(xfrm_if_register_cb);
4165 
xfrm_if_unregister_cb(void)4166 void xfrm_if_unregister_cb(void)
4167 {
4168 	RCU_INIT_POINTER(xfrm_if_cb, NULL);
4169 	synchronize_rcu();
4170 }
4171 EXPORT_SYMBOL(xfrm_if_unregister_cb);
4172 
4173 #ifdef CONFIG_XFRM_STATISTICS
xfrm_statistics_init(struct net * net)4174 static int __net_init xfrm_statistics_init(struct net *net)
4175 {
4176 	int rv;
4177 	net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
4178 	if (!net->mib.xfrm_statistics)
4179 		return -ENOMEM;
4180 	rv = xfrm_proc_init(net);
4181 	if (rv < 0)
4182 		free_percpu(net->mib.xfrm_statistics);
4183 	return rv;
4184 }
4185 
xfrm_statistics_fini(struct net * net)4186 static void xfrm_statistics_fini(struct net *net)
4187 {
4188 	xfrm_proc_fini(net);
4189 	free_percpu(net->mib.xfrm_statistics);
4190 }
4191 #else
xfrm_statistics_init(struct net * net)4192 static int __net_init xfrm_statistics_init(struct net *net)
4193 {
4194 	return 0;
4195 }
4196 
xfrm_statistics_fini(struct net * net)4197 static void xfrm_statistics_fini(struct net *net)
4198 {
4199 }
4200 #endif
4201 
xfrm_policy_init(struct net * net)4202 static int __net_init xfrm_policy_init(struct net *net)
4203 {
4204 	unsigned int hmask, sz;
4205 	int dir, err;
4206 
4207 	if (net_eq(net, &init_net)) {
4208 		xfrm_dst_cache = KMEM_CACHE(xfrm_dst, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
4209 		err = rhashtable_init(&xfrm_policy_inexact_table,
4210 				      &xfrm_pol_inexact_params);
4211 		BUG_ON(err);
4212 	}
4213 
4214 	hmask = 8 - 1;
4215 	sz = (hmask+1) * sizeof(struct hlist_head);
4216 
4217 	net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
4218 	if (!net->xfrm.policy_byidx)
4219 		goto out_byidx;
4220 	net->xfrm.policy_idx_hmask = hmask;
4221 
4222 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4223 		struct xfrm_policy_hash *htab;
4224 
4225 		net->xfrm.policy_count[dir] = 0;
4226 		net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
4227 
4228 		htab = &net->xfrm.policy_bydst[dir];
4229 		htab->table = xfrm_hash_alloc(sz);
4230 		if (!htab->table)
4231 			goto out_bydst;
4232 		htab->hmask = hmask;
4233 		htab->dbits4 = 32;
4234 		htab->sbits4 = 32;
4235 		htab->dbits6 = 128;
4236 		htab->sbits6 = 128;
4237 	}
4238 	net->xfrm.policy_hthresh.lbits4 = 32;
4239 	net->xfrm.policy_hthresh.rbits4 = 32;
4240 	net->xfrm.policy_hthresh.lbits6 = 128;
4241 	net->xfrm.policy_hthresh.rbits6 = 128;
4242 
4243 	seqlock_init(&net->xfrm.policy_hthresh.lock);
4244 
4245 	INIT_LIST_HEAD(&net->xfrm.policy_all);
4246 	INIT_LIST_HEAD(&net->xfrm.inexact_bins);
4247 	INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
4248 	INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
4249 	return 0;
4250 
4251 out_bydst:
4252 	for (dir--; dir >= 0; dir--) {
4253 		struct xfrm_policy_hash *htab;
4254 
4255 		htab = &net->xfrm.policy_bydst[dir];
4256 		xfrm_hash_free(htab->table, sz);
4257 	}
4258 	xfrm_hash_free(net->xfrm.policy_byidx, sz);
4259 out_byidx:
4260 	return -ENOMEM;
4261 }
4262 
xfrm_policy_fini(struct net * net)4263 static void xfrm_policy_fini(struct net *net)
4264 {
4265 	struct xfrm_pol_inexact_bin *b, *t;
4266 	unsigned int sz;
4267 	int dir;
4268 
4269 	flush_work(&net->xfrm.policy_hash_work);
4270 #ifdef CONFIG_XFRM_SUB_POLICY
4271 	xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
4272 #endif
4273 	xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
4274 
4275 	WARN_ON(!list_empty(&net->xfrm.policy_all));
4276 
4277 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4278 		struct xfrm_policy_hash *htab;
4279 
4280 		htab = &net->xfrm.policy_bydst[dir];
4281 		sz = (htab->hmask + 1) * sizeof(struct hlist_head);
4282 		WARN_ON(!hlist_empty(htab->table));
4283 		xfrm_hash_free(htab->table, sz);
4284 	}
4285 
4286 	sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
4287 	WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
4288 	xfrm_hash_free(net->xfrm.policy_byidx, sz);
4289 
4290 	spin_lock_bh(&net->xfrm.xfrm_policy_lock);
4291 	list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins)
4292 		__xfrm_policy_inexact_prune_bin(b, true);
4293 	spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
4294 }
4295 
xfrm_net_init(struct net * net)4296 static int __net_init xfrm_net_init(struct net *net)
4297 {
4298 	int rv;
4299 
4300 	/* Initialize the per-net locks here */
4301 	spin_lock_init(&net->xfrm.xfrm_state_lock);
4302 	spin_lock_init(&net->xfrm.xfrm_policy_lock);
4303 	seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock);
4304 	mutex_init(&net->xfrm.xfrm_cfg_mutex);
4305 	net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT;
4306 	net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT;
4307 	net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT;
4308 
4309 	rv = xfrm_statistics_init(net);
4310 	if (rv < 0)
4311 		goto out_statistics;
4312 	rv = xfrm_state_init(net);
4313 	if (rv < 0)
4314 		goto out_state;
4315 	rv = xfrm_policy_init(net);
4316 	if (rv < 0)
4317 		goto out_policy;
4318 	rv = xfrm_sysctl_init(net);
4319 	if (rv < 0)
4320 		goto out_sysctl;
4321 
4322 	rv = xfrm_nat_keepalive_net_init(net);
4323 	if (rv < 0)
4324 		goto out_nat_keepalive;
4325 
4326 	return 0;
4327 
4328 out_nat_keepalive:
4329 	xfrm_sysctl_fini(net);
4330 out_sysctl:
4331 	xfrm_policy_fini(net);
4332 out_policy:
4333 	xfrm_state_fini(net);
4334 out_state:
4335 	xfrm_statistics_fini(net);
4336 out_statistics:
4337 	return rv;
4338 }
4339 
xfrm_net_exit(struct net * net)4340 static void __net_exit xfrm_net_exit(struct net *net)
4341 {
4342 	xfrm_nat_keepalive_net_fini(net);
4343 	xfrm_sysctl_fini(net);
4344 	xfrm_policy_fini(net);
4345 	xfrm_state_fini(net);
4346 	xfrm_statistics_fini(net);
4347 }
4348 
4349 static struct pernet_operations __net_initdata xfrm_net_ops = {
4350 	.init = xfrm_net_init,
4351 	.exit = xfrm_net_exit,
4352 };
4353 
4354 static const struct flow_dissector_key xfrm_flow_dissector_keys[] = {
4355 	{
4356 		.key_id = FLOW_DISSECTOR_KEY_CONTROL,
4357 		.offset = offsetof(struct xfrm_flow_keys, control),
4358 	},
4359 	{
4360 		.key_id = FLOW_DISSECTOR_KEY_BASIC,
4361 		.offset = offsetof(struct xfrm_flow_keys, basic),
4362 	},
4363 	{
4364 		.key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
4365 		.offset = offsetof(struct xfrm_flow_keys, addrs.ipv4),
4366 	},
4367 	{
4368 		.key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
4369 		.offset = offsetof(struct xfrm_flow_keys, addrs.ipv6),
4370 	},
4371 	{
4372 		.key_id = FLOW_DISSECTOR_KEY_PORTS,
4373 		.offset = offsetof(struct xfrm_flow_keys, ports),
4374 	},
4375 	{
4376 		.key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
4377 		.offset = offsetof(struct xfrm_flow_keys, gre),
4378 	},
4379 	{
4380 		.key_id = FLOW_DISSECTOR_KEY_IP,
4381 		.offset = offsetof(struct xfrm_flow_keys, ip),
4382 	},
4383 	{
4384 		.key_id = FLOW_DISSECTOR_KEY_ICMP,
4385 		.offset = offsetof(struct xfrm_flow_keys, icmp),
4386 	},
4387 };
4388 
xfrm_init(void)4389 void __init xfrm_init(void)
4390 {
4391 	skb_flow_dissector_init(&xfrm_session_dissector,
4392 				xfrm_flow_dissector_keys,
4393 				ARRAY_SIZE(xfrm_flow_dissector_keys));
4394 
4395 	register_pernet_subsys(&xfrm_net_ops);
4396 	xfrm_dev_init();
4397 	xfrm_input_init();
4398 
4399 #ifdef CONFIG_XFRM_ESPINTCP
4400 	espintcp_init();
4401 #endif
4402 
4403 	register_xfrm_state_bpf();
4404 	xfrm_nat_keepalive_init(AF_INET);
4405 }
4406 
4407 #ifdef CONFIG_AUDITSYSCALL
xfrm_audit_common_policyinfo(struct xfrm_policy * xp,struct audit_buffer * audit_buf)4408 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
4409 					 struct audit_buffer *audit_buf)
4410 {
4411 	struct xfrm_sec_ctx *ctx = xp->security;
4412 	struct xfrm_selector *sel = &xp->selector;
4413 
4414 	if (ctx)
4415 		audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
4416 				 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
4417 
4418 	switch (sel->family) {
4419 	case AF_INET:
4420 		audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
4421 		if (sel->prefixlen_s != 32)
4422 			audit_log_format(audit_buf, " src_prefixlen=%d",
4423 					 sel->prefixlen_s);
4424 		audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
4425 		if (sel->prefixlen_d != 32)
4426 			audit_log_format(audit_buf, " dst_prefixlen=%d",
4427 					 sel->prefixlen_d);
4428 		break;
4429 	case AF_INET6:
4430 		audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
4431 		if (sel->prefixlen_s != 128)
4432 			audit_log_format(audit_buf, " src_prefixlen=%d",
4433 					 sel->prefixlen_s);
4434 		audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
4435 		if (sel->prefixlen_d != 128)
4436 			audit_log_format(audit_buf, " dst_prefixlen=%d",
4437 					 sel->prefixlen_d);
4438 		break;
4439 	}
4440 }
4441 
xfrm_audit_policy_add(struct xfrm_policy * xp,int result,bool task_valid)4442 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
4443 {
4444 	struct audit_buffer *audit_buf;
4445 
4446 	audit_buf = xfrm_audit_start("SPD-add");
4447 	if (audit_buf == NULL)
4448 		return;
4449 	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4450 	audit_log_format(audit_buf, " res=%u", result);
4451 	xfrm_audit_common_policyinfo(xp, audit_buf);
4452 	audit_log_end(audit_buf);
4453 }
4454 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
4455 
xfrm_audit_policy_delete(struct xfrm_policy * xp,int result,bool task_valid)4456 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
4457 			      bool task_valid)
4458 {
4459 	struct audit_buffer *audit_buf;
4460 
4461 	audit_buf = xfrm_audit_start("SPD-delete");
4462 	if (audit_buf == NULL)
4463 		return;
4464 	xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4465 	audit_log_format(audit_buf, " res=%u", result);
4466 	xfrm_audit_common_policyinfo(xp, audit_buf);
4467 	audit_log_end(audit_buf);
4468 }
4469 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
4470 #endif
4471 
4472 #ifdef CONFIG_XFRM_MIGRATE
xfrm_migrate_policy_find(const struct xfrm_selector * sel,u8 dir,u8 type,struct net * net,u32 if_id)4473 static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
4474 						    u8 dir, u8 type, struct net *net, u32 if_id)
4475 {
4476 	struct xfrm_policy *pol;
4477 	struct flowi fl;
4478 
4479 	memset(&fl, 0, sizeof(fl));
4480 
4481 	fl.flowi_proto = sel->proto;
4482 
4483 	switch (sel->family) {
4484 	case AF_INET:
4485 		fl.u.ip4.saddr = sel->saddr.a4;
4486 		fl.u.ip4.daddr = sel->daddr.a4;
4487 		if (sel->proto == IPSEC_ULPROTO_ANY)
4488 			break;
4489 		fl.u.flowi4_oif = sel->ifindex;
4490 		fl.u.ip4.fl4_sport = sel->sport;
4491 		fl.u.ip4.fl4_dport = sel->dport;
4492 		break;
4493 	case AF_INET6:
4494 		fl.u.ip6.saddr = sel->saddr.in6;
4495 		fl.u.ip6.daddr = sel->daddr.in6;
4496 		if (sel->proto == IPSEC_ULPROTO_ANY)
4497 			break;
4498 		fl.u.flowi6_oif = sel->ifindex;
4499 		fl.u.ip6.fl4_sport = sel->sport;
4500 		fl.u.ip6.fl4_dport = sel->dport;
4501 		break;
4502 	default:
4503 		return ERR_PTR(-EAFNOSUPPORT);
4504 	}
4505 
4506 	rcu_read_lock();
4507 
4508 	pol = xfrm_policy_lookup_bytype(net, type, &fl, sel->family, dir, if_id);
4509 	if (IS_ERR_OR_NULL(pol))
4510 		goto out_unlock;
4511 
4512 	if (!xfrm_pol_hold_rcu(pol))
4513 		pol = NULL;
4514 out_unlock:
4515 	rcu_read_unlock();
4516 	return pol;
4517 }
4518 
migrate_tmpl_match(const struct xfrm_migrate * m,const struct xfrm_tmpl * t)4519 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
4520 {
4521 	int match = 0;
4522 
4523 	if (t->mode == m->mode && t->id.proto == m->proto &&
4524 	    (m->reqid == 0 || t->reqid == m->reqid)) {
4525 		switch (t->mode) {
4526 		case XFRM_MODE_TUNNEL:
4527 		case XFRM_MODE_BEET:
4528 		case XFRM_MODE_IPTFS:
4529 			if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
4530 					    m->old_family) &&
4531 			    xfrm_addr_equal(&t->saddr, &m->old_saddr,
4532 					    m->old_family)) {
4533 				match = 1;
4534 			}
4535 			break;
4536 		case XFRM_MODE_TRANSPORT:
4537 			/* in case of transport mode, template does not store
4538 			   any IP addresses, hence we just compare mode and
4539 			   protocol */
4540 			match = 1;
4541 			break;
4542 		default:
4543 			break;
4544 		}
4545 	}
4546 	return match;
4547 }
4548 
4549 /* update endpoint address(es) of template(s) */
xfrm_policy_migrate(struct xfrm_policy * pol,struct xfrm_migrate * m,int num_migrate,struct netlink_ext_ack * extack)4550 static int xfrm_policy_migrate(struct xfrm_policy *pol,
4551 			       struct xfrm_migrate *m, int num_migrate,
4552 			       struct netlink_ext_ack *extack)
4553 {
4554 	struct xfrm_migrate *mp;
4555 	int i, j, n = 0;
4556 
4557 	write_lock_bh(&pol->lock);
4558 	if (unlikely(pol->walk.dead)) {
4559 		/* target policy has been deleted */
4560 		NL_SET_ERR_MSG(extack, "Target policy not found");
4561 		write_unlock_bh(&pol->lock);
4562 		return -ENOENT;
4563 	}
4564 
4565 	for (i = 0; i < pol->xfrm_nr; i++) {
4566 		for (j = 0, mp = m; j < num_migrate; j++, mp++) {
4567 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
4568 				continue;
4569 			n++;
4570 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
4571 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET &&
4572 			    pol->xfrm_vec[i].mode != XFRM_MODE_IPTFS)
4573 				continue;
4574 			/* update endpoints */
4575 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
4576 			       sizeof(pol->xfrm_vec[i].id.daddr));
4577 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
4578 			       sizeof(pol->xfrm_vec[i].saddr));
4579 			pol->xfrm_vec[i].encap_family = mp->new_family;
4580 			/* flush bundles */
4581 			atomic_inc(&pol->genid);
4582 		}
4583 	}
4584 
4585 	write_unlock_bh(&pol->lock);
4586 
4587 	if (!n)
4588 		return -ENODATA;
4589 
4590 	return 0;
4591 }
4592 
xfrm_migrate_check(const struct xfrm_migrate * m,int num_migrate,struct netlink_ext_ack * extack)4593 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate,
4594 			      struct netlink_ext_ack *extack)
4595 {
4596 	int i, j;
4597 
4598 	if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH) {
4599 		NL_SET_ERR_MSG(extack, "Invalid number of SAs to migrate, must be 0 < num <= XFRM_MAX_DEPTH (6)");
4600 		return -EINVAL;
4601 	}
4602 
4603 	for (i = 0; i < num_migrate; i++) {
4604 		if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
4605 		    xfrm_addr_any(&m[i].new_saddr, m[i].new_family)) {
4606 			NL_SET_ERR_MSG(extack, "Addresses in the MIGRATE attribute's list cannot be null");
4607 			return -EINVAL;
4608 		}
4609 
4610 		/* check if there is any duplicated entry */
4611 		for (j = i + 1; j < num_migrate; j++) {
4612 			if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
4613 				    sizeof(m[i].old_daddr)) &&
4614 			    !memcmp(&m[i].old_saddr, &m[j].old_saddr,
4615 				    sizeof(m[i].old_saddr)) &&
4616 			    m[i].proto == m[j].proto &&
4617 			    m[i].mode == m[j].mode &&
4618 			    m[i].reqid == m[j].reqid &&
4619 			    m[i].old_family == m[j].old_family) {
4620 				NL_SET_ERR_MSG(extack, "Entries in the MIGRATE attribute's list must be unique");
4621 				return -EINVAL;
4622 			}
4623 		}
4624 	}
4625 
4626 	return 0;
4627 }
4628 
xfrm_migrate(const struct xfrm_selector * sel,u8 dir,u8 type,struct xfrm_migrate * m,int num_migrate,struct xfrm_kmaddress * k,struct net * net,struct xfrm_encap_tmpl * encap,u32 if_id,struct netlink_ext_ack * extack)4629 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
4630 		 struct xfrm_migrate *m, int num_migrate,
4631 		 struct xfrm_kmaddress *k, struct net *net,
4632 		 struct xfrm_encap_tmpl *encap, u32 if_id,
4633 		 struct netlink_ext_ack *extack)
4634 {
4635 	int i, err, nx_cur = 0, nx_new = 0;
4636 	struct xfrm_policy *pol = NULL;
4637 	struct xfrm_state *x, *xc;
4638 	struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
4639 	struct xfrm_state *x_new[XFRM_MAX_DEPTH];
4640 	struct xfrm_migrate *mp;
4641 
4642 	/* Stage 0 - sanity checks */
4643 	err = xfrm_migrate_check(m, num_migrate, extack);
4644 	if (err < 0)
4645 		goto out;
4646 
4647 	if (dir >= XFRM_POLICY_MAX) {
4648 		NL_SET_ERR_MSG(extack, "Invalid policy direction");
4649 		err = -EINVAL;
4650 		goto out;
4651 	}
4652 
4653 	/* Stage 1 - find policy */
4654 	pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id);
4655 	if (IS_ERR_OR_NULL(pol)) {
4656 		NL_SET_ERR_MSG(extack, "Target policy not found");
4657 		err = IS_ERR(pol) ? PTR_ERR(pol) : -ENOENT;
4658 		goto out;
4659 	}
4660 
4661 	/* Stage 2 - find and update state(s) */
4662 	for (i = 0, mp = m; i < num_migrate; i++, mp++) {
4663 		if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
4664 			x_cur[nx_cur] = x;
4665 			nx_cur++;
4666 			xc = xfrm_state_migrate(x, mp, encap);
4667 			if (xc) {
4668 				x_new[nx_new] = xc;
4669 				nx_new++;
4670 			} else {
4671 				err = -ENODATA;
4672 				goto restore_state;
4673 			}
4674 		}
4675 	}
4676 
4677 	/* Stage 3 - update policy */
4678 	err = xfrm_policy_migrate(pol, m, num_migrate, extack);
4679 	if (err < 0)
4680 		goto restore_state;
4681 
4682 	/* Stage 4 - delete old state(s) */
4683 	if (nx_cur) {
4684 		xfrm_states_put(x_cur, nx_cur);
4685 		xfrm_states_delete(x_cur, nx_cur);
4686 	}
4687 
4688 	/* Stage 5 - announce */
4689 	km_migrate(sel, dir, type, m, num_migrate, k, encap);
4690 
4691 	xfrm_pol_put(pol);
4692 
4693 	return 0;
4694 out:
4695 	return err;
4696 
4697 restore_state:
4698 	if (pol)
4699 		xfrm_pol_put(pol);
4700 	if (nx_cur)
4701 		xfrm_states_put(x_cur, nx_cur);
4702 	if (nx_new)
4703 		xfrm_states_delete(x_new, nx_new);
4704 
4705 	return err;
4706 }
4707 EXPORT_SYMBOL(xfrm_migrate);
4708 #endif
4709