xref: /linux/net/xfrm/xfrm_policy.c (revision 4413e16d9d21673bb5048a2e542f1aaa00015c2e)
1 /*
2  * xfrm_policy.c
3  *
4  * Changes:
5  *	Mitsuru KANDA @USAGI
6  * 	Kazunori MIYAZAWA @USAGI
7  * 	Kunihiro Ishiguro <kunihiro@ipinfusion.com>
8  * 		IPv6 support
9  * 	Kazunori MIYAZAWA @USAGI
10  * 	YOSHIFUJI Hideaki
11  * 		Split up af-specific portion
12  *	Derek Atkins <derek@ihtfp.com>		Add the post_input processor
13  *
14  */
15 
16 #include <linux/err.h>
17 #include <linux/slab.h>
18 #include <linux/kmod.h>
19 #include <linux/list.h>
20 #include <linux/spinlock.h>
21 #include <linux/workqueue.h>
22 #include <linux/notifier.h>
23 #include <linux/netdevice.h>
24 #include <linux/netfilter.h>
25 #include <linux/module.h>
26 #include <linux/cache.h>
27 #include <linux/audit.h>
28 #include <net/dst.h>
29 #include <net/flow.h>
30 #include <net/xfrm.h>
31 #include <net/ip.h>
32 #ifdef CONFIG_XFRM_STATISTICS
33 #include <net/snmp.h>
34 #endif
35 
36 #include "xfrm_hash.h"
37 
38 DEFINE_MUTEX(xfrm_cfg_mutex);
39 EXPORT_SYMBOL(xfrm_cfg_mutex);
40 
41 static DEFINE_SPINLOCK(xfrm_policy_sk_bundle_lock);
42 static struct dst_entry *xfrm_policy_sk_bundles;
43 static DEFINE_RWLOCK(xfrm_policy_lock);
44 
45 static DEFINE_RWLOCK(xfrm_policy_afinfo_lock);
46 static struct xfrm_policy_afinfo *xfrm_policy_afinfo[NPROTO];
47 
48 static struct kmem_cache *xfrm_dst_cache __read_mostly;
49 
50 static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family);
51 static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo);
52 static void xfrm_init_pmtu(struct dst_entry *dst);
53 static int stale_bundle(struct dst_entry *dst);
54 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
55 
56 
57 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
58 						int dir);
59 
60 static inline bool
61 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
62 {
63 	const struct flowi4 *fl4 = &fl->u.ip4;
64 
65 	return  addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
66 		addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
67 		!((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
68 		!((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
69 		(fl4->flowi4_proto == sel->proto || !sel->proto) &&
70 		(fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
71 }
72 
73 static inline bool
74 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
75 {
76 	const struct flowi6 *fl6 = &fl->u.ip6;
77 
78 	return  addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
79 		addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
80 		!((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
81 		!((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
82 		(fl6->flowi6_proto == sel->proto || !sel->proto) &&
83 		(fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
84 }
85 
86 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
87 			 unsigned short family)
88 {
89 	switch (family) {
90 	case AF_INET:
91 		return __xfrm4_selector_match(sel, fl);
92 	case AF_INET6:
93 		return __xfrm6_selector_match(sel, fl);
94 	}
95 	return false;
96 }
97 
98 static inline struct dst_entry *__xfrm_dst_lookup(struct net *net, int tos,
99 						  const xfrm_address_t *saddr,
100 						  const xfrm_address_t *daddr,
101 						  int family)
102 {
103 	struct xfrm_policy_afinfo *afinfo;
104 	struct dst_entry *dst;
105 
106 	afinfo = xfrm_policy_get_afinfo(family);
107 	if (unlikely(afinfo == NULL))
108 		return ERR_PTR(-EAFNOSUPPORT);
109 
110 	dst = afinfo->dst_lookup(net, tos, saddr, daddr);
111 
112 	xfrm_policy_put_afinfo(afinfo);
113 
114 	return dst;
115 }
116 
117 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, int tos,
118 						xfrm_address_t *prev_saddr,
119 						xfrm_address_t *prev_daddr,
120 						int family)
121 {
122 	struct net *net = xs_net(x);
123 	xfrm_address_t *saddr = &x->props.saddr;
124 	xfrm_address_t *daddr = &x->id.daddr;
125 	struct dst_entry *dst;
126 
127 	if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
128 		saddr = x->coaddr;
129 		daddr = prev_daddr;
130 	}
131 	if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
132 		saddr = prev_saddr;
133 		daddr = x->coaddr;
134 	}
135 
136 	dst = __xfrm_dst_lookup(net, tos, saddr, daddr, family);
137 
138 	if (!IS_ERR(dst)) {
139 		if (prev_saddr != saddr)
140 			memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
141 		if (prev_daddr != daddr)
142 			memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
143 	}
144 
145 	return dst;
146 }
147 
148 static inline unsigned long make_jiffies(long secs)
149 {
150 	if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
151 		return MAX_SCHEDULE_TIMEOUT-1;
152 	else
153 		return secs*HZ;
154 }
155 
156 static void xfrm_policy_timer(unsigned long data)
157 {
158 	struct xfrm_policy *xp = (struct xfrm_policy*)data;
159 	unsigned long now = get_seconds();
160 	long next = LONG_MAX;
161 	int warn = 0;
162 	int dir;
163 
164 	read_lock(&xp->lock);
165 
166 	if (unlikely(xp->walk.dead))
167 		goto out;
168 
169 	dir = xfrm_policy_id2dir(xp->index);
170 
171 	if (xp->lft.hard_add_expires_seconds) {
172 		long tmo = xp->lft.hard_add_expires_seconds +
173 			xp->curlft.add_time - now;
174 		if (tmo <= 0)
175 			goto expired;
176 		if (tmo < next)
177 			next = tmo;
178 	}
179 	if (xp->lft.hard_use_expires_seconds) {
180 		long tmo = xp->lft.hard_use_expires_seconds +
181 			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
182 		if (tmo <= 0)
183 			goto expired;
184 		if (tmo < next)
185 			next = tmo;
186 	}
187 	if (xp->lft.soft_add_expires_seconds) {
188 		long tmo = xp->lft.soft_add_expires_seconds +
189 			xp->curlft.add_time - now;
190 		if (tmo <= 0) {
191 			warn = 1;
192 			tmo = XFRM_KM_TIMEOUT;
193 		}
194 		if (tmo < next)
195 			next = tmo;
196 	}
197 	if (xp->lft.soft_use_expires_seconds) {
198 		long tmo = xp->lft.soft_use_expires_seconds +
199 			(xp->curlft.use_time ? : xp->curlft.add_time) - now;
200 		if (tmo <= 0) {
201 			warn = 1;
202 			tmo = XFRM_KM_TIMEOUT;
203 		}
204 		if (tmo < next)
205 			next = tmo;
206 	}
207 
208 	if (warn)
209 		km_policy_expired(xp, dir, 0, 0);
210 	if (next != LONG_MAX &&
211 	    !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
212 		xfrm_pol_hold(xp);
213 
214 out:
215 	read_unlock(&xp->lock);
216 	xfrm_pol_put(xp);
217 	return;
218 
219 expired:
220 	read_unlock(&xp->lock);
221 	if (!xfrm_policy_delete(xp, dir))
222 		km_policy_expired(xp, dir, 1, 0);
223 	xfrm_pol_put(xp);
224 }
225 
226 static struct flow_cache_object *xfrm_policy_flo_get(struct flow_cache_object *flo)
227 {
228 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
229 
230 	if (unlikely(pol->walk.dead))
231 		flo = NULL;
232 	else
233 		xfrm_pol_hold(pol);
234 
235 	return flo;
236 }
237 
238 static int xfrm_policy_flo_check(struct flow_cache_object *flo)
239 {
240 	struct xfrm_policy *pol = container_of(flo, struct xfrm_policy, flo);
241 
242 	return !pol->walk.dead;
243 }
244 
245 static void xfrm_policy_flo_delete(struct flow_cache_object *flo)
246 {
247 	xfrm_pol_put(container_of(flo, struct xfrm_policy, flo));
248 }
249 
250 static const struct flow_cache_ops xfrm_policy_fc_ops = {
251 	.get = xfrm_policy_flo_get,
252 	.check = xfrm_policy_flo_check,
253 	.delete = xfrm_policy_flo_delete,
254 };
255 
256 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
257  * SPD calls.
258  */
259 
260 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
261 {
262 	struct xfrm_policy *policy;
263 
264 	policy = kzalloc(sizeof(struct xfrm_policy), gfp);
265 
266 	if (policy) {
267 		write_pnet(&policy->xp_net, net);
268 		INIT_LIST_HEAD(&policy->walk.all);
269 		INIT_HLIST_NODE(&policy->bydst);
270 		INIT_HLIST_NODE(&policy->byidx);
271 		rwlock_init(&policy->lock);
272 		atomic_set(&policy->refcnt, 1);
273 		setup_timer(&policy->timer, xfrm_policy_timer,
274 				(unsigned long)policy);
275 		policy->flo.ops = &xfrm_policy_fc_ops;
276 	}
277 	return policy;
278 }
279 EXPORT_SYMBOL(xfrm_policy_alloc);
280 
281 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
282 
283 void xfrm_policy_destroy(struct xfrm_policy *policy)
284 {
285 	BUG_ON(!policy->walk.dead);
286 
287 	if (del_timer(&policy->timer))
288 		BUG();
289 
290 	security_xfrm_policy_free(policy->security);
291 	kfree(policy);
292 }
293 EXPORT_SYMBOL(xfrm_policy_destroy);
294 
295 /* Rule must be locked. Release descentant resources, announce
296  * entry dead. The rule must be unlinked from lists to the moment.
297  */
298 
299 static void xfrm_policy_kill(struct xfrm_policy *policy)
300 {
301 	policy->walk.dead = 1;
302 
303 	atomic_inc(&policy->genid);
304 
305 	if (del_timer(&policy->timer))
306 		xfrm_pol_put(policy);
307 
308 	xfrm_pol_put(policy);
309 }
310 
311 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
312 
313 static inline unsigned int idx_hash(struct net *net, u32 index)
314 {
315 	return __idx_hash(index, net->xfrm.policy_idx_hmask);
316 }
317 
318 static struct hlist_head *policy_hash_bysel(struct net *net,
319 					    const struct xfrm_selector *sel,
320 					    unsigned short family, int dir)
321 {
322 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
323 	unsigned int hash = __sel_hash(sel, family, hmask);
324 
325 	return (hash == hmask + 1 ?
326 		&net->xfrm.policy_inexact[dir] :
327 		net->xfrm.policy_bydst[dir].table + hash);
328 }
329 
330 static struct hlist_head *policy_hash_direct(struct net *net,
331 					     const xfrm_address_t *daddr,
332 					     const xfrm_address_t *saddr,
333 					     unsigned short family, int dir)
334 {
335 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
336 	unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
337 
338 	return net->xfrm.policy_bydst[dir].table + hash;
339 }
340 
341 static void xfrm_dst_hash_transfer(struct hlist_head *list,
342 				   struct hlist_head *ndsttable,
343 				   unsigned int nhashmask)
344 {
345 	struct hlist_node *entry, *tmp, *entry0 = NULL;
346 	struct xfrm_policy *pol;
347 	unsigned int h0 = 0;
348 
349 redo:
350 	hlist_for_each_entry_safe(pol, entry, tmp, list, bydst) {
351 		unsigned int h;
352 
353 		h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
354 				pol->family, nhashmask);
355 		if (!entry0) {
356 			hlist_del(entry);
357 			hlist_add_head(&pol->bydst, ndsttable+h);
358 			h0 = h;
359 		} else {
360 			if (h != h0)
361 				continue;
362 			hlist_del(entry);
363 			hlist_add_after(entry0, &pol->bydst);
364 		}
365 		entry0 = entry;
366 	}
367 	if (!hlist_empty(list)) {
368 		entry0 = NULL;
369 		goto redo;
370 	}
371 }
372 
373 static void xfrm_idx_hash_transfer(struct hlist_head *list,
374 				   struct hlist_head *nidxtable,
375 				   unsigned int nhashmask)
376 {
377 	struct hlist_node *entry, *tmp;
378 	struct xfrm_policy *pol;
379 
380 	hlist_for_each_entry_safe(pol, entry, tmp, list, byidx) {
381 		unsigned int h;
382 
383 		h = __idx_hash(pol->index, nhashmask);
384 		hlist_add_head(&pol->byidx, nidxtable+h);
385 	}
386 }
387 
388 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
389 {
390 	return ((old_hmask + 1) << 1) - 1;
391 }
392 
393 static void xfrm_bydst_resize(struct net *net, int dir)
394 {
395 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
396 	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
397 	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
398 	struct hlist_head *odst = net->xfrm.policy_bydst[dir].table;
399 	struct hlist_head *ndst = xfrm_hash_alloc(nsize);
400 	int i;
401 
402 	if (!ndst)
403 		return;
404 
405 	write_lock_bh(&xfrm_policy_lock);
406 
407 	for (i = hmask; i >= 0; i--)
408 		xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
409 
410 	net->xfrm.policy_bydst[dir].table = ndst;
411 	net->xfrm.policy_bydst[dir].hmask = nhashmask;
412 
413 	write_unlock_bh(&xfrm_policy_lock);
414 
415 	xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
416 }
417 
418 static void xfrm_byidx_resize(struct net *net, int total)
419 {
420 	unsigned int hmask = net->xfrm.policy_idx_hmask;
421 	unsigned int nhashmask = xfrm_new_hash_mask(hmask);
422 	unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
423 	struct hlist_head *oidx = net->xfrm.policy_byidx;
424 	struct hlist_head *nidx = xfrm_hash_alloc(nsize);
425 	int i;
426 
427 	if (!nidx)
428 		return;
429 
430 	write_lock_bh(&xfrm_policy_lock);
431 
432 	for (i = hmask; i >= 0; i--)
433 		xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
434 
435 	net->xfrm.policy_byidx = nidx;
436 	net->xfrm.policy_idx_hmask = nhashmask;
437 
438 	write_unlock_bh(&xfrm_policy_lock);
439 
440 	xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
441 }
442 
443 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
444 {
445 	unsigned int cnt = net->xfrm.policy_count[dir];
446 	unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
447 
448 	if (total)
449 		*total += cnt;
450 
451 	if ((hmask + 1) < xfrm_policy_hashmax &&
452 	    cnt > hmask)
453 		return 1;
454 
455 	return 0;
456 }
457 
458 static inline int xfrm_byidx_should_resize(struct net *net, int total)
459 {
460 	unsigned int hmask = net->xfrm.policy_idx_hmask;
461 
462 	if ((hmask + 1) < xfrm_policy_hashmax &&
463 	    total > hmask)
464 		return 1;
465 
466 	return 0;
467 }
468 
469 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
470 {
471 	read_lock_bh(&xfrm_policy_lock);
472 	si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
473 	si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
474 	si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
475 	si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
476 	si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
477 	si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
478 	si->spdhcnt = net->xfrm.policy_idx_hmask;
479 	si->spdhmcnt = xfrm_policy_hashmax;
480 	read_unlock_bh(&xfrm_policy_lock);
481 }
482 EXPORT_SYMBOL(xfrm_spd_getinfo);
483 
484 static DEFINE_MUTEX(hash_resize_mutex);
485 static void xfrm_hash_resize(struct work_struct *work)
486 {
487 	struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
488 	int dir, total;
489 
490 	mutex_lock(&hash_resize_mutex);
491 
492 	total = 0;
493 	for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
494 		if (xfrm_bydst_should_resize(net, dir, &total))
495 			xfrm_bydst_resize(net, dir);
496 	}
497 	if (xfrm_byidx_should_resize(net, total))
498 		xfrm_byidx_resize(net, total);
499 
500 	mutex_unlock(&hash_resize_mutex);
501 }
502 
503 /* Generate new index... KAME seems to generate them ordered by cost
504  * of an absolute inpredictability of ordering of rules. This will not pass. */
505 static u32 xfrm_gen_index(struct net *net, int dir)
506 {
507 	static u32 idx_generator;
508 
509 	for (;;) {
510 		struct hlist_node *entry;
511 		struct hlist_head *list;
512 		struct xfrm_policy *p;
513 		u32 idx;
514 		int found;
515 
516 		idx = (idx_generator | dir);
517 		idx_generator += 8;
518 		if (idx == 0)
519 			idx = 8;
520 		list = net->xfrm.policy_byidx + idx_hash(net, idx);
521 		found = 0;
522 		hlist_for_each_entry(p, entry, list, byidx) {
523 			if (p->index == idx) {
524 				found = 1;
525 				break;
526 			}
527 		}
528 		if (!found)
529 			return idx;
530 	}
531 }
532 
533 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
534 {
535 	u32 *p1 = (u32 *) s1;
536 	u32 *p2 = (u32 *) s2;
537 	int len = sizeof(struct xfrm_selector) / sizeof(u32);
538 	int i;
539 
540 	for (i = 0; i < len; i++) {
541 		if (p1[i] != p2[i])
542 			return 1;
543 	}
544 
545 	return 0;
546 }
547 
548 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
549 {
550 	struct net *net = xp_net(policy);
551 	struct xfrm_policy *pol;
552 	struct xfrm_policy *delpol;
553 	struct hlist_head *chain;
554 	struct hlist_node *entry, *newpos;
555 	u32 mark = policy->mark.v & policy->mark.m;
556 
557 	write_lock_bh(&xfrm_policy_lock);
558 	chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
559 	delpol = NULL;
560 	newpos = NULL;
561 	hlist_for_each_entry(pol, entry, chain, bydst) {
562 		if (pol->type == policy->type &&
563 		    !selector_cmp(&pol->selector, &policy->selector) &&
564 		    (mark & pol->mark.m) == pol->mark.v &&
565 		    xfrm_sec_ctx_match(pol->security, policy->security) &&
566 		    !WARN_ON(delpol)) {
567 			if (excl) {
568 				write_unlock_bh(&xfrm_policy_lock);
569 				return -EEXIST;
570 			}
571 			delpol = pol;
572 			if (policy->priority > pol->priority)
573 				continue;
574 		} else if (policy->priority >= pol->priority) {
575 			newpos = &pol->bydst;
576 			continue;
577 		}
578 		if (delpol)
579 			break;
580 	}
581 	if (newpos)
582 		hlist_add_after(newpos, &policy->bydst);
583 	else
584 		hlist_add_head(&policy->bydst, chain);
585 	xfrm_pol_hold(policy);
586 	net->xfrm.policy_count[dir]++;
587 	atomic_inc(&flow_cache_genid);
588 	rt_genid_bump(net);
589 	if (delpol)
590 		__xfrm_policy_unlink(delpol, dir);
591 	policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir);
592 	hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
593 	policy->curlft.add_time = get_seconds();
594 	policy->curlft.use_time = 0;
595 	if (!mod_timer(&policy->timer, jiffies + HZ))
596 		xfrm_pol_hold(policy);
597 	list_add(&policy->walk.all, &net->xfrm.policy_all);
598 	write_unlock_bh(&xfrm_policy_lock);
599 
600 	if (delpol)
601 		xfrm_policy_kill(delpol);
602 	else if (xfrm_bydst_should_resize(net, dir, NULL))
603 		schedule_work(&net->xfrm.policy_hash_work);
604 
605 	return 0;
606 }
607 EXPORT_SYMBOL(xfrm_policy_insert);
608 
609 struct xfrm_policy *xfrm_policy_bysel_ctx(struct net *net, u32 mark, u8 type,
610 					  int dir, struct xfrm_selector *sel,
611 					  struct xfrm_sec_ctx *ctx, int delete,
612 					  int *err)
613 {
614 	struct xfrm_policy *pol, *ret;
615 	struct hlist_head *chain;
616 	struct hlist_node *entry;
617 
618 	*err = 0;
619 	write_lock_bh(&xfrm_policy_lock);
620 	chain = policy_hash_bysel(net, sel, sel->family, dir);
621 	ret = NULL;
622 	hlist_for_each_entry(pol, entry, chain, bydst) {
623 		if (pol->type == type &&
624 		    (mark & pol->mark.m) == pol->mark.v &&
625 		    !selector_cmp(sel, &pol->selector) &&
626 		    xfrm_sec_ctx_match(ctx, pol->security)) {
627 			xfrm_pol_hold(pol);
628 			if (delete) {
629 				*err = security_xfrm_policy_delete(
630 								pol->security);
631 				if (*err) {
632 					write_unlock_bh(&xfrm_policy_lock);
633 					return pol;
634 				}
635 				__xfrm_policy_unlink(pol, dir);
636 			}
637 			ret = pol;
638 			break;
639 		}
640 	}
641 	write_unlock_bh(&xfrm_policy_lock);
642 
643 	if (ret && delete)
644 		xfrm_policy_kill(ret);
645 	return ret;
646 }
647 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
648 
649 struct xfrm_policy *xfrm_policy_byid(struct net *net, u32 mark, u8 type,
650 				     int dir, u32 id, int delete, int *err)
651 {
652 	struct xfrm_policy *pol, *ret;
653 	struct hlist_head *chain;
654 	struct hlist_node *entry;
655 
656 	*err = -ENOENT;
657 	if (xfrm_policy_id2dir(id) != dir)
658 		return NULL;
659 
660 	*err = 0;
661 	write_lock_bh(&xfrm_policy_lock);
662 	chain = net->xfrm.policy_byidx + idx_hash(net, id);
663 	ret = NULL;
664 	hlist_for_each_entry(pol, entry, chain, byidx) {
665 		if (pol->type == type && pol->index == id &&
666 		    (mark & pol->mark.m) == pol->mark.v) {
667 			xfrm_pol_hold(pol);
668 			if (delete) {
669 				*err = security_xfrm_policy_delete(
670 								pol->security);
671 				if (*err) {
672 					write_unlock_bh(&xfrm_policy_lock);
673 					return pol;
674 				}
675 				__xfrm_policy_unlink(pol, dir);
676 			}
677 			ret = pol;
678 			break;
679 		}
680 	}
681 	write_unlock_bh(&xfrm_policy_lock);
682 
683 	if (ret && delete)
684 		xfrm_policy_kill(ret);
685 	return ret;
686 }
687 EXPORT_SYMBOL(xfrm_policy_byid);
688 
689 #ifdef CONFIG_SECURITY_NETWORK_XFRM
690 static inline int
691 xfrm_policy_flush_secctx_check(struct net *net, u8 type, struct xfrm_audit *audit_info)
692 {
693 	int dir, err = 0;
694 
695 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
696 		struct xfrm_policy *pol;
697 		struct hlist_node *entry;
698 		int i;
699 
700 		hlist_for_each_entry(pol, entry,
701 				     &net->xfrm.policy_inexact[dir], bydst) {
702 			if (pol->type != type)
703 				continue;
704 			err = security_xfrm_policy_delete(pol->security);
705 			if (err) {
706 				xfrm_audit_policy_delete(pol, 0,
707 							 audit_info->loginuid,
708 							 audit_info->sessionid,
709 							 audit_info->secid);
710 				return err;
711 			}
712 		}
713 		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
714 			hlist_for_each_entry(pol, entry,
715 					     net->xfrm.policy_bydst[dir].table + i,
716 					     bydst) {
717 				if (pol->type != type)
718 					continue;
719 				err = security_xfrm_policy_delete(
720 								pol->security);
721 				if (err) {
722 					xfrm_audit_policy_delete(pol, 0,
723 							audit_info->loginuid,
724 							audit_info->sessionid,
725 							audit_info->secid);
726 					return err;
727 				}
728 			}
729 		}
730 	}
731 	return err;
732 }
733 #else
734 static inline int
735 xfrm_policy_flush_secctx_check(struct net *net, u8 type, struct xfrm_audit *audit_info)
736 {
737 	return 0;
738 }
739 #endif
740 
741 int xfrm_policy_flush(struct net *net, u8 type, struct xfrm_audit *audit_info)
742 {
743 	int dir, err = 0, cnt = 0;
744 
745 	write_lock_bh(&xfrm_policy_lock);
746 
747 	err = xfrm_policy_flush_secctx_check(net, type, audit_info);
748 	if (err)
749 		goto out;
750 
751 	for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
752 		struct xfrm_policy *pol;
753 		struct hlist_node *entry;
754 		int i;
755 
756 	again1:
757 		hlist_for_each_entry(pol, entry,
758 				     &net->xfrm.policy_inexact[dir], bydst) {
759 			if (pol->type != type)
760 				continue;
761 			__xfrm_policy_unlink(pol, dir);
762 			write_unlock_bh(&xfrm_policy_lock);
763 			cnt++;
764 
765 			xfrm_audit_policy_delete(pol, 1, audit_info->loginuid,
766 						 audit_info->sessionid,
767 						 audit_info->secid);
768 
769 			xfrm_policy_kill(pol);
770 
771 			write_lock_bh(&xfrm_policy_lock);
772 			goto again1;
773 		}
774 
775 		for (i = net->xfrm.policy_bydst[dir].hmask; i >= 0; i--) {
776 	again2:
777 			hlist_for_each_entry(pol, entry,
778 					     net->xfrm.policy_bydst[dir].table + i,
779 					     bydst) {
780 				if (pol->type != type)
781 					continue;
782 				__xfrm_policy_unlink(pol, dir);
783 				write_unlock_bh(&xfrm_policy_lock);
784 				cnt++;
785 
786 				xfrm_audit_policy_delete(pol, 1,
787 							 audit_info->loginuid,
788 							 audit_info->sessionid,
789 							 audit_info->secid);
790 				xfrm_policy_kill(pol);
791 
792 				write_lock_bh(&xfrm_policy_lock);
793 				goto again2;
794 			}
795 		}
796 
797 	}
798 	if (!cnt)
799 		err = -ESRCH;
800 out:
801 	write_unlock_bh(&xfrm_policy_lock);
802 	return err;
803 }
804 EXPORT_SYMBOL(xfrm_policy_flush);
805 
806 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
807 		     int (*func)(struct xfrm_policy *, int, int, void*),
808 		     void *data)
809 {
810 	struct xfrm_policy *pol;
811 	struct xfrm_policy_walk_entry *x;
812 	int error = 0;
813 
814 	if (walk->type >= XFRM_POLICY_TYPE_MAX &&
815 	    walk->type != XFRM_POLICY_TYPE_ANY)
816 		return -EINVAL;
817 
818 	if (list_empty(&walk->walk.all) && walk->seq != 0)
819 		return 0;
820 
821 	write_lock_bh(&xfrm_policy_lock);
822 	if (list_empty(&walk->walk.all))
823 		x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
824 	else
825 		x = list_entry(&walk->walk.all, struct xfrm_policy_walk_entry, all);
826 	list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
827 		if (x->dead)
828 			continue;
829 		pol = container_of(x, struct xfrm_policy, walk);
830 		if (walk->type != XFRM_POLICY_TYPE_ANY &&
831 		    walk->type != pol->type)
832 			continue;
833 		error = func(pol, xfrm_policy_id2dir(pol->index),
834 			     walk->seq, data);
835 		if (error) {
836 			list_move_tail(&walk->walk.all, &x->all);
837 			goto out;
838 		}
839 		walk->seq++;
840 	}
841 	if (walk->seq == 0) {
842 		error = -ENOENT;
843 		goto out;
844 	}
845 	list_del_init(&walk->walk.all);
846 out:
847 	write_unlock_bh(&xfrm_policy_lock);
848 	return error;
849 }
850 EXPORT_SYMBOL(xfrm_policy_walk);
851 
852 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
853 {
854 	INIT_LIST_HEAD(&walk->walk.all);
855 	walk->walk.dead = 1;
856 	walk->type = type;
857 	walk->seq = 0;
858 }
859 EXPORT_SYMBOL(xfrm_policy_walk_init);
860 
861 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk)
862 {
863 	if (list_empty(&walk->walk.all))
864 		return;
865 
866 	write_lock_bh(&xfrm_policy_lock);
867 	list_del(&walk->walk.all);
868 	write_unlock_bh(&xfrm_policy_lock);
869 }
870 EXPORT_SYMBOL(xfrm_policy_walk_done);
871 
872 /*
873  * Find policy to apply to this flow.
874  *
875  * Returns 0 if policy found, else an -errno.
876  */
877 static int xfrm_policy_match(const struct xfrm_policy *pol,
878 			     const struct flowi *fl,
879 			     u8 type, u16 family, int dir)
880 {
881 	const struct xfrm_selector *sel = &pol->selector;
882 	int ret = -ESRCH;
883 	bool match;
884 
885 	if (pol->family != family ||
886 	    (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
887 	    pol->type != type)
888 		return ret;
889 
890 	match = xfrm_selector_match(sel, fl, family);
891 	if (match)
892 		ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid,
893 						  dir);
894 
895 	return ret;
896 }
897 
898 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
899 						     const struct flowi *fl,
900 						     u16 family, u8 dir)
901 {
902 	int err;
903 	struct xfrm_policy *pol, *ret;
904 	const xfrm_address_t *daddr, *saddr;
905 	struct hlist_node *entry;
906 	struct hlist_head *chain;
907 	u32 priority = ~0U;
908 
909 	daddr = xfrm_flowi_daddr(fl, family);
910 	saddr = xfrm_flowi_saddr(fl, family);
911 	if (unlikely(!daddr || !saddr))
912 		return NULL;
913 
914 	read_lock_bh(&xfrm_policy_lock);
915 	chain = policy_hash_direct(net, daddr, saddr, family, dir);
916 	ret = NULL;
917 	hlist_for_each_entry(pol, entry, chain, bydst) {
918 		err = xfrm_policy_match(pol, fl, type, family, dir);
919 		if (err) {
920 			if (err == -ESRCH)
921 				continue;
922 			else {
923 				ret = ERR_PTR(err);
924 				goto fail;
925 			}
926 		} else {
927 			ret = pol;
928 			priority = ret->priority;
929 			break;
930 		}
931 	}
932 	chain = &net->xfrm.policy_inexact[dir];
933 	hlist_for_each_entry(pol, entry, chain, bydst) {
934 		err = xfrm_policy_match(pol, fl, type, family, dir);
935 		if (err) {
936 			if (err == -ESRCH)
937 				continue;
938 			else {
939 				ret = ERR_PTR(err);
940 				goto fail;
941 			}
942 		} else if (pol->priority < priority) {
943 			ret = pol;
944 			break;
945 		}
946 	}
947 	if (ret)
948 		xfrm_pol_hold(ret);
949 fail:
950 	read_unlock_bh(&xfrm_policy_lock);
951 
952 	return ret;
953 }
954 
955 static struct xfrm_policy *
956 __xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir)
957 {
958 #ifdef CONFIG_XFRM_SUB_POLICY
959 	struct xfrm_policy *pol;
960 
961 	pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family, dir);
962 	if (pol != NULL)
963 		return pol;
964 #endif
965 	return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family, dir);
966 }
967 
968 static struct flow_cache_object *
969 xfrm_policy_lookup(struct net *net, const struct flowi *fl, u16 family,
970 		   u8 dir, struct flow_cache_object *old_obj, void *ctx)
971 {
972 	struct xfrm_policy *pol;
973 
974 	if (old_obj)
975 		xfrm_pol_put(container_of(old_obj, struct xfrm_policy, flo));
976 
977 	pol = __xfrm_policy_lookup(net, fl, family, dir);
978 	if (IS_ERR_OR_NULL(pol))
979 		return ERR_CAST(pol);
980 
981 	/* Resolver returns two references:
982 	 * one for cache and one for caller of flow_cache_lookup() */
983 	xfrm_pol_hold(pol);
984 
985 	return &pol->flo;
986 }
987 
988 static inline int policy_to_flow_dir(int dir)
989 {
990 	if (XFRM_POLICY_IN == FLOW_DIR_IN &&
991 	    XFRM_POLICY_OUT == FLOW_DIR_OUT &&
992 	    XFRM_POLICY_FWD == FLOW_DIR_FWD)
993 		return dir;
994 	switch (dir) {
995 	default:
996 	case XFRM_POLICY_IN:
997 		return FLOW_DIR_IN;
998 	case XFRM_POLICY_OUT:
999 		return FLOW_DIR_OUT;
1000 	case XFRM_POLICY_FWD:
1001 		return FLOW_DIR_FWD;
1002 	}
1003 }
1004 
1005 static struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir,
1006 						 const struct flowi *fl)
1007 {
1008 	struct xfrm_policy *pol;
1009 
1010 	read_lock_bh(&xfrm_policy_lock);
1011 	if ((pol = sk->sk_policy[dir]) != NULL) {
1012 		bool match = xfrm_selector_match(&pol->selector, fl,
1013 						 sk->sk_family);
1014 		int err = 0;
1015 
1016 		if (match) {
1017 			if ((sk->sk_mark & pol->mark.m) != pol->mark.v) {
1018 				pol = NULL;
1019 				goto out;
1020 			}
1021 			err = security_xfrm_policy_lookup(pol->security,
1022 						      fl->flowi_secid,
1023 						      policy_to_flow_dir(dir));
1024 			if (!err)
1025 				xfrm_pol_hold(pol);
1026 			else if (err == -ESRCH)
1027 				pol = NULL;
1028 			else
1029 				pol = ERR_PTR(err);
1030 		} else
1031 			pol = NULL;
1032 	}
1033 out:
1034 	read_unlock_bh(&xfrm_policy_lock);
1035 	return pol;
1036 }
1037 
1038 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
1039 {
1040 	struct net *net = xp_net(pol);
1041 	struct hlist_head *chain = policy_hash_bysel(net, &pol->selector,
1042 						     pol->family, dir);
1043 
1044 	list_add(&pol->walk.all, &net->xfrm.policy_all);
1045 	hlist_add_head(&pol->bydst, chain);
1046 	hlist_add_head(&pol->byidx, net->xfrm.policy_byidx+idx_hash(net, pol->index));
1047 	net->xfrm.policy_count[dir]++;
1048 	xfrm_pol_hold(pol);
1049 
1050 	if (xfrm_bydst_should_resize(net, dir, NULL))
1051 		schedule_work(&net->xfrm.policy_hash_work);
1052 }
1053 
1054 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
1055 						int dir)
1056 {
1057 	struct net *net = xp_net(pol);
1058 
1059 	if (hlist_unhashed(&pol->bydst))
1060 		return NULL;
1061 
1062 	hlist_del(&pol->bydst);
1063 	hlist_del(&pol->byidx);
1064 	list_del(&pol->walk.all);
1065 	net->xfrm.policy_count[dir]--;
1066 
1067 	return pol;
1068 }
1069 
1070 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
1071 {
1072 	write_lock_bh(&xfrm_policy_lock);
1073 	pol = __xfrm_policy_unlink(pol, dir);
1074 	write_unlock_bh(&xfrm_policy_lock);
1075 	if (pol) {
1076 		xfrm_policy_kill(pol);
1077 		return 0;
1078 	}
1079 	return -ENOENT;
1080 }
1081 EXPORT_SYMBOL(xfrm_policy_delete);
1082 
1083 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
1084 {
1085 	struct net *net = xp_net(pol);
1086 	struct xfrm_policy *old_pol;
1087 
1088 #ifdef CONFIG_XFRM_SUB_POLICY
1089 	if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
1090 		return -EINVAL;
1091 #endif
1092 
1093 	write_lock_bh(&xfrm_policy_lock);
1094 	old_pol = sk->sk_policy[dir];
1095 	sk->sk_policy[dir] = pol;
1096 	if (pol) {
1097 		pol->curlft.add_time = get_seconds();
1098 		pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir);
1099 		__xfrm_policy_link(pol, XFRM_POLICY_MAX+dir);
1100 	}
1101 	if (old_pol)
1102 		/* Unlinking succeeds always. This is the only function
1103 		 * allowed to delete or replace socket policy.
1104 		 */
1105 		__xfrm_policy_unlink(old_pol, XFRM_POLICY_MAX+dir);
1106 	write_unlock_bh(&xfrm_policy_lock);
1107 
1108 	if (old_pol) {
1109 		xfrm_policy_kill(old_pol);
1110 	}
1111 	return 0;
1112 }
1113 
1114 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
1115 {
1116 	struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
1117 
1118 	if (newp) {
1119 		newp->selector = old->selector;
1120 		if (security_xfrm_policy_clone(old->security,
1121 					       &newp->security)) {
1122 			kfree(newp);
1123 			return NULL;  /* ENOMEM */
1124 		}
1125 		newp->lft = old->lft;
1126 		newp->curlft = old->curlft;
1127 		newp->mark = old->mark;
1128 		newp->action = old->action;
1129 		newp->flags = old->flags;
1130 		newp->xfrm_nr = old->xfrm_nr;
1131 		newp->index = old->index;
1132 		newp->type = old->type;
1133 		memcpy(newp->xfrm_vec, old->xfrm_vec,
1134 		       newp->xfrm_nr*sizeof(struct xfrm_tmpl));
1135 		write_lock_bh(&xfrm_policy_lock);
1136 		__xfrm_policy_link(newp, XFRM_POLICY_MAX+dir);
1137 		write_unlock_bh(&xfrm_policy_lock);
1138 		xfrm_pol_put(newp);
1139 	}
1140 	return newp;
1141 }
1142 
1143 int __xfrm_sk_clone_policy(struct sock *sk)
1144 {
1145 	struct xfrm_policy *p0 = sk->sk_policy[0],
1146 			   *p1 = sk->sk_policy[1];
1147 
1148 	sk->sk_policy[0] = sk->sk_policy[1] = NULL;
1149 	if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL)
1150 		return -ENOMEM;
1151 	if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL)
1152 		return -ENOMEM;
1153 	return 0;
1154 }
1155 
1156 static int
1157 xfrm_get_saddr(struct net *net, xfrm_address_t *local, xfrm_address_t *remote,
1158 	       unsigned short family)
1159 {
1160 	int err;
1161 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1162 
1163 	if (unlikely(afinfo == NULL))
1164 		return -EINVAL;
1165 	err = afinfo->get_saddr(net, local, remote);
1166 	xfrm_policy_put_afinfo(afinfo);
1167 	return err;
1168 }
1169 
1170 /* Resolve list of templates for the flow, given policy. */
1171 
1172 static int
1173 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
1174 		      struct xfrm_state **xfrm, unsigned short family)
1175 {
1176 	struct net *net = xp_net(policy);
1177 	int nx;
1178 	int i, error;
1179 	xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
1180 	xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
1181 	xfrm_address_t tmp;
1182 
1183 	for (nx=0, i = 0; i < policy->xfrm_nr; i++) {
1184 		struct xfrm_state *x;
1185 		xfrm_address_t *remote = daddr;
1186 		xfrm_address_t *local  = saddr;
1187 		struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
1188 
1189 		if (tmpl->mode == XFRM_MODE_TUNNEL ||
1190 		    tmpl->mode == XFRM_MODE_BEET) {
1191 			remote = &tmpl->id.daddr;
1192 			local = &tmpl->saddr;
1193 			if (xfrm_addr_any(local, tmpl->encap_family)) {
1194 				error = xfrm_get_saddr(net, &tmp, remote, tmpl->encap_family);
1195 				if (error)
1196 					goto fail;
1197 				local = &tmp;
1198 			}
1199 		}
1200 
1201 		x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);
1202 
1203 		if (x && x->km.state == XFRM_STATE_VALID) {
1204 			xfrm[nx++] = x;
1205 			daddr = remote;
1206 			saddr = local;
1207 			continue;
1208 		}
1209 		if (x) {
1210 			error = (x->km.state == XFRM_STATE_ERROR ?
1211 				 -EINVAL : -EAGAIN);
1212 			xfrm_state_put(x);
1213 		}
1214 		else if (error == -ESRCH)
1215 			error = -EAGAIN;
1216 
1217 		if (!tmpl->optional)
1218 			goto fail;
1219 	}
1220 	return nx;
1221 
1222 fail:
1223 	for (nx--; nx>=0; nx--)
1224 		xfrm_state_put(xfrm[nx]);
1225 	return error;
1226 }
1227 
1228 static int
1229 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
1230 		  struct xfrm_state **xfrm, unsigned short family)
1231 {
1232 	struct xfrm_state *tp[XFRM_MAX_DEPTH];
1233 	struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
1234 	int cnx = 0;
1235 	int error;
1236 	int ret;
1237 	int i;
1238 
1239 	for (i = 0; i < npols; i++) {
1240 		if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
1241 			error = -ENOBUFS;
1242 			goto fail;
1243 		}
1244 
1245 		ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
1246 		if (ret < 0) {
1247 			error = ret;
1248 			goto fail;
1249 		} else
1250 			cnx += ret;
1251 	}
1252 
1253 	/* found states are sorted for outbound processing */
1254 	if (npols > 1)
1255 		xfrm_state_sort(xfrm, tpp, cnx, family);
1256 
1257 	return cnx;
1258 
1259  fail:
1260 	for (cnx--; cnx>=0; cnx--)
1261 		xfrm_state_put(tpp[cnx]);
1262 	return error;
1263 
1264 }
1265 
1266 /* Check that the bundle accepts the flow and its components are
1267  * still valid.
1268  */
1269 
1270 static inline int xfrm_get_tos(const struct flowi *fl, int family)
1271 {
1272 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1273 	int tos;
1274 
1275 	if (!afinfo)
1276 		return -EINVAL;
1277 
1278 	tos = afinfo->get_tos(fl);
1279 
1280 	xfrm_policy_put_afinfo(afinfo);
1281 
1282 	return tos;
1283 }
1284 
1285 static struct flow_cache_object *xfrm_bundle_flo_get(struct flow_cache_object *flo)
1286 {
1287 	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1288 	struct dst_entry *dst = &xdst->u.dst;
1289 
1290 	if (xdst->route == NULL) {
1291 		/* Dummy bundle - if it has xfrms we were not
1292 		 * able to build bundle as template resolution failed.
1293 		 * It means we need to try again resolving. */
1294 		if (xdst->num_xfrms > 0)
1295 			return NULL;
1296 	} else {
1297 		/* Real bundle */
1298 		if (stale_bundle(dst))
1299 			return NULL;
1300 	}
1301 
1302 	dst_hold(dst);
1303 	return flo;
1304 }
1305 
1306 static int xfrm_bundle_flo_check(struct flow_cache_object *flo)
1307 {
1308 	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1309 	struct dst_entry *dst = &xdst->u.dst;
1310 
1311 	if (!xdst->route)
1312 		return 0;
1313 	if (stale_bundle(dst))
1314 		return 0;
1315 
1316 	return 1;
1317 }
1318 
1319 static void xfrm_bundle_flo_delete(struct flow_cache_object *flo)
1320 {
1321 	struct xfrm_dst *xdst = container_of(flo, struct xfrm_dst, flo);
1322 	struct dst_entry *dst = &xdst->u.dst;
1323 
1324 	dst_free(dst);
1325 }
1326 
1327 static const struct flow_cache_ops xfrm_bundle_fc_ops = {
1328 	.get = xfrm_bundle_flo_get,
1329 	.check = xfrm_bundle_flo_check,
1330 	.delete = xfrm_bundle_flo_delete,
1331 };
1332 
1333 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
1334 {
1335 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1336 	struct dst_ops *dst_ops;
1337 	struct xfrm_dst *xdst;
1338 
1339 	if (!afinfo)
1340 		return ERR_PTR(-EINVAL);
1341 
1342 	switch (family) {
1343 	case AF_INET:
1344 		dst_ops = &net->xfrm.xfrm4_dst_ops;
1345 		break;
1346 #if IS_ENABLED(CONFIG_IPV6)
1347 	case AF_INET6:
1348 		dst_ops = &net->xfrm.xfrm6_dst_ops;
1349 		break;
1350 #endif
1351 	default:
1352 		BUG();
1353 	}
1354 	xdst = dst_alloc(dst_ops, NULL, 0, DST_OBSOLETE_NONE, 0);
1355 
1356 	if (likely(xdst)) {
1357 		struct dst_entry *dst = &xdst->u.dst;
1358 
1359 		memset(dst + 1, 0, sizeof(*xdst) - sizeof(*dst));
1360 		xdst->flo.ops = &xfrm_bundle_fc_ops;
1361 		if (afinfo->init_dst)
1362 			afinfo->init_dst(net, xdst);
1363 	} else
1364 		xdst = ERR_PTR(-ENOBUFS);
1365 
1366 	xfrm_policy_put_afinfo(afinfo);
1367 
1368 	return xdst;
1369 }
1370 
1371 static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
1372 				 int nfheader_len)
1373 {
1374 	struct xfrm_policy_afinfo *afinfo =
1375 		xfrm_policy_get_afinfo(dst->ops->family);
1376 	int err;
1377 
1378 	if (!afinfo)
1379 		return -EINVAL;
1380 
1381 	err = afinfo->init_path(path, dst, nfheader_len);
1382 
1383 	xfrm_policy_put_afinfo(afinfo);
1384 
1385 	return err;
1386 }
1387 
1388 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
1389 				const struct flowi *fl)
1390 {
1391 	struct xfrm_policy_afinfo *afinfo =
1392 		xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
1393 	int err;
1394 
1395 	if (!afinfo)
1396 		return -EINVAL;
1397 
1398 	err = afinfo->fill_dst(xdst, dev, fl);
1399 
1400 	xfrm_policy_put_afinfo(afinfo);
1401 
1402 	return err;
1403 }
1404 
1405 
1406 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
1407  * all the metrics... Shortly, bundle a bundle.
1408  */
1409 
1410 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
1411 					    struct xfrm_state **xfrm, int nx,
1412 					    const struct flowi *fl,
1413 					    struct dst_entry *dst)
1414 {
1415 	struct net *net = xp_net(policy);
1416 	unsigned long now = jiffies;
1417 	struct net_device *dev;
1418 	struct xfrm_mode *inner_mode;
1419 	struct dst_entry *dst_prev = NULL;
1420 	struct dst_entry *dst0 = NULL;
1421 	int i = 0;
1422 	int err;
1423 	int header_len = 0;
1424 	int nfheader_len = 0;
1425 	int trailer_len = 0;
1426 	int tos;
1427 	int family = policy->selector.family;
1428 	xfrm_address_t saddr, daddr;
1429 
1430 	xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
1431 
1432 	tos = xfrm_get_tos(fl, family);
1433 	err = tos;
1434 	if (tos < 0)
1435 		goto put_states;
1436 
1437 	dst_hold(dst);
1438 
1439 	for (; i < nx; i++) {
1440 		struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
1441 		struct dst_entry *dst1 = &xdst->u.dst;
1442 
1443 		err = PTR_ERR(xdst);
1444 		if (IS_ERR(xdst)) {
1445 			dst_release(dst);
1446 			goto put_states;
1447 		}
1448 
1449 		if (xfrm[i]->sel.family == AF_UNSPEC) {
1450 			inner_mode = xfrm_ip2inner_mode(xfrm[i],
1451 							xfrm_af2proto(family));
1452 			if (!inner_mode) {
1453 				err = -EAFNOSUPPORT;
1454 				dst_release(dst);
1455 				goto put_states;
1456 			}
1457 		} else
1458 			inner_mode = xfrm[i]->inner_mode;
1459 
1460 		if (!dst_prev)
1461 			dst0 = dst1;
1462 		else {
1463 			dst_prev->child = dst_clone(dst1);
1464 			dst1->flags |= DST_NOHASH;
1465 		}
1466 
1467 		xdst->route = dst;
1468 		dst_copy_metrics(dst1, dst);
1469 
1470 		if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
1471 			family = xfrm[i]->props.family;
1472 			dst = xfrm_dst_lookup(xfrm[i], tos, &saddr, &daddr,
1473 					      family);
1474 			err = PTR_ERR(dst);
1475 			if (IS_ERR(dst))
1476 				goto put_states;
1477 		} else
1478 			dst_hold(dst);
1479 
1480 		dst1->xfrm = xfrm[i];
1481 		xdst->xfrm_genid = xfrm[i]->genid;
1482 
1483 		dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
1484 		dst1->flags |= DST_HOST;
1485 		dst1->lastuse = now;
1486 
1487 		dst1->input = dst_discard;
1488 		dst1->output = inner_mode->afinfo->output;
1489 
1490 		dst1->next = dst_prev;
1491 		dst_prev = dst1;
1492 
1493 		header_len += xfrm[i]->props.header_len;
1494 		if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
1495 			nfheader_len += xfrm[i]->props.header_len;
1496 		trailer_len += xfrm[i]->props.trailer_len;
1497 	}
1498 
1499 	dst_prev->child = dst;
1500 	dst0->path = dst;
1501 
1502 	err = -ENODEV;
1503 	dev = dst->dev;
1504 	if (!dev)
1505 		goto free_dst;
1506 
1507 	xfrm_init_path((struct xfrm_dst *)dst0, dst, nfheader_len);
1508 	xfrm_init_pmtu(dst_prev);
1509 
1510 	for (dst_prev = dst0; dst_prev != dst; dst_prev = dst_prev->child) {
1511 		struct xfrm_dst *xdst = (struct xfrm_dst *)dst_prev;
1512 
1513 		err = xfrm_fill_dst(xdst, dev, fl);
1514 		if (err)
1515 			goto free_dst;
1516 
1517 		dst_prev->header_len = header_len;
1518 		dst_prev->trailer_len = trailer_len;
1519 		header_len -= xdst->u.dst.xfrm->props.header_len;
1520 		trailer_len -= xdst->u.dst.xfrm->props.trailer_len;
1521 	}
1522 
1523 out:
1524 	return dst0;
1525 
1526 put_states:
1527 	for (; i < nx; i++)
1528 		xfrm_state_put(xfrm[i]);
1529 free_dst:
1530 	if (dst0)
1531 		dst_free(dst0);
1532 	dst0 = ERR_PTR(err);
1533 	goto out;
1534 }
1535 
1536 static int inline
1537 xfrm_dst_alloc_copy(void **target, const void *src, int size)
1538 {
1539 	if (!*target) {
1540 		*target = kmalloc(size, GFP_ATOMIC);
1541 		if (!*target)
1542 			return -ENOMEM;
1543 	}
1544 	memcpy(*target, src, size);
1545 	return 0;
1546 }
1547 
1548 static int inline
1549 xfrm_dst_update_parent(struct dst_entry *dst, const struct xfrm_selector *sel)
1550 {
1551 #ifdef CONFIG_XFRM_SUB_POLICY
1552 	struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1553 	return xfrm_dst_alloc_copy((void **)&(xdst->partner),
1554 				   sel, sizeof(*sel));
1555 #else
1556 	return 0;
1557 #endif
1558 }
1559 
1560 static int inline
1561 xfrm_dst_update_origin(struct dst_entry *dst, const struct flowi *fl)
1562 {
1563 #ifdef CONFIG_XFRM_SUB_POLICY
1564 	struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1565 	return xfrm_dst_alloc_copy((void **)&(xdst->origin), fl, sizeof(*fl));
1566 #else
1567 	return 0;
1568 #endif
1569 }
1570 
1571 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
1572 				struct xfrm_policy **pols,
1573 				int *num_pols, int *num_xfrms)
1574 {
1575 	int i;
1576 
1577 	if (*num_pols == 0 || !pols[0]) {
1578 		*num_pols = 0;
1579 		*num_xfrms = 0;
1580 		return 0;
1581 	}
1582 	if (IS_ERR(pols[0]))
1583 		return PTR_ERR(pols[0]);
1584 
1585 	*num_xfrms = pols[0]->xfrm_nr;
1586 
1587 #ifdef CONFIG_XFRM_SUB_POLICY
1588 	if (pols[0] && pols[0]->action == XFRM_POLICY_ALLOW &&
1589 	    pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1590 		pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
1591 						    XFRM_POLICY_TYPE_MAIN,
1592 						    fl, family,
1593 						    XFRM_POLICY_OUT);
1594 		if (pols[1]) {
1595 			if (IS_ERR(pols[1])) {
1596 				xfrm_pols_put(pols, *num_pols);
1597 				return PTR_ERR(pols[1]);
1598 			}
1599 			(*num_pols) ++;
1600 			(*num_xfrms) += pols[1]->xfrm_nr;
1601 		}
1602 	}
1603 #endif
1604 	for (i = 0; i < *num_pols; i++) {
1605 		if (pols[i]->action != XFRM_POLICY_ALLOW) {
1606 			*num_xfrms = -1;
1607 			break;
1608 		}
1609 	}
1610 
1611 	return 0;
1612 
1613 }
1614 
1615 static struct xfrm_dst *
1616 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
1617 			       const struct flowi *fl, u16 family,
1618 			       struct dst_entry *dst_orig)
1619 {
1620 	struct net *net = xp_net(pols[0]);
1621 	struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
1622 	struct dst_entry *dst;
1623 	struct xfrm_dst *xdst;
1624 	int err;
1625 
1626 	/* Try to instantiate a bundle */
1627 	err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
1628 	if (err <= 0) {
1629 		if (err != 0 && err != -EAGAIN)
1630 			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
1631 		return ERR_PTR(err);
1632 	}
1633 
1634 	dst = xfrm_bundle_create(pols[0], xfrm, err, fl, dst_orig);
1635 	if (IS_ERR(dst)) {
1636 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
1637 		return ERR_CAST(dst);
1638 	}
1639 
1640 	xdst = (struct xfrm_dst *)dst;
1641 	xdst->num_xfrms = err;
1642 	if (num_pols > 1)
1643 		err = xfrm_dst_update_parent(dst, &pols[1]->selector);
1644 	else
1645 		err = xfrm_dst_update_origin(dst, fl);
1646 	if (unlikely(err)) {
1647 		dst_free(dst);
1648 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1649 		return ERR_PTR(err);
1650 	}
1651 
1652 	xdst->num_pols = num_pols;
1653 	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy*) * num_pols);
1654 	xdst->policy_genid = atomic_read(&pols[0]->genid);
1655 
1656 	return xdst;
1657 }
1658 
1659 static struct flow_cache_object *
1660 xfrm_bundle_lookup(struct net *net, const struct flowi *fl, u16 family, u8 dir,
1661 		   struct flow_cache_object *oldflo, void *ctx)
1662 {
1663 	struct dst_entry *dst_orig = (struct dst_entry *)ctx;
1664 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
1665 	struct xfrm_dst *xdst, *new_xdst;
1666 	int num_pols = 0, num_xfrms = 0, i, err, pol_dead;
1667 
1668 	/* Check if the policies from old bundle are usable */
1669 	xdst = NULL;
1670 	if (oldflo) {
1671 		xdst = container_of(oldflo, struct xfrm_dst, flo);
1672 		num_pols = xdst->num_pols;
1673 		num_xfrms = xdst->num_xfrms;
1674 		pol_dead = 0;
1675 		for (i = 0; i < num_pols; i++) {
1676 			pols[i] = xdst->pols[i];
1677 			pol_dead |= pols[i]->walk.dead;
1678 		}
1679 		if (pol_dead) {
1680 			dst_free(&xdst->u.dst);
1681 			xdst = NULL;
1682 			num_pols = 0;
1683 			num_xfrms = 0;
1684 			oldflo = NULL;
1685 		}
1686 	}
1687 
1688 	/* Resolve policies to use if we couldn't get them from
1689 	 * previous cache entry */
1690 	if (xdst == NULL) {
1691 		num_pols = 1;
1692 		pols[0] = __xfrm_policy_lookup(net, fl, family, dir);
1693 		err = xfrm_expand_policies(fl, family, pols,
1694 					   &num_pols, &num_xfrms);
1695 		if (err < 0)
1696 			goto inc_error;
1697 		if (num_pols == 0)
1698 			return NULL;
1699 		if (num_xfrms <= 0)
1700 			goto make_dummy_bundle;
1701 	}
1702 
1703 	new_xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family, dst_orig);
1704 	if (IS_ERR(new_xdst)) {
1705 		err = PTR_ERR(new_xdst);
1706 		if (err != -EAGAIN)
1707 			goto error;
1708 		if (oldflo == NULL)
1709 			goto make_dummy_bundle;
1710 		dst_hold(&xdst->u.dst);
1711 		return oldflo;
1712 	} else if (new_xdst == NULL) {
1713 		num_xfrms = 0;
1714 		if (oldflo == NULL)
1715 			goto make_dummy_bundle;
1716 		xdst->num_xfrms = 0;
1717 		dst_hold(&xdst->u.dst);
1718 		return oldflo;
1719 	}
1720 
1721 	/* Kill the previous bundle */
1722 	if (xdst) {
1723 		/* The policies were stolen for newly generated bundle */
1724 		xdst->num_pols = 0;
1725 		dst_free(&xdst->u.dst);
1726 	}
1727 
1728 	/* Flow cache does not have reference, it dst_free()'s,
1729 	 * but we do need to return one reference for original caller */
1730 	dst_hold(&new_xdst->u.dst);
1731 	return &new_xdst->flo;
1732 
1733 make_dummy_bundle:
1734 	/* We found policies, but there's no bundles to instantiate:
1735 	 * either because the policy blocks, has no transformations or
1736 	 * we could not build template (no xfrm_states).*/
1737 	xdst = xfrm_alloc_dst(net, family);
1738 	if (IS_ERR(xdst)) {
1739 		xfrm_pols_put(pols, num_pols);
1740 		return ERR_CAST(xdst);
1741 	}
1742 	xdst->num_pols = num_pols;
1743 	xdst->num_xfrms = num_xfrms;
1744 	memcpy(xdst->pols, pols, sizeof(struct xfrm_policy*) * num_pols);
1745 
1746 	dst_hold(&xdst->u.dst);
1747 	return &xdst->flo;
1748 
1749 inc_error:
1750 	XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
1751 error:
1752 	if (xdst != NULL)
1753 		dst_free(&xdst->u.dst);
1754 	else
1755 		xfrm_pols_put(pols, num_pols);
1756 	return ERR_PTR(err);
1757 }
1758 
1759 static struct dst_entry *make_blackhole(struct net *net, u16 family,
1760 					struct dst_entry *dst_orig)
1761 {
1762 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1763 	struct dst_entry *ret;
1764 
1765 	if (!afinfo) {
1766 		dst_release(dst_orig);
1767 		return ERR_PTR(-EINVAL);
1768 	} else {
1769 		ret = afinfo->blackhole_route(net, dst_orig);
1770 	}
1771 	xfrm_policy_put_afinfo(afinfo);
1772 
1773 	return ret;
1774 }
1775 
1776 /* Main function: finds/creates a bundle for given flow.
1777  *
1778  * At the moment we eat a raw IP route. Mostly to speed up lookups
1779  * on interfaces with disabled IPsec.
1780  */
1781 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
1782 			      const struct flowi *fl,
1783 			      struct sock *sk, int flags)
1784 {
1785 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
1786 	struct flow_cache_object *flo;
1787 	struct xfrm_dst *xdst;
1788 	struct dst_entry *dst, *route;
1789 	u16 family = dst_orig->ops->family;
1790 	u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
1791 	int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
1792 
1793 restart:
1794 	dst = NULL;
1795 	xdst = NULL;
1796 	route = NULL;
1797 
1798 	if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
1799 		num_pols = 1;
1800 		pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl);
1801 		err = xfrm_expand_policies(fl, family, pols,
1802 					   &num_pols, &num_xfrms);
1803 		if (err < 0)
1804 			goto dropdst;
1805 
1806 		if (num_pols) {
1807 			if (num_xfrms <= 0) {
1808 				drop_pols = num_pols;
1809 				goto no_transform;
1810 			}
1811 
1812 			xdst = xfrm_resolve_and_create_bundle(
1813 					pols, num_pols, fl,
1814 					family, dst_orig);
1815 			if (IS_ERR(xdst)) {
1816 				xfrm_pols_put(pols, num_pols);
1817 				err = PTR_ERR(xdst);
1818 				goto dropdst;
1819 			} else if (xdst == NULL) {
1820 				num_xfrms = 0;
1821 				drop_pols = num_pols;
1822 				goto no_transform;
1823 			}
1824 
1825 			dst_hold(&xdst->u.dst);
1826 
1827 			spin_lock_bh(&xfrm_policy_sk_bundle_lock);
1828 			xdst->u.dst.next = xfrm_policy_sk_bundles;
1829 			xfrm_policy_sk_bundles = &xdst->u.dst;
1830 			spin_unlock_bh(&xfrm_policy_sk_bundle_lock);
1831 
1832 			route = xdst->route;
1833 		}
1834 	}
1835 
1836 	if (xdst == NULL) {
1837 		/* To accelerate a bit...  */
1838 		if ((dst_orig->flags & DST_NOXFRM) ||
1839 		    !net->xfrm.policy_count[XFRM_POLICY_OUT])
1840 			goto nopol;
1841 
1842 		flo = flow_cache_lookup(net, fl, family, dir,
1843 					xfrm_bundle_lookup, dst_orig);
1844 		if (flo == NULL)
1845 			goto nopol;
1846 		if (IS_ERR(flo)) {
1847 			err = PTR_ERR(flo);
1848 			goto dropdst;
1849 		}
1850 		xdst = container_of(flo, struct xfrm_dst, flo);
1851 
1852 		num_pols = xdst->num_pols;
1853 		num_xfrms = xdst->num_xfrms;
1854 		memcpy(pols, xdst->pols, sizeof(struct xfrm_policy*) * num_pols);
1855 		route = xdst->route;
1856 	}
1857 
1858 	dst = &xdst->u.dst;
1859 	if (route == NULL && num_xfrms > 0) {
1860 		/* The only case when xfrm_bundle_lookup() returns a
1861 		 * bundle with null route, is when the template could
1862 		 * not be resolved. It means policies are there, but
1863 		 * bundle could not be created, since we don't yet
1864 		 * have the xfrm_state's. We need to wait for KM to
1865 		 * negotiate new SA's or bail out with error.*/
1866 		if (net->xfrm.sysctl_larval_drop) {
1867 			/* EREMOTE tells the caller to generate
1868 			 * a one-shot blackhole route. */
1869 			dst_release(dst);
1870 			xfrm_pols_put(pols, drop_pols);
1871 			XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
1872 
1873 			return make_blackhole(net, family, dst_orig);
1874 		}
1875 		if (fl->flowi_flags & FLOWI_FLAG_CAN_SLEEP) {
1876 			DECLARE_WAITQUEUE(wait, current);
1877 
1878 			add_wait_queue(&net->xfrm.km_waitq, &wait);
1879 			set_current_state(TASK_INTERRUPTIBLE);
1880 			schedule();
1881 			set_current_state(TASK_RUNNING);
1882 			remove_wait_queue(&net->xfrm.km_waitq, &wait);
1883 
1884 			if (!signal_pending(current)) {
1885 				dst_release(dst);
1886 				goto restart;
1887 			}
1888 
1889 			err = -ERESTART;
1890 		} else
1891 			err = -EAGAIN;
1892 
1893 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
1894 		goto error;
1895 	}
1896 
1897 no_transform:
1898 	if (num_pols == 0)
1899 		goto nopol;
1900 
1901 	if ((flags & XFRM_LOOKUP_ICMP) &&
1902 	    !(pols[0]->flags & XFRM_POLICY_ICMP)) {
1903 		err = -ENOENT;
1904 		goto error;
1905 	}
1906 
1907 	for (i = 0; i < num_pols; i++)
1908 		pols[i]->curlft.use_time = get_seconds();
1909 
1910 	if (num_xfrms < 0) {
1911 		/* Prohibit the flow */
1912 		XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
1913 		err = -EPERM;
1914 		goto error;
1915 	} else if (num_xfrms > 0) {
1916 		/* Flow transformed */
1917 		dst_release(dst_orig);
1918 	} else {
1919 		/* Flow passes untransformed */
1920 		dst_release(dst);
1921 		dst = dst_orig;
1922 	}
1923 ok:
1924 	xfrm_pols_put(pols, drop_pols);
1925 	if (dst && dst->xfrm &&
1926 	    dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
1927 		dst->flags |= DST_XFRM_TUNNEL;
1928 	return dst;
1929 
1930 nopol:
1931 	if (!(flags & XFRM_LOOKUP_ICMP)) {
1932 		dst = dst_orig;
1933 		goto ok;
1934 	}
1935 	err = -ENOENT;
1936 error:
1937 	dst_release(dst);
1938 dropdst:
1939 	dst_release(dst_orig);
1940 	xfrm_pols_put(pols, drop_pols);
1941 	return ERR_PTR(err);
1942 }
1943 EXPORT_SYMBOL(xfrm_lookup);
1944 
1945 static inline int
1946 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
1947 {
1948 	struct xfrm_state *x;
1949 
1950 	if (!skb->sp || idx < 0 || idx >= skb->sp->len)
1951 		return 0;
1952 	x = skb->sp->xvec[idx];
1953 	if (!x->type->reject)
1954 		return 0;
1955 	return x->type->reject(x, skb, fl);
1956 }
1957 
1958 /* When skb is transformed back to its "native" form, we have to
1959  * check policy restrictions. At the moment we make this in maximally
1960  * stupid way. Shame on me. :-) Of course, connected sockets must
1961  * have policy cached at them.
1962  */
1963 
1964 static inline int
1965 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
1966 	      unsigned short family)
1967 {
1968 	if (xfrm_state_kern(x))
1969 		return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
1970 	return	x->id.proto == tmpl->id.proto &&
1971 		(x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
1972 		(x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
1973 		x->props.mode == tmpl->mode &&
1974 		(tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
1975 		 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
1976 		!(x->props.mode != XFRM_MODE_TRANSPORT &&
1977 		  xfrm_state_addr_cmp(tmpl, x, family));
1978 }
1979 
1980 /*
1981  * 0 or more than 0 is returned when validation is succeeded (either bypass
1982  * because of optional transport mode, or next index of the mathced secpath
1983  * state with the template.
1984  * -1 is returned when no matching template is found.
1985  * Otherwise "-2 - errored_index" is returned.
1986  */
1987 static inline int
1988 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
1989 	       unsigned short family)
1990 {
1991 	int idx = start;
1992 
1993 	if (tmpl->optional) {
1994 		if (tmpl->mode == XFRM_MODE_TRANSPORT)
1995 			return start;
1996 	} else
1997 		start = -1;
1998 	for (; idx < sp->len; idx++) {
1999 		if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
2000 			return ++idx;
2001 		if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
2002 			if (start == -1)
2003 				start = -2-idx;
2004 			break;
2005 		}
2006 	}
2007 	return start;
2008 }
2009 
2010 int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
2011 			  unsigned int family, int reverse)
2012 {
2013 	struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2014 	int err;
2015 
2016 	if (unlikely(afinfo == NULL))
2017 		return -EAFNOSUPPORT;
2018 
2019 	afinfo->decode_session(skb, fl, reverse);
2020 	err = security_xfrm_decode_session(skb, &fl->flowi_secid);
2021 	xfrm_policy_put_afinfo(afinfo);
2022 	return err;
2023 }
2024 EXPORT_SYMBOL(__xfrm_decode_session);
2025 
2026 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
2027 {
2028 	for (; k < sp->len; k++) {
2029 		if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
2030 			*idxp = k;
2031 			return 1;
2032 		}
2033 	}
2034 
2035 	return 0;
2036 }
2037 
2038 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
2039 			unsigned short family)
2040 {
2041 	struct net *net = dev_net(skb->dev);
2042 	struct xfrm_policy *pol;
2043 	struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
2044 	int npols = 0;
2045 	int xfrm_nr;
2046 	int pi;
2047 	int reverse;
2048 	struct flowi fl;
2049 	u8 fl_dir;
2050 	int xerr_idx = -1;
2051 
2052 	reverse = dir & ~XFRM_POLICY_MASK;
2053 	dir &= XFRM_POLICY_MASK;
2054 	fl_dir = policy_to_flow_dir(dir);
2055 
2056 	if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
2057 		XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
2058 		return 0;
2059 	}
2060 
2061 	nf_nat_decode_session(skb, &fl, family);
2062 
2063 	/* First, check used SA against their selectors. */
2064 	if (skb->sp) {
2065 		int i;
2066 
2067 		for (i=skb->sp->len-1; i>=0; i--) {
2068 			struct xfrm_state *x = skb->sp->xvec[i];
2069 			if (!xfrm_selector_match(&x->sel, &fl, family)) {
2070 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
2071 				return 0;
2072 			}
2073 		}
2074 	}
2075 
2076 	pol = NULL;
2077 	if (sk && sk->sk_policy[dir]) {
2078 		pol = xfrm_sk_policy_lookup(sk, dir, &fl);
2079 		if (IS_ERR(pol)) {
2080 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2081 			return 0;
2082 		}
2083 	}
2084 
2085 	if (!pol) {
2086 		struct flow_cache_object *flo;
2087 
2088 		flo = flow_cache_lookup(net, &fl, family, fl_dir,
2089 					xfrm_policy_lookup, NULL);
2090 		if (IS_ERR_OR_NULL(flo))
2091 			pol = ERR_CAST(flo);
2092 		else
2093 			pol = container_of(flo, struct xfrm_policy, flo);
2094 	}
2095 
2096 	if (IS_ERR(pol)) {
2097 		XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2098 		return 0;
2099 	}
2100 
2101 	if (!pol) {
2102 		if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
2103 			xfrm_secpath_reject(xerr_idx, skb, &fl);
2104 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
2105 			return 0;
2106 		}
2107 		return 1;
2108 	}
2109 
2110 	pol->curlft.use_time = get_seconds();
2111 
2112 	pols[0] = pol;
2113 	npols ++;
2114 #ifdef CONFIG_XFRM_SUB_POLICY
2115 	if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2116 		pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
2117 						    &fl, family,
2118 						    XFRM_POLICY_IN);
2119 		if (pols[1]) {
2120 			if (IS_ERR(pols[1])) {
2121 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
2122 				return 0;
2123 			}
2124 			pols[1]->curlft.use_time = get_seconds();
2125 			npols ++;
2126 		}
2127 	}
2128 #endif
2129 
2130 	if (pol->action == XFRM_POLICY_ALLOW) {
2131 		struct sec_path *sp;
2132 		static struct sec_path dummy;
2133 		struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
2134 		struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
2135 		struct xfrm_tmpl **tpp = tp;
2136 		int ti = 0;
2137 		int i, k;
2138 
2139 		if ((sp = skb->sp) == NULL)
2140 			sp = &dummy;
2141 
2142 		for (pi = 0; pi < npols; pi++) {
2143 			if (pols[pi] != pol &&
2144 			    pols[pi]->action != XFRM_POLICY_ALLOW) {
2145 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2146 				goto reject;
2147 			}
2148 			if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
2149 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
2150 				goto reject_error;
2151 			}
2152 			for (i = 0; i < pols[pi]->xfrm_nr; i++)
2153 				tpp[ti++] = &pols[pi]->xfrm_vec[i];
2154 		}
2155 		xfrm_nr = ti;
2156 		if (npols > 1) {
2157 			xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
2158 			tpp = stp;
2159 		}
2160 
2161 		/* For each tunnel xfrm, find the first matching tmpl.
2162 		 * For each tmpl before that, find corresponding xfrm.
2163 		 * Order is _important_. Later we will implement
2164 		 * some barriers, but at the moment barriers
2165 		 * are implied between each two transformations.
2166 		 */
2167 		for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
2168 			k = xfrm_policy_ok(tpp[i], sp, k, family);
2169 			if (k < 0) {
2170 				if (k < -1)
2171 					/* "-2 - errored_index" returned */
2172 					xerr_idx = -(2+k);
2173 				XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2174 				goto reject;
2175 			}
2176 		}
2177 
2178 		if (secpath_has_nontransport(sp, k, &xerr_idx)) {
2179 			XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
2180 			goto reject;
2181 		}
2182 
2183 		xfrm_pols_put(pols, npols);
2184 		return 1;
2185 	}
2186 	XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
2187 
2188 reject:
2189 	xfrm_secpath_reject(xerr_idx, skb, &fl);
2190 reject_error:
2191 	xfrm_pols_put(pols, npols);
2192 	return 0;
2193 }
2194 EXPORT_SYMBOL(__xfrm_policy_check);
2195 
2196 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
2197 {
2198 	struct net *net = dev_net(skb->dev);
2199 	struct flowi fl;
2200 	struct dst_entry *dst;
2201 	int res = 1;
2202 
2203 	if (xfrm_decode_session(skb, &fl, family) < 0) {
2204 		XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
2205 		return 0;
2206 	}
2207 
2208 	skb_dst_force(skb);
2209 
2210 	dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, 0);
2211 	if (IS_ERR(dst)) {
2212 		res = 0;
2213 		dst = NULL;
2214 	}
2215 	skb_dst_set(skb, dst);
2216 	return res;
2217 }
2218 EXPORT_SYMBOL(__xfrm_route_forward);
2219 
2220 /* Optimize later using cookies and generation ids. */
2221 
2222 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
2223 {
2224 	/* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
2225 	 * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
2226 	 * get validated by dst_ops->check on every use.  We do this
2227 	 * because when a normal route referenced by an XFRM dst is
2228 	 * obsoleted we do not go looking around for all parent
2229 	 * referencing XFRM dsts so that we can invalidate them.  It
2230 	 * is just too much work.  Instead we make the checks here on
2231 	 * every use.  For example:
2232 	 *
2233 	 *	XFRM dst A --> IPv4 dst X
2234 	 *
2235 	 * X is the "xdst->route" of A (X is also the "dst->path" of A
2236 	 * in this example).  If X is marked obsolete, "A" will not
2237 	 * notice.  That's what we are validating here via the
2238 	 * stale_bundle() check.
2239 	 *
2240 	 * When a policy's bundle is pruned, we dst_free() the XFRM
2241 	 * dst which causes it's ->obsolete field to be set to
2242 	 * DST_OBSOLETE_DEAD.  If an XFRM dst has been pruned like
2243 	 * this, we want to force a new route lookup.
2244 	 */
2245 	if (dst->obsolete < 0 && !stale_bundle(dst))
2246 		return dst;
2247 
2248 	return NULL;
2249 }
2250 
2251 static int stale_bundle(struct dst_entry *dst)
2252 {
2253 	return !xfrm_bundle_ok((struct xfrm_dst *)dst);
2254 }
2255 
2256 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
2257 {
2258 	while ((dst = dst->child) && dst->xfrm && dst->dev == dev) {
2259 		dst->dev = dev_net(dev)->loopback_dev;
2260 		dev_hold(dst->dev);
2261 		dev_put(dev);
2262 	}
2263 }
2264 EXPORT_SYMBOL(xfrm_dst_ifdown);
2265 
2266 static void xfrm_link_failure(struct sk_buff *skb)
2267 {
2268 	/* Impossible. Such dst must be popped before reaches point of failure. */
2269 }
2270 
2271 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
2272 {
2273 	if (dst) {
2274 		if (dst->obsolete) {
2275 			dst_release(dst);
2276 			dst = NULL;
2277 		}
2278 	}
2279 	return dst;
2280 }
2281 
2282 static void __xfrm_garbage_collect(struct net *net)
2283 {
2284 	struct dst_entry *head, *next;
2285 
2286 	spin_lock_bh(&xfrm_policy_sk_bundle_lock);
2287 	head = xfrm_policy_sk_bundles;
2288 	xfrm_policy_sk_bundles = NULL;
2289 	spin_unlock_bh(&xfrm_policy_sk_bundle_lock);
2290 
2291 	while (head) {
2292 		next = head->next;
2293 		dst_free(head);
2294 		head = next;
2295 	}
2296 }
2297 
2298 static void xfrm_garbage_collect(struct net *net)
2299 {
2300 	flow_cache_flush();
2301 	__xfrm_garbage_collect(net);
2302 }
2303 
2304 static void xfrm_garbage_collect_deferred(struct net *net)
2305 {
2306 	flow_cache_flush_deferred();
2307 	__xfrm_garbage_collect(net);
2308 }
2309 
2310 static void xfrm_init_pmtu(struct dst_entry *dst)
2311 {
2312 	do {
2313 		struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2314 		u32 pmtu, route_mtu_cached;
2315 
2316 		pmtu = dst_mtu(dst->child);
2317 		xdst->child_mtu_cached = pmtu;
2318 
2319 		pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
2320 
2321 		route_mtu_cached = dst_mtu(xdst->route);
2322 		xdst->route_mtu_cached = route_mtu_cached;
2323 
2324 		if (pmtu > route_mtu_cached)
2325 			pmtu = route_mtu_cached;
2326 
2327 		dst_metric_set(dst, RTAX_MTU, pmtu);
2328 	} while ((dst = dst->next));
2329 }
2330 
2331 /* Check that the bundle accepts the flow and its components are
2332  * still valid.
2333  */
2334 
2335 static int xfrm_bundle_ok(struct xfrm_dst *first)
2336 {
2337 	struct dst_entry *dst = &first->u.dst;
2338 	struct xfrm_dst *last;
2339 	u32 mtu;
2340 
2341 	if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) ||
2342 	    (dst->dev && !netif_running(dst->dev)))
2343 		return 0;
2344 
2345 	last = NULL;
2346 
2347 	do {
2348 		struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2349 
2350 		if (dst->xfrm->km.state != XFRM_STATE_VALID)
2351 			return 0;
2352 		if (xdst->xfrm_genid != dst->xfrm->genid)
2353 			return 0;
2354 		if (xdst->num_pols > 0 &&
2355 		    xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
2356 			return 0;
2357 
2358 		mtu = dst_mtu(dst->child);
2359 		if (xdst->child_mtu_cached != mtu) {
2360 			last = xdst;
2361 			xdst->child_mtu_cached = mtu;
2362 		}
2363 
2364 		if (!dst_check(xdst->route, xdst->route_cookie))
2365 			return 0;
2366 		mtu = dst_mtu(xdst->route);
2367 		if (xdst->route_mtu_cached != mtu) {
2368 			last = xdst;
2369 			xdst->route_mtu_cached = mtu;
2370 		}
2371 
2372 		dst = dst->child;
2373 	} while (dst->xfrm);
2374 
2375 	if (likely(!last))
2376 		return 1;
2377 
2378 	mtu = last->child_mtu_cached;
2379 	for (;;) {
2380 		dst = &last->u.dst;
2381 
2382 		mtu = xfrm_state_mtu(dst->xfrm, mtu);
2383 		if (mtu > last->route_mtu_cached)
2384 			mtu = last->route_mtu_cached;
2385 		dst_metric_set(dst, RTAX_MTU, mtu);
2386 
2387 		if (last == first)
2388 			break;
2389 
2390 		last = (struct xfrm_dst *)last->u.dst.next;
2391 		last->child_mtu_cached = mtu;
2392 	}
2393 
2394 	return 1;
2395 }
2396 
2397 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
2398 {
2399 	return dst_metric_advmss(dst->path);
2400 }
2401 
2402 static unsigned int xfrm_mtu(const struct dst_entry *dst)
2403 {
2404 	unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
2405 
2406 	return mtu ? : dst_mtu(dst->path);
2407 }
2408 
2409 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
2410 					   struct sk_buff *skb,
2411 					   const void *daddr)
2412 {
2413 	return dst->path->ops->neigh_lookup(dst, skb, daddr);
2414 }
2415 
2416 int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
2417 {
2418 	struct net *net;
2419 	int err = 0;
2420 	if (unlikely(afinfo == NULL))
2421 		return -EINVAL;
2422 	if (unlikely(afinfo->family >= NPROTO))
2423 		return -EAFNOSUPPORT;
2424 	write_lock_bh(&xfrm_policy_afinfo_lock);
2425 	if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
2426 		err = -ENOBUFS;
2427 	else {
2428 		struct dst_ops *dst_ops = afinfo->dst_ops;
2429 		if (likely(dst_ops->kmem_cachep == NULL))
2430 			dst_ops->kmem_cachep = xfrm_dst_cache;
2431 		if (likely(dst_ops->check == NULL))
2432 			dst_ops->check = xfrm_dst_check;
2433 		if (likely(dst_ops->default_advmss == NULL))
2434 			dst_ops->default_advmss = xfrm_default_advmss;
2435 		if (likely(dst_ops->mtu == NULL))
2436 			dst_ops->mtu = xfrm_mtu;
2437 		if (likely(dst_ops->negative_advice == NULL))
2438 			dst_ops->negative_advice = xfrm_negative_advice;
2439 		if (likely(dst_ops->link_failure == NULL))
2440 			dst_ops->link_failure = xfrm_link_failure;
2441 		if (likely(dst_ops->neigh_lookup == NULL))
2442 			dst_ops->neigh_lookup = xfrm_neigh_lookup;
2443 		if (likely(afinfo->garbage_collect == NULL))
2444 			afinfo->garbage_collect = xfrm_garbage_collect_deferred;
2445 		xfrm_policy_afinfo[afinfo->family] = afinfo;
2446 	}
2447 	write_unlock_bh(&xfrm_policy_afinfo_lock);
2448 
2449 	rtnl_lock();
2450 	for_each_net(net) {
2451 		struct dst_ops *xfrm_dst_ops;
2452 
2453 		switch (afinfo->family) {
2454 		case AF_INET:
2455 			xfrm_dst_ops = &net->xfrm.xfrm4_dst_ops;
2456 			break;
2457 #if IS_ENABLED(CONFIG_IPV6)
2458 		case AF_INET6:
2459 			xfrm_dst_ops = &net->xfrm.xfrm6_dst_ops;
2460 			break;
2461 #endif
2462 		default:
2463 			BUG();
2464 		}
2465 		*xfrm_dst_ops = *afinfo->dst_ops;
2466 	}
2467 	rtnl_unlock();
2468 
2469 	return err;
2470 }
2471 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
2472 
2473 int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
2474 {
2475 	int err = 0;
2476 	if (unlikely(afinfo == NULL))
2477 		return -EINVAL;
2478 	if (unlikely(afinfo->family >= NPROTO))
2479 		return -EAFNOSUPPORT;
2480 	write_lock_bh(&xfrm_policy_afinfo_lock);
2481 	if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
2482 		if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
2483 			err = -EINVAL;
2484 		else {
2485 			struct dst_ops *dst_ops = afinfo->dst_ops;
2486 			xfrm_policy_afinfo[afinfo->family] = NULL;
2487 			dst_ops->kmem_cachep = NULL;
2488 			dst_ops->check = NULL;
2489 			dst_ops->negative_advice = NULL;
2490 			dst_ops->link_failure = NULL;
2491 			afinfo->garbage_collect = NULL;
2492 		}
2493 	}
2494 	write_unlock_bh(&xfrm_policy_afinfo_lock);
2495 	return err;
2496 }
2497 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
2498 
2499 static void __net_init xfrm_dst_ops_init(struct net *net)
2500 {
2501 	struct xfrm_policy_afinfo *afinfo;
2502 
2503 	read_lock_bh(&xfrm_policy_afinfo_lock);
2504 	afinfo = xfrm_policy_afinfo[AF_INET];
2505 	if (afinfo)
2506 		net->xfrm.xfrm4_dst_ops = *afinfo->dst_ops;
2507 #if IS_ENABLED(CONFIG_IPV6)
2508 	afinfo = xfrm_policy_afinfo[AF_INET6];
2509 	if (afinfo)
2510 		net->xfrm.xfrm6_dst_ops = *afinfo->dst_ops;
2511 #endif
2512 	read_unlock_bh(&xfrm_policy_afinfo_lock);
2513 }
2514 
2515 static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
2516 {
2517 	struct xfrm_policy_afinfo *afinfo;
2518 	if (unlikely(family >= NPROTO))
2519 		return NULL;
2520 	read_lock(&xfrm_policy_afinfo_lock);
2521 	afinfo = xfrm_policy_afinfo[family];
2522 	if (unlikely(!afinfo))
2523 		read_unlock(&xfrm_policy_afinfo_lock);
2524 	return afinfo;
2525 }
2526 
2527 static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
2528 {
2529 	read_unlock(&xfrm_policy_afinfo_lock);
2530 }
2531 
2532 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
2533 {
2534 	struct net_device *dev = ptr;
2535 
2536 	switch (event) {
2537 	case NETDEV_DOWN:
2538 		xfrm_garbage_collect(dev_net(dev));
2539 	}
2540 	return NOTIFY_DONE;
2541 }
2542 
2543 static struct notifier_block xfrm_dev_notifier = {
2544 	.notifier_call	= xfrm_dev_event,
2545 };
2546 
2547 #ifdef CONFIG_XFRM_STATISTICS
2548 static int __net_init xfrm_statistics_init(struct net *net)
2549 {
2550 	int rv;
2551 
2552 	if (snmp_mib_init((void __percpu **)net->mib.xfrm_statistics,
2553 			  sizeof(struct linux_xfrm_mib),
2554 			  __alignof__(struct linux_xfrm_mib)) < 0)
2555 		return -ENOMEM;
2556 	rv = xfrm_proc_init(net);
2557 	if (rv < 0)
2558 		snmp_mib_free((void __percpu **)net->mib.xfrm_statistics);
2559 	return rv;
2560 }
2561 
2562 static void xfrm_statistics_fini(struct net *net)
2563 {
2564 	xfrm_proc_fini(net);
2565 	snmp_mib_free((void __percpu **)net->mib.xfrm_statistics);
2566 }
2567 #else
2568 static int __net_init xfrm_statistics_init(struct net *net)
2569 {
2570 	return 0;
2571 }
2572 
2573 static void xfrm_statistics_fini(struct net *net)
2574 {
2575 }
2576 #endif
2577 
2578 static int __net_init xfrm_policy_init(struct net *net)
2579 {
2580 	unsigned int hmask, sz;
2581 	int dir;
2582 
2583 	if (net_eq(net, &init_net))
2584 		xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
2585 					   sizeof(struct xfrm_dst),
2586 					   0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2587 					   NULL);
2588 
2589 	hmask = 8 - 1;
2590 	sz = (hmask+1) * sizeof(struct hlist_head);
2591 
2592 	net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
2593 	if (!net->xfrm.policy_byidx)
2594 		goto out_byidx;
2595 	net->xfrm.policy_idx_hmask = hmask;
2596 
2597 	for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2598 		struct xfrm_policy_hash *htab;
2599 
2600 		net->xfrm.policy_count[dir] = 0;
2601 		INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
2602 
2603 		htab = &net->xfrm.policy_bydst[dir];
2604 		htab->table = xfrm_hash_alloc(sz);
2605 		if (!htab->table)
2606 			goto out_bydst;
2607 		htab->hmask = hmask;
2608 	}
2609 
2610 	INIT_LIST_HEAD(&net->xfrm.policy_all);
2611 	INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
2612 	if (net_eq(net, &init_net))
2613 		register_netdevice_notifier(&xfrm_dev_notifier);
2614 	return 0;
2615 
2616 out_bydst:
2617 	for (dir--; dir >= 0; dir--) {
2618 		struct xfrm_policy_hash *htab;
2619 
2620 		htab = &net->xfrm.policy_bydst[dir];
2621 		xfrm_hash_free(htab->table, sz);
2622 	}
2623 	xfrm_hash_free(net->xfrm.policy_byidx, sz);
2624 out_byidx:
2625 	return -ENOMEM;
2626 }
2627 
2628 static void xfrm_policy_fini(struct net *net)
2629 {
2630 	struct xfrm_audit audit_info;
2631 	unsigned int sz;
2632 	int dir;
2633 
2634 	flush_work(&net->xfrm.policy_hash_work);
2635 #ifdef CONFIG_XFRM_SUB_POLICY
2636 	audit_info.loginuid = -1;
2637 	audit_info.sessionid = -1;
2638 	audit_info.secid = 0;
2639 	xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, &audit_info);
2640 #endif
2641 	audit_info.loginuid = -1;
2642 	audit_info.sessionid = -1;
2643 	audit_info.secid = 0;
2644 	xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, &audit_info);
2645 
2646 	WARN_ON(!list_empty(&net->xfrm.policy_all));
2647 
2648 	for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2649 		struct xfrm_policy_hash *htab;
2650 
2651 		WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
2652 
2653 		htab = &net->xfrm.policy_bydst[dir];
2654 		sz = (htab->hmask + 1);
2655 		WARN_ON(!hlist_empty(htab->table));
2656 		xfrm_hash_free(htab->table, sz);
2657 	}
2658 
2659 	sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
2660 	WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
2661 	xfrm_hash_free(net->xfrm.policy_byidx, sz);
2662 }
2663 
2664 static int __net_init xfrm_net_init(struct net *net)
2665 {
2666 	int rv;
2667 
2668 	rv = xfrm_statistics_init(net);
2669 	if (rv < 0)
2670 		goto out_statistics;
2671 	rv = xfrm_state_init(net);
2672 	if (rv < 0)
2673 		goto out_state;
2674 	rv = xfrm_policy_init(net);
2675 	if (rv < 0)
2676 		goto out_policy;
2677 	xfrm_dst_ops_init(net);
2678 	rv = xfrm_sysctl_init(net);
2679 	if (rv < 0)
2680 		goto out_sysctl;
2681 	return 0;
2682 
2683 out_sysctl:
2684 	xfrm_policy_fini(net);
2685 out_policy:
2686 	xfrm_state_fini(net);
2687 out_state:
2688 	xfrm_statistics_fini(net);
2689 out_statistics:
2690 	return rv;
2691 }
2692 
2693 static void __net_exit xfrm_net_exit(struct net *net)
2694 {
2695 	xfrm_sysctl_fini(net);
2696 	xfrm_policy_fini(net);
2697 	xfrm_state_fini(net);
2698 	xfrm_statistics_fini(net);
2699 }
2700 
2701 static struct pernet_operations __net_initdata xfrm_net_ops = {
2702 	.init = xfrm_net_init,
2703 	.exit = xfrm_net_exit,
2704 };
2705 
2706 void __init xfrm_init(void)
2707 {
2708 	register_pernet_subsys(&xfrm_net_ops);
2709 	xfrm_input_init();
2710 }
2711 
2712 #ifdef CONFIG_AUDITSYSCALL
2713 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
2714 					 struct audit_buffer *audit_buf)
2715 {
2716 	struct xfrm_sec_ctx *ctx = xp->security;
2717 	struct xfrm_selector *sel = &xp->selector;
2718 
2719 	if (ctx)
2720 		audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
2721 				 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
2722 
2723 	switch(sel->family) {
2724 	case AF_INET:
2725 		audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
2726 		if (sel->prefixlen_s != 32)
2727 			audit_log_format(audit_buf, " src_prefixlen=%d",
2728 					 sel->prefixlen_s);
2729 		audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
2730 		if (sel->prefixlen_d != 32)
2731 			audit_log_format(audit_buf, " dst_prefixlen=%d",
2732 					 sel->prefixlen_d);
2733 		break;
2734 	case AF_INET6:
2735 		audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
2736 		if (sel->prefixlen_s != 128)
2737 			audit_log_format(audit_buf, " src_prefixlen=%d",
2738 					 sel->prefixlen_s);
2739 		audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
2740 		if (sel->prefixlen_d != 128)
2741 			audit_log_format(audit_buf, " dst_prefixlen=%d",
2742 					 sel->prefixlen_d);
2743 		break;
2744 	}
2745 }
2746 
2747 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result,
2748 			   uid_t auid, u32 sessionid, u32 secid)
2749 {
2750 	struct audit_buffer *audit_buf;
2751 
2752 	audit_buf = xfrm_audit_start("SPD-add");
2753 	if (audit_buf == NULL)
2754 		return;
2755 	xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
2756 	audit_log_format(audit_buf, " res=%u", result);
2757 	xfrm_audit_common_policyinfo(xp, audit_buf);
2758 	audit_log_end(audit_buf);
2759 }
2760 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
2761 
2762 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
2763 			      uid_t auid, u32 sessionid, u32 secid)
2764 {
2765 	struct audit_buffer *audit_buf;
2766 
2767 	audit_buf = xfrm_audit_start("SPD-delete");
2768 	if (audit_buf == NULL)
2769 		return;
2770 	xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
2771 	audit_log_format(audit_buf, " res=%u", result);
2772 	xfrm_audit_common_policyinfo(xp, audit_buf);
2773 	audit_log_end(audit_buf);
2774 }
2775 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
2776 #endif
2777 
2778 #ifdef CONFIG_XFRM_MIGRATE
2779 static bool xfrm_migrate_selector_match(const struct xfrm_selector *sel_cmp,
2780 					const struct xfrm_selector *sel_tgt)
2781 {
2782 	if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
2783 		if (sel_tgt->family == sel_cmp->family &&
2784 		    xfrm_addr_cmp(&sel_tgt->daddr, &sel_cmp->daddr,
2785 				  sel_cmp->family) == 0 &&
2786 		    xfrm_addr_cmp(&sel_tgt->saddr, &sel_cmp->saddr,
2787 				  sel_cmp->family) == 0 &&
2788 		    sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
2789 		    sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
2790 			return true;
2791 		}
2792 	} else {
2793 		if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
2794 			return true;
2795 		}
2796 	}
2797 	return false;
2798 }
2799 
2800 static struct xfrm_policy * xfrm_migrate_policy_find(const struct xfrm_selector *sel,
2801 						     u8 dir, u8 type)
2802 {
2803 	struct xfrm_policy *pol, *ret = NULL;
2804 	struct hlist_node *entry;
2805 	struct hlist_head *chain;
2806 	u32 priority = ~0U;
2807 
2808 	read_lock_bh(&xfrm_policy_lock);
2809 	chain = policy_hash_direct(&init_net, &sel->daddr, &sel->saddr, sel->family, dir);
2810 	hlist_for_each_entry(pol, entry, chain, bydst) {
2811 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&
2812 		    pol->type == type) {
2813 			ret = pol;
2814 			priority = ret->priority;
2815 			break;
2816 		}
2817 	}
2818 	chain = &init_net.xfrm.policy_inexact[dir];
2819 	hlist_for_each_entry(pol, entry, chain, bydst) {
2820 		if (xfrm_migrate_selector_match(sel, &pol->selector) &&
2821 		    pol->type == type &&
2822 		    pol->priority < priority) {
2823 			ret = pol;
2824 			break;
2825 		}
2826 	}
2827 
2828 	if (ret)
2829 		xfrm_pol_hold(ret);
2830 
2831 	read_unlock_bh(&xfrm_policy_lock);
2832 
2833 	return ret;
2834 }
2835 
2836 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
2837 {
2838 	int match = 0;
2839 
2840 	if (t->mode == m->mode && t->id.proto == m->proto &&
2841 	    (m->reqid == 0 || t->reqid == m->reqid)) {
2842 		switch (t->mode) {
2843 		case XFRM_MODE_TUNNEL:
2844 		case XFRM_MODE_BEET:
2845 			if (xfrm_addr_cmp(&t->id.daddr, &m->old_daddr,
2846 					  m->old_family) == 0 &&
2847 			    xfrm_addr_cmp(&t->saddr, &m->old_saddr,
2848 					  m->old_family) == 0) {
2849 				match = 1;
2850 			}
2851 			break;
2852 		case XFRM_MODE_TRANSPORT:
2853 			/* in case of transport mode, template does not store
2854 			   any IP addresses, hence we just compare mode and
2855 			   protocol */
2856 			match = 1;
2857 			break;
2858 		default:
2859 			break;
2860 		}
2861 	}
2862 	return match;
2863 }
2864 
2865 /* update endpoint address(es) of template(s) */
2866 static int xfrm_policy_migrate(struct xfrm_policy *pol,
2867 			       struct xfrm_migrate *m, int num_migrate)
2868 {
2869 	struct xfrm_migrate *mp;
2870 	int i, j, n = 0;
2871 
2872 	write_lock_bh(&pol->lock);
2873 	if (unlikely(pol->walk.dead)) {
2874 		/* target policy has been deleted */
2875 		write_unlock_bh(&pol->lock);
2876 		return -ENOENT;
2877 	}
2878 
2879 	for (i = 0; i < pol->xfrm_nr; i++) {
2880 		for (j = 0, mp = m; j < num_migrate; j++, mp++) {
2881 			if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
2882 				continue;
2883 			n++;
2884 			if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
2885 			    pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
2886 				continue;
2887 			/* update endpoints */
2888 			memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
2889 			       sizeof(pol->xfrm_vec[i].id.daddr));
2890 			memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
2891 			       sizeof(pol->xfrm_vec[i].saddr));
2892 			pol->xfrm_vec[i].encap_family = mp->new_family;
2893 			/* flush bundles */
2894 			atomic_inc(&pol->genid);
2895 		}
2896 	}
2897 
2898 	write_unlock_bh(&pol->lock);
2899 
2900 	if (!n)
2901 		return -ENODATA;
2902 
2903 	return 0;
2904 }
2905 
2906 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate)
2907 {
2908 	int i, j;
2909 
2910 	if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
2911 		return -EINVAL;
2912 
2913 	for (i = 0; i < num_migrate; i++) {
2914 		if ((xfrm_addr_cmp(&m[i].old_daddr, &m[i].new_daddr,
2915 				   m[i].old_family) == 0) &&
2916 		    (xfrm_addr_cmp(&m[i].old_saddr, &m[i].new_saddr,
2917 				   m[i].old_family) == 0))
2918 			return -EINVAL;
2919 		if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
2920 		    xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
2921 			return -EINVAL;
2922 
2923 		/* check if there is any duplicated entry */
2924 		for (j = i + 1; j < num_migrate; j++) {
2925 			if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
2926 				    sizeof(m[i].old_daddr)) &&
2927 			    !memcmp(&m[i].old_saddr, &m[j].old_saddr,
2928 				    sizeof(m[i].old_saddr)) &&
2929 			    m[i].proto == m[j].proto &&
2930 			    m[i].mode == m[j].mode &&
2931 			    m[i].reqid == m[j].reqid &&
2932 			    m[i].old_family == m[j].old_family)
2933 				return -EINVAL;
2934 		}
2935 	}
2936 
2937 	return 0;
2938 }
2939 
2940 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
2941 		 struct xfrm_migrate *m, int num_migrate,
2942 		 struct xfrm_kmaddress *k)
2943 {
2944 	int i, err, nx_cur = 0, nx_new = 0;
2945 	struct xfrm_policy *pol = NULL;
2946 	struct xfrm_state *x, *xc;
2947 	struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
2948 	struct xfrm_state *x_new[XFRM_MAX_DEPTH];
2949 	struct xfrm_migrate *mp;
2950 
2951 	if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
2952 		goto out;
2953 
2954 	/* Stage 1 - find policy */
2955 	if ((pol = xfrm_migrate_policy_find(sel, dir, type)) == NULL) {
2956 		err = -ENOENT;
2957 		goto out;
2958 	}
2959 
2960 	/* Stage 2 - find and update state(s) */
2961 	for (i = 0, mp = m; i < num_migrate; i++, mp++) {
2962 		if ((x = xfrm_migrate_state_find(mp))) {
2963 			x_cur[nx_cur] = x;
2964 			nx_cur++;
2965 			if ((xc = xfrm_state_migrate(x, mp))) {
2966 				x_new[nx_new] = xc;
2967 				nx_new++;
2968 			} else {
2969 				err = -ENODATA;
2970 				goto restore_state;
2971 			}
2972 		}
2973 	}
2974 
2975 	/* Stage 3 - update policy */
2976 	if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
2977 		goto restore_state;
2978 
2979 	/* Stage 4 - delete old state(s) */
2980 	if (nx_cur) {
2981 		xfrm_states_put(x_cur, nx_cur);
2982 		xfrm_states_delete(x_cur, nx_cur);
2983 	}
2984 
2985 	/* Stage 5 - announce */
2986 	km_migrate(sel, dir, type, m, num_migrate, k);
2987 
2988 	xfrm_pol_put(pol);
2989 
2990 	return 0;
2991 out:
2992 	return err;
2993 
2994 restore_state:
2995 	if (pol)
2996 		xfrm_pol_put(pol);
2997 	if (nx_cur)
2998 		xfrm_states_put(x_cur, nx_cur);
2999 	if (nx_new)
3000 		xfrm_states_delete(x_new, nx_new);
3001 
3002 	return err;
3003 }
3004 EXPORT_SYMBOL(xfrm_migrate);
3005 #endif
3006