1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * xfrm_policy.c
4 *
5 * Changes:
6 * Mitsuru KANDA @USAGI
7 * Kazunori MIYAZAWA @USAGI
8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9 * IPv6 support
10 * Kazunori MIYAZAWA @USAGI
11 * YOSHIFUJI Hideaki
12 * Split up af-specific portion
13 * Derek Atkins <derek@ihtfp.com> Add the post_input processor
14 *
15 */
16
17 #include <linux/err.h>
18 #include <linux/slab.h>
19 #include <linux/kmod.h>
20 #include <linux/list.h>
21 #include <linux/spinlock.h>
22 #include <linux/workqueue.h>
23 #include <linux/notifier.h>
24 #include <linux/netdevice.h>
25 #include <linux/netfilter.h>
26 #include <linux/module.h>
27 #include <linux/cache.h>
28 #include <linux/cpu.h>
29 #include <linux/audit.h>
30 #include <linux/rhashtable.h>
31 #include <linux/if_tunnel.h>
32 #include <linux/icmp.h>
33 #include <net/dst.h>
34 #include <net/flow.h>
35 #include <net/inet_ecn.h>
36 #include <net/xfrm.h>
37 #include <net/ip.h>
38 #include <net/gre.h>
39 #if IS_ENABLED(CONFIG_IPV6_MIP6)
40 #include <net/mip6.h>
41 #endif
42 #ifdef CONFIG_XFRM_STATISTICS
43 #include <net/snmp.h>
44 #endif
45 #ifdef CONFIG_XFRM_ESPINTCP
46 #include <net/espintcp.h>
47 #endif
48 #include <net/inet_dscp.h>
49
50 #include "xfrm_hash.h"
51
52 #define XFRM_QUEUE_TMO_MIN ((unsigned)(HZ/10))
53 #define XFRM_QUEUE_TMO_MAX ((unsigned)(60*HZ))
54 #define XFRM_MAX_QUEUE_LEN 100
55
56 struct xfrm_flo {
57 struct dst_entry *dst_orig;
58 u8 flags;
59 };
60
61 /* prefixes smaller than this are stored in lists, not trees. */
62 #define INEXACT_PREFIXLEN_IPV4 16
63 #define INEXACT_PREFIXLEN_IPV6 48
64
65 struct xfrm_pol_inexact_node {
66 struct rb_node node;
67 union {
68 xfrm_address_t addr;
69 struct rcu_head rcu;
70 };
71 u8 prefixlen;
72
73 struct rb_root root;
74
75 /* the policies matching this node, can be empty list */
76 struct hlist_head hhead;
77 };
78
79 /* xfrm inexact policy search tree:
80 * xfrm_pol_inexact_bin = hash(dir,type,family,if_id);
81 * |
82 * +---- root_d: sorted by daddr:prefix
83 * | |
84 * | xfrm_pol_inexact_node
85 * | |
86 * | +- root: sorted by saddr/prefix
87 * | | |
88 * | | xfrm_pol_inexact_node
89 * | | |
90 * | | + root: unused
91 * | | |
92 * | | + hhead: saddr:daddr policies
93 * | |
94 * | +- coarse policies and all any:daddr policies
95 * |
96 * +---- root_s: sorted by saddr:prefix
97 * | |
98 * | xfrm_pol_inexact_node
99 * | |
100 * | + root: unused
101 * | |
102 * | + hhead: saddr:any policies
103 * |
104 * +---- coarse policies and all any:any policies
105 *
106 * Lookups return four candidate lists:
107 * 1. any:any list from top-level xfrm_pol_inexact_bin
108 * 2. any:daddr list from daddr tree
109 * 3. saddr:daddr list from 2nd level daddr tree
110 * 4. saddr:any list from saddr tree
111 *
112 * This result set then needs to be searched for the policy with
113 * the lowest priority. If two candidates have the same priority, the
114 * struct xfrm_policy pos member with the lower number is used.
115 *
116 * This replicates previous single-list-search algorithm which would
117 * return first matching policy in the (ordered-by-priority) list.
118 */
119
120 struct xfrm_pol_inexact_key {
121 possible_net_t net;
122 u32 if_id;
123 u16 family;
124 u8 dir, type;
125 };
126
127 struct xfrm_pol_inexact_bin {
128 struct xfrm_pol_inexact_key k;
129 struct rhash_head head;
130 /* list containing '*:*' policies */
131 struct hlist_head hhead;
132
133 seqcount_spinlock_t count;
134 /* tree sorted by daddr/prefix */
135 struct rb_root root_d;
136
137 /* tree sorted by saddr/prefix */
138 struct rb_root root_s;
139
140 /* slow path below */
141 struct list_head inexact_bins;
142 struct rcu_head rcu;
143 };
144
145 enum xfrm_pol_inexact_candidate_type {
146 XFRM_POL_CAND_BOTH,
147 XFRM_POL_CAND_SADDR,
148 XFRM_POL_CAND_DADDR,
149 XFRM_POL_CAND_ANY,
150
151 XFRM_POL_CAND_MAX,
152 };
153
154 struct xfrm_pol_inexact_candidates {
155 struct hlist_head *res[XFRM_POL_CAND_MAX];
156 };
157
158 struct xfrm_flow_keys {
159 struct flow_dissector_key_basic basic;
160 struct flow_dissector_key_control control;
161 union {
162 struct flow_dissector_key_ipv4_addrs ipv4;
163 struct flow_dissector_key_ipv6_addrs ipv6;
164 } addrs;
165 struct flow_dissector_key_ip ip;
166 struct flow_dissector_key_icmp icmp;
167 struct flow_dissector_key_ports ports;
168 struct flow_dissector_key_keyid gre;
169 };
170
171 static struct flow_dissector xfrm_session_dissector __ro_after_init;
172
173 static DEFINE_SPINLOCK(xfrm_if_cb_lock);
174 static struct xfrm_if_cb const __rcu *xfrm_if_cb __read_mostly;
175
176 static DEFINE_SPINLOCK(xfrm_policy_afinfo_lock);
177 static struct xfrm_policy_afinfo const __rcu *xfrm_policy_afinfo[AF_INET6 + 1]
178 __read_mostly;
179
180 static struct kmem_cache *xfrm_dst_cache __ro_after_init;
181
182 static struct rhashtable xfrm_policy_inexact_table;
183 static const struct rhashtable_params xfrm_pol_inexact_params;
184
185 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr);
186 static int stale_bundle(struct dst_entry *dst);
187 static int xfrm_bundle_ok(struct xfrm_dst *xdst);
188 static void xfrm_policy_queue_process(struct timer_list *t);
189
190 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir);
191 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
192 int dir);
193
194 static struct xfrm_pol_inexact_bin *
195 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family, u8 dir,
196 u32 if_id);
197
198 static struct xfrm_pol_inexact_bin *
199 xfrm_policy_inexact_lookup_rcu(struct net *net,
200 u8 type, u16 family, u8 dir, u32 if_id);
201 static struct xfrm_policy *
202 xfrm_policy_insert_list(struct hlist_head *chain, struct xfrm_policy *policy,
203 bool excl);
204
205 static bool
206 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
207 struct xfrm_pol_inexact_bin *b,
208 const xfrm_address_t *saddr,
209 const xfrm_address_t *daddr);
210
xfrm_pol_hold_rcu(struct xfrm_policy * policy)211 static inline bool xfrm_pol_hold_rcu(struct xfrm_policy *policy)
212 {
213 return refcount_inc_not_zero(&policy->refcnt);
214 }
215
216 static inline bool
__xfrm4_selector_match(const struct xfrm_selector * sel,const struct flowi * fl)217 __xfrm4_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
218 {
219 const struct flowi4 *fl4 = &fl->u.ip4;
220
221 return addr4_match(fl4->daddr, sel->daddr.a4, sel->prefixlen_d) &&
222 addr4_match(fl4->saddr, sel->saddr.a4, sel->prefixlen_s) &&
223 !((xfrm_flowi_dport(fl, &fl4->uli) ^ sel->dport) & sel->dport_mask) &&
224 !((xfrm_flowi_sport(fl, &fl4->uli) ^ sel->sport) & sel->sport_mask) &&
225 (fl4->flowi4_proto == sel->proto || !sel->proto) &&
226 (fl4->flowi4_oif == sel->ifindex || !sel->ifindex);
227 }
228
229 static inline bool
__xfrm6_selector_match(const struct xfrm_selector * sel,const struct flowi * fl)230 __xfrm6_selector_match(const struct xfrm_selector *sel, const struct flowi *fl)
231 {
232 const struct flowi6 *fl6 = &fl->u.ip6;
233
234 return addr_match(&fl6->daddr, &sel->daddr, sel->prefixlen_d) &&
235 addr_match(&fl6->saddr, &sel->saddr, sel->prefixlen_s) &&
236 !((xfrm_flowi_dport(fl, &fl6->uli) ^ sel->dport) & sel->dport_mask) &&
237 !((xfrm_flowi_sport(fl, &fl6->uli) ^ sel->sport) & sel->sport_mask) &&
238 (fl6->flowi6_proto == sel->proto || !sel->proto) &&
239 (fl6->flowi6_oif == sel->ifindex || !sel->ifindex);
240 }
241
xfrm_selector_match(const struct xfrm_selector * sel,const struct flowi * fl,unsigned short family)242 bool xfrm_selector_match(const struct xfrm_selector *sel, const struct flowi *fl,
243 unsigned short family)
244 {
245 switch (family) {
246 case AF_INET:
247 return __xfrm4_selector_match(sel, fl);
248 case AF_INET6:
249 return __xfrm6_selector_match(sel, fl);
250 }
251 return false;
252 }
253
xfrm_policy_get_afinfo(unsigned short family)254 static const struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
255 {
256 const struct xfrm_policy_afinfo *afinfo;
257
258 if (unlikely(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
259 return NULL;
260 rcu_read_lock();
261 afinfo = rcu_dereference(xfrm_policy_afinfo[family]);
262 if (unlikely(!afinfo))
263 rcu_read_unlock();
264 return afinfo;
265 }
266
267 /* Called with rcu_read_lock(). */
xfrm_if_get_cb(void)268 static const struct xfrm_if_cb *xfrm_if_get_cb(void)
269 {
270 return rcu_dereference(xfrm_if_cb);
271 }
272
__xfrm_dst_lookup(int family,const struct xfrm_dst_lookup_params * params)273 struct dst_entry *__xfrm_dst_lookup(int family,
274 const struct xfrm_dst_lookup_params *params)
275 {
276 const struct xfrm_policy_afinfo *afinfo;
277 struct dst_entry *dst;
278
279 afinfo = xfrm_policy_get_afinfo(family);
280 if (unlikely(afinfo == NULL))
281 return ERR_PTR(-EAFNOSUPPORT);
282
283 dst = afinfo->dst_lookup(params);
284
285 rcu_read_unlock();
286
287 return dst;
288 }
289 EXPORT_SYMBOL(__xfrm_dst_lookup);
290
xfrm_dst_lookup(struct xfrm_state * x,int tos,int oif,xfrm_address_t * prev_saddr,xfrm_address_t * prev_daddr,int family,u32 mark)291 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x,
292 int tos, int oif,
293 xfrm_address_t *prev_saddr,
294 xfrm_address_t *prev_daddr,
295 int family, u32 mark)
296 {
297 struct xfrm_dst_lookup_params params;
298 struct net *net = xs_net(x);
299 xfrm_address_t *saddr = &x->props.saddr;
300 xfrm_address_t *daddr = &x->id.daddr;
301 struct dst_entry *dst;
302
303 if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
304 saddr = x->coaddr;
305 daddr = prev_daddr;
306 }
307 if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
308 saddr = prev_saddr;
309 daddr = x->coaddr;
310 }
311
312 params.net = net;
313 params.saddr = saddr;
314 params.daddr = daddr;
315 params.tos = tos;
316 params.oif = oif;
317 params.mark = mark;
318 params.ipproto = x->id.proto;
319 if (x->encap) {
320 switch (x->encap->encap_type) {
321 case UDP_ENCAP_ESPINUDP:
322 params.ipproto = IPPROTO_UDP;
323 params.uli.ports.sport = x->encap->encap_sport;
324 params.uli.ports.dport = x->encap->encap_dport;
325 break;
326 case TCP_ENCAP_ESPINTCP:
327 params.ipproto = IPPROTO_TCP;
328 params.uli.ports.sport = x->encap->encap_sport;
329 params.uli.ports.dport = x->encap->encap_dport;
330 break;
331 }
332 }
333
334 dst = __xfrm_dst_lookup(family, ¶ms);
335
336 if (!IS_ERR(dst)) {
337 if (prev_saddr != saddr)
338 memcpy(prev_saddr, saddr, sizeof(*prev_saddr));
339 if (prev_daddr != daddr)
340 memcpy(prev_daddr, daddr, sizeof(*prev_daddr));
341 }
342
343 return dst;
344 }
345
make_jiffies(long secs)346 static inline unsigned long make_jiffies(long secs)
347 {
348 if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
349 return MAX_SCHEDULE_TIMEOUT-1;
350 else
351 return secs*HZ;
352 }
353
xfrm_policy_timer(struct timer_list * t)354 static void xfrm_policy_timer(struct timer_list *t)
355 {
356 struct xfrm_policy *xp = from_timer(xp, t, timer);
357 time64_t now = ktime_get_real_seconds();
358 time64_t next = TIME64_MAX;
359 int warn = 0;
360 int dir;
361
362 read_lock(&xp->lock);
363
364 if (unlikely(xp->walk.dead))
365 goto out;
366
367 dir = xfrm_policy_id2dir(xp->index);
368
369 if (xp->lft.hard_add_expires_seconds) {
370 time64_t tmo = xp->lft.hard_add_expires_seconds +
371 xp->curlft.add_time - now;
372 if (tmo <= 0)
373 goto expired;
374 if (tmo < next)
375 next = tmo;
376 }
377 if (xp->lft.hard_use_expires_seconds) {
378 time64_t tmo = xp->lft.hard_use_expires_seconds +
379 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
380 if (tmo <= 0)
381 goto expired;
382 if (tmo < next)
383 next = tmo;
384 }
385 if (xp->lft.soft_add_expires_seconds) {
386 time64_t tmo = xp->lft.soft_add_expires_seconds +
387 xp->curlft.add_time - now;
388 if (tmo <= 0) {
389 warn = 1;
390 tmo = XFRM_KM_TIMEOUT;
391 }
392 if (tmo < next)
393 next = tmo;
394 }
395 if (xp->lft.soft_use_expires_seconds) {
396 time64_t tmo = xp->lft.soft_use_expires_seconds +
397 (READ_ONCE(xp->curlft.use_time) ? : xp->curlft.add_time) - now;
398 if (tmo <= 0) {
399 warn = 1;
400 tmo = XFRM_KM_TIMEOUT;
401 }
402 if (tmo < next)
403 next = tmo;
404 }
405
406 if (warn)
407 km_policy_expired(xp, dir, 0, 0);
408 if (next != TIME64_MAX &&
409 !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
410 xfrm_pol_hold(xp);
411
412 out:
413 read_unlock(&xp->lock);
414 xfrm_pol_put(xp);
415 return;
416
417 expired:
418 read_unlock(&xp->lock);
419 if (!xfrm_policy_delete(xp, dir))
420 km_policy_expired(xp, dir, 1, 0);
421 xfrm_pol_put(xp);
422 }
423
424 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
425 * SPD calls.
426 */
427
xfrm_policy_alloc(struct net * net,gfp_t gfp)428 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
429 {
430 struct xfrm_policy *policy;
431
432 policy = kzalloc(sizeof(struct xfrm_policy), gfp);
433
434 if (policy) {
435 write_pnet(&policy->xp_net, net);
436 INIT_LIST_HEAD(&policy->walk.all);
437 INIT_HLIST_NODE(&policy->bydst);
438 INIT_HLIST_NODE(&policy->byidx);
439 rwlock_init(&policy->lock);
440 refcount_set(&policy->refcnt, 1);
441 skb_queue_head_init(&policy->polq.hold_queue);
442 timer_setup(&policy->timer, xfrm_policy_timer, 0);
443 timer_setup(&policy->polq.hold_timer,
444 xfrm_policy_queue_process, 0);
445 }
446 return policy;
447 }
448 EXPORT_SYMBOL(xfrm_policy_alloc);
449
xfrm_policy_destroy_rcu(struct rcu_head * head)450 static void xfrm_policy_destroy_rcu(struct rcu_head *head)
451 {
452 struct xfrm_policy *policy = container_of(head, struct xfrm_policy, rcu);
453
454 security_xfrm_policy_free(policy->security);
455 kfree(policy);
456 }
457
458 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
459
xfrm_policy_destroy(struct xfrm_policy * policy)460 void xfrm_policy_destroy(struct xfrm_policy *policy)
461 {
462 BUG_ON(!policy->walk.dead);
463
464 if (del_timer(&policy->timer) || del_timer(&policy->polq.hold_timer))
465 BUG();
466
467 xfrm_dev_policy_free(policy);
468 call_rcu(&policy->rcu, xfrm_policy_destroy_rcu);
469 }
470 EXPORT_SYMBOL(xfrm_policy_destroy);
471
472 /* Rule must be locked. Release descendant resources, announce
473 * entry dead. The rule must be unlinked from lists to the moment.
474 */
475
xfrm_policy_kill(struct xfrm_policy * policy)476 static void xfrm_policy_kill(struct xfrm_policy *policy)
477 {
478 xfrm_dev_policy_delete(policy);
479
480 write_lock_bh(&policy->lock);
481 policy->walk.dead = 1;
482 write_unlock_bh(&policy->lock);
483
484 atomic_inc(&policy->genid);
485
486 if (del_timer(&policy->polq.hold_timer))
487 xfrm_pol_put(policy);
488 skb_queue_purge(&policy->polq.hold_queue);
489
490 if (del_timer(&policy->timer))
491 xfrm_pol_put(policy);
492
493 xfrm_pol_put(policy);
494 }
495
496 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
497
idx_hash(struct net * net,u32 index)498 static inline unsigned int idx_hash(struct net *net, u32 index)
499 {
500 return __idx_hash(index, net->xfrm.policy_idx_hmask);
501 }
502
503 /* calculate policy hash thresholds */
__get_hash_thresh(struct net * net,unsigned short family,int dir,u8 * dbits,u8 * sbits)504 static void __get_hash_thresh(struct net *net,
505 unsigned short family, int dir,
506 u8 *dbits, u8 *sbits)
507 {
508 switch (family) {
509 case AF_INET:
510 *dbits = net->xfrm.policy_bydst[dir].dbits4;
511 *sbits = net->xfrm.policy_bydst[dir].sbits4;
512 break;
513
514 case AF_INET6:
515 *dbits = net->xfrm.policy_bydst[dir].dbits6;
516 *sbits = net->xfrm.policy_bydst[dir].sbits6;
517 break;
518
519 default:
520 *dbits = 0;
521 *sbits = 0;
522 }
523 }
524
policy_hash_bysel(struct net * net,const struct xfrm_selector * sel,unsigned short family,int dir)525 static struct hlist_head *policy_hash_bysel(struct net *net,
526 const struct xfrm_selector *sel,
527 unsigned short family, int dir)
528 {
529 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
530 unsigned int hash;
531 u8 dbits;
532 u8 sbits;
533
534 __get_hash_thresh(net, family, dir, &dbits, &sbits);
535 hash = __sel_hash(sel, family, hmask, dbits, sbits);
536
537 if (hash == hmask + 1)
538 return NULL;
539
540 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
541 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
542 }
543
policy_hash_direct(struct net * net,const xfrm_address_t * daddr,const xfrm_address_t * saddr,unsigned short family,int dir)544 static struct hlist_head *policy_hash_direct(struct net *net,
545 const xfrm_address_t *daddr,
546 const xfrm_address_t *saddr,
547 unsigned short family, int dir)
548 {
549 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
550 unsigned int hash;
551 u8 dbits;
552 u8 sbits;
553
554 __get_hash_thresh(net, family, dir, &dbits, &sbits);
555 hash = __addr_hash(daddr, saddr, family, hmask, dbits, sbits);
556
557 return rcu_dereference_check(net->xfrm.policy_bydst[dir].table,
558 lockdep_is_held(&net->xfrm.xfrm_policy_lock)) + hash;
559 }
560
xfrm_dst_hash_transfer(struct net * net,struct hlist_head * list,struct hlist_head * ndsttable,unsigned int nhashmask,int dir)561 static void xfrm_dst_hash_transfer(struct net *net,
562 struct hlist_head *list,
563 struct hlist_head *ndsttable,
564 unsigned int nhashmask,
565 int dir)
566 {
567 struct hlist_node *tmp, *entry0 = NULL;
568 struct xfrm_policy *pol;
569 unsigned int h0 = 0;
570 u8 dbits;
571 u8 sbits;
572
573 redo:
574 hlist_for_each_entry_safe(pol, tmp, list, bydst) {
575 unsigned int h;
576
577 __get_hash_thresh(net, pol->family, dir, &dbits, &sbits);
578 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
579 pol->family, nhashmask, dbits, sbits);
580 if (!entry0 || pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
581 hlist_del_rcu(&pol->bydst);
582 hlist_add_head_rcu(&pol->bydst, ndsttable + h);
583 h0 = h;
584 } else {
585 if (h != h0)
586 continue;
587 hlist_del_rcu(&pol->bydst);
588 hlist_add_behind_rcu(&pol->bydst, entry0);
589 }
590 entry0 = &pol->bydst;
591 }
592 if (!hlist_empty(list)) {
593 entry0 = NULL;
594 goto redo;
595 }
596 }
597
xfrm_idx_hash_transfer(struct hlist_head * list,struct hlist_head * nidxtable,unsigned int nhashmask)598 static void xfrm_idx_hash_transfer(struct hlist_head *list,
599 struct hlist_head *nidxtable,
600 unsigned int nhashmask)
601 {
602 struct hlist_node *tmp;
603 struct xfrm_policy *pol;
604
605 hlist_for_each_entry_safe(pol, tmp, list, byidx) {
606 unsigned int h;
607
608 h = __idx_hash(pol->index, nhashmask);
609 hlist_add_head(&pol->byidx, nidxtable+h);
610 }
611 }
612
xfrm_new_hash_mask(unsigned int old_hmask)613 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
614 {
615 return ((old_hmask + 1) << 1) - 1;
616 }
617
xfrm_bydst_resize(struct net * net,int dir)618 static void xfrm_bydst_resize(struct net *net, int dir)
619 {
620 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
621 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
622 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
623 struct hlist_head *ndst = xfrm_hash_alloc(nsize);
624 struct hlist_head *odst;
625 int i;
626
627 if (!ndst)
628 return;
629
630 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
631 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
632
633 odst = rcu_dereference_protected(net->xfrm.policy_bydst[dir].table,
634 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
635
636 for (i = hmask; i >= 0; i--)
637 xfrm_dst_hash_transfer(net, odst + i, ndst, nhashmask, dir);
638
639 rcu_assign_pointer(net->xfrm.policy_bydst[dir].table, ndst);
640 net->xfrm.policy_bydst[dir].hmask = nhashmask;
641
642 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
643 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
644
645 synchronize_rcu();
646
647 xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
648 }
649
xfrm_byidx_resize(struct net * net)650 static void xfrm_byidx_resize(struct net *net)
651 {
652 unsigned int hmask = net->xfrm.policy_idx_hmask;
653 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
654 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
655 struct hlist_head *oidx = net->xfrm.policy_byidx;
656 struct hlist_head *nidx = xfrm_hash_alloc(nsize);
657 int i;
658
659 if (!nidx)
660 return;
661
662 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
663
664 for (i = hmask; i >= 0; i--)
665 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
666
667 net->xfrm.policy_byidx = nidx;
668 net->xfrm.policy_idx_hmask = nhashmask;
669
670 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
671
672 xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
673 }
674
xfrm_bydst_should_resize(struct net * net,int dir,int * total)675 static inline int xfrm_bydst_should_resize(struct net *net, int dir, int *total)
676 {
677 unsigned int cnt = net->xfrm.policy_count[dir];
678 unsigned int hmask = net->xfrm.policy_bydst[dir].hmask;
679
680 if (total)
681 *total += cnt;
682
683 if ((hmask + 1) < xfrm_policy_hashmax &&
684 cnt > hmask)
685 return 1;
686
687 return 0;
688 }
689
xfrm_byidx_should_resize(struct net * net,int total)690 static inline int xfrm_byidx_should_resize(struct net *net, int total)
691 {
692 unsigned int hmask = net->xfrm.policy_idx_hmask;
693
694 if ((hmask + 1) < xfrm_policy_hashmax &&
695 total > hmask)
696 return 1;
697
698 return 0;
699 }
700
xfrm_spd_getinfo(struct net * net,struct xfrmk_spdinfo * si)701 void xfrm_spd_getinfo(struct net *net, struct xfrmk_spdinfo *si)
702 {
703 si->incnt = net->xfrm.policy_count[XFRM_POLICY_IN];
704 si->outcnt = net->xfrm.policy_count[XFRM_POLICY_OUT];
705 si->fwdcnt = net->xfrm.policy_count[XFRM_POLICY_FWD];
706 si->inscnt = net->xfrm.policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
707 si->outscnt = net->xfrm.policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
708 si->fwdscnt = net->xfrm.policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
709 si->spdhcnt = net->xfrm.policy_idx_hmask;
710 si->spdhmcnt = xfrm_policy_hashmax;
711 }
712 EXPORT_SYMBOL(xfrm_spd_getinfo);
713
714 static DEFINE_MUTEX(hash_resize_mutex);
xfrm_hash_resize(struct work_struct * work)715 static void xfrm_hash_resize(struct work_struct *work)
716 {
717 struct net *net = container_of(work, struct net, xfrm.policy_hash_work);
718 int dir, total;
719
720 mutex_lock(&hash_resize_mutex);
721
722 total = 0;
723 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
724 if (xfrm_bydst_should_resize(net, dir, &total))
725 xfrm_bydst_resize(net, dir);
726 }
727 if (xfrm_byidx_should_resize(net, total))
728 xfrm_byidx_resize(net);
729
730 mutex_unlock(&hash_resize_mutex);
731 }
732
733 /* Make sure *pol can be inserted into fastbin.
734 * Useful to check that later insert requests will be successful
735 * (provided xfrm_policy_lock is held throughout).
736 */
737 static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_alloc_bin(const struct xfrm_policy * pol,u8 dir)738 xfrm_policy_inexact_alloc_bin(const struct xfrm_policy *pol, u8 dir)
739 {
740 struct xfrm_pol_inexact_bin *bin, *prev;
741 struct xfrm_pol_inexact_key k = {
742 .family = pol->family,
743 .type = pol->type,
744 .dir = dir,
745 .if_id = pol->if_id,
746 };
747 struct net *net = xp_net(pol);
748
749 lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
750
751 write_pnet(&k.net, net);
752 bin = rhashtable_lookup_fast(&xfrm_policy_inexact_table, &k,
753 xfrm_pol_inexact_params);
754 if (bin)
755 return bin;
756
757 bin = kzalloc(sizeof(*bin), GFP_ATOMIC);
758 if (!bin)
759 return NULL;
760
761 bin->k = k;
762 INIT_HLIST_HEAD(&bin->hhead);
763 bin->root_d = RB_ROOT;
764 bin->root_s = RB_ROOT;
765 seqcount_spinlock_init(&bin->count, &net->xfrm.xfrm_policy_lock);
766
767 prev = rhashtable_lookup_get_insert_key(&xfrm_policy_inexact_table,
768 &bin->k, &bin->head,
769 xfrm_pol_inexact_params);
770 if (!prev) {
771 list_add(&bin->inexact_bins, &net->xfrm.inexact_bins);
772 return bin;
773 }
774
775 kfree(bin);
776
777 return IS_ERR(prev) ? NULL : prev;
778 }
779
xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t * addr,int family,u8 prefixlen)780 static bool xfrm_pol_inexact_addr_use_any_list(const xfrm_address_t *addr,
781 int family, u8 prefixlen)
782 {
783 if (xfrm_addr_any(addr, family))
784 return true;
785
786 if (family == AF_INET6 && prefixlen < INEXACT_PREFIXLEN_IPV6)
787 return true;
788
789 if (family == AF_INET && prefixlen < INEXACT_PREFIXLEN_IPV4)
790 return true;
791
792 return false;
793 }
794
795 static bool
xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy * policy)796 xfrm_policy_inexact_insert_use_any_list(const struct xfrm_policy *policy)
797 {
798 const xfrm_address_t *addr;
799 bool saddr_any, daddr_any;
800 u8 prefixlen;
801
802 addr = &policy->selector.saddr;
803 prefixlen = policy->selector.prefixlen_s;
804
805 saddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
806 policy->family,
807 prefixlen);
808 addr = &policy->selector.daddr;
809 prefixlen = policy->selector.prefixlen_d;
810 daddr_any = xfrm_pol_inexact_addr_use_any_list(addr,
811 policy->family,
812 prefixlen);
813 return saddr_any && daddr_any;
814 }
815
xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node * node,const xfrm_address_t * addr,u8 prefixlen)816 static void xfrm_pol_inexact_node_init(struct xfrm_pol_inexact_node *node,
817 const xfrm_address_t *addr, u8 prefixlen)
818 {
819 node->addr = *addr;
820 node->prefixlen = prefixlen;
821 }
822
823 static struct xfrm_pol_inexact_node *
xfrm_pol_inexact_node_alloc(const xfrm_address_t * addr,u8 prefixlen)824 xfrm_pol_inexact_node_alloc(const xfrm_address_t *addr, u8 prefixlen)
825 {
826 struct xfrm_pol_inexact_node *node;
827
828 node = kzalloc(sizeof(*node), GFP_ATOMIC);
829 if (node)
830 xfrm_pol_inexact_node_init(node, addr, prefixlen);
831
832 return node;
833 }
834
xfrm_policy_addr_delta(const xfrm_address_t * a,const xfrm_address_t * b,u8 prefixlen,u16 family)835 static int xfrm_policy_addr_delta(const xfrm_address_t *a,
836 const xfrm_address_t *b,
837 u8 prefixlen, u16 family)
838 {
839 u32 ma, mb, mask;
840 unsigned int pdw, pbi;
841 int delta = 0;
842
843 switch (family) {
844 case AF_INET:
845 if (prefixlen == 0)
846 return 0;
847 mask = ~0U << (32 - prefixlen);
848 ma = ntohl(a->a4) & mask;
849 mb = ntohl(b->a4) & mask;
850 if (ma < mb)
851 delta = -1;
852 else if (ma > mb)
853 delta = 1;
854 break;
855 case AF_INET6:
856 pdw = prefixlen >> 5;
857 pbi = prefixlen & 0x1f;
858
859 if (pdw) {
860 delta = memcmp(a->a6, b->a6, pdw << 2);
861 if (delta)
862 return delta;
863 }
864 if (pbi) {
865 mask = ~0U << (32 - pbi);
866 ma = ntohl(a->a6[pdw]) & mask;
867 mb = ntohl(b->a6[pdw]) & mask;
868 if (ma < mb)
869 delta = -1;
870 else if (ma > mb)
871 delta = 1;
872 }
873 break;
874 default:
875 break;
876 }
877
878 return delta;
879 }
880
xfrm_policy_inexact_list_reinsert(struct net * net,struct xfrm_pol_inexact_node * n,u16 family)881 static void xfrm_policy_inexact_list_reinsert(struct net *net,
882 struct xfrm_pol_inexact_node *n,
883 u16 family)
884 {
885 unsigned int matched_s, matched_d;
886 struct xfrm_policy *policy, *p;
887
888 matched_s = 0;
889 matched_d = 0;
890
891 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
892 struct hlist_node *newpos = NULL;
893 bool matches_s, matches_d;
894
895 if (policy->walk.dead || !policy->bydst_reinsert)
896 continue;
897
898 WARN_ON_ONCE(policy->family != family);
899
900 policy->bydst_reinsert = false;
901 hlist_for_each_entry(p, &n->hhead, bydst) {
902 if (policy->priority > p->priority)
903 newpos = &p->bydst;
904 else if (policy->priority == p->priority &&
905 policy->pos > p->pos)
906 newpos = &p->bydst;
907 else
908 break;
909 }
910
911 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
912 hlist_add_behind_rcu(&policy->bydst, newpos);
913 else
914 hlist_add_head_rcu(&policy->bydst, &n->hhead);
915
916 /* paranoia checks follow.
917 * Check that the reinserted policy matches at least
918 * saddr or daddr for current node prefix.
919 *
920 * Matching both is fine, matching saddr in one policy
921 * (but not daddr) and then matching only daddr in another
922 * is a bug.
923 */
924 matches_s = xfrm_policy_addr_delta(&policy->selector.saddr,
925 &n->addr,
926 n->prefixlen,
927 family) == 0;
928 matches_d = xfrm_policy_addr_delta(&policy->selector.daddr,
929 &n->addr,
930 n->prefixlen,
931 family) == 0;
932 if (matches_s && matches_d)
933 continue;
934
935 WARN_ON_ONCE(!matches_s && !matches_d);
936 if (matches_s)
937 matched_s++;
938 if (matches_d)
939 matched_d++;
940 WARN_ON_ONCE(matched_s && matched_d);
941 }
942 }
943
xfrm_policy_inexact_node_reinsert(struct net * net,struct xfrm_pol_inexact_node * n,struct rb_root * new,u16 family)944 static void xfrm_policy_inexact_node_reinsert(struct net *net,
945 struct xfrm_pol_inexact_node *n,
946 struct rb_root *new,
947 u16 family)
948 {
949 struct xfrm_pol_inexact_node *node;
950 struct rb_node **p, *parent;
951
952 /* we should not have another subtree here */
953 WARN_ON_ONCE(!RB_EMPTY_ROOT(&n->root));
954 restart:
955 parent = NULL;
956 p = &new->rb_node;
957 while (*p) {
958 u8 prefixlen;
959 int delta;
960
961 parent = *p;
962 node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
963
964 prefixlen = min(node->prefixlen, n->prefixlen);
965
966 delta = xfrm_policy_addr_delta(&n->addr, &node->addr,
967 prefixlen, family);
968 if (delta < 0) {
969 p = &parent->rb_left;
970 } else if (delta > 0) {
971 p = &parent->rb_right;
972 } else {
973 bool same_prefixlen = node->prefixlen == n->prefixlen;
974 struct xfrm_policy *tmp;
975
976 hlist_for_each_entry(tmp, &n->hhead, bydst) {
977 tmp->bydst_reinsert = true;
978 hlist_del_rcu(&tmp->bydst);
979 }
980
981 node->prefixlen = prefixlen;
982
983 xfrm_policy_inexact_list_reinsert(net, node, family);
984
985 if (same_prefixlen) {
986 kfree_rcu(n, rcu);
987 return;
988 }
989
990 rb_erase(*p, new);
991 kfree_rcu(n, rcu);
992 n = node;
993 goto restart;
994 }
995 }
996
997 rb_link_node_rcu(&n->node, parent, p);
998 rb_insert_color(&n->node, new);
999 }
1000
1001 /* merge nodes v and n */
xfrm_policy_inexact_node_merge(struct net * net,struct xfrm_pol_inexact_node * v,struct xfrm_pol_inexact_node * n,u16 family)1002 static void xfrm_policy_inexact_node_merge(struct net *net,
1003 struct xfrm_pol_inexact_node *v,
1004 struct xfrm_pol_inexact_node *n,
1005 u16 family)
1006 {
1007 struct xfrm_pol_inexact_node *node;
1008 struct xfrm_policy *tmp;
1009 struct rb_node *rnode;
1010
1011 /* To-be-merged node v has a subtree.
1012 *
1013 * Dismantle it and insert its nodes to n->root.
1014 */
1015 while ((rnode = rb_first(&v->root)) != NULL) {
1016 node = rb_entry(rnode, struct xfrm_pol_inexact_node, node);
1017 rb_erase(&node->node, &v->root);
1018 xfrm_policy_inexact_node_reinsert(net, node, &n->root,
1019 family);
1020 }
1021
1022 hlist_for_each_entry(tmp, &v->hhead, bydst) {
1023 tmp->bydst_reinsert = true;
1024 hlist_del_rcu(&tmp->bydst);
1025 }
1026
1027 xfrm_policy_inexact_list_reinsert(net, n, family);
1028 }
1029
1030 static struct xfrm_pol_inexact_node *
xfrm_policy_inexact_insert_node(struct net * net,struct rb_root * root,xfrm_address_t * addr,u16 family,u8 prefixlen,u8 dir)1031 xfrm_policy_inexact_insert_node(struct net *net,
1032 struct rb_root *root,
1033 xfrm_address_t *addr,
1034 u16 family, u8 prefixlen, u8 dir)
1035 {
1036 struct xfrm_pol_inexact_node *cached = NULL;
1037 struct rb_node **p, *parent = NULL;
1038 struct xfrm_pol_inexact_node *node;
1039
1040 p = &root->rb_node;
1041 while (*p) {
1042 int delta;
1043
1044 parent = *p;
1045 node = rb_entry(*p, struct xfrm_pol_inexact_node, node);
1046
1047 delta = xfrm_policy_addr_delta(addr, &node->addr,
1048 node->prefixlen,
1049 family);
1050 if (delta == 0 && prefixlen >= node->prefixlen) {
1051 WARN_ON_ONCE(cached); /* ipsec policies got lost */
1052 return node;
1053 }
1054
1055 if (delta < 0)
1056 p = &parent->rb_left;
1057 else
1058 p = &parent->rb_right;
1059
1060 if (prefixlen < node->prefixlen) {
1061 delta = xfrm_policy_addr_delta(addr, &node->addr,
1062 prefixlen,
1063 family);
1064 if (delta)
1065 continue;
1066
1067 /* This node is a subnet of the new prefix. It needs
1068 * to be removed and re-inserted with the smaller
1069 * prefix and all nodes that are now also covered
1070 * by the reduced prefixlen.
1071 */
1072 rb_erase(&node->node, root);
1073
1074 if (!cached) {
1075 xfrm_pol_inexact_node_init(node, addr,
1076 prefixlen);
1077 cached = node;
1078 } else {
1079 /* This node also falls within the new
1080 * prefixlen. Merge the to-be-reinserted
1081 * node and this one.
1082 */
1083 xfrm_policy_inexact_node_merge(net, node,
1084 cached, family);
1085 kfree_rcu(node, rcu);
1086 }
1087
1088 /* restart */
1089 p = &root->rb_node;
1090 parent = NULL;
1091 }
1092 }
1093
1094 node = cached;
1095 if (!node) {
1096 node = xfrm_pol_inexact_node_alloc(addr, prefixlen);
1097 if (!node)
1098 return NULL;
1099 }
1100
1101 rb_link_node_rcu(&node->node, parent, p);
1102 rb_insert_color(&node->node, root);
1103
1104 return node;
1105 }
1106
xfrm_policy_inexact_gc_tree(struct rb_root * r,bool rm)1107 static void xfrm_policy_inexact_gc_tree(struct rb_root *r, bool rm)
1108 {
1109 struct xfrm_pol_inexact_node *node;
1110 struct rb_node *rn = rb_first(r);
1111
1112 while (rn) {
1113 node = rb_entry(rn, struct xfrm_pol_inexact_node, node);
1114
1115 xfrm_policy_inexact_gc_tree(&node->root, rm);
1116 rn = rb_next(rn);
1117
1118 if (!hlist_empty(&node->hhead) || !RB_EMPTY_ROOT(&node->root)) {
1119 WARN_ON_ONCE(rm);
1120 continue;
1121 }
1122
1123 rb_erase(&node->node, r);
1124 kfree_rcu(node, rcu);
1125 }
1126 }
1127
__xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin * b,bool net_exit)1128 static void __xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b, bool net_exit)
1129 {
1130 write_seqcount_begin(&b->count);
1131 xfrm_policy_inexact_gc_tree(&b->root_d, net_exit);
1132 xfrm_policy_inexact_gc_tree(&b->root_s, net_exit);
1133 write_seqcount_end(&b->count);
1134
1135 if (!RB_EMPTY_ROOT(&b->root_d) || !RB_EMPTY_ROOT(&b->root_s) ||
1136 !hlist_empty(&b->hhead)) {
1137 WARN_ON_ONCE(net_exit);
1138 return;
1139 }
1140
1141 if (rhashtable_remove_fast(&xfrm_policy_inexact_table, &b->head,
1142 xfrm_pol_inexact_params) == 0) {
1143 list_del(&b->inexact_bins);
1144 kfree_rcu(b, rcu);
1145 }
1146 }
1147
xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin * b)1148 static void xfrm_policy_inexact_prune_bin(struct xfrm_pol_inexact_bin *b)
1149 {
1150 struct net *net = read_pnet(&b->k.net);
1151
1152 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1153 __xfrm_policy_inexact_prune_bin(b, false);
1154 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1155 }
1156
__xfrm_policy_inexact_flush(struct net * net)1157 static void __xfrm_policy_inexact_flush(struct net *net)
1158 {
1159 struct xfrm_pol_inexact_bin *bin, *t;
1160
1161 lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1162
1163 list_for_each_entry_safe(bin, t, &net->xfrm.inexact_bins, inexact_bins)
1164 __xfrm_policy_inexact_prune_bin(bin, false);
1165 }
1166
1167 static struct hlist_head *
xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin * bin,struct xfrm_policy * policy,u8 dir)1168 xfrm_policy_inexact_alloc_chain(struct xfrm_pol_inexact_bin *bin,
1169 struct xfrm_policy *policy, u8 dir)
1170 {
1171 struct xfrm_pol_inexact_node *n;
1172 struct net *net;
1173
1174 net = xp_net(policy);
1175 lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1176
1177 if (xfrm_policy_inexact_insert_use_any_list(policy))
1178 return &bin->hhead;
1179
1180 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.daddr,
1181 policy->family,
1182 policy->selector.prefixlen_d)) {
1183 write_seqcount_begin(&bin->count);
1184 n = xfrm_policy_inexact_insert_node(net,
1185 &bin->root_s,
1186 &policy->selector.saddr,
1187 policy->family,
1188 policy->selector.prefixlen_s,
1189 dir);
1190 write_seqcount_end(&bin->count);
1191 if (!n)
1192 return NULL;
1193
1194 return &n->hhead;
1195 }
1196
1197 /* daddr is fixed */
1198 write_seqcount_begin(&bin->count);
1199 n = xfrm_policy_inexact_insert_node(net,
1200 &bin->root_d,
1201 &policy->selector.daddr,
1202 policy->family,
1203 policy->selector.prefixlen_d, dir);
1204 write_seqcount_end(&bin->count);
1205 if (!n)
1206 return NULL;
1207
1208 /* saddr is wildcard */
1209 if (xfrm_pol_inexact_addr_use_any_list(&policy->selector.saddr,
1210 policy->family,
1211 policy->selector.prefixlen_s))
1212 return &n->hhead;
1213
1214 write_seqcount_begin(&bin->count);
1215 n = xfrm_policy_inexact_insert_node(net,
1216 &n->root,
1217 &policy->selector.saddr,
1218 policy->family,
1219 policy->selector.prefixlen_s, dir);
1220 write_seqcount_end(&bin->count);
1221 if (!n)
1222 return NULL;
1223
1224 return &n->hhead;
1225 }
1226
1227 static struct xfrm_policy *
xfrm_policy_inexact_insert(struct xfrm_policy * policy,u8 dir,int excl)1228 xfrm_policy_inexact_insert(struct xfrm_policy *policy, u8 dir, int excl)
1229 {
1230 struct xfrm_pol_inexact_bin *bin;
1231 struct xfrm_policy *delpol;
1232 struct hlist_head *chain;
1233 struct net *net;
1234
1235 bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1236 if (!bin)
1237 return ERR_PTR(-ENOMEM);
1238
1239 net = xp_net(policy);
1240 lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
1241
1242 chain = xfrm_policy_inexact_alloc_chain(bin, policy, dir);
1243 if (!chain) {
1244 __xfrm_policy_inexact_prune_bin(bin, false);
1245 return ERR_PTR(-ENOMEM);
1246 }
1247
1248 delpol = xfrm_policy_insert_list(chain, policy, excl);
1249 if (delpol && excl) {
1250 __xfrm_policy_inexact_prune_bin(bin, false);
1251 return ERR_PTR(-EEXIST);
1252 }
1253
1254 if (delpol)
1255 __xfrm_policy_inexact_prune_bin(bin, false);
1256
1257 return delpol;
1258 }
1259
xfrm_policy_is_dead_or_sk(const struct xfrm_policy * policy)1260 static bool xfrm_policy_is_dead_or_sk(const struct xfrm_policy *policy)
1261 {
1262 int dir;
1263
1264 if (policy->walk.dead)
1265 return true;
1266
1267 dir = xfrm_policy_id2dir(policy->index);
1268 return dir >= XFRM_POLICY_MAX;
1269 }
1270
xfrm_hash_rebuild(struct work_struct * work)1271 static void xfrm_hash_rebuild(struct work_struct *work)
1272 {
1273 struct net *net = container_of(work, struct net,
1274 xfrm.policy_hthresh.work);
1275 struct xfrm_policy *pol;
1276 struct xfrm_policy *policy;
1277 struct hlist_head *chain;
1278 struct hlist_node *newpos;
1279 int dir;
1280 unsigned seq;
1281 u8 lbits4, rbits4, lbits6, rbits6;
1282
1283 mutex_lock(&hash_resize_mutex);
1284
1285 /* read selector prefixlen thresholds */
1286 do {
1287 seq = read_seqbegin(&net->xfrm.policy_hthresh.lock);
1288
1289 lbits4 = net->xfrm.policy_hthresh.lbits4;
1290 rbits4 = net->xfrm.policy_hthresh.rbits4;
1291 lbits6 = net->xfrm.policy_hthresh.lbits6;
1292 rbits6 = net->xfrm.policy_hthresh.rbits6;
1293 } while (read_seqretry(&net->xfrm.policy_hthresh.lock, seq));
1294
1295 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1296 write_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
1297
1298 /* make sure that we can insert the indirect policies again before
1299 * we start with destructive action.
1300 */
1301 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
1302 struct xfrm_pol_inexact_bin *bin;
1303 u8 dbits, sbits;
1304
1305 if (xfrm_policy_is_dead_or_sk(policy))
1306 continue;
1307
1308 dir = xfrm_policy_id2dir(policy->index);
1309 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1310 if (policy->family == AF_INET) {
1311 dbits = rbits4;
1312 sbits = lbits4;
1313 } else {
1314 dbits = rbits6;
1315 sbits = lbits6;
1316 }
1317 } else {
1318 if (policy->family == AF_INET) {
1319 dbits = lbits4;
1320 sbits = rbits4;
1321 } else {
1322 dbits = lbits6;
1323 sbits = rbits6;
1324 }
1325 }
1326
1327 if (policy->selector.prefixlen_d < dbits ||
1328 policy->selector.prefixlen_s < sbits)
1329 continue;
1330
1331 bin = xfrm_policy_inexact_alloc_bin(policy, dir);
1332 if (!bin)
1333 goto out_unlock;
1334
1335 if (!xfrm_policy_inexact_alloc_chain(bin, policy, dir))
1336 goto out_unlock;
1337 }
1338
1339 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
1340 if ((dir & XFRM_POLICY_MASK) == XFRM_POLICY_OUT) {
1341 /* dir out => dst = remote, src = local */
1342 net->xfrm.policy_bydst[dir].dbits4 = rbits4;
1343 net->xfrm.policy_bydst[dir].sbits4 = lbits4;
1344 net->xfrm.policy_bydst[dir].dbits6 = rbits6;
1345 net->xfrm.policy_bydst[dir].sbits6 = lbits6;
1346 } else {
1347 /* dir in/fwd => dst = local, src = remote */
1348 net->xfrm.policy_bydst[dir].dbits4 = lbits4;
1349 net->xfrm.policy_bydst[dir].sbits4 = rbits4;
1350 net->xfrm.policy_bydst[dir].dbits6 = lbits6;
1351 net->xfrm.policy_bydst[dir].sbits6 = rbits6;
1352 }
1353 }
1354
1355 /* re-insert all policies by order of creation */
1356 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
1357 if (xfrm_policy_is_dead_or_sk(policy))
1358 continue;
1359
1360 hlist_del_rcu(&policy->bydst);
1361
1362 newpos = NULL;
1363 dir = xfrm_policy_id2dir(policy->index);
1364 chain = policy_hash_bysel(net, &policy->selector,
1365 policy->family, dir);
1366
1367 if (!chain) {
1368 void *p = xfrm_policy_inexact_insert(policy, dir, 0);
1369
1370 WARN_ONCE(IS_ERR(p), "reinsert: %ld\n", PTR_ERR(p));
1371 continue;
1372 }
1373
1374 hlist_for_each_entry(pol, chain, bydst) {
1375 if (policy->priority >= pol->priority)
1376 newpos = &pol->bydst;
1377 else
1378 break;
1379 }
1380 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1381 hlist_add_behind_rcu(&policy->bydst, newpos);
1382 else
1383 hlist_add_head_rcu(&policy->bydst, chain);
1384 }
1385
1386 out_unlock:
1387 __xfrm_policy_inexact_flush(net);
1388 write_seqcount_end(&net->xfrm.xfrm_policy_hash_generation);
1389 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1390
1391 mutex_unlock(&hash_resize_mutex);
1392 }
1393
xfrm_policy_hash_rebuild(struct net * net)1394 void xfrm_policy_hash_rebuild(struct net *net)
1395 {
1396 schedule_work(&net->xfrm.policy_hthresh.work);
1397 }
1398 EXPORT_SYMBOL(xfrm_policy_hash_rebuild);
1399
1400 /* Generate new index... KAME seems to generate them ordered by cost
1401 * of an absolute inpredictability of ordering of rules. This will not pass. */
xfrm_gen_index(struct net * net,int dir,u32 index)1402 static u32 xfrm_gen_index(struct net *net, int dir, u32 index)
1403 {
1404 for (;;) {
1405 struct hlist_head *list;
1406 struct xfrm_policy *p;
1407 u32 idx;
1408 int found;
1409
1410 if (!index) {
1411 idx = (net->xfrm.idx_generator | dir);
1412 net->xfrm.idx_generator += 8;
1413 } else {
1414 idx = index;
1415 index = 0;
1416 }
1417
1418 if (idx == 0)
1419 idx = 8;
1420 list = net->xfrm.policy_byidx + idx_hash(net, idx);
1421 found = 0;
1422 hlist_for_each_entry(p, list, byidx) {
1423 if (p->index == idx) {
1424 found = 1;
1425 break;
1426 }
1427 }
1428 if (!found)
1429 return idx;
1430 }
1431 }
1432
selector_cmp(struct xfrm_selector * s1,struct xfrm_selector * s2)1433 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
1434 {
1435 u32 *p1 = (u32 *) s1;
1436 u32 *p2 = (u32 *) s2;
1437 int len = sizeof(struct xfrm_selector) / sizeof(u32);
1438 int i;
1439
1440 for (i = 0; i < len; i++) {
1441 if (p1[i] != p2[i])
1442 return 1;
1443 }
1444
1445 return 0;
1446 }
1447
xfrm_policy_requeue(struct xfrm_policy * old,struct xfrm_policy * new)1448 static void xfrm_policy_requeue(struct xfrm_policy *old,
1449 struct xfrm_policy *new)
1450 {
1451 struct xfrm_policy_queue *pq = &old->polq;
1452 struct sk_buff_head list;
1453
1454 if (skb_queue_empty(&pq->hold_queue))
1455 return;
1456
1457 __skb_queue_head_init(&list);
1458
1459 spin_lock_bh(&pq->hold_queue.lock);
1460 skb_queue_splice_init(&pq->hold_queue, &list);
1461 if (del_timer(&pq->hold_timer))
1462 xfrm_pol_put(old);
1463 spin_unlock_bh(&pq->hold_queue.lock);
1464
1465 pq = &new->polq;
1466
1467 spin_lock_bh(&pq->hold_queue.lock);
1468 skb_queue_splice(&list, &pq->hold_queue);
1469 pq->timeout = XFRM_QUEUE_TMO_MIN;
1470 if (!mod_timer(&pq->hold_timer, jiffies))
1471 xfrm_pol_hold(new);
1472 spin_unlock_bh(&pq->hold_queue.lock);
1473 }
1474
xfrm_policy_mark_match(const struct xfrm_mark * mark,struct xfrm_policy * pol)1475 static inline bool xfrm_policy_mark_match(const struct xfrm_mark *mark,
1476 struct xfrm_policy *pol)
1477 {
1478 return mark->v == pol->mark.v && mark->m == pol->mark.m;
1479 }
1480
xfrm_pol_bin_key(const void * data,u32 len,u32 seed)1481 static u32 xfrm_pol_bin_key(const void *data, u32 len, u32 seed)
1482 {
1483 const struct xfrm_pol_inexact_key *k = data;
1484 u32 a = k->type << 24 | k->dir << 16 | k->family;
1485
1486 return jhash_3words(a, k->if_id, net_hash_mix(read_pnet(&k->net)),
1487 seed);
1488 }
1489
xfrm_pol_bin_obj(const void * data,u32 len,u32 seed)1490 static u32 xfrm_pol_bin_obj(const void *data, u32 len, u32 seed)
1491 {
1492 const struct xfrm_pol_inexact_bin *b = data;
1493
1494 return xfrm_pol_bin_key(&b->k, 0, seed);
1495 }
1496
xfrm_pol_bin_cmp(struct rhashtable_compare_arg * arg,const void * ptr)1497 static int xfrm_pol_bin_cmp(struct rhashtable_compare_arg *arg,
1498 const void *ptr)
1499 {
1500 const struct xfrm_pol_inexact_key *key = arg->key;
1501 const struct xfrm_pol_inexact_bin *b = ptr;
1502 int ret;
1503
1504 if (!net_eq(read_pnet(&b->k.net), read_pnet(&key->net)))
1505 return -1;
1506
1507 ret = b->k.dir ^ key->dir;
1508 if (ret)
1509 return ret;
1510
1511 ret = b->k.type ^ key->type;
1512 if (ret)
1513 return ret;
1514
1515 ret = b->k.family ^ key->family;
1516 if (ret)
1517 return ret;
1518
1519 return b->k.if_id ^ key->if_id;
1520 }
1521
1522 static const struct rhashtable_params xfrm_pol_inexact_params = {
1523 .head_offset = offsetof(struct xfrm_pol_inexact_bin, head),
1524 .hashfn = xfrm_pol_bin_key,
1525 .obj_hashfn = xfrm_pol_bin_obj,
1526 .obj_cmpfn = xfrm_pol_bin_cmp,
1527 .automatic_shrinking = true,
1528 };
1529
xfrm_policy_insert_list(struct hlist_head * chain,struct xfrm_policy * policy,bool excl)1530 static struct xfrm_policy *xfrm_policy_insert_list(struct hlist_head *chain,
1531 struct xfrm_policy *policy,
1532 bool excl)
1533 {
1534 struct xfrm_policy *pol, *newpos = NULL, *delpol = NULL;
1535
1536 hlist_for_each_entry(pol, chain, bydst) {
1537 if (pol->type == policy->type &&
1538 pol->if_id == policy->if_id &&
1539 !selector_cmp(&pol->selector, &policy->selector) &&
1540 xfrm_policy_mark_match(&policy->mark, pol) &&
1541 xfrm_sec_ctx_match(pol->security, policy->security) &&
1542 !WARN_ON(delpol)) {
1543 if (excl)
1544 return ERR_PTR(-EEXIST);
1545 delpol = pol;
1546 if (policy->priority > pol->priority)
1547 continue;
1548 } else if (policy->priority >= pol->priority) {
1549 newpos = pol;
1550 continue;
1551 }
1552 if (delpol)
1553 break;
1554 }
1555
1556 if (newpos && policy->xdo.type != XFRM_DEV_OFFLOAD_PACKET)
1557 hlist_add_behind_rcu(&policy->bydst, &newpos->bydst);
1558 else
1559 /* Packet offload policies enter to the head
1560 * to speed-up lookups.
1561 */
1562 hlist_add_head_rcu(&policy->bydst, chain);
1563
1564 return delpol;
1565 }
1566
xfrm_policy_insert(int dir,struct xfrm_policy * policy,int excl)1567 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
1568 {
1569 struct net *net = xp_net(policy);
1570 struct xfrm_policy *delpol;
1571 struct hlist_head *chain;
1572
1573 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1574 chain = policy_hash_bysel(net, &policy->selector, policy->family, dir);
1575 if (chain)
1576 delpol = xfrm_policy_insert_list(chain, policy, excl);
1577 else
1578 delpol = xfrm_policy_inexact_insert(policy, dir, excl);
1579
1580 if (IS_ERR(delpol)) {
1581 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1582 return PTR_ERR(delpol);
1583 }
1584
1585 __xfrm_policy_link(policy, dir);
1586
1587 /* After previous checking, family can either be AF_INET or AF_INET6 */
1588 if (policy->family == AF_INET)
1589 rt_genid_bump_ipv4(net);
1590 else
1591 rt_genid_bump_ipv6(net);
1592
1593 if (delpol) {
1594 xfrm_policy_requeue(delpol, policy);
1595 __xfrm_policy_unlink(delpol, dir);
1596 }
1597 policy->index = delpol ? delpol->index : xfrm_gen_index(net, dir, policy->index);
1598 hlist_add_head(&policy->byidx, net->xfrm.policy_byidx+idx_hash(net, policy->index));
1599 policy->curlft.add_time = ktime_get_real_seconds();
1600 policy->curlft.use_time = 0;
1601 if (!mod_timer(&policy->timer, jiffies + HZ))
1602 xfrm_pol_hold(policy);
1603 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1604
1605 if (delpol)
1606 xfrm_policy_kill(delpol);
1607 else if (xfrm_bydst_should_resize(net, dir, NULL))
1608 schedule_work(&net->xfrm.policy_hash_work);
1609
1610 return 0;
1611 }
1612 EXPORT_SYMBOL(xfrm_policy_insert);
1613
1614 static struct xfrm_policy *
__xfrm_policy_bysel_ctx(struct hlist_head * chain,const struct xfrm_mark * mark,u32 if_id,u8 type,int dir,struct xfrm_selector * sel,struct xfrm_sec_ctx * ctx)1615 __xfrm_policy_bysel_ctx(struct hlist_head *chain, const struct xfrm_mark *mark,
1616 u32 if_id, u8 type, int dir, struct xfrm_selector *sel,
1617 struct xfrm_sec_ctx *ctx)
1618 {
1619 struct xfrm_policy *pol;
1620
1621 if (!chain)
1622 return NULL;
1623
1624 hlist_for_each_entry(pol, chain, bydst) {
1625 if (pol->type == type &&
1626 pol->if_id == if_id &&
1627 xfrm_policy_mark_match(mark, pol) &&
1628 !selector_cmp(sel, &pol->selector) &&
1629 xfrm_sec_ctx_match(ctx, pol->security))
1630 return pol;
1631 }
1632
1633 return NULL;
1634 }
1635
1636 struct xfrm_policy *
xfrm_policy_bysel_ctx(struct net * net,const struct xfrm_mark * mark,u32 if_id,u8 type,int dir,struct xfrm_selector * sel,struct xfrm_sec_ctx * ctx,int delete,int * err)1637 xfrm_policy_bysel_ctx(struct net *net, const struct xfrm_mark *mark, u32 if_id,
1638 u8 type, int dir, struct xfrm_selector *sel,
1639 struct xfrm_sec_ctx *ctx, int delete, int *err)
1640 {
1641 struct xfrm_pol_inexact_bin *bin = NULL;
1642 struct xfrm_policy *pol, *ret = NULL;
1643 struct hlist_head *chain;
1644
1645 *err = 0;
1646 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1647 chain = policy_hash_bysel(net, sel, sel->family, dir);
1648 if (!chain) {
1649 struct xfrm_pol_inexact_candidates cand;
1650 int i;
1651
1652 bin = xfrm_policy_inexact_lookup(net, type,
1653 sel->family, dir, if_id);
1654 if (!bin) {
1655 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1656 return NULL;
1657 }
1658
1659 if (!xfrm_policy_find_inexact_candidates(&cand, bin,
1660 &sel->saddr,
1661 &sel->daddr)) {
1662 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1663 return NULL;
1664 }
1665
1666 pol = NULL;
1667 for (i = 0; i < ARRAY_SIZE(cand.res); i++) {
1668 struct xfrm_policy *tmp;
1669
1670 tmp = __xfrm_policy_bysel_ctx(cand.res[i], mark,
1671 if_id, type, dir,
1672 sel, ctx);
1673 if (!tmp)
1674 continue;
1675
1676 if (!pol || tmp->pos < pol->pos)
1677 pol = tmp;
1678 }
1679 } else {
1680 pol = __xfrm_policy_bysel_ctx(chain, mark, if_id, type, dir,
1681 sel, ctx);
1682 }
1683
1684 if (pol) {
1685 xfrm_pol_hold(pol);
1686 if (delete) {
1687 *err = security_xfrm_policy_delete(pol->security);
1688 if (*err) {
1689 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1690 return pol;
1691 }
1692 __xfrm_policy_unlink(pol, dir);
1693 }
1694 ret = pol;
1695 }
1696 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1697
1698 if (ret && delete)
1699 xfrm_policy_kill(ret);
1700 if (bin && delete)
1701 xfrm_policy_inexact_prune_bin(bin);
1702 return ret;
1703 }
1704 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
1705
1706 struct xfrm_policy *
xfrm_policy_byid(struct net * net,const struct xfrm_mark * mark,u32 if_id,u8 type,int dir,u32 id,int delete,int * err)1707 xfrm_policy_byid(struct net *net, const struct xfrm_mark *mark, u32 if_id,
1708 u8 type, int dir, u32 id, int delete, int *err)
1709 {
1710 struct xfrm_policy *pol, *ret;
1711 struct hlist_head *chain;
1712
1713 *err = -ENOENT;
1714 if (xfrm_policy_id2dir(id) != dir)
1715 return NULL;
1716
1717 *err = 0;
1718 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1719 chain = net->xfrm.policy_byidx + idx_hash(net, id);
1720 ret = NULL;
1721 hlist_for_each_entry(pol, chain, byidx) {
1722 if (pol->type == type && pol->index == id &&
1723 pol->if_id == if_id && xfrm_policy_mark_match(mark, pol)) {
1724 xfrm_pol_hold(pol);
1725 if (delete) {
1726 *err = security_xfrm_policy_delete(
1727 pol->security);
1728 if (*err) {
1729 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1730 return pol;
1731 }
1732 __xfrm_policy_unlink(pol, dir);
1733 }
1734 ret = pol;
1735 break;
1736 }
1737 }
1738 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1739
1740 if (ret && delete)
1741 xfrm_policy_kill(ret);
1742 return ret;
1743 }
1744 EXPORT_SYMBOL(xfrm_policy_byid);
1745
1746 #ifdef CONFIG_SECURITY_NETWORK_XFRM
1747 static inline int
xfrm_policy_flush_secctx_check(struct net * net,u8 type,bool task_valid)1748 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1749 {
1750 struct xfrm_policy *pol;
1751 int err = 0;
1752
1753 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1754 if (pol->walk.dead ||
1755 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1756 pol->type != type)
1757 continue;
1758
1759 err = security_xfrm_policy_delete(pol->security);
1760 if (err) {
1761 xfrm_audit_policy_delete(pol, 0, task_valid);
1762 return err;
1763 }
1764 }
1765 return err;
1766 }
1767
xfrm_dev_policy_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)1768 static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1769 struct net_device *dev,
1770 bool task_valid)
1771 {
1772 struct xfrm_policy *pol;
1773 int err = 0;
1774
1775 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1776 if (pol->walk.dead ||
1777 xfrm_policy_id2dir(pol->index) >= XFRM_POLICY_MAX ||
1778 pol->xdo.dev != dev)
1779 continue;
1780
1781 err = security_xfrm_policy_delete(pol->security);
1782 if (err) {
1783 xfrm_audit_policy_delete(pol, 0, task_valid);
1784 return err;
1785 }
1786 }
1787 return err;
1788 }
1789 #else
1790 static inline int
xfrm_policy_flush_secctx_check(struct net * net,u8 type,bool task_valid)1791 xfrm_policy_flush_secctx_check(struct net *net, u8 type, bool task_valid)
1792 {
1793 return 0;
1794 }
1795
xfrm_dev_policy_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)1796 static inline int xfrm_dev_policy_flush_secctx_check(struct net *net,
1797 struct net_device *dev,
1798 bool task_valid)
1799 {
1800 return 0;
1801 }
1802 #endif
1803
xfrm_policy_flush(struct net * net,u8 type,bool task_valid)1804 int xfrm_policy_flush(struct net *net, u8 type, bool task_valid)
1805 {
1806 int dir, err = 0, cnt = 0;
1807 struct xfrm_policy *pol;
1808
1809 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1810
1811 err = xfrm_policy_flush_secctx_check(net, type, task_valid);
1812 if (err)
1813 goto out;
1814
1815 again:
1816 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1817 if (pol->walk.dead)
1818 continue;
1819
1820 dir = xfrm_policy_id2dir(pol->index);
1821 if (dir >= XFRM_POLICY_MAX ||
1822 pol->type != type)
1823 continue;
1824
1825 __xfrm_policy_unlink(pol, dir);
1826 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1827 cnt++;
1828 xfrm_audit_policy_delete(pol, 1, task_valid);
1829 xfrm_policy_kill(pol);
1830 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1831 goto again;
1832 }
1833 if (cnt)
1834 __xfrm_policy_inexact_flush(net);
1835 else
1836 err = -ESRCH;
1837 out:
1838 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1839 return err;
1840 }
1841 EXPORT_SYMBOL(xfrm_policy_flush);
1842
xfrm_dev_policy_flush(struct net * net,struct net_device * dev,bool task_valid)1843 int xfrm_dev_policy_flush(struct net *net, struct net_device *dev,
1844 bool task_valid)
1845 {
1846 int dir, err = 0, cnt = 0;
1847 struct xfrm_policy *pol;
1848
1849 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1850
1851 err = xfrm_dev_policy_flush_secctx_check(net, dev, task_valid);
1852 if (err)
1853 goto out;
1854
1855 again:
1856 list_for_each_entry(pol, &net->xfrm.policy_all, walk.all) {
1857 if (pol->walk.dead)
1858 continue;
1859
1860 dir = xfrm_policy_id2dir(pol->index);
1861 if (dir >= XFRM_POLICY_MAX ||
1862 pol->xdo.dev != dev)
1863 continue;
1864
1865 __xfrm_policy_unlink(pol, dir);
1866 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1867 cnt++;
1868 xfrm_audit_policy_delete(pol, 1, task_valid);
1869 xfrm_policy_kill(pol);
1870 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1871 goto again;
1872 }
1873 if (cnt)
1874 __xfrm_policy_inexact_flush(net);
1875 else
1876 err = -ESRCH;
1877 out:
1878 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1879 return err;
1880 }
1881 EXPORT_SYMBOL(xfrm_dev_policy_flush);
1882
xfrm_policy_walk(struct net * net,struct xfrm_policy_walk * walk,int (* func)(struct xfrm_policy *,int,int,void *),void * data)1883 int xfrm_policy_walk(struct net *net, struct xfrm_policy_walk *walk,
1884 int (*func)(struct xfrm_policy *, int, int, void*),
1885 void *data)
1886 {
1887 struct xfrm_policy *pol;
1888 struct xfrm_policy_walk_entry *x;
1889 int error = 0;
1890
1891 if (walk->type >= XFRM_POLICY_TYPE_MAX &&
1892 walk->type != XFRM_POLICY_TYPE_ANY)
1893 return -EINVAL;
1894
1895 if (list_empty(&walk->walk.all) && walk->seq != 0)
1896 return 0;
1897
1898 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
1899 if (list_empty(&walk->walk.all))
1900 x = list_first_entry(&net->xfrm.policy_all, struct xfrm_policy_walk_entry, all);
1901 else
1902 x = list_first_entry(&walk->walk.all,
1903 struct xfrm_policy_walk_entry, all);
1904
1905 list_for_each_entry_from(x, &net->xfrm.policy_all, all) {
1906 if (x->dead)
1907 continue;
1908 pol = container_of(x, struct xfrm_policy, walk);
1909 if (walk->type != XFRM_POLICY_TYPE_ANY &&
1910 walk->type != pol->type)
1911 continue;
1912 error = func(pol, xfrm_policy_id2dir(pol->index),
1913 walk->seq, data);
1914 if (error) {
1915 list_move_tail(&walk->walk.all, &x->all);
1916 goto out;
1917 }
1918 walk->seq++;
1919 }
1920 if (walk->seq == 0) {
1921 error = -ENOENT;
1922 goto out;
1923 }
1924 list_del_init(&walk->walk.all);
1925 out:
1926 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1927 return error;
1928 }
1929 EXPORT_SYMBOL(xfrm_policy_walk);
1930
xfrm_policy_walk_init(struct xfrm_policy_walk * walk,u8 type)1931 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1932 {
1933 INIT_LIST_HEAD(&walk->walk.all);
1934 walk->walk.dead = 1;
1935 walk->type = type;
1936 walk->seq = 0;
1937 }
1938 EXPORT_SYMBOL(xfrm_policy_walk_init);
1939
xfrm_policy_walk_done(struct xfrm_policy_walk * walk,struct net * net)1940 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk, struct net *net)
1941 {
1942 if (list_empty(&walk->walk.all))
1943 return;
1944
1945 spin_lock_bh(&net->xfrm.xfrm_policy_lock); /*FIXME where is net? */
1946 list_del(&walk->walk.all);
1947 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
1948 }
1949 EXPORT_SYMBOL(xfrm_policy_walk_done);
1950
1951 /*
1952 * Find policy to apply to this flow.
1953 *
1954 * Returns 0 if policy found, else an -errno.
1955 */
xfrm_policy_match(const struct xfrm_policy * pol,const struct flowi * fl,u8 type,u16 family,u32 if_id)1956 static int xfrm_policy_match(const struct xfrm_policy *pol,
1957 const struct flowi *fl,
1958 u8 type, u16 family, u32 if_id)
1959 {
1960 const struct xfrm_selector *sel = &pol->selector;
1961 int ret = -ESRCH;
1962 bool match;
1963
1964 if (pol->family != family ||
1965 pol->if_id != if_id ||
1966 (fl->flowi_mark & pol->mark.m) != pol->mark.v ||
1967 pol->type != type)
1968 return ret;
1969
1970 match = xfrm_selector_match(sel, fl, family);
1971 if (match)
1972 ret = security_xfrm_policy_lookup(pol->security, fl->flowi_secid);
1973 return ret;
1974 }
1975
1976 static struct xfrm_pol_inexact_node *
xfrm_policy_lookup_inexact_addr(const struct rb_root * r,seqcount_spinlock_t * count,const xfrm_address_t * addr,u16 family)1977 xfrm_policy_lookup_inexact_addr(const struct rb_root *r,
1978 seqcount_spinlock_t *count,
1979 const xfrm_address_t *addr, u16 family)
1980 {
1981 const struct rb_node *parent;
1982 int seq;
1983
1984 again:
1985 seq = read_seqcount_begin(count);
1986
1987 parent = rcu_dereference_raw(r->rb_node);
1988 while (parent) {
1989 struct xfrm_pol_inexact_node *node;
1990 int delta;
1991
1992 node = rb_entry(parent, struct xfrm_pol_inexact_node, node);
1993
1994 delta = xfrm_policy_addr_delta(addr, &node->addr,
1995 node->prefixlen, family);
1996 if (delta < 0) {
1997 parent = rcu_dereference_raw(parent->rb_left);
1998 continue;
1999 } else if (delta > 0) {
2000 parent = rcu_dereference_raw(parent->rb_right);
2001 continue;
2002 }
2003
2004 return node;
2005 }
2006
2007 if (read_seqcount_retry(count, seq))
2008 goto again;
2009
2010 return NULL;
2011 }
2012
2013 static bool
xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates * cand,struct xfrm_pol_inexact_bin * b,const xfrm_address_t * saddr,const xfrm_address_t * daddr)2014 xfrm_policy_find_inexact_candidates(struct xfrm_pol_inexact_candidates *cand,
2015 struct xfrm_pol_inexact_bin *b,
2016 const xfrm_address_t *saddr,
2017 const xfrm_address_t *daddr)
2018 {
2019 struct xfrm_pol_inexact_node *n;
2020 u16 family;
2021
2022 if (!b)
2023 return false;
2024
2025 family = b->k.family;
2026 memset(cand, 0, sizeof(*cand));
2027 cand->res[XFRM_POL_CAND_ANY] = &b->hhead;
2028
2029 n = xfrm_policy_lookup_inexact_addr(&b->root_d, &b->count, daddr,
2030 family);
2031 if (n) {
2032 cand->res[XFRM_POL_CAND_DADDR] = &n->hhead;
2033 n = xfrm_policy_lookup_inexact_addr(&n->root, &b->count, saddr,
2034 family);
2035 if (n)
2036 cand->res[XFRM_POL_CAND_BOTH] = &n->hhead;
2037 }
2038
2039 n = xfrm_policy_lookup_inexact_addr(&b->root_s, &b->count, saddr,
2040 family);
2041 if (n)
2042 cand->res[XFRM_POL_CAND_SADDR] = &n->hhead;
2043
2044 return true;
2045 }
2046
2047 static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup_rcu(struct net * net,u8 type,u16 family,u8 dir,u32 if_id)2048 xfrm_policy_inexact_lookup_rcu(struct net *net, u8 type, u16 family,
2049 u8 dir, u32 if_id)
2050 {
2051 struct xfrm_pol_inexact_key k = {
2052 .family = family,
2053 .type = type,
2054 .dir = dir,
2055 .if_id = if_id,
2056 };
2057
2058 write_pnet(&k.net, net);
2059
2060 return rhashtable_lookup(&xfrm_policy_inexact_table, &k,
2061 xfrm_pol_inexact_params);
2062 }
2063
2064 static struct xfrm_pol_inexact_bin *
xfrm_policy_inexact_lookup(struct net * net,u8 type,u16 family,u8 dir,u32 if_id)2065 xfrm_policy_inexact_lookup(struct net *net, u8 type, u16 family,
2066 u8 dir, u32 if_id)
2067 {
2068 struct xfrm_pol_inexact_bin *bin;
2069
2070 lockdep_assert_held(&net->xfrm.xfrm_policy_lock);
2071
2072 rcu_read_lock();
2073 bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2074 rcu_read_unlock();
2075
2076 return bin;
2077 }
2078
2079 static struct xfrm_policy *
__xfrm_policy_eval_candidates(struct hlist_head * chain,struct xfrm_policy * prefer,const struct flowi * fl,u8 type,u16 family,u32 if_id)2080 __xfrm_policy_eval_candidates(struct hlist_head *chain,
2081 struct xfrm_policy *prefer,
2082 const struct flowi *fl,
2083 u8 type, u16 family, u32 if_id)
2084 {
2085 u32 priority = prefer ? prefer->priority : ~0u;
2086 struct xfrm_policy *pol;
2087
2088 if (!chain)
2089 return NULL;
2090
2091 hlist_for_each_entry_rcu(pol, chain, bydst) {
2092 int err;
2093
2094 if (pol->priority > priority)
2095 break;
2096
2097 err = xfrm_policy_match(pol, fl, type, family, if_id);
2098 if (err) {
2099 if (err != -ESRCH)
2100 return ERR_PTR(err);
2101
2102 continue;
2103 }
2104
2105 if (prefer) {
2106 /* matches. Is it older than *prefer? */
2107 if (pol->priority == priority &&
2108 prefer->pos < pol->pos)
2109 return prefer;
2110 }
2111
2112 return pol;
2113 }
2114
2115 return NULL;
2116 }
2117
2118 static struct xfrm_policy *
xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates * cand,struct xfrm_policy * prefer,const struct flowi * fl,u8 type,u16 family,u32 if_id)2119 xfrm_policy_eval_candidates(struct xfrm_pol_inexact_candidates *cand,
2120 struct xfrm_policy *prefer,
2121 const struct flowi *fl,
2122 u8 type, u16 family, u32 if_id)
2123 {
2124 struct xfrm_policy *tmp;
2125 int i;
2126
2127 for (i = 0; i < ARRAY_SIZE(cand->res); i++) {
2128 tmp = __xfrm_policy_eval_candidates(cand->res[i],
2129 prefer,
2130 fl, type, family, if_id);
2131 if (!tmp)
2132 continue;
2133
2134 if (IS_ERR(tmp))
2135 return tmp;
2136 prefer = tmp;
2137 }
2138
2139 return prefer;
2140 }
2141
xfrm_policy_lookup_bytype(struct net * net,u8 type,const struct flowi * fl,u16 family,u8 dir,u32 if_id)2142 static struct xfrm_policy *xfrm_policy_lookup_bytype(struct net *net, u8 type,
2143 const struct flowi *fl,
2144 u16 family, u8 dir,
2145 u32 if_id)
2146 {
2147 struct xfrm_pol_inexact_candidates cand;
2148 const xfrm_address_t *daddr, *saddr;
2149 struct xfrm_pol_inexact_bin *bin;
2150 struct xfrm_policy *pol, *ret;
2151 struct hlist_head *chain;
2152 unsigned int sequence;
2153 int err;
2154
2155 daddr = xfrm_flowi_daddr(fl, family);
2156 saddr = xfrm_flowi_saddr(fl, family);
2157 if (unlikely(!daddr || !saddr))
2158 return NULL;
2159
2160 rcu_read_lock();
2161 retry:
2162 do {
2163 sequence = read_seqcount_begin(&net->xfrm.xfrm_policy_hash_generation);
2164 chain = policy_hash_direct(net, daddr, saddr, family, dir);
2165 } while (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence));
2166
2167 ret = NULL;
2168 hlist_for_each_entry_rcu(pol, chain, bydst) {
2169 err = xfrm_policy_match(pol, fl, type, family, if_id);
2170 if (err) {
2171 if (err == -ESRCH)
2172 continue;
2173 else {
2174 ret = ERR_PTR(err);
2175 goto fail;
2176 }
2177 } else {
2178 ret = pol;
2179 break;
2180 }
2181 }
2182 if (ret && ret->xdo.type == XFRM_DEV_OFFLOAD_PACKET)
2183 goto skip_inexact;
2184
2185 bin = xfrm_policy_inexact_lookup_rcu(net, type, family, dir, if_id);
2186 if (!bin || !xfrm_policy_find_inexact_candidates(&cand, bin, saddr,
2187 daddr))
2188 goto skip_inexact;
2189
2190 pol = xfrm_policy_eval_candidates(&cand, ret, fl, type,
2191 family, if_id);
2192 if (pol) {
2193 ret = pol;
2194 if (IS_ERR(pol))
2195 goto fail;
2196 }
2197
2198 skip_inexact:
2199 if (read_seqcount_retry(&net->xfrm.xfrm_policy_hash_generation, sequence))
2200 goto retry;
2201
2202 if (ret && !xfrm_pol_hold_rcu(ret))
2203 goto retry;
2204 fail:
2205 rcu_read_unlock();
2206
2207 return ret;
2208 }
2209
xfrm_policy_lookup(struct net * net,const struct flowi * fl,u16 family,u8 dir,u32 if_id)2210 static struct xfrm_policy *xfrm_policy_lookup(struct net *net,
2211 const struct flowi *fl,
2212 u16 family, u8 dir, u32 if_id)
2213 {
2214 #ifdef CONFIG_XFRM_SUB_POLICY
2215 struct xfrm_policy *pol;
2216
2217 pol = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_SUB, fl, family,
2218 dir, if_id);
2219 if (pol != NULL)
2220 return pol;
2221 #endif
2222 return xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN, fl, family,
2223 dir, if_id);
2224 }
2225
xfrm_sk_policy_lookup(const struct sock * sk,int dir,const struct flowi * fl,u16 family,u32 if_id)2226 static struct xfrm_policy *xfrm_sk_policy_lookup(const struct sock *sk, int dir,
2227 const struct flowi *fl,
2228 u16 family, u32 if_id)
2229 {
2230 struct xfrm_policy *pol;
2231
2232 rcu_read_lock();
2233 again:
2234 pol = rcu_dereference(sk->sk_policy[dir]);
2235 if (pol != NULL) {
2236 bool match;
2237 int err = 0;
2238
2239 if (pol->family != family) {
2240 pol = NULL;
2241 goto out;
2242 }
2243
2244 match = xfrm_selector_match(&pol->selector, fl, family);
2245 if (match) {
2246 if ((READ_ONCE(sk->sk_mark) & pol->mark.m) != pol->mark.v ||
2247 pol->if_id != if_id) {
2248 pol = NULL;
2249 goto out;
2250 }
2251 err = security_xfrm_policy_lookup(pol->security,
2252 fl->flowi_secid);
2253 if (!err) {
2254 if (!xfrm_pol_hold_rcu(pol))
2255 goto again;
2256 } else if (err == -ESRCH) {
2257 pol = NULL;
2258 } else {
2259 pol = ERR_PTR(err);
2260 }
2261 } else
2262 pol = NULL;
2263 }
2264 out:
2265 rcu_read_unlock();
2266 return pol;
2267 }
2268
xfrm_gen_pos_slow(struct net * net)2269 static u32 xfrm_gen_pos_slow(struct net *net)
2270 {
2271 struct xfrm_policy *policy;
2272 u32 i = 0;
2273
2274 /* oldest entry is last in list */
2275 list_for_each_entry_reverse(policy, &net->xfrm.policy_all, walk.all) {
2276 if (!xfrm_policy_is_dead_or_sk(policy))
2277 policy->pos = ++i;
2278 }
2279
2280 return i;
2281 }
2282
xfrm_gen_pos(struct net * net)2283 static u32 xfrm_gen_pos(struct net *net)
2284 {
2285 const struct xfrm_policy *policy;
2286 u32 i = 0;
2287
2288 /* most recently added policy is at the head of the list */
2289 list_for_each_entry(policy, &net->xfrm.policy_all, walk.all) {
2290 if (xfrm_policy_is_dead_or_sk(policy))
2291 continue;
2292
2293 if (policy->pos == UINT_MAX)
2294 return xfrm_gen_pos_slow(net);
2295
2296 i = policy->pos + 1;
2297 break;
2298 }
2299
2300 return i;
2301 }
2302
__xfrm_policy_link(struct xfrm_policy * pol,int dir)2303 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
2304 {
2305 struct net *net = xp_net(pol);
2306
2307 switch (dir) {
2308 case XFRM_POLICY_IN:
2309 case XFRM_POLICY_FWD:
2310 case XFRM_POLICY_OUT:
2311 pol->pos = xfrm_gen_pos(net);
2312 break;
2313 }
2314
2315 list_add(&pol->walk.all, &net->xfrm.policy_all);
2316 net->xfrm.policy_count[dir]++;
2317 xfrm_pol_hold(pol);
2318 }
2319
__xfrm_policy_unlink(struct xfrm_policy * pol,int dir)2320 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
2321 int dir)
2322 {
2323 struct net *net = xp_net(pol);
2324
2325 if (list_empty(&pol->walk.all))
2326 return NULL;
2327
2328 /* Socket policies are not hashed. */
2329 if (!hlist_unhashed(&pol->bydst)) {
2330 hlist_del_rcu(&pol->bydst);
2331 hlist_del(&pol->byidx);
2332 }
2333
2334 list_del_init(&pol->walk.all);
2335 net->xfrm.policy_count[dir]--;
2336
2337 return pol;
2338 }
2339
xfrm_sk_policy_link(struct xfrm_policy * pol,int dir)2340 static void xfrm_sk_policy_link(struct xfrm_policy *pol, int dir)
2341 {
2342 __xfrm_policy_link(pol, XFRM_POLICY_MAX + dir);
2343 }
2344
xfrm_sk_policy_unlink(struct xfrm_policy * pol,int dir)2345 static void xfrm_sk_policy_unlink(struct xfrm_policy *pol, int dir)
2346 {
2347 __xfrm_policy_unlink(pol, XFRM_POLICY_MAX + dir);
2348 }
2349
xfrm_policy_delete(struct xfrm_policy * pol,int dir)2350 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
2351 {
2352 struct net *net = xp_net(pol);
2353
2354 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2355 pol = __xfrm_policy_unlink(pol, dir);
2356 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2357 if (pol) {
2358 xfrm_policy_kill(pol);
2359 return 0;
2360 }
2361 return -ENOENT;
2362 }
2363 EXPORT_SYMBOL(xfrm_policy_delete);
2364
xfrm_sk_policy_insert(struct sock * sk,int dir,struct xfrm_policy * pol)2365 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
2366 {
2367 struct net *net = sock_net(sk);
2368 struct xfrm_policy *old_pol;
2369
2370 #ifdef CONFIG_XFRM_SUB_POLICY
2371 if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
2372 return -EINVAL;
2373 #endif
2374
2375 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2376 old_pol = rcu_dereference_protected(sk->sk_policy[dir],
2377 lockdep_is_held(&net->xfrm.xfrm_policy_lock));
2378 if (pol) {
2379 pol->curlft.add_time = ktime_get_real_seconds();
2380 pol->index = xfrm_gen_index(net, XFRM_POLICY_MAX+dir, 0);
2381 xfrm_sk_policy_link(pol, dir);
2382 }
2383 rcu_assign_pointer(sk->sk_policy[dir], pol);
2384 if (old_pol) {
2385 if (pol)
2386 xfrm_policy_requeue(old_pol, pol);
2387
2388 /* Unlinking succeeds always. This is the only function
2389 * allowed to delete or replace socket policy.
2390 */
2391 xfrm_sk_policy_unlink(old_pol, dir);
2392 }
2393 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2394
2395 if (old_pol) {
2396 xfrm_policy_kill(old_pol);
2397 }
2398 return 0;
2399 }
2400
clone_policy(const struct xfrm_policy * old,int dir)2401 static struct xfrm_policy *clone_policy(const struct xfrm_policy *old, int dir)
2402 {
2403 struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
2404 struct net *net = xp_net(old);
2405
2406 if (newp) {
2407 newp->selector = old->selector;
2408 if (security_xfrm_policy_clone(old->security,
2409 &newp->security)) {
2410 kfree(newp);
2411 return NULL; /* ENOMEM */
2412 }
2413 newp->lft = old->lft;
2414 newp->curlft = old->curlft;
2415 newp->mark = old->mark;
2416 newp->if_id = old->if_id;
2417 newp->action = old->action;
2418 newp->flags = old->flags;
2419 newp->xfrm_nr = old->xfrm_nr;
2420 newp->index = old->index;
2421 newp->type = old->type;
2422 newp->family = old->family;
2423 memcpy(newp->xfrm_vec, old->xfrm_vec,
2424 newp->xfrm_nr*sizeof(struct xfrm_tmpl));
2425 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
2426 xfrm_sk_policy_link(newp, dir);
2427 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
2428 xfrm_pol_put(newp);
2429 }
2430 return newp;
2431 }
2432
__xfrm_sk_clone_policy(struct sock * sk,const struct sock * osk)2433 int __xfrm_sk_clone_policy(struct sock *sk, const struct sock *osk)
2434 {
2435 const struct xfrm_policy *p;
2436 struct xfrm_policy *np;
2437 int i, ret = 0;
2438
2439 rcu_read_lock();
2440 for (i = 0; i < 2; i++) {
2441 p = rcu_dereference(osk->sk_policy[i]);
2442 if (p) {
2443 np = clone_policy(p, i);
2444 if (unlikely(!np)) {
2445 ret = -ENOMEM;
2446 break;
2447 }
2448 rcu_assign_pointer(sk->sk_policy[i], np);
2449 }
2450 }
2451 rcu_read_unlock();
2452 return ret;
2453 }
2454
2455 static int
xfrm_get_saddr(unsigned short family,xfrm_address_t * saddr,const struct xfrm_dst_lookup_params * params)2456 xfrm_get_saddr(unsigned short family, xfrm_address_t *saddr,
2457 const struct xfrm_dst_lookup_params *params)
2458 {
2459 int err;
2460 const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2461
2462 if (unlikely(afinfo == NULL))
2463 return -EINVAL;
2464 err = afinfo->get_saddr(saddr, params);
2465 rcu_read_unlock();
2466 return err;
2467 }
2468
2469 /* Resolve list of templates for the flow, given policy. */
2470
2471 static int
xfrm_tmpl_resolve_one(struct xfrm_policy * policy,const struct flowi * fl,struct xfrm_state ** xfrm,unsigned short family)2472 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, const struct flowi *fl,
2473 struct xfrm_state **xfrm, unsigned short family)
2474 {
2475 struct net *net = xp_net(policy);
2476 int nx;
2477 int i, error;
2478 xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
2479 xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
2480 xfrm_address_t tmp;
2481
2482 for (nx = 0, i = 0; i < policy->xfrm_nr; i++) {
2483 struct xfrm_state *x;
2484 xfrm_address_t *remote = daddr;
2485 xfrm_address_t *local = saddr;
2486 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
2487
2488 if (tmpl->mode == XFRM_MODE_TUNNEL ||
2489 tmpl->mode == XFRM_MODE_BEET) {
2490 remote = &tmpl->id.daddr;
2491 local = &tmpl->saddr;
2492 if (xfrm_addr_any(local, tmpl->encap_family)) {
2493 struct xfrm_dst_lookup_params params;
2494
2495 memset(¶ms, 0, sizeof(params));
2496 params.net = net;
2497 params.oif = fl->flowi_oif;
2498 params.daddr = remote;
2499 error = xfrm_get_saddr(tmpl->encap_family, &tmp,
2500 ¶ms);
2501 if (error)
2502 goto fail;
2503 local = &tmp;
2504 }
2505 }
2506
2507 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error,
2508 family, policy->if_id);
2509 if (x && x->dir && x->dir != XFRM_SA_DIR_OUT) {
2510 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTSTATEDIRERROR);
2511 xfrm_state_put(x);
2512 error = -EINVAL;
2513 goto fail;
2514 }
2515
2516 if (x && x->km.state == XFRM_STATE_VALID) {
2517 xfrm[nx++] = x;
2518 daddr = remote;
2519 saddr = local;
2520 continue;
2521 }
2522 if (x) {
2523 error = (x->km.state == XFRM_STATE_ERROR ?
2524 -EINVAL : -EAGAIN);
2525 xfrm_state_put(x);
2526 } else if (error == -ESRCH) {
2527 error = -EAGAIN;
2528 }
2529
2530 if (!tmpl->optional)
2531 goto fail;
2532 }
2533 return nx;
2534
2535 fail:
2536 for (nx--; nx >= 0; nx--)
2537 xfrm_state_put(xfrm[nx]);
2538 return error;
2539 }
2540
2541 static int
xfrm_tmpl_resolve(struct xfrm_policy ** pols,int npols,const struct flowi * fl,struct xfrm_state ** xfrm,unsigned short family)2542 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, const struct flowi *fl,
2543 struct xfrm_state **xfrm, unsigned short family)
2544 {
2545 struct xfrm_state *tp[XFRM_MAX_DEPTH];
2546 struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
2547 int cnx = 0;
2548 int error;
2549 int ret;
2550 int i;
2551
2552 for (i = 0; i < npols; i++) {
2553 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
2554 error = -ENOBUFS;
2555 goto fail;
2556 }
2557
2558 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
2559 if (ret < 0) {
2560 error = ret;
2561 goto fail;
2562 } else
2563 cnx += ret;
2564 }
2565
2566 /* found states are sorted for outbound processing */
2567 if (npols > 1)
2568 xfrm_state_sort(xfrm, tpp, cnx, family);
2569
2570 return cnx;
2571
2572 fail:
2573 for (cnx--; cnx >= 0; cnx--)
2574 xfrm_state_put(tpp[cnx]);
2575 return error;
2576
2577 }
2578
xfrm_get_tos(const struct flowi * fl,int family)2579 static int xfrm_get_tos(const struct flowi *fl, int family)
2580 {
2581 if (family == AF_INET)
2582 return fl->u.ip4.flowi4_tos & INET_DSCP_MASK;
2583
2584 return 0;
2585 }
2586
xfrm_alloc_dst(struct net * net,int family)2587 static inline struct xfrm_dst *xfrm_alloc_dst(struct net *net, int family)
2588 {
2589 const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
2590 struct dst_ops *dst_ops;
2591 struct xfrm_dst *xdst;
2592
2593 if (!afinfo)
2594 return ERR_PTR(-EINVAL);
2595
2596 switch (family) {
2597 case AF_INET:
2598 dst_ops = &net->xfrm.xfrm4_dst_ops;
2599 break;
2600 #if IS_ENABLED(CONFIG_IPV6)
2601 case AF_INET6:
2602 dst_ops = &net->xfrm.xfrm6_dst_ops;
2603 break;
2604 #endif
2605 default:
2606 BUG();
2607 }
2608 xdst = dst_alloc(dst_ops, NULL, DST_OBSOLETE_NONE, 0);
2609
2610 if (likely(xdst)) {
2611 memset_after(xdst, 0, u.dst);
2612 } else
2613 xdst = ERR_PTR(-ENOBUFS);
2614
2615 rcu_read_unlock();
2616
2617 return xdst;
2618 }
2619
xfrm_init_path(struct xfrm_dst * path,struct dst_entry * dst,int nfheader_len)2620 static void xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
2621 int nfheader_len)
2622 {
2623 if (dst->ops->family == AF_INET6) {
2624 path->path_cookie = rt6_get_cookie(dst_rt6_info(dst));
2625 path->u.rt6.rt6i_nfheader_len = nfheader_len;
2626 }
2627 }
2628
xfrm_fill_dst(struct xfrm_dst * xdst,struct net_device * dev,const struct flowi * fl)2629 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
2630 const struct flowi *fl)
2631 {
2632 const struct xfrm_policy_afinfo *afinfo =
2633 xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
2634 int err;
2635
2636 if (!afinfo)
2637 return -EINVAL;
2638
2639 err = afinfo->fill_dst(xdst, dev, fl);
2640
2641 rcu_read_unlock();
2642
2643 return err;
2644 }
2645
2646
2647 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
2648 * all the metrics... Shortly, bundle a bundle.
2649 */
2650
xfrm_bundle_create(struct xfrm_policy * policy,struct xfrm_state ** xfrm,struct xfrm_dst ** bundle,int nx,const struct flowi * fl,struct dst_entry * dst)2651 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
2652 struct xfrm_state **xfrm,
2653 struct xfrm_dst **bundle,
2654 int nx,
2655 const struct flowi *fl,
2656 struct dst_entry *dst)
2657 {
2658 const struct xfrm_state_afinfo *afinfo;
2659 const struct xfrm_mode *inner_mode;
2660 struct net *net = xp_net(policy);
2661 unsigned long now = jiffies;
2662 struct net_device *dev;
2663 struct xfrm_dst *xdst_prev = NULL;
2664 struct xfrm_dst *xdst0 = NULL;
2665 int i = 0;
2666 int err;
2667 int header_len = 0;
2668 int nfheader_len = 0;
2669 int trailer_len = 0;
2670 int tos;
2671 int family = policy->selector.family;
2672 xfrm_address_t saddr, daddr;
2673
2674 xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
2675
2676 tos = xfrm_get_tos(fl, family);
2677
2678 dst_hold(dst);
2679
2680 for (; i < nx; i++) {
2681 struct xfrm_dst *xdst = xfrm_alloc_dst(net, family);
2682 struct dst_entry *dst1 = &xdst->u.dst;
2683
2684 err = PTR_ERR(xdst);
2685 if (IS_ERR(xdst)) {
2686 dst_release(dst);
2687 goto put_states;
2688 }
2689
2690 bundle[i] = xdst;
2691 if (!xdst_prev)
2692 xdst0 = xdst;
2693 else
2694 /* Ref count is taken during xfrm_alloc_dst()
2695 * No need to do dst_clone() on dst1
2696 */
2697 xfrm_dst_set_child(xdst_prev, &xdst->u.dst);
2698
2699 if (xfrm[i]->sel.family == AF_UNSPEC) {
2700 inner_mode = xfrm_ip2inner_mode(xfrm[i],
2701 xfrm_af2proto(family));
2702 if (!inner_mode) {
2703 err = -EAFNOSUPPORT;
2704 dst_release(dst);
2705 goto put_states;
2706 }
2707 } else
2708 inner_mode = &xfrm[i]->inner_mode;
2709
2710 xdst->route = dst;
2711 dst_copy_metrics(dst1, dst);
2712
2713 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
2714 __u32 mark = 0;
2715 int oif;
2716
2717 if (xfrm[i]->props.smark.v || xfrm[i]->props.smark.m)
2718 mark = xfrm_smark_get(fl->flowi_mark, xfrm[i]);
2719
2720 if (xfrm[i]->xso.type != XFRM_DEV_OFFLOAD_PACKET)
2721 family = xfrm[i]->props.family;
2722
2723 oif = fl->flowi_oif ? : fl->flowi_l3mdev;
2724 dst = xfrm_dst_lookup(xfrm[i], tos, oif,
2725 &saddr, &daddr, family, mark);
2726 err = PTR_ERR(dst);
2727 if (IS_ERR(dst))
2728 goto put_states;
2729 } else
2730 dst_hold(dst);
2731
2732 dst1->xfrm = xfrm[i];
2733 xdst->xfrm_genid = xfrm[i]->genid;
2734
2735 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
2736 dst1->lastuse = now;
2737
2738 dst1->input = dst_discard;
2739
2740 rcu_read_lock();
2741 afinfo = xfrm_state_afinfo_get_rcu(inner_mode->family);
2742 if (likely(afinfo))
2743 dst1->output = afinfo->output;
2744 else
2745 dst1->output = dst_discard_out;
2746 rcu_read_unlock();
2747
2748 xdst_prev = xdst;
2749
2750 header_len += xfrm[i]->props.header_len;
2751 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
2752 nfheader_len += xfrm[i]->props.header_len;
2753 trailer_len += xfrm[i]->props.trailer_len;
2754 }
2755
2756 xfrm_dst_set_child(xdst_prev, dst);
2757 xdst0->path = dst;
2758
2759 err = -ENODEV;
2760 dev = dst->dev;
2761 if (!dev)
2762 goto free_dst;
2763
2764 xfrm_init_path(xdst0, dst, nfheader_len);
2765 xfrm_init_pmtu(bundle, nx);
2766
2767 for (xdst_prev = xdst0; xdst_prev != (struct xfrm_dst *)dst;
2768 xdst_prev = (struct xfrm_dst *) xfrm_dst_child(&xdst_prev->u.dst)) {
2769 err = xfrm_fill_dst(xdst_prev, dev, fl);
2770 if (err)
2771 goto free_dst;
2772
2773 xdst_prev->u.dst.header_len = header_len;
2774 xdst_prev->u.dst.trailer_len = trailer_len;
2775 header_len -= xdst_prev->u.dst.xfrm->props.header_len;
2776 trailer_len -= xdst_prev->u.dst.xfrm->props.trailer_len;
2777 }
2778
2779 return &xdst0->u.dst;
2780
2781 put_states:
2782 for (; i < nx; i++)
2783 xfrm_state_put(xfrm[i]);
2784 free_dst:
2785 if (xdst0)
2786 dst_release_immediate(&xdst0->u.dst);
2787
2788 return ERR_PTR(err);
2789 }
2790
xfrm_expand_policies(const struct flowi * fl,u16 family,struct xfrm_policy ** pols,int * num_pols,int * num_xfrms)2791 static int xfrm_expand_policies(const struct flowi *fl, u16 family,
2792 struct xfrm_policy **pols,
2793 int *num_pols, int *num_xfrms)
2794 {
2795 int i;
2796
2797 if (*num_pols == 0 || !pols[0]) {
2798 *num_pols = 0;
2799 *num_xfrms = 0;
2800 return 0;
2801 }
2802 if (IS_ERR(pols[0])) {
2803 *num_pols = 0;
2804 return PTR_ERR(pols[0]);
2805 }
2806
2807 *num_xfrms = pols[0]->xfrm_nr;
2808
2809 #ifdef CONFIG_XFRM_SUB_POLICY
2810 if (pols[0]->action == XFRM_POLICY_ALLOW &&
2811 pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
2812 pols[1] = xfrm_policy_lookup_bytype(xp_net(pols[0]),
2813 XFRM_POLICY_TYPE_MAIN,
2814 fl, family,
2815 XFRM_POLICY_OUT,
2816 pols[0]->if_id);
2817 if (pols[1]) {
2818 if (IS_ERR(pols[1])) {
2819 xfrm_pols_put(pols, *num_pols);
2820 *num_pols = 0;
2821 return PTR_ERR(pols[1]);
2822 }
2823 (*num_pols)++;
2824 (*num_xfrms) += pols[1]->xfrm_nr;
2825 }
2826 }
2827 #endif
2828 for (i = 0; i < *num_pols; i++) {
2829 if (pols[i]->action != XFRM_POLICY_ALLOW) {
2830 *num_xfrms = -1;
2831 break;
2832 }
2833 }
2834
2835 return 0;
2836
2837 }
2838
2839 static struct xfrm_dst *
xfrm_resolve_and_create_bundle(struct xfrm_policy ** pols,int num_pols,const struct flowi * fl,u16 family,struct dst_entry * dst_orig)2840 xfrm_resolve_and_create_bundle(struct xfrm_policy **pols, int num_pols,
2841 const struct flowi *fl, u16 family,
2842 struct dst_entry *dst_orig)
2843 {
2844 struct net *net = xp_net(pols[0]);
2845 struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
2846 struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
2847 struct xfrm_dst *xdst;
2848 struct dst_entry *dst;
2849 int err;
2850
2851 /* Try to instantiate a bundle */
2852 err = xfrm_tmpl_resolve(pols, num_pols, fl, xfrm, family);
2853 if (err <= 0) {
2854 if (err == 0)
2855 return NULL;
2856
2857 if (err != -EAGAIN)
2858 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
2859 return ERR_PTR(err);
2860 }
2861
2862 dst = xfrm_bundle_create(pols[0], xfrm, bundle, err, fl, dst_orig);
2863 if (IS_ERR(dst)) {
2864 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTBUNDLEGENERROR);
2865 return ERR_CAST(dst);
2866 }
2867
2868 xdst = (struct xfrm_dst *)dst;
2869 xdst->num_xfrms = err;
2870 xdst->num_pols = num_pols;
2871 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
2872 xdst->policy_genid = atomic_read(&pols[0]->genid);
2873
2874 return xdst;
2875 }
2876
xfrm_policy_queue_process(struct timer_list * t)2877 static void xfrm_policy_queue_process(struct timer_list *t)
2878 {
2879 struct sk_buff *skb;
2880 struct sock *sk;
2881 struct dst_entry *dst;
2882 struct xfrm_policy *pol = from_timer(pol, t, polq.hold_timer);
2883 struct net *net = xp_net(pol);
2884 struct xfrm_policy_queue *pq = &pol->polq;
2885 struct flowi fl;
2886 struct sk_buff_head list;
2887 __u32 skb_mark;
2888
2889 spin_lock(&pq->hold_queue.lock);
2890 skb = skb_peek(&pq->hold_queue);
2891 if (!skb) {
2892 spin_unlock(&pq->hold_queue.lock);
2893 goto out;
2894 }
2895 dst = skb_dst(skb);
2896 sk = skb->sk;
2897
2898 /* Fixup the mark to support VTI. */
2899 skb_mark = skb->mark;
2900 skb->mark = pol->mark.v;
2901 xfrm_decode_session(net, skb, &fl, dst->ops->family);
2902 skb->mark = skb_mark;
2903 spin_unlock(&pq->hold_queue.lock);
2904
2905 dst_hold(xfrm_dst_path(dst));
2906 dst = xfrm_lookup(net, xfrm_dst_path(dst), &fl, sk, XFRM_LOOKUP_QUEUE);
2907 if (IS_ERR(dst))
2908 goto purge_queue;
2909
2910 if (dst->flags & DST_XFRM_QUEUE) {
2911 dst_release(dst);
2912
2913 if (pq->timeout >= XFRM_QUEUE_TMO_MAX)
2914 goto purge_queue;
2915
2916 pq->timeout = pq->timeout << 1;
2917 if (!mod_timer(&pq->hold_timer, jiffies + pq->timeout))
2918 xfrm_pol_hold(pol);
2919 goto out;
2920 }
2921
2922 dst_release(dst);
2923
2924 __skb_queue_head_init(&list);
2925
2926 spin_lock(&pq->hold_queue.lock);
2927 pq->timeout = 0;
2928 skb_queue_splice_init(&pq->hold_queue, &list);
2929 spin_unlock(&pq->hold_queue.lock);
2930
2931 while (!skb_queue_empty(&list)) {
2932 skb = __skb_dequeue(&list);
2933
2934 /* Fixup the mark to support VTI. */
2935 skb_mark = skb->mark;
2936 skb->mark = pol->mark.v;
2937 xfrm_decode_session(net, skb, &fl, skb_dst(skb)->ops->family);
2938 skb->mark = skb_mark;
2939
2940 dst_hold(xfrm_dst_path(skb_dst(skb)));
2941 dst = xfrm_lookup(net, xfrm_dst_path(skb_dst(skb)), &fl, skb->sk, 0);
2942 if (IS_ERR(dst)) {
2943 kfree_skb(skb);
2944 continue;
2945 }
2946
2947 nf_reset_ct(skb);
2948 skb_dst_drop(skb);
2949 skb_dst_set(skb, dst);
2950
2951 dst_output(net, skb->sk, skb);
2952 }
2953
2954 out:
2955 xfrm_pol_put(pol);
2956 return;
2957
2958 purge_queue:
2959 pq->timeout = 0;
2960 skb_queue_purge(&pq->hold_queue);
2961 xfrm_pol_put(pol);
2962 }
2963
xdst_queue_output(struct net * net,struct sock * sk,struct sk_buff * skb)2964 static int xdst_queue_output(struct net *net, struct sock *sk, struct sk_buff *skb)
2965 {
2966 unsigned long sched_next;
2967 struct dst_entry *dst = skb_dst(skb);
2968 struct xfrm_dst *xdst = (struct xfrm_dst *) dst;
2969 struct xfrm_policy *pol = xdst->pols[0];
2970 struct xfrm_policy_queue *pq = &pol->polq;
2971
2972 if (unlikely(skb_fclone_busy(sk, skb))) {
2973 kfree_skb(skb);
2974 return 0;
2975 }
2976
2977 if (pq->hold_queue.qlen > XFRM_MAX_QUEUE_LEN) {
2978 kfree_skb(skb);
2979 return -EAGAIN;
2980 }
2981
2982 skb_dst_force(skb);
2983
2984 spin_lock_bh(&pq->hold_queue.lock);
2985
2986 if (!pq->timeout)
2987 pq->timeout = XFRM_QUEUE_TMO_MIN;
2988
2989 sched_next = jiffies + pq->timeout;
2990
2991 if (del_timer(&pq->hold_timer)) {
2992 if (time_before(pq->hold_timer.expires, sched_next))
2993 sched_next = pq->hold_timer.expires;
2994 xfrm_pol_put(pol);
2995 }
2996
2997 __skb_queue_tail(&pq->hold_queue, skb);
2998 if (!mod_timer(&pq->hold_timer, sched_next))
2999 xfrm_pol_hold(pol);
3000
3001 spin_unlock_bh(&pq->hold_queue.lock);
3002
3003 return 0;
3004 }
3005
xfrm_create_dummy_bundle(struct net * net,struct xfrm_flo * xflo,const struct flowi * fl,int num_xfrms,u16 family)3006 static struct xfrm_dst *xfrm_create_dummy_bundle(struct net *net,
3007 struct xfrm_flo *xflo,
3008 const struct flowi *fl,
3009 int num_xfrms,
3010 u16 family)
3011 {
3012 int err;
3013 struct net_device *dev;
3014 struct dst_entry *dst;
3015 struct dst_entry *dst1;
3016 struct xfrm_dst *xdst;
3017
3018 xdst = xfrm_alloc_dst(net, family);
3019 if (IS_ERR(xdst))
3020 return xdst;
3021
3022 if (!(xflo->flags & XFRM_LOOKUP_QUEUE) ||
3023 net->xfrm.sysctl_larval_drop ||
3024 num_xfrms <= 0)
3025 return xdst;
3026
3027 dst = xflo->dst_orig;
3028 dst1 = &xdst->u.dst;
3029 dst_hold(dst);
3030 xdst->route = dst;
3031
3032 dst_copy_metrics(dst1, dst);
3033
3034 dst1->obsolete = DST_OBSOLETE_FORCE_CHK;
3035 dst1->flags |= DST_XFRM_QUEUE;
3036 dst1->lastuse = jiffies;
3037
3038 dst1->input = dst_discard;
3039 dst1->output = xdst_queue_output;
3040
3041 dst_hold(dst);
3042 xfrm_dst_set_child(xdst, dst);
3043 xdst->path = dst;
3044
3045 xfrm_init_path((struct xfrm_dst *)dst1, dst, 0);
3046
3047 err = -ENODEV;
3048 dev = dst->dev;
3049 if (!dev)
3050 goto free_dst;
3051
3052 err = xfrm_fill_dst(xdst, dev, fl);
3053 if (err)
3054 goto free_dst;
3055
3056 out:
3057 return xdst;
3058
3059 free_dst:
3060 dst_release(dst1);
3061 xdst = ERR_PTR(err);
3062 goto out;
3063 }
3064
xfrm_bundle_lookup(struct net * net,const struct flowi * fl,u16 family,u8 dir,struct xfrm_flo * xflo,u32 if_id)3065 static struct xfrm_dst *xfrm_bundle_lookup(struct net *net,
3066 const struct flowi *fl,
3067 u16 family, u8 dir,
3068 struct xfrm_flo *xflo, u32 if_id)
3069 {
3070 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3071 int num_pols = 0, num_xfrms = 0, err;
3072 struct xfrm_dst *xdst;
3073
3074 /* Resolve policies to use if we couldn't get them from
3075 * previous cache entry */
3076 num_pols = 1;
3077 pols[0] = xfrm_policy_lookup(net, fl, family, dir, if_id);
3078 err = xfrm_expand_policies(fl, family, pols,
3079 &num_pols, &num_xfrms);
3080 if (err < 0)
3081 goto inc_error;
3082 if (num_pols == 0)
3083 return NULL;
3084 if (num_xfrms <= 0)
3085 goto make_dummy_bundle;
3086
3087 xdst = xfrm_resolve_and_create_bundle(pols, num_pols, fl, family,
3088 xflo->dst_orig);
3089 if (IS_ERR(xdst)) {
3090 err = PTR_ERR(xdst);
3091 if (err == -EREMOTE) {
3092 xfrm_pols_put(pols, num_pols);
3093 return NULL;
3094 }
3095
3096 if (err != -EAGAIN)
3097 goto error;
3098 goto make_dummy_bundle;
3099 } else if (xdst == NULL) {
3100 num_xfrms = 0;
3101 goto make_dummy_bundle;
3102 }
3103
3104 return xdst;
3105
3106 make_dummy_bundle:
3107 /* We found policies, but there's no bundles to instantiate:
3108 * either because the policy blocks, has no transformations or
3109 * we could not build template (no xfrm_states).*/
3110 xdst = xfrm_create_dummy_bundle(net, xflo, fl, num_xfrms, family);
3111 if (IS_ERR(xdst)) {
3112 xfrm_pols_put(pols, num_pols);
3113 return ERR_CAST(xdst);
3114 }
3115 xdst->num_pols = num_pols;
3116 xdst->num_xfrms = num_xfrms;
3117 memcpy(xdst->pols, pols, sizeof(struct xfrm_policy *) * num_pols);
3118
3119 return xdst;
3120
3121 inc_error:
3122 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLERROR);
3123 error:
3124 xfrm_pols_put(pols, num_pols);
3125 return ERR_PTR(err);
3126 }
3127
make_blackhole(struct net * net,u16 family,struct dst_entry * dst_orig)3128 static struct dst_entry *make_blackhole(struct net *net, u16 family,
3129 struct dst_entry *dst_orig)
3130 {
3131 const struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
3132 struct dst_entry *ret;
3133
3134 if (!afinfo) {
3135 dst_release(dst_orig);
3136 return ERR_PTR(-EINVAL);
3137 } else {
3138 ret = afinfo->blackhole_route(net, dst_orig);
3139 }
3140 rcu_read_unlock();
3141
3142 return ret;
3143 }
3144
3145 /* Finds/creates a bundle for given flow and if_id
3146 *
3147 * At the moment we eat a raw IP route. Mostly to speed up lookups
3148 * on interfaces with disabled IPsec.
3149 *
3150 * xfrm_lookup uses an if_id of 0 by default, and is provided for
3151 * compatibility
3152 */
xfrm_lookup_with_ifid(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags,u32 if_id)3153 struct dst_entry *xfrm_lookup_with_ifid(struct net *net,
3154 struct dst_entry *dst_orig,
3155 const struct flowi *fl,
3156 const struct sock *sk,
3157 int flags, u32 if_id)
3158 {
3159 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3160 struct xfrm_dst *xdst;
3161 struct dst_entry *dst, *route;
3162 u16 family = dst_orig->ops->family;
3163 u8 dir = XFRM_POLICY_OUT;
3164 int i, err, num_pols, num_xfrms = 0, drop_pols = 0;
3165
3166 dst = NULL;
3167 xdst = NULL;
3168 route = NULL;
3169
3170 sk = sk_const_to_full_sk(sk);
3171 if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
3172 num_pols = 1;
3173 pols[0] = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl, family,
3174 if_id);
3175 err = xfrm_expand_policies(fl, family, pols,
3176 &num_pols, &num_xfrms);
3177 if (err < 0)
3178 goto dropdst;
3179
3180 if (num_pols) {
3181 if (num_xfrms <= 0) {
3182 drop_pols = num_pols;
3183 goto no_transform;
3184 }
3185
3186 xdst = xfrm_resolve_and_create_bundle(
3187 pols, num_pols, fl,
3188 family, dst_orig);
3189
3190 if (IS_ERR(xdst)) {
3191 xfrm_pols_put(pols, num_pols);
3192 err = PTR_ERR(xdst);
3193 if (err == -EREMOTE)
3194 goto nopol;
3195
3196 goto dropdst;
3197 } else if (xdst == NULL) {
3198 num_xfrms = 0;
3199 drop_pols = num_pols;
3200 goto no_transform;
3201 }
3202
3203 route = xdst->route;
3204 }
3205 }
3206
3207 if (xdst == NULL) {
3208 struct xfrm_flo xflo;
3209
3210 xflo.dst_orig = dst_orig;
3211 xflo.flags = flags;
3212
3213 /* To accelerate a bit... */
3214 if (!if_id && ((dst_orig->flags & DST_NOXFRM) ||
3215 !net->xfrm.policy_count[XFRM_POLICY_OUT]))
3216 goto nopol;
3217
3218 xdst = xfrm_bundle_lookup(net, fl, family, dir, &xflo, if_id);
3219 if (xdst == NULL)
3220 goto nopol;
3221 if (IS_ERR(xdst)) {
3222 err = PTR_ERR(xdst);
3223 goto dropdst;
3224 }
3225
3226 num_pols = xdst->num_pols;
3227 num_xfrms = xdst->num_xfrms;
3228 memcpy(pols, xdst->pols, sizeof(struct xfrm_policy *) * num_pols);
3229 route = xdst->route;
3230 }
3231
3232 dst = &xdst->u.dst;
3233 if (route == NULL && num_xfrms > 0) {
3234 /* The only case when xfrm_bundle_lookup() returns a
3235 * bundle with null route, is when the template could
3236 * not be resolved. It means policies are there, but
3237 * bundle could not be created, since we don't yet
3238 * have the xfrm_state's. We need to wait for KM to
3239 * negotiate new SA's or bail out with error.*/
3240 if (net->xfrm.sysctl_larval_drop) {
3241 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3242 err = -EREMOTE;
3243 goto error;
3244 }
3245
3246 err = -EAGAIN;
3247
3248 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTNOSTATES);
3249 goto error;
3250 }
3251
3252 no_transform:
3253 if (num_pols == 0)
3254 goto nopol;
3255
3256 if ((flags & XFRM_LOOKUP_ICMP) &&
3257 !(pols[0]->flags & XFRM_POLICY_ICMP)) {
3258 err = -ENOENT;
3259 goto error;
3260 }
3261
3262 for (i = 0; i < num_pols; i++)
3263 WRITE_ONCE(pols[i]->curlft.use_time, ktime_get_real_seconds());
3264
3265 if (num_xfrms < 0) {
3266 /* Prohibit the flow */
3267 XFRM_INC_STATS(net, LINUX_MIB_XFRMOUTPOLBLOCK);
3268 err = -EPERM;
3269 goto error;
3270 } else if (num_xfrms > 0) {
3271 /* Flow transformed */
3272 dst_release(dst_orig);
3273 } else {
3274 /* Flow passes untransformed */
3275 dst_release(dst);
3276 dst = dst_orig;
3277 }
3278 ok:
3279 xfrm_pols_put(pols, drop_pols);
3280 if (dst && dst->xfrm &&
3281 dst->xfrm->props.mode == XFRM_MODE_TUNNEL)
3282 dst->flags |= DST_XFRM_TUNNEL;
3283 return dst;
3284
3285 nopol:
3286 if ((!dst_orig->dev || !(dst_orig->dev->flags & IFF_LOOPBACK)) &&
3287 net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3288 err = -EPERM;
3289 goto error;
3290 }
3291 if (!(flags & XFRM_LOOKUP_ICMP)) {
3292 dst = dst_orig;
3293 goto ok;
3294 }
3295 err = -ENOENT;
3296 error:
3297 dst_release(dst);
3298 dropdst:
3299 if (!(flags & XFRM_LOOKUP_KEEP_DST_REF))
3300 dst_release(dst_orig);
3301 xfrm_pols_put(pols, drop_pols);
3302 return ERR_PTR(err);
3303 }
3304 EXPORT_SYMBOL(xfrm_lookup_with_ifid);
3305
3306 /* Main function: finds/creates a bundle for given flow.
3307 *
3308 * At the moment we eat a raw IP route. Mostly to speed up lookups
3309 * on interfaces with disabled IPsec.
3310 */
xfrm_lookup(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)3311 struct dst_entry *xfrm_lookup(struct net *net, struct dst_entry *dst_orig,
3312 const struct flowi *fl, const struct sock *sk,
3313 int flags)
3314 {
3315 return xfrm_lookup_with_ifid(net, dst_orig, fl, sk, flags, 0);
3316 }
3317 EXPORT_SYMBOL(xfrm_lookup);
3318
3319 /* Callers of xfrm_lookup_route() must ensure a call to dst_output().
3320 * Otherwise we may send out blackholed packets.
3321 */
xfrm_lookup_route(struct net * net,struct dst_entry * dst_orig,const struct flowi * fl,const struct sock * sk,int flags)3322 struct dst_entry *xfrm_lookup_route(struct net *net, struct dst_entry *dst_orig,
3323 const struct flowi *fl,
3324 const struct sock *sk, int flags)
3325 {
3326 struct dst_entry *dst = xfrm_lookup(net, dst_orig, fl, sk,
3327 flags | XFRM_LOOKUP_QUEUE |
3328 XFRM_LOOKUP_KEEP_DST_REF);
3329
3330 if (PTR_ERR(dst) == -EREMOTE)
3331 return make_blackhole(net, dst_orig->ops->family, dst_orig);
3332
3333 if (IS_ERR(dst))
3334 dst_release(dst_orig);
3335
3336 return dst;
3337 }
3338 EXPORT_SYMBOL(xfrm_lookup_route);
3339
3340 static inline int
xfrm_secpath_reject(int idx,struct sk_buff * skb,const struct flowi * fl)3341 xfrm_secpath_reject(int idx, struct sk_buff *skb, const struct flowi *fl)
3342 {
3343 struct sec_path *sp = skb_sec_path(skb);
3344 struct xfrm_state *x;
3345
3346 if (!sp || idx < 0 || idx >= sp->len)
3347 return 0;
3348 x = sp->xvec[idx];
3349 if (!x->type->reject)
3350 return 0;
3351 return x->type->reject(x, skb, fl);
3352 }
3353
3354 /* When skb is transformed back to its "native" form, we have to
3355 * check policy restrictions. At the moment we make this in maximally
3356 * stupid way. Shame on me. :-) Of course, connected sockets must
3357 * have policy cached at them.
3358 */
3359
3360 static inline int
xfrm_state_ok(const struct xfrm_tmpl * tmpl,const struct xfrm_state * x,unsigned short family,u32 if_id)3361 xfrm_state_ok(const struct xfrm_tmpl *tmpl, const struct xfrm_state *x,
3362 unsigned short family, u32 if_id)
3363 {
3364 if (xfrm_state_kern(x))
3365 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
3366 return x->id.proto == tmpl->id.proto &&
3367 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
3368 (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
3369 x->props.mode == tmpl->mode &&
3370 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
3371 !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
3372 !(x->props.mode != XFRM_MODE_TRANSPORT &&
3373 xfrm_state_addr_cmp(tmpl, x, family)) &&
3374 (if_id == 0 || if_id == x->if_id);
3375 }
3376
3377 /*
3378 * 0 or more than 0 is returned when validation is succeeded (either bypass
3379 * because of optional transport mode, or next index of the matched secpath
3380 * state with the template.
3381 * -1 is returned when no matching template is found.
3382 * Otherwise "-2 - errored_index" is returned.
3383 */
3384 static inline int
xfrm_policy_ok(const struct xfrm_tmpl * tmpl,const struct sec_path * sp,int start,unsigned short family,u32 if_id)3385 xfrm_policy_ok(const struct xfrm_tmpl *tmpl, const struct sec_path *sp, int start,
3386 unsigned short family, u32 if_id)
3387 {
3388 int idx = start;
3389
3390 if (tmpl->optional) {
3391 if (tmpl->mode == XFRM_MODE_TRANSPORT)
3392 return start;
3393 } else
3394 start = -1;
3395 for (; idx < sp->len; idx++) {
3396 if (xfrm_state_ok(tmpl, sp->xvec[idx], family, if_id))
3397 return ++idx;
3398 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
3399 if (idx < sp->verified_cnt) {
3400 /* Secpath entry previously verified, consider optional and
3401 * continue searching
3402 */
3403 continue;
3404 }
3405
3406 if (start == -1)
3407 start = -2-idx;
3408 break;
3409 }
3410 }
3411 return start;
3412 }
3413
3414 static void
decode_session4(const struct xfrm_flow_keys * flkeys,struct flowi * fl,bool reverse)3415 decode_session4(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
3416 {
3417 struct flowi4 *fl4 = &fl->u.ip4;
3418
3419 memset(fl4, 0, sizeof(struct flowi4));
3420
3421 if (reverse) {
3422 fl4->saddr = flkeys->addrs.ipv4.dst;
3423 fl4->daddr = flkeys->addrs.ipv4.src;
3424 fl4->fl4_sport = flkeys->ports.dst;
3425 fl4->fl4_dport = flkeys->ports.src;
3426 } else {
3427 fl4->saddr = flkeys->addrs.ipv4.src;
3428 fl4->daddr = flkeys->addrs.ipv4.dst;
3429 fl4->fl4_sport = flkeys->ports.src;
3430 fl4->fl4_dport = flkeys->ports.dst;
3431 }
3432
3433 switch (flkeys->basic.ip_proto) {
3434 case IPPROTO_GRE:
3435 fl4->fl4_gre_key = flkeys->gre.keyid;
3436 break;
3437 case IPPROTO_ICMP:
3438 fl4->fl4_icmp_type = flkeys->icmp.type;
3439 fl4->fl4_icmp_code = flkeys->icmp.code;
3440 break;
3441 }
3442
3443 fl4->flowi4_proto = flkeys->basic.ip_proto;
3444 fl4->flowi4_tos = flkeys->ip.tos & ~INET_ECN_MASK;
3445 }
3446
3447 #if IS_ENABLED(CONFIG_IPV6)
3448 static void
decode_session6(const struct xfrm_flow_keys * flkeys,struct flowi * fl,bool reverse)3449 decode_session6(const struct xfrm_flow_keys *flkeys, struct flowi *fl, bool reverse)
3450 {
3451 struct flowi6 *fl6 = &fl->u.ip6;
3452
3453 memset(fl6, 0, sizeof(struct flowi6));
3454
3455 if (reverse) {
3456 fl6->saddr = flkeys->addrs.ipv6.dst;
3457 fl6->daddr = flkeys->addrs.ipv6.src;
3458 fl6->fl6_sport = flkeys->ports.dst;
3459 fl6->fl6_dport = flkeys->ports.src;
3460 } else {
3461 fl6->saddr = flkeys->addrs.ipv6.src;
3462 fl6->daddr = flkeys->addrs.ipv6.dst;
3463 fl6->fl6_sport = flkeys->ports.src;
3464 fl6->fl6_dport = flkeys->ports.dst;
3465 }
3466
3467 switch (flkeys->basic.ip_proto) {
3468 case IPPROTO_GRE:
3469 fl6->fl6_gre_key = flkeys->gre.keyid;
3470 break;
3471 case IPPROTO_ICMPV6:
3472 fl6->fl6_icmp_type = flkeys->icmp.type;
3473 fl6->fl6_icmp_code = flkeys->icmp.code;
3474 break;
3475 }
3476
3477 fl6->flowi6_proto = flkeys->basic.ip_proto;
3478 }
3479 #endif
3480
__xfrm_decode_session(struct net * net,struct sk_buff * skb,struct flowi * fl,unsigned int family,int reverse)3481 int __xfrm_decode_session(struct net *net, struct sk_buff *skb, struct flowi *fl,
3482 unsigned int family, int reverse)
3483 {
3484 struct xfrm_flow_keys flkeys;
3485
3486 memset(&flkeys, 0, sizeof(flkeys));
3487 __skb_flow_dissect(net, skb, &xfrm_session_dissector, &flkeys,
3488 NULL, 0, 0, 0, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
3489
3490 switch (family) {
3491 case AF_INET:
3492 decode_session4(&flkeys, fl, reverse);
3493 break;
3494 #if IS_ENABLED(CONFIG_IPV6)
3495 case AF_INET6:
3496 decode_session6(&flkeys, fl, reverse);
3497 break;
3498 #endif
3499 default:
3500 return -EAFNOSUPPORT;
3501 }
3502
3503 fl->flowi_mark = skb->mark;
3504 if (reverse) {
3505 fl->flowi_oif = skb->skb_iif;
3506 } else {
3507 int oif = 0;
3508
3509 if (skb_dst(skb) && skb_dst(skb)->dev)
3510 oif = skb_dst(skb)->dev->ifindex;
3511
3512 fl->flowi_oif = oif;
3513 }
3514
3515 return security_xfrm_decode_session(skb, &fl->flowi_secid);
3516 }
3517 EXPORT_SYMBOL(__xfrm_decode_session);
3518
secpath_has_nontransport(const struct sec_path * sp,int k,int * idxp)3519 static inline int secpath_has_nontransport(const struct sec_path *sp, int k, int *idxp)
3520 {
3521 for (; k < sp->len; k++) {
3522 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
3523 *idxp = k;
3524 return 1;
3525 }
3526 }
3527
3528 return 0;
3529 }
3530
icmp_err_packet(const struct flowi * fl,unsigned short family)3531 static bool icmp_err_packet(const struct flowi *fl, unsigned short family)
3532 {
3533 const struct flowi4 *fl4 = &fl->u.ip4;
3534
3535 if (family == AF_INET &&
3536 fl4->flowi4_proto == IPPROTO_ICMP &&
3537 (fl4->fl4_icmp_type == ICMP_DEST_UNREACH ||
3538 fl4->fl4_icmp_type == ICMP_TIME_EXCEEDED))
3539 return true;
3540
3541 #if IS_ENABLED(CONFIG_IPV6)
3542 if (family == AF_INET6) {
3543 const struct flowi6 *fl6 = &fl->u.ip6;
3544
3545 if (fl6->flowi6_proto == IPPROTO_ICMPV6 &&
3546 (fl6->fl6_icmp_type == ICMPV6_DEST_UNREACH ||
3547 fl6->fl6_icmp_type == ICMPV6_PKT_TOOBIG ||
3548 fl6->fl6_icmp_type == ICMPV6_TIME_EXCEED))
3549 return true;
3550 }
3551 #endif
3552 return false;
3553 }
3554
xfrm_icmp_flow_decode(struct sk_buff * skb,unsigned short family,const struct flowi * fl,struct flowi * fl1)3555 static bool xfrm_icmp_flow_decode(struct sk_buff *skb, unsigned short family,
3556 const struct flowi *fl, struct flowi *fl1)
3557 {
3558 bool ret = true;
3559 struct sk_buff *newskb = skb_clone(skb, GFP_ATOMIC);
3560 int hl = family == AF_INET ? (sizeof(struct iphdr) + sizeof(struct icmphdr)) :
3561 (sizeof(struct ipv6hdr) + sizeof(struct icmp6hdr));
3562
3563 if (!newskb)
3564 return true;
3565
3566 if (!pskb_pull(newskb, hl))
3567 goto out;
3568
3569 skb_reset_network_header(newskb);
3570
3571 if (xfrm_decode_session_reverse(dev_net(skb->dev), newskb, fl1, family) < 0)
3572 goto out;
3573
3574 fl1->flowi_oif = fl->flowi_oif;
3575 fl1->flowi_mark = fl->flowi_mark;
3576 fl1->flowi_tos = fl->flowi_tos;
3577 nf_nat_decode_session(newskb, fl1, family);
3578 ret = false;
3579
3580 out:
3581 consume_skb(newskb);
3582 return ret;
3583 }
3584
xfrm_selector_inner_icmp_match(struct sk_buff * skb,unsigned short family,const struct xfrm_selector * sel,const struct flowi * fl)3585 static bool xfrm_selector_inner_icmp_match(struct sk_buff *skb, unsigned short family,
3586 const struct xfrm_selector *sel,
3587 const struct flowi *fl)
3588 {
3589 bool ret = false;
3590
3591 if (icmp_err_packet(fl, family)) {
3592 struct flowi fl1;
3593
3594 if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3595 return ret;
3596
3597 ret = xfrm_selector_match(sel, &fl1, family);
3598 }
3599
3600 return ret;
3601 }
3602
3603 static inline struct
xfrm_in_fwd_icmp(struct sk_buff * skb,const struct flowi * fl,unsigned short family,u32 if_id)3604 xfrm_policy *xfrm_in_fwd_icmp(struct sk_buff *skb,
3605 const struct flowi *fl, unsigned short family,
3606 u32 if_id)
3607 {
3608 struct xfrm_policy *pol = NULL;
3609
3610 if (icmp_err_packet(fl, family)) {
3611 struct flowi fl1;
3612 struct net *net = dev_net(skb->dev);
3613
3614 if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3615 return pol;
3616
3617 pol = xfrm_policy_lookup(net, &fl1, family, XFRM_POLICY_FWD, if_id);
3618 if (IS_ERR(pol))
3619 pol = NULL;
3620 }
3621
3622 return pol;
3623 }
3624
3625 static inline struct
xfrm_out_fwd_icmp(struct sk_buff * skb,struct flowi * fl,unsigned short family,struct dst_entry * dst)3626 dst_entry *xfrm_out_fwd_icmp(struct sk_buff *skb, struct flowi *fl,
3627 unsigned short family, struct dst_entry *dst)
3628 {
3629 if (icmp_err_packet(fl, family)) {
3630 struct net *net = dev_net(skb->dev);
3631 struct dst_entry *dst2;
3632 struct flowi fl1;
3633
3634 if (xfrm_icmp_flow_decode(skb, family, fl, &fl1))
3635 return dst;
3636
3637 dst_hold(dst);
3638
3639 dst2 = xfrm_lookup(net, dst, &fl1, NULL, (XFRM_LOOKUP_QUEUE | XFRM_LOOKUP_ICMP));
3640
3641 if (IS_ERR(dst2))
3642 return dst;
3643
3644 if (dst2->xfrm) {
3645 dst_release(dst);
3646 dst = dst2;
3647 } else {
3648 dst_release(dst2);
3649 }
3650 }
3651
3652 return dst;
3653 }
3654
__xfrm_policy_check(struct sock * sk,int dir,struct sk_buff * skb,unsigned short family)3655 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
3656 unsigned short family)
3657 {
3658 struct net *net = dev_net(skb->dev);
3659 struct xfrm_policy *pol;
3660 struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
3661 int npols = 0;
3662 int xfrm_nr;
3663 int pi;
3664 int reverse;
3665 struct flowi fl;
3666 int xerr_idx = -1;
3667 const struct xfrm_if_cb *ifcb;
3668 struct sec_path *sp;
3669 u32 if_id = 0;
3670
3671 rcu_read_lock();
3672 ifcb = xfrm_if_get_cb();
3673
3674 if (ifcb) {
3675 struct xfrm_if_decode_session_result r;
3676
3677 if (ifcb->decode_session(skb, family, &r)) {
3678 if_id = r.if_id;
3679 net = r.net;
3680 }
3681 }
3682 rcu_read_unlock();
3683
3684 reverse = dir & ~XFRM_POLICY_MASK;
3685 dir &= XFRM_POLICY_MASK;
3686
3687 if (__xfrm_decode_session(net, skb, &fl, family, reverse) < 0) {
3688 XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
3689 return 0;
3690 }
3691
3692 nf_nat_decode_session(skb, &fl, family);
3693
3694 /* First, check used SA against their selectors. */
3695 sp = skb_sec_path(skb);
3696 if (sp) {
3697 int i;
3698
3699 for (i = sp->len - 1; i >= 0; i--) {
3700 struct xfrm_state *x = sp->xvec[i];
3701 int ret = 0;
3702
3703 if (!xfrm_selector_match(&x->sel, &fl, family)) {
3704 ret = 1;
3705 if (x->props.flags & XFRM_STATE_ICMP &&
3706 xfrm_selector_inner_icmp_match(skb, family, &x->sel, &fl))
3707 ret = 0;
3708 if (ret) {
3709 XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
3710 return 0;
3711 }
3712 }
3713 }
3714 }
3715
3716 pol = NULL;
3717 sk = sk_to_full_sk(sk);
3718 if (sk && sk->sk_policy[dir]) {
3719 pol = xfrm_sk_policy_lookup(sk, dir, &fl, family, if_id);
3720 if (IS_ERR(pol)) {
3721 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3722 return 0;
3723 }
3724 }
3725
3726 if (!pol)
3727 pol = xfrm_policy_lookup(net, &fl, family, dir, if_id);
3728
3729 if (IS_ERR(pol)) {
3730 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3731 return 0;
3732 }
3733
3734 if (!pol && dir == XFRM_POLICY_FWD)
3735 pol = xfrm_in_fwd_icmp(skb, &fl, family, if_id);
3736
3737 if (!pol) {
3738 const bool is_crypto_offload = sp &&
3739 (xfrm_input_state(skb)->xso.type == XFRM_DEV_OFFLOAD_CRYPTO);
3740
3741 if (net->xfrm.policy_default[dir] == XFRM_USERPOLICY_BLOCK) {
3742 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3743 return 0;
3744 }
3745
3746 if (sp && secpath_has_nontransport(sp, 0, &xerr_idx) && !is_crypto_offload) {
3747 xfrm_secpath_reject(xerr_idx, skb, &fl);
3748 XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOPOLS);
3749 return 0;
3750 }
3751 return 1;
3752 }
3753
3754 /* This lockless write can happen from different cpus. */
3755 WRITE_ONCE(pol->curlft.use_time, ktime_get_real_seconds());
3756
3757 pols[0] = pol;
3758 npols++;
3759 #ifdef CONFIG_XFRM_SUB_POLICY
3760 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
3761 pols[1] = xfrm_policy_lookup_bytype(net, XFRM_POLICY_TYPE_MAIN,
3762 &fl, family,
3763 XFRM_POLICY_IN, if_id);
3764 if (pols[1]) {
3765 if (IS_ERR(pols[1])) {
3766 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLERROR);
3767 xfrm_pol_put(pols[0]);
3768 return 0;
3769 }
3770 /* This write can happen from different cpus. */
3771 WRITE_ONCE(pols[1]->curlft.use_time,
3772 ktime_get_real_seconds());
3773 npols++;
3774 }
3775 }
3776 #endif
3777
3778 if (pol->action == XFRM_POLICY_ALLOW) {
3779 static struct sec_path dummy;
3780 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
3781 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
3782 struct xfrm_tmpl **tpp = tp;
3783 int ti = 0;
3784 int i, k;
3785
3786 sp = skb_sec_path(skb);
3787 if (!sp)
3788 sp = &dummy;
3789
3790 for (pi = 0; pi < npols; pi++) {
3791 if (pols[pi] != pol &&
3792 pols[pi]->action != XFRM_POLICY_ALLOW) {
3793 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3794 goto reject;
3795 }
3796 if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
3797 XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
3798 goto reject_error;
3799 }
3800 for (i = 0; i < pols[pi]->xfrm_nr; i++)
3801 tpp[ti++] = &pols[pi]->xfrm_vec[i];
3802 }
3803 xfrm_nr = ti;
3804
3805 if (npols > 1) {
3806 xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
3807 tpp = stp;
3808 }
3809
3810 /* For each tunnel xfrm, find the first matching tmpl.
3811 * For each tmpl before that, find corresponding xfrm.
3812 * Order is _important_. Later we will implement
3813 * some barriers, but at the moment barriers
3814 * are implied between each two transformations.
3815 * Upon success, marks secpath entries as having been
3816 * verified to allow them to be skipped in future policy
3817 * checks (e.g. nested tunnels).
3818 */
3819 for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
3820 k = xfrm_policy_ok(tpp[i], sp, k, family, if_id);
3821 if (k < 0) {
3822 if (k < -1)
3823 /* "-2 - errored_index" returned */
3824 xerr_idx = -(2+k);
3825 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3826 goto reject;
3827 }
3828 }
3829
3830 if (secpath_has_nontransport(sp, k, &xerr_idx)) {
3831 XFRM_INC_STATS(net, LINUX_MIB_XFRMINTMPLMISMATCH);
3832 goto reject;
3833 }
3834
3835 xfrm_pols_put(pols, npols);
3836 sp->verified_cnt = k;
3837
3838 return 1;
3839 }
3840 XFRM_INC_STATS(net, LINUX_MIB_XFRMINPOLBLOCK);
3841
3842 reject:
3843 xfrm_secpath_reject(xerr_idx, skb, &fl);
3844 reject_error:
3845 xfrm_pols_put(pols, npols);
3846 return 0;
3847 }
3848 EXPORT_SYMBOL(__xfrm_policy_check);
3849
__xfrm_route_forward(struct sk_buff * skb,unsigned short family)3850 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
3851 {
3852 struct net *net = dev_net(skb->dev);
3853 struct flowi fl;
3854 struct dst_entry *dst;
3855 int res = 1;
3856
3857 if (xfrm_decode_session(net, skb, &fl, family) < 0) {
3858 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3859 return 0;
3860 }
3861
3862 skb_dst_force(skb);
3863 if (!skb_dst(skb)) {
3864 XFRM_INC_STATS(net, LINUX_MIB_XFRMFWDHDRERROR);
3865 return 0;
3866 }
3867
3868 dst = xfrm_lookup(net, skb_dst(skb), &fl, NULL, XFRM_LOOKUP_QUEUE);
3869 if (IS_ERR(dst)) {
3870 res = 0;
3871 dst = NULL;
3872 }
3873
3874 if (dst && !dst->xfrm)
3875 dst = xfrm_out_fwd_icmp(skb, &fl, family, dst);
3876
3877 skb_dst_set(skb, dst);
3878 return res;
3879 }
3880 EXPORT_SYMBOL(__xfrm_route_forward);
3881
3882 /* Optimize later using cookies and generation ids. */
3883
xfrm_dst_check(struct dst_entry * dst,u32 cookie)3884 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
3885 {
3886 /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
3887 * to DST_OBSOLETE_FORCE_CHK to force all XFRM destinations to
3888 * get validated by dst_ops->check on every use. We do this
3889 * because when a normal route referenced by an XFRM dst is
3890 * obsoleted we do not go looking around for all parent
3891 * referencing XFRM dsts so that we can invalidate them. It
3892 * is just too much work. Instead we make the checks here on
3893 * every use. For example:
3894 *
3895 * XFRM dst A --> IPv4 dst X
3896 *
3897 * X is the "xdst->route" of A (X is also the "dst->path" of A
3898 * in this example). If X is marked obsolete, "A" will not
3899 * notice. That's what we are validating here via the
3900 * stale_bundle() check.
3901 *
3902 * When a dst is removed from the fib tree, DST_OBSOLETE_DEAD will
3903 * be marked on it.
3904 * This will force stale_bundle() to fail on any xdst bundle with
3905 * this dst linked in it.
3906 */
3907 if (dst->obsolete < 0 && !stale_bundle(dst))
3908 return dst;
3909
3910 return NULL;
3911 }
3912
stale_bundle(struct dst_entry * dst)3913 static int stale_bundle(struct dst_entry *dst)
3914 {
3915 return !xfrm_bundle_ok((struct xfrm_dst *)dst);
3916 }
3917
xfrm_dst_ifdown(struct dst_entry * dst,struct net_device * dev)3918 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
3919 {
3920 while ((dst = xfrm_dst_child(dst)) && dst->xfrm && dst->dev == dev) {
3921 dst->dev = blackhole_netdev;
3922 dev_hold(dst->dev);
3923 dev_put(dev);
3924 }
3925 }
3926 EXPORT_SYMBOL(xfrm_dst_ifdown);
3927
xfrm_link_failure(struct sk_buff * skb)3928 static void xfrm_link_failure(struct sk_buff *skb)
3929 {
3930 /* Impossible. Such dst must be popped before reaches point of failure. */
3931 }
3932
xfrm_negative_advice(struct sock * sk,struct dst_entry * dst)3933 static void xfrm_negative_advice(struct sock *sk, struct dst_entry *dst)
3934 {
3935 if (dst->obsolete)
3936 sk_dst_reset(sk);
3937 }
3938
xfrm_init_pmtu(struct xfrm_dst ** bundle,int nr)3939 static void xfrm_init_pmtu(struct xfrm_dst **bundle, int nr)
3940 {
3941 while (nr--) {
3942 struct xfrm_dst *xdst = bundle[nr];
3943 u32 pmtu, route_mtu_cached;
3944 struct dst_entry *dst;
3945
3946 dst = &xdst->u.dst;
3947 pmtu = dst_mtu(xfrm_dst_child(dst));
3948 xdst->child_mtu_cached = pmtu;
3949
3950 pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
3951
3952 route_mtu_cached = dst_mtu(xdst->route);
3953 xdst->route_mtu_cached = route_mtu_cached;
3954
3955 if (pmtu > route_mtu_cached)
3956 pmtu = route_mtu_cached;
3957
3958 dst_metric_set(dst, RTAX_MTU, pmtu);
3959 }
3960 }
3961
3962 /* Check that the bundle accepts the flow and its components are
3963 * still valid.
3964 */
3965
xfrm_bundle_ok(struct xfrm_dst * first)3966 static int xfrm_bundle_ok(struct xfrm_dst *first)
3967 {
3968 struct xfrm_dst *bundle[XFRM_MAX_DEPTH];
3969 struct dst_entry *dst = &first->u.dst;
3970 struct xfrm_dst *xdst;
3971 int start_from, nr;
3972 u32 mtu;
3973
3974 if (!dst_check(xfrm_dst_path(dst), ((struct xfrm_dst *)dst)->path_cookie) ||
3975 (dst->dev && !netif_running(dst->dev)))
3976 return 0;
3977
3978 if (dst->flags & DST_XFRM_QUEUE)
3979 return 1;
3980
3981 start_from = nr = 0;
3982 do {
3983 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
3984
3985 if (dst->xfrm->km.state != XFRM_STATE_VALID)
3986 return 0;
3987 if (xdst->xfrm_genid != dst->xfrm->genid)
3988 return 0;
3989 if (xdst->num_pols > 0 &&
3990 xdst->policy_genid != atomic_read(&xdst->pols[0]->genid))
3991 return 0;
3992
3993 bundle[nr++] = xdst;
3994
3995 mtu = dst_mtu(xfrm_dst_child(dst));
3996 if (xdst->child_mtu_cached != mtu) {
3997 start_from = nr;
3998 xdst->child_mtu_cached = mtu;
3999 }
4000
4001 if (!dst_check(xdst->route, xdst->route_cookie))
4002 return 0;
4003 mtu = dst_mtu(xdst->route);
4004 if (xdst->route_mtu_cached != mtu) {
4005 start_from = nr;
4006 xdst->route_mtu_cached = mtu;
4007 }
4008
4009 dst = xfrm_dst_child(dst);
4010 } while (dst->xfrm);
4011
4012 if (likely(!start_from))
4013 return 1;
4014
4015 xdst = bundle[start_from - 1];
4016 mtu = xdst->child_mtu_cached;
4017 while (start_from--) {
4018 dst = &xdst->u.dst;
4019
4020 mtu = xfrm_state_mtu(dst->xfrm, mtu);
4021 if (mtu > xdst->route_mtu_cached)
4022 mtu = xdst->route_mtu_cached;
4023 dst_metric_set(dst, RTAX_MTU, mtu);
4024 if (!start_from)
4025 break;
4026
4027 xdst = bundle[start_from - 1];
4028 xdst->child_mtu_cached = mtu;
4029 }
4030
4031 return 1;
4032 }
4033
xfrm_default_advmss(const struct dst_entry * dst)4034 static unsigned int xfrm_default_advmss(const struct dst_entry *dst)
4035 {
4036 return dst_metric_advmss(xfrm_dst_path(dst));
4037 }
4038
xfrm_mtu(const struct dst_entry * dst)4039 static unsigned int xfrm_mtu(const struct dst_entry *dst)
4040 {
4041 unsigned int mtu = dst_metric_raw(dst, RTAX_MTU);
4042
4043 return mtu ? : dst_mtu(xfrm_dst_path(dst));
4044 }
4045
xfrm_get_dst_nexthop(const struct dst_entry * dst,const void * daddr)4046 static const void *xfrm_get_dst_nexthop(const struct dst_entry *dst,
4047 const void *daddr)
4048 {
4049 while (dst->xfrm) {
4050 const struct xfrm_state *xfrm = dst->xfrm;
4051
4052 dst = xfrm_dst_child(dst);
4053
4054 if (xfrm->props.mode == XFRM_MODE_TRANSPORT)
4055 continue;
4056 if (xfrm->type->flags & XFRM_TYPE_REMOTE_COADDR)
4057 daddr = xfrm->coaddr;
4058 else if (!(xfrm->type->flags & XFRM_TYPE_LOCAL_COADDR))
4059 daddr = &xfrm->id.daddr;
4060 }
4061 return daddr;
4062 }
4063
xfrm_neigh_lookup(const struct dst_entry * dst,struct sk_buff * skb,const void * daddr)4064 static struct neighbour *xfrm_neigh_lookup(const struct dst_entry *dst,
4065 struct sk_buff *skb,
4066 const void *daddr)
4067 {
4068 const struct dst_entry *path = xfrm_dst_path(dst);
4069
4070 if (!skb)
4071 daddr = xfrm_get_dst_nexthop(dst, daddr);
4072 return path->ops->neigh_lookup(path, skb, daddr);
4073 }
4074
xfrm_confirm_neigh(const struct dst_entry * dst,const void * daddr)4075 static void xfrm_confirm_neigh(const struct dst_entry *dst, const void *daddr)
4076 {
4077 const struct dst_entry *path = xfrm_dst_path(dst);
4078
4079 daddr = xfrm_get_dst_nexthop(dst, daddr);
4080 path->ops->confirm_neigh(path, daddr);
4081 }
4082
xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo * afinfo,int family)4083 int xfrm_policy_register_afinfo(const struct xfrm_policy_afinfo *afinfo, int family)
4084 {
4085 int err = 0;
4086
4087 if (WARN_ON(family >= ARRAY_SIZE(xfrm_policy_afinfo)))
4088 return -EAFNOSUPPORT;
4089
4090 spin_lock(&xfrm_policy_afinfo_lock);
4091 if (unlikely(xfrm_policy_afinfo[family] != NULL))
4092 err = -EEXIST;
4093 else {
4094 struct dst_ops *dst_ops = afinfo->dst_ops;
4095 if (likely(dst_ops->kmem_cachep == NULL))
4096 dst_ops->kmem_cachep = xfrm_dst_cache;
4097 if (likely(dst_ops->check == NULL))
4098 dst_ops->check = xfrm_dst_check;
4099 if (likely(dst_ops->default_advmss == NULL))
4100 dst_ops->default_advmss = xfrm_default_advmss;
4101 if (likely(dst_ops->mtu == NULL))
4102 dst_ops->mtu = xfrm_mtu;
4103 if (likely(dst_ops->negative_advice == NULL))
4104 dst_ops->negative_advice = xfrm_negative_advice;
4105 if (likely(dst_ops->link_failure == NULL))
4106 dst_ops->link_failure = xfrm_link_failure;
4107 if (likely(dst_ops->neigh_lookup == NULL))
4108 dst_ops->neigh_lookup = xfrm_neigh_lookup;
4109 if (likely(!dst_ops->confirm_neigh))
4110 dst_ops->confirm_neigh = xfrm_confirm_neigh;
4111 rcu_assign_pointer(xfrm_policy_afinfo[family], afinfo);
4112 }
4113 spin_unlock(&xfrm_policy_afinfo_lock);
4114
4115 return err;
4116 }
4117 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
4118
xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo * afinfo)4119 void xfrm_policy_unregister_afinfo(const struct xfrm_policy_afinfo *afinfo)
4120 {
4121 struct dst_ops *dst_ops = afinfo->dst_ops;
4122 int i;
4123
4124 for (i = 0; i < ARRAY_SIZE(xfrm_policy_afinfo); i++) {
4125 if (xfrm_policy_afinfo[i] != afinfo)
4126 continue;
4127 RCU_INIT_POINTER(xfrm_policy_afinfo[i], NULL);
4128 break;
4129 }
4130
4131 synchronize_rcu();
4132
4133 dst_ops->kmem_cachep = NULL;
4134 dst_ops->check = NULL;
4135 dst_ops->negative_advice = NULL;
4136 dst_ops->link_failure = NULL;
4137 }
4138 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
4139
xfrm_if_register_cb(const struct xfrm_if_cb * ifcb)4140 void xfrm_if_register_cb(const struct xfrm_if_cb *ifcb)
4141 {
4142 spin_lock(&xfrm_if_cb_lock);
4143 rcu_assign_pointer(xfrm_if_cb, ifcb);
4144 spin_unlock(&xfrm_if_cb_lock);
4145 }
4146 EXPORT_SYMBOL(xfrm_if_register_cb);
4147
xfrm_if_unregister_cb(void)4148 void xfrm_if_unregister_cb(void)
4149 {
4150 RCU_INIT_POINTER(xfrm_if_cb, NULL);
4151 synchronize_rcu();
4152 }
4153 EXPORT_SYMBOL(xfrm_if_unregister_cb);
4154
4155 #ifdef CONFIG_XFRM_STATISTICS
xfrm_statistics_init(struct net * net)4156 static int __net_init xfrm_statistics_init(struct net *net)
4157 {
4158 int rv;
4159 net->mib.xfrm_statistics = alloc_percpu(struct linux_xfrm_mib);
4160 if (!net->mib.xfrm_statistics)
4161 return -ENOMEM;
4162 rv = xfrm_proc_init(net);
4163 if (rv < 0)
4164 free_percpu(net->mib.xfrm_statistics);
4165 return rv;
4166 }
4167
xfrm_statistics_fini(struct net * net)4168 static void xfrm_statistics_fini(struct net *net)
4169 {
4170 xfrm_proc_fini(net);
4171 free_percpu(net->mib.xfrm_statistics);
4172 }
4173 #else
xfrm_statistics_init(struct net * net)4174 static int __net_init xfrm_statistics_init(struct net *net)
4175 {
4176 return 0;
4177 }
4178
xfrm_statistics_fini(struct net * net)4179 static void xfrm_statistics_fini(struct net *net)
4180 {
4181 }
4182 #endif
4183
xfrm_policy_init(struct net * net)4184 static int __net_init xfrm_policy_init(struct net *net)
4185 {
4186 unsigned int hmask, sz;
4187 int dir, err;
4188
4189 if (net_eq(net, &init_net)) {
4190 xfrm_dst_cache = KMEM_CACHE(xfrm_dst, SLAB_HWCACHE_ALIGN | SLAB_PANIC);
4191 err = rhashtable_init(&xfrm_policy_inexact_table,
4192 &xfrm_pol_inexact_params);
4193 BUG_ON(err);
4194 }
4195
4196 hmask = 8 - 1;
4197 sz = (hmask+1) * sizeof(struct hlist_head);
4198
4199 net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
4200 if (!net->xfrm.policy_byidx)
4201 goto out_byidx;
4202 net->xfrm.policy_idx_hmask = hmask;
4203
4204 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4205 struct xfrm_policy_hash *htab;
4206
4207 net->xfrm.policy_count[dir] = 0;
4208 net->xfrm.policy_count[XFRM_POLICY_MAX + dir] = 0;
4209
4210 htab = &net->xfrm.policy_bydst[dir];
4211 htab->table = xfrm_hash_alloc(sz);
4212 if (!htab->table)
4213 goto out_bydst;
4214 htab->hmask = hmask;
4215 htab->dbits4 = 32;
4216 htab->sbits4 = 32;
4217 htab->dbits6 = 128;
4218 htab->sbits6 = 128;
4219 }
4220 net->xfrm.policy_hthresh.lbits4 = 32;
4221 net->xfrm.policy_hthresh.rbits4 = 32;
4222 net->xfrm.policy_hthresh.lbits6 = 128;
4223 net->xfrm.policy_hthresh.rbits6 = 128;
4224
4225 seqlock_init(&net->xfrm.policy_hthresh.lock);
4226
4227 INIT_LIST_HEAD(&net->xfrm.policy_all);
4228 INIT_LIST_HEAD(&net->xfrm.inexact_bins);
4229 INIT_WORK(&net->xfrm.policy_hash_work, xfrm_hash_resize);
4230 INIT_WORK(&net->xfrm.policy_hthresh.work, xfrm_hash_rebuild);
4231 return 0;
4232
4233 out_bydst:
4234 for (dir--; dir >= 0; dir--) {
4235 struct xfrm_policy_hash *htab;
4236
4237 htab = &net->xfrm.policy_bydst[dir];
4238 xfrm_hash_free(htab->table, sz);
4239 }
4240 xfrm_hash_free(net->xfrm.policy_byidx, sz);
4241 out_byidx:
4242 return -ENOMEM;
4243 }
4244
xfrm_policy_fini(struct net * net)4245 static void xfrm_policy_fini(struct net *net)
4246 {
4247 struct xfrm_pol_inexact_bin *b, *t;
4248 unsigned int sz;
4249 int dir;
4250
4251 flush_work(&net->xfrm.policy_hash_work);
4252 #ifdef CONFIG_XFRM_SUB_POLICY
4253 xfrm_policy_flush(net, XFRM_POLICY_TYPE_SUB, false);
4254 #endif
4255 xfrm_policy_flush(net, XFRM_POLICY_TYPE_MAIN, false);
4256
4257 WARN_ON(!list_empty(&net->xfrm.policy_all));
4258
4259 for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
4260 struct xfrm_policy_hash *htab;
4261
4262 htab = &net->xfrm.policy_bydst[dir];
4263 sz = (htab->hmask + 1) * sizeof(struct hlist_head);
4264 WARN_ON(!hlist_empty(htab->table));
4265 xfrm_hash_free(htab->table, sz);
4266 }
4267
4268 sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
4269 WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
4270 xfrm_hash_free(net->xfrm.policy_byidx, sz);
4271
4272 spin_lock_bh(&net->xfrm.xfrm_policy_lock);
4273 list_for_each_entry_safe(b, t, &net->xfrm.inexact_bins, inexact_bins)
4274 __xfrm_policy_inexact_prune_bin(b, true);
4275 spin_unlock_bh(&net->xfrm.xfrm_policy_lock);
4276 }
4277
xfrm_net_init(struct net * net)4278 static int __net_init xfrm_net_init(struct net *net)
4279 {
4280 int rv;
4281
4282 /* Initialize the per-net locks here */
4283 spin_lock_init(&net->xfrm.xfrm_state_lock);
4284 spin_lock_init(&net->xfrm.xfrm_policy_lock);
4285 seqcount_spinlock_init(&net->xfrm.xfrm_policy_hash_generation, &net->xfrm.xfrm_policy_lock);
4286 mutex_init(&net->xfrm.xfrm_cfg_mutex);
4287 net->xfrm.policy_default[XFRM_POLICY_IN] = XFRM_USERPOLICY_ACCEPT;
4288 net->xfrm.policy_default[XFRM_POLICY_FWD] = XFRM_USERPOLICY_ACCEPT;
4289 net->xfrm.policy_default[XFRM_POLICY_OUT] = XFRM_USERPOLICY_ACCEPT;
4290
4291 rv = xfrm_statistics_init(net);
4292 if (rv < 0)
4293 goto out_statistics;
4294 rv = xfrm_state_init(net);
4295 if (rv < 0)
4296 goto out_state;
4297 rv = xfrm_policy_init(net);
4298 if (rv < 0)
4299 goto out_policy;
4300 rv = xfrm_sysctl_init(net);
4301 if (rv < 0)
4302 goto out_sysctl;
4303
4304 rv = xfrm_nat_keepalive_net_init(net);
4305 if (rv < 0)
4306 goto out_nat_keepalive;
4307
4308 return 0;
4309
4310 out_nat_keepalive:
4311 xfrm_sysctl_fini(net);
4312 out_sysctl:
4313 xfrm_policy_fini(net);
4314 out_policy:
4315 xfrm_state_fini(net);
4316 out_state:
4317 xfrm_statistics_fini(net);
4318 out_statistics:
4319 return rv;
4320 }
4321
xfrm_net_exit(struct net * net)4322 static void __net_exit xfrm_net_exit(struct net *net)
4323 {
4324 xfrm_nat_keepalive_net_fini(net);
4325 xfrm_sysctl_fini(net);
4326 xfrm_policy_fini(net);
4327 xfrm_state_fini(net);
4328 xfrm_statistics_fini(net);
4329 }
4330
4331 static struct pernet_operations __net_initdata xfrm_net_ops = {
4332 .init = xfrm_net_init,
4333 .exit = xfrm_net_exit,
4334 };
4335
4336 static const struct flow_dissector_key xfrm_flow_dissector_keys[] = {
4337 {
4338 .key_id = FLOW_DISSECTOR_KEY_CONTROL,
4339 .offset = offsetof(struct xfrm_flow_keys, control),
4340 },
4341 {
4342 .key_id = FLOW_DISSECTOR_KEY_BASIC,
4343 .offset = offsetof(struct xfrm_flow_keys, basic),
4344 },
4345 {
4346 .key_id = FLOW_DISSECTOR_KEY_IPV4_ADDRS,
4347 .offset = offsetof(struct xfrm_flow_keys, addrs.ipv4),
4348 },
4349 {
4350 .key_id = FLOW_DISSECTOR_KEY_IPV6_ADDRS,
4351 .offset = offsetof(struct xfrm_flow_keys, addrs.ipv6),
4352 },
4353 {
4354 .key_id = FLOW_DISSECTOR_KEY_PORTS,
4355 .offset = offsetof(struct xfrm_flow_keys, ports),
4356 },
4357 {
4358 .key_id = FLOW_DISSECTOR_KEY_GRE_KEYID,
4359 .offset = offsetof(struct xfrm_flow_keys, gre),
4360 },
4361 {
4362 .key_id = FLOW_DISSECTOR_KEY_IP,
4363 .offset = offsetof(struct xfrm_flow_keys, ip),
4364 },
4365 {
4366 .key_id = FLOW_DISSECTOR_KEY_ICMP,
4367 .offset = offsetof(struct xfrm_flow_keys, icmp),
4368 },
4369 };
4370
xfrm_init(void)4371 void __init xfrm_init(void)
4372 {
4373 skb_flow_dissector_init(&xfrm_session_dissector,
4374 xfrm_flow_dissector_keys,
4375 ARRAY_SIZE(xfrm_flow_dissector_keys));
4376
4377 register_pernet_subsys(&xfrm_net_ops);
4378 xfrm_dev_init();
4379 xfrm_input_init();
4380
4381 #ifdef CONFIG_XFRM_ESPINTCP
4382 espintcp_init();
4383 #endif
4384
4385 register_xfrm_state_bpf();
4386 xfrm_nat_keepalive_init(AF_INET);
4387 }
4388
4389 #ifdef CONFIG_AUDITSYSCALL
xfrm_audit_common_policyinfo(struct xfrm_policy * xp,struct audit_buffer * audit_buf)4390 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
4391 struct audit_buffer *audit_buf)
4392 {
4393 struct xfrm_sec_ctx *ctx = xp->security;
4394 struct xfrm_selector *sel = &xp->selector;
4395
4396 if (ctx)
4397 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
4398 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
4399
4400 switch (sel->family) {
4401 case AF_INET:
4402 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
4403 if (sel->prefixlen_s != 32)
4404 audit_log_format(audit_buf, " src_prefixlen=%d",
4405 sel->prefixlen_s);
4406 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
4407 if (sel->prefixlen_d != 32)
4408 audit_log_format(audit_buf, " dst_prefixlen=%d",
4409 sel->prefixlen_d);
4410 break;
4411 case AF_INET6:
4412 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
4413 if (sel->prefixlen_s != 128)
4414 audit_log_format(audit_buf, " src_prefixlen=%d",
4415 sel->prefixlen_s);
4416 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
4417 if (sel->prefixlen_d != 128)
4418 audit_log_format(audit_buf, " dst_prefixlen=%d",
4419 sel->prefixlen_d);
4420 break;
4421 }
4422 }
4423
xfrm_audit_policy_add(struct xfrm_policy * xp,int result,bool task_valid)4424 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result, bool task_valid)
4425 {
4426 struct audit_buffer *audit_buf;
4427
4428 audit_buf = xfrm_audit_start("SPD-add");
4429 if (audit_buf == NULL)
4430 return;
4431 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4432 audit_log_format(audit_buf, " res=%u", result);
4433 xfrm_audit_common_policyinfo(xp, audit_buf);
4434 audit_log_end(audit_buf);
4435 }
4436 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
4437
xfrm_audit_policy_delete(struct xfrm_policy * xp,int result,bool task_valid)4438 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
4439 bool task_valid)
4440 {
4441 struct audit_buffer *audit_buf;
4442
4443 audit_buf = xfrm_audit_start("SPD-delete");
4444 if (audit_buf == NULL)
4445 return;
4446 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
4447 audit_log_format(audit_buf, " res=%u", result);
4448 xfrm_audit_common_policyinfo(xp, audit_buf);
4449 audit_log_end(audit_buf);
4450 }
4451 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
4452 #endif
4453
4454 #ifdef CONFIG_XFRM_MIGRATE
xfrm_migrate_policy_find(const struct xfrm_selector * sel,u8 dir,u8 type,struct net * net,u32 if_id)4455 static struct xfrm_policy *xfrm_migrate_policy_find(const struct xfrm_selector *sel,
4456 u8 dir, u8 type, struct net *net, u32 if_id)
4457 {
4458 struct xfrm_policy *pol;
4459 struct flowi fl;
4460
4461 memset(&fl, 0, sizeof(fl));
4462
4463 fl.flowi_proto = sel->proto;
4464
4465 switch (sel->family) {
4466 case AF_INET:
4467 fl.u.ip4.saddr = sel->saddr.a4;
4468 fl.u.ip4.daddr = sel->daddr.a4;
4469 if (sel->proto == IPSEC_ULPROTO_ANY)
4470 break;
4471 fl.u.flowi4_oif = sel->ifindex;
4472 fl.u.ip4.fl4_sport = sel->sport;
4473 fl.u.ip4.fl4_dport = sel->dport;
4474 break;
4475 case AF_INET6:
4476 fl.u.ip6.saddr = sel->saddr.in6;
4477 fl.u.ip6.daddr = sel->daddr.in6;
4478 if (sel->proto == IPSEC_ULPROTO_ANY)
4479 break;
4480 fl.u.flowi6_oif = sel->ifindex;
4481 fl.u.ip6.fl4_sport = sel->sport;
4482 fl.u.ip6.fl4_dport = sel->dport;
4483 break;
4484 default:
4485 return ERR_PTR(-EAFNOSUPPORT);
4486 }
4487
4488 rcu_read_lock();
4489
4490 pol = xfrm_policy_lookup_bytype(net, type, &fl, sel->family, dir, if_id);
4491 if (IS_ERR_OR_NULL(pol))
4492 goto out_unlock;
4493
4494 if (!xfrm_pol_hold_rcu(pol))
4495 pol = NULL;
4496 out_unlock:
4497 rcu_read_unlock();
4498 return pol;
4499 }
4500
migrate_tmpl_match(const struct xfrm_migrate * m,const struct xfrm_tmpl * t)4501 static int migrate_tmpl_match(const struct xfrm_migrate *m, const struct xfrm_tmpl *t)
4502 {
4503 int match = 0;
4504
4505 if (t->mode == m->mode && t->id.proto == m->proto &&
4506 (m->reqid == 0 || t->reqid == m->reqid)) {
4507 switch (t->mode) {
4508 case XFRM_MODE_TUNNEL:
4509 case XFRM_MODE_BEET:
4510 if (xfrm_addr_equal(&t->id.daddr, &m->old_daddr,
4511 m->old_family) &&
4512 xfrm_addr_equal(&t->saddr, &m->old_saddr,
4513 m->old_family)) {
4514 match = 1;
4515 }
4516 break;
4517 case XFRM_MODE_TRANSPORT:
4518 /* in case of transport mode, template does not store
4519 any IP addresses, hence we just compare mode and
4520 protocol */
4521 match = 1;
4522 break;
4523 default:
4524 break;
4525 }
4526 }
4527 return match;
4528 }
4529
4530 /* update endpoint address(es) of template(s) */
xfrm_policy_migrate(struct xfrm_policy * pol,struct xfrm_migrate * m,int num_migrate,struct netlink_ext_ack * extack)4531 static int xfrm_policy_migrate(struct xfrm_policy *pol,
4532 struct xfrm_migrate *m, int num_migrate,
4533 struct netlink_ext_ack *extack)
4534 {
4535 struct xfrm_migrate *mp;
4536 int i, j, n = 0;
4537
4538 write_lock_bh(&pol->lock);
4539 if (unlikely(pol->walk.dead)) {
4540 /* target policy has been deleted */
4541 NL_SET_ERR_MSG(extack, "Target policy not found");
4542 write_unlock_bh(&pol->lock);
4543 return -ENOENT;
4544 }
4545
4546 for (i = 0; i < pol->xfrm_nr; i++) {
4547 for (j = 0, mp = m; j < num_migrate; j++, mp++) {
4548 if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
4549 continue;
4550 n++;
4551 if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
4552 pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
4553 continue;
4554 /* update endpoints */
4555 memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
4556 sizeof(pol->xfrm_vec[i].id.daddr));
4557 memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
4558 sizeof(pol->xfrm_vec[i].saddr));
4559 pol->xfrm_vec[i].encap_family = mp->new_family;
4560 /* flush bundles */
4561 atomic_inc(&pol->genid);
4562 }
4563 }
4564
4565 write_unlock_bh(&pol->lock);
4566
4567 if (!n)
4568 return -ENODATA;
4569
4570 return 0;
4571 }
4572
xfrm_migrate_check(const struct xfrm_migrate * m,int num_migrate,struct netlink_ext_ack * extack)4573 static int xfrm_migrate_check(const struct xfrm_migrate *m, int num_migrate,
4574 struct netlink_ext_ack *extack)
4575 {
4576 int i, j;
4577
4578 if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH) {
4579 NL_SET_ERR_MSG(extack, "Invalid number of SAs to migrate, must be 0 < num <= XFRM_MAX_DEPTH (6)");
4580 return -EINVAL;
4581 }
4582
4583 for (i = 0; i < num_migrate; i++) {
4584 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
4585 xfrm_addr_any(&m[i].new_saddr, m[i].new_family)) {
4586 NL_SET_ERR_MSG(extack, "Addresses in the MIGRATE attribute's list cannot be null");
4587 return -EINVAL;
4588 }
4589
4590 /* check if there is any duplicated entry */
4591 for (j = i + 1; j < num_migrate; j++) {
4592 if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
4593 sizeof(m[i].old_daddr)) &&
4594 !memcmp(&m[i].old_saddr, &m[j].old_saddr,
4595 sizeof(m[i].old_saddr)) &&
4596 m[i].proto == m[j].proto &&
4597 m[i].mode == m[j].mode &&
4598 m[i].reqid == m[j].reqid &&
4599 m[i].old_family == m[j].old_family) {
4600 NL_SET_ERR_MSG(extack, "Entries in the MIGRATE attribute's list must be unique");
4601 return -EINVAL;
4602 }
4603 }
4604 }
4605
4606 return 0;
4607 }
4608
xfrm_migrate(const struct xfrm_selector * sel,u8 dir,u8 type,struct xfrm_migrate * m,int num_migrate,struct xfrm_kmaddress * k,struct net * net,struct xfrm_encap_tmpl * encap,u32 if_id,struct netlink_ext_ack * extack)4609 int xfrm_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
4610 struct xfrm_migrate *m, int num_migrate,
4611 struct xfrm_kmaddress *k, struct net *net,
4612 struct xfrm_encap_tmpl *encap, u32 if_id,
4613 struct netlink_ext_ack *extack)
4614 {
4615 int i, err, nx_cur = 0, nx_new = 0;
4616 struct xfrm_policy *pol = NULL;
4617 struct xfrm_state *x, *xc;
4618 struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
4619 struct xfrm_state *x_new[XFRM_MAX_DEPTH];
4620 struct xfrm_migrate *mp;
4621
4622 /* Stage 0 - sanity checks */
4623 err = xfrm_migrate_check(m, num_migrate, extack);
4624 if (err < 0)
4625 goto out;
4626
4627 if (dir >= XFRM_POLICY_MAX) {
4628 NL_SET_ERR_MSG(extack, "Invalid policy direction");
4629 err = -EINVAL;
4630 goto out;
4631 }
4632
4633 /* Stage 1 - find policy */
4634 pol = xfrm_migrate_policy_find(sel, dir, type, net, if_id);
4635 if (IS_ERR_OR_NULL(pol)) {
4636 NL_SET_ERR_MSG(extack, "Target policy not found");
4637 err = IS_ERR(pol) ? PTR_ERR(pol) : -ENOENT;
4638 goto out;
4639 }
4640
4641 /* Stage 2 - find and update state(s) */
4642 for (i = 0, mp = m; i < num_migrate; i++, mp++) {
4643 if ((x = xfrm_migrate_state_find(mp, net, if_id))) {
4644 x_cur[nx_cur] = x;
4645 nx_cur++;
4646 xc = xfrm_state_migrate(x, mp, encap);
4647 if (xc) {
4648 x_new[nx_new] = xc;
4649 nx_new++;
4650 } else {
4651 err = -ENODATA;
4652 goto restore_state;
4653 }
4654 }
4655 }
4656
4657 /* Stage 3 - update policy */
4658 err = xfrm_policy_migrate(pol, m, num_migrate, extack);
4659 if (err < 0)
4660 goto restore_state;
4661
4662 /* Stage 4 - delete old state(s) */
4663 if (nx_cur) {
4664 xfrm_states_put(x_cur, nx_cur);
4665 xfrm_states_delete(x_cur, nx_cur);
4666 }
4667
4668 /* Stage 5 - announce */
4669 km_migrate(sel, dir, type, m, num_migrate, k, encap);
4670
4671 xfrm_pol_put(pol);
4672
4673 return 0;
4674 out:
4675 return err;
4676
4677 restore_state:
4678 if (pol)
4679 xfrm_pol_put(pol);
4680 if (nx_cur)
4681 xfrm_states_put(x_cur, nx_cur);
4682 if (nx_new)
4683 xfrm_states_delete(x_new, nx_new);
4684
4685 return err;
4686 }
4687 EXPORT_SYMBOL(xfrm_migrate);
4688 #endif
4689