1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * xfrm_state.c
4 *
5 * Changes:
6 * Mitsuru KANDA @USAGI
7 * Kazunori MIYAZAWA @USAGI
8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9 * IPv6 support
10 * YOSHIFUJI Hideaki @USAGI
11 * Split up af-specific functions
12 * Derek Atkins <derek@ihtfp.com>
13 * Add UDP Encapsulation
14 *
15 */
16
17 #include <linux/compat.h>
18 #include <linux/workqueue.h>
19 #include <net/xfrm.h>
20 #include <linux/pfkeyv2.h>
21 #include <linux/ipsec.h>
22 #include <linux/module.h>
23 #include <linux/cache.h>
24 #include <linux/audit.h>
25 #include <linux/uaccess.h>
26 #include <linux/ktime.h>
27 #include <linux/slab.h>
28 #include <linux/interrupt.h>
29 #include <linux/kernel.h>
30
31 #include <crypto/aead.h>
32
33 #include "xfrm_hash.h"
34
35 #define xfrm_state_deref_prot(table, net) \
36 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37 #define xfrm_state_deref_check(table, net) \
38 rcu_dereference_check((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
39
40 static void xfrm_state_gc_task(struct work_struct *work);
41
42 /* Each xfrm_state may be linked to two tables:
43
44 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
45 2. Hash table by (daddr,family,reqid) to find what SAs exist for given
46 destination/tunnel endpoint. (output)
47 */
48
49 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
50 static struct kmem_cache *xfrm_state_cache __ro_after_init;
51
52 static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task);
53 static HLIST_HEAD(xfrm_state_gc_list);
54 static HLIST_HEAD(xfrm_state_dev_gc_list);
55
xfrm_state_hold_rcu(struct xfrm_state * x)56 static inline bool xfrm_state_hold_rcu(struct xfrm_state *x)
57 {
58 return refcount_inc_not_zero(&x->refcnt);
59 }
60
xfrm_dst_hash(struct net * net,const xfrm_address_t * daddr,const xfrm_address_t * saddr,u32 reqid,unsigned short family)61 static inline unsigned int xfrm_dst_hash(struct net *net,
62 const xfrm_address_t *daddr,
63 const xfrm_address_t *saddr,
64 u32 reqid,
65 unsigned short family)
66 {
67 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
68
69 return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);
70 }
71
xfrm_src_hash(struct net * net,const xfrm_address_t * daddr,const xfrm_address_t * saddr,unsigned short family)72 static inline unsigned int xfrm_src_hash(struct net *net,
73 const xfrm_address_t *daddr,
74 const xfrm_address_t *saddr,
75 unsigned short family)
76 {
77 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
78
79 return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);
80 }
81
82 static inline unsigned int
xfrm_spi_hash(struct net * net,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)83 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,
84 __be32 spi, u8 proto, unsigned short family)
85 {
86 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
87
88 return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);
89 }
90
xfrm_seq_hash(struct net * net,u32 seq)91 static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
92 {
93 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
94
95 return __xfrm_seq_hash(seq, net->xfrm.state_hmask);
96 }
97
98 #define XFRM_STATE_INSERT(by, _n, _h, _type) \
99 { \
100 struct xfrm_state *_x = NULL; \
101 \
102 if (_type != XFRM_DEV_OFFLOAD_PACKET) { \
103 hlist_for_each_entry_rcu(_x, _h, by) { \
104 if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
105 continue; \
106 break; \
107 } \
108 } \
109 \
110 if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
111 /* SAD is empty or consist from HW SAs only */ \
112 hlist_add_head_rcu(_n, _h); \
113 else \
114 hlist_add_before_rcu(_n, &_x->by); \
115 }
116
xfrm_hash_transfer(struct hlist_head * list,struct hlist_head * ndsttable,struct hlist_head * nsrctable,struct hlist_head * nspitable,struct hlist_head * nseqtable,unsigned int nhashmask)117 static void xfrm_hash_transfer(struct hlist_head *list,
118 struct hlist_head *ndsttable,
119 struct hlist_head *nsrctable,
120 struct hlist_head *nspitable,
121 struct hlist_head *nseqtable,
122 unsigned int nhashmask)
123 {
124 struct hlist_node *tmp;
125 struct xfrm_state *x;
126
127 hlist_for_each_entry_safe(x, tmp, list, bydst) {
128 unsigned int h;
129
130 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
131 x->props.reqid, x->props.family,
132 nhashmask);
133 XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);
134
135 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,
136 x->props.family,
137 nhashmask);
138 XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);
139
140 if (x->id.spi) {
141 h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,
142 x->id.proto, x->props.family,
143 nhashmask);
144 XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,
145 x->xso.type);
146 }
147
148 if (x->km.seq) {
149 h = __xfrm_seq_hash(x->km.seq, nhashmask);
150 XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
151 x->xso.type);
152 }
153 }
154 }
155
xfrm_hash_new_size(unsigned int state_hmask)156 static unsigned long xfrm_hash_new_size(unsigned int state_hmask)
157 {
158 return ((state_hmask + 1) << 1) * sizeof(struct hlist_head);
159 }
160
xfrm_hash_resize(struct work_struct * work)161 static void xfrm_hash_resize(struct work_struct *work)
162 {
163 struct net *net = container_of(work, struct net, xfrm.state_hash_work);
164 struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq;
165 unsigned long nsize, osize;
166 unsigned int nhashmask, ohashmask;
167 int i;
168
169 nsize = xfrm_hash_new_size(net->xfrm.state_hmask);
170 ndst = xfrm_hash_alloc(nsize);
171 if (!ndst)
172 return;
173 nsrc = xfrm_hash_alloc(nsize);
174 if (!nsrc) {
175 xfrm_hash_free(ndst, nsize);
176 return;
177 }
178 nspi = xfrm_hash_alloc(nsize);
179 if (!nspi) {
180 xfrm_hash_free(ndst, nsize);
181 xfrm_hash_free(nsrc, nsize);
182 return;
183 }
184 nseq = xfrm_hash_alloc(nsize);
185 if (!nseq) {
186 xfrm_hash_free(ndst, nsize);
187 xfrm_hash_free(nsrc, nsize);
188 xfrm_hash_free(nspi, nsize);
189 return;
190 }
191
192 spin_lock_bh(&net->xfrm.xfrm_state_lock);
193 write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
194
195 nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
196 odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
197 for (i = net->xfrm.state_hmask; i >= 0; i--)
198 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask);
199
200 osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
201 ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
202 oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);
203 ohashmask = net->xfrm.state_hmask;
204
205 rcu_assign_pointer(net->xfrm.state_bydst, ndst);
206 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
207 rcu_assign_pointer(net->xfrm.state_byspi, nspi);
208 rcu_assign_pointer(net->xfrm.state_byseq, nseq);
209 net->xfrm.state_hmask = nhashmask;
210
211 write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
212 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
213
214 osize = (ohashmask + 1) * sizeof(struct hlist_head);
215
216 synchronize_rcu();
217
218 xfrm_hash_free(odst, osize);
219 xfrm_hash_free(osrc, osize);
220 xfrm_hash_free(ospi, osize);
221 xfrm_hash_free(oseq, osize);
222 }
223
224 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
225 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO];
226
227 static DEFINE_SPINLOCK(xfrm_state_gc_lock);
228 static DEFINE_SPINLOCK(xfrm_state_dev_gc_lock);
229
230 int __xfrm_state_delete(struct xfrm_state *x);
231
232 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
233 static bool km_is_alive(const struct km_event *c);
234 void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
235
xfrm_register_type(const struct xfrm_type * type,unsigned short family)236 int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
237 {
238 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
239 int err = 0;
240
241 if (!afinfo)
242 return -EAFNOSUPPORT;
243
244 #define X(afi, T, name) do { \
245 WARN_ON((afi)->type_ ## name); \
246 (afi)->type_ ## name = (T); \
247 } while (0)
248
249 switch (type->proto) {
250 case IPPROTO_COMP:
251 X(afinfo, type, comp);
252 break;
253 case IPPROTO_AH:
254 X(afinfo, type, ah);
255 break;
256 case IPPROTO_ESP:
257 X(afinfo, type, esp);
258 break;
259 case IPPROTO_IPIP:
260 X(afinfo, type, ipip);
261 break;
262 case IPPROTO_DSTOPTS:
263 X(afinfo, type, dstopts);
264 break;
265 case IPPROTO_ROUTING:
266 X(afinfo, type, routing);
267 break;
268 case IPPROTO_IPV6:
269 X(afinfo, type, ipip6);
270 break;
271 default:
272 WARN_ON(1);
273 err = -EPROTONOSUPPORT;
274 break;
275 }
276 #undef X
277 rcu_read_unlock();
278 return err;
279 }
280 EXPORT_SYMBOL(xfrm_register_type);
281
xfrm_unregister_type(const struct xfrm_type * type,unsigned short family)282 void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
283 {
284 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
285
286 if (unlikely(afinfo == NULL))
287 return;
288
289 #define X(afi, T, name) do { \
290 WARN_ON((afi)->type_ ## name != (T)); \
291 (afi)->type_ ## name = NULL; \
292 } while (0)
293
294 switch (type->proto) {
295 case IPPROTO_COMP:
296 X(afinfo, type, comp);
297 break;
298 case IPPROTO_AH:
299 X(afinfo, type, ah);
300 break;
301 case IPPROTO_ESP:
302 X(afinfo, type, esp);
303 break;
304 case IPPROTO_IPIP:
305 X(afinfo, type, ipip);
306 break;
307 case IPPROTO_DSTOPTS:
308 X(afinfo, type, dstopts);
309 break;
310 case IPPROTO_ROUTING:
311 X(afinfo, type, routing);
312 break;
313 case IPPROTO_IPV6:
314 X(afinfo, type, ipip6);
315 break;
316 default:
317 WARN_ON(1);
318 break;
319 }
320 #undef X
321 rcu_read_unlock();
322 }
323 EXPORT_SYMBOL(xfrm_unregister_type);
324
xfrm_get_type(u8 proto,unsigned short family)325 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
326 {
327 const struct xfrm_type *type = NULL;
328 struct xfrm_state_afinfo *afinfo;
329 int modload_attempted = 0;
330
331 retry:
332 afinfo = xfrm_state_get_afinfo(family);
333 if (unlikely(afinfo == NULL))
334 return NULL;
335
336 switch (proto) {
337 case IPPROTO_COMP:
338 type = afinfo->type_comp;
339 break;
340 case IPPROTO_AH:
341 type = afinfo->type_ah;
342 break;
343 case IPPROTO_ESP:
344 type = afinfo->type_esp;
345 break;
346 case IPPROTO_IPIP:
347 type = afinfo->type_ipip;
348 break;
349 case IPPROTO_DSTOPTS:
350 type = afinfo->type_dstopts;
351 break;
352 case IPPROTO_ROUTING:
353 type = afinfo->type_routing;
354 break;
355 case IPPROTO_IPV6:
356 type = afinfo->type_ipip6;
357 break;
358 default:
359 break;
360 }
361
362 if (unlikely(type && !try_module_get(type->owner)))
363 type = NULL;
364
365 rcu_read_unlock();
366
367 if (!type && !modload_attempted) {
368 request_module("xfrm-type-%d-%d", family, proto);
369 modload_attempted = 1;
370 goto retry;
371 }
372
373 return type;
374 }
375
xfrm_put_type(const struct xfrm_type * type)376 static void xfrm_put_type(const struct xfrm_type *type)
377 {
378 module_put(type->owner);
379 }
380
xfrm_register_type_offload(const struct xfrm_type_offload * type,unsigned short family)381 int xfrm_register_type_offload(const struct xfrm_type_offload *type,
382 unsigned short family)
383 {
384 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
385 int err = 0;
386
387 if (unlikely(afinfo == NULL))
388 return -EAFNOSUPPORT;
389
390 switch (type->proto) {
391 case IPPROTO_ESP:
392 WARN_ON(afinfo->type_offload_esp);
393 afinfo->type_offload_esp = type;
394 break;
395 default:
396 WARN_ON(1);
397 err = -EPROTONOSUPPORT;
398 break;
399 }
400
401 rcu_read_unlock();
402 return err;
403 }
404 EXPORT_SYMBOL(xfrm_register_type_offload);
405
xfrm_unregister_type_offload(const struct xfrm_type_offload * type,unsigned short family)406 void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
407 unsigned short family)
408 {
409 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
410
411 if (unlikely(afinfo == NULL))
412 return;
413
414 switch (type->proto) {
415 case IPPROTO_ESP:
416 WARN_ON(afinfo->type_offload_esp != type);
417 afinfo->type_offload_esp = NULL;
418 break;
419 default:
420 WARN_ON(1);
421 break;
422 }
423 rcu_read_unlock();
424 }
425 EXPORT_SYMBOL(xfrm_unregister_type_offload);
426
xfrm_set_type_offload(struct xfrm_state * x,bool try_load)427 void xfrm_set_type_offload(struct xfrm_state *x, bool try_load)
428 {
429 const struct xfrm_type_offload *type = NULL;
430 struct xfrm_state_afinfo *afinfo;
431
432 retry:
433 afinfo = xfrm_state_get_afinfo(x->props.family);
434 if (unlikely(afinfo == NULL))
435 goto out;
436
437 switch (x->id.proto) {
438 case IPPROTO_ESP:
439 type = afinfo->type_offload_esp;
440 break;
441 default:
442 break;
443 }
444
445 if ((type && !try_module_get(type->owner)))
446 type = NULL;
447
448 rcu_read_unlock();
449
450 if (!type && try_load) {
451 request_module("xfrm-offload-%d-%d", x->props.family,
452 x->id.proto);
453 try_load = false;
454 goto retry;
455 }
456
457 out:
458 x->type_offload = type;
459 }
460 EXPORT_SYMBOL(xfrm_set_type_offload);
461
462 static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
463 [XFRM_MODE_BEET] = {
464 .encap = XFRM_MODE_BEET,
465 .flags = XFRM_MODE_FLAG_TUNNEL,
466 .family = AF_INET,
467 },
468 [XFRM_MODE_TRANSPORT] = {
469 .encap = XFRM_MODE_TRANSPORT,
470 .family = AF_INET,
471 },
472 [XFRM_MODE_TUNNEL] = {
473 .encap = XFRM_MODE_TUNNEL,
474 .flags = XFRM_MODE_FLAG_TUNNEL,
475 .family = AF_INET,
476 },
477 [XFRM_MODE_IPTFS] = {
478 .encap = XFRM_MODE_IPTFS,
479 .flags = XFRM_MODE_FLAG_TUNNEL,
480 .family = AF_INET,
481 },
482 };
483
484 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = {
485 [XFRM_MODE_BEET] = {
486 .encap = XFRM_MODE_BEET,
487 .flags = XFRM_MODE_FLAG_TUNNEL,
488 .family = AF_INET6,
489 },
490 [XFRM_MODE_ROUTEOPTIMIZATION] = {
491 .encap = XFRM_MODE_ROUTEOPTIMIZATION,
492 .family = AF_INET6,
493 },
494 [XFRM_MODE_TRANSPORT] = {
495 .encap = XFRM_MODE_TRANSPORT,
496 .family = AF_INET6,
497 },
498 [XFRM_MODE_TUNNEL] = {
499 .encap = XFRM_MODE_TUNNEL,
500 .flags = XFRM_MODE_FLAG_TUNNEL,
501 .family = AF_INET6,
502 },
503 [XFRM_MODE_IPTFS] = {
504 .encap = XFRM_MODE_IPTFS,
505 .flags = XFRM_MODE_FLAG_TUNNEL,
506 .family = AF_INET6,
507 },
508 };
509
xfrm_get_mode(unsigned int encap,int family)510 static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
511 {
512 const struct xfrm_mode *mode;
513
514 if (unlikely(encap >= XFRM_MODE_MAX))
515 return NULL;
516
517 switch (family) {
518 case AF_INET:
519 mode = &xfrm4_mode_map[encap];
520 if (mode->family == family)
521 return mode;
522 break;
523 case AF_INET6:
524 mode = &xfrm6_mode_map[encap];
525 if (mode->family == family)
526 return mode;
527 break;
528 default:
529 break;
530 }
531
532 return NULL;
533 }
534
535 static const struct xfrm_mode_cbs __rcu *xfrm_mode_cbs_map[XFRM_MODE_MAX];
536 static DEFINE_SPINLOCK(xfrm_mode_cbs_map_lock);
537
xfrm_register_mode_cbs(u8 mode,const struct xfrm_mode_cbs * mode_cbs)538 int xfrm_register_mode_cbs(u8 mode, const struct xfrm_mode_cbs *mode_cbs)
539 {
540 if (mode >= XFRM_MODE_MAX)
541 return -EINVAL;
542
543 spin_lock_bh(&xfrm_mode_cbs_map_lock);
544 rcu_assign_pointer(xfrm_mode_cbs_map[mode], mode_cbs);
545 spin_unlock_bh(&xfrm_mode_cbs_map_lock);
546
547 return 0;
548 }
549 EXPORT_SYMBOL(xfrm_register_mode_cbs);
550
xfrm_unregister_mode_cbs(u8 mode)551 void xfrm_unregister_mode_cbs(u8 mode)
552 {
553 if (mode >= XFRM_MODE_MAX)
554 return;
555
556 spin_lock_bh(&xfrm_mode_cbs_map_lock);
557 RCU_INIT_POINTER(xfrm_mode_cbs_map[mode], NULL);
558 spin_unlock_bh(&xfrm_mode_cbs_map_lock);
559 synchronize_rcu();
560 }
561 EXPORT_SYMBOL(xfrm_unregister_mode_cbs);
562
xfrm_get_mode_cbs(u8 mode)563 static const struct xfrm_mode_cbs *xfrm_get_mode_cbs(u8 mode)
564 {
565 const struct xfrm_mode_cbs *cbs;
566 bool try_load = true;
567
568 if (mode >= XFRM_MODE_MAX)
569 return NULL;
570
571 retry:
572 rcu_read_lock();
573
574 cbs = rcu_dereference(xfrm_mode_cbs_map[mode]);
575 if (cbs && !try_module_get(cbs->owner))
576 cbs = NULL;
577
578 rcu_read_unlock();
579
580 if (mode == XFRM_MODE_IPTFS && !cbs && try_load) {
581 request_module("xfrm-iptfs");
582 try_load = false;
583 goto retry;
584 }
585
586 return cbs;
587 }
588
xfrm_state_free(struct xfrm_state * x)589 void xfrm_state_free(struct xfrm_state *x)
590 {
591 kmem_cache_free(xfrm_state_cache, x);
592 }
593 EXPORT_SYMBOL(xfrm_state_free);
594
595 static void xfrm_state_delete_tunnel(struct xfrm_state *x);
xfrm_state_gc_destroy(struct xfrm_state * x)596 static void xfrm_state_gc_destroy(struct xfrm_state *x)
597 {
598 if (x->mode_cbs && x->mode_cbs->destroy_state)
599 x->mode_cbs->destroy_state(x);
600 hrtimer_cancel(&x->mtimer);
601 timer_delete_sync(&x->rtimer);
602 kfree_sensitive(x->aead);
603 kfree_sensitive(x->aalg);
604 kfree_sensitive(x->ealg);
605 kfree(x->calg);
606 kfree(x->encap);
607 kfree(x->coaddr);
608 kfree(x->replay_esn);
609 kfree(x->preplay_esn);
610 xfrm_unset_type_offload(x);
611 xfrm_state_delete_tunnel(x);
612 if (x->type) {
613 x->type->destructor(x);
614 xfrm_put_type(x->type);
615 }
616 if (x->xfrag.page)
617 put_page(x->xfrag.page);
618 xfrm_dev_state_free(x);
619 security_xfrm_state_free(x);
620 xfrm_state_free(x);
621 }
622
xfrm_state_gc_task(struct work_struct * work)623 static void xfrm_state_gc_task(struct work_struct *work)
624 {
625 struct xfrm_state *x;
626 struct hlist_node *tmp;
627 struct hlist_head gc_list;
628
629 spin_lock_bh(&xfrm_state_gc_lock);
630 hlist_move_list(&xfrm_state_gc_list, &gc_list);
631 spin_unlock_bh(&xfrm_state_gc_lock);
632
633 synchronize_rcu();
634
635 hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
636 xfrm_state_gc_destroy(x);
637 }
638
xfrm_timer_handler(struct hrtimer * me)639 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
640 {
641 struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);
642 enum hrtimer_restart ret = HRTIMER_NORESTART;
643 time64_t now = ktime_get_real_seconds();
644 time64_t next = TIME64_MAX;
645 int warn = 0;
646 int err = 0;
647
648 spin_lock(&x->lock);
649 xfrm_dev_state_update_stats(x);
650
651 if (x->km.state == XFRM_STATE_DEAD)
652 goto out;
653 if (x->km.state == XFRM_STATE_EXPIRED)
654 goto expired;
655 if (x->lft.hard_add_expires_seconds) {
656 time64_t tmo = x->lft.hard_add_expires_seconds +
657 x->curlft.add_time - now;
658 if (tmo <= 0) {
659 if (x->xflags & XFRM_SOFT_EXPIRE) {
660 /* enter hard expire without soft expire first?!
661 * setting a new date could trigger this.
662 * workaround: fix x->curflt.add_time by below:
663 */
664 x->curlft.add_time = now - x->saved_tmo - 1;
665 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;
666 } else
667 goto expired;
668 }
669 if (tmo < next)
670 next = tmo;
671 }
672 if (x->lft.hard_use_expires_seconds) {
673 time64_t tmo = x->lft.hard_use_expires_seconds +
674 (READ_ONCE(x->curlft.use_time) ? : now) - now;
675 if (tmo <= 0)
676 goto expired;
677 if (tmo < next)
678 next = tmo;
679 }
680 if (x->km.dying)
681 goto resched;
682 if (x->lft.soft_add_expires_seconds) {
683 time64_t tmo = x->lft.soft_add_expires_seconds +
684 x->curlft.add_time - now;
685 if (tmo <= 0) {
686 warn = 1;
687 x->xflags &= ~XFRM_SOFT_EXPIRE;
688 } else if (tmo < next) {
689 next = tmo;
690 x->xflags |= XFRM_SOFT_EXPIRE;
691 x->saved_tmo = tmo;
692 }
693 }
694 if (x->lft.soft_use_expires_seconds) {
695 time64_t tmo = x->lft.soft_use_expires_seconds +
696 (READ_ONCE(x->curlft.use_time) ? : now) - now;
697 if (tmo <= 0)
698 warn = 1;
699 else if (tmo < next)
700 next = tmo;
701 }
702
703 x->km.dying = warn;
704 if (warn)
705 km_state_expired(x, 0, 0);
706 resched:
707 if (next != TIME64_MAX) {
708 hrtimer_forward_now(&x->mtimer, ktime_set(next, 0));
709 ret = HRTIMER_RESTART;
710 }
711
712 goto out;
713
714 expired:
715 if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)
716 x->km.state = XFRM_STATE_EXPIRED;
717
718 err = __xfrm_state_delete(x);
719 if (!err)
720 km_state_expired(x, 1, 0);
721
722 xfrm_audit_state_delete(x, err ? 0 : 1, true);
723
724 out:
725 spin_unlock(&x->lock);
726 return ret;
727 }
728
729 static void xfrm_replay_timer_handler(struct timer_list *t);
730
xfrm_state_alloc(struct net * net)731 struct xfrm_state *xfrm_state_alloc(struct net *net)
732 {
733 struct xfrm_state *x;
734
735 x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC);
736
737 if (x) {
738 write_pnet(&x->xs_net, net);
739 refcount_set(&x->refcnt, 1);
740 atomic_set(&x->tunnel_users, 0);
741 INIT_LIST_HEAD(&x->km.all);
742 INIT_HLIST_NODE(&x->state_cache);
743 INIT_HLIST_NODE(&x->bydst);
744 INIT_HLIST_NODE(&x->bysrc);
745 INIT_HLIST_NODE(&x->byspi);
746 INIT_HLIST_NODE(&x->byseq);
747 hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME,
748 HRTIMER_MODE_ABS_SOFT);
749 timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
750 x->curlft.add_time = ktime_get_real_seconds();
751 x->lft.soft_byte_limit = XFRM_INF;
752 x->lft.soft_packet_limit = XFRM_INF;
753 x->lft.hard_byte_limit = XFRM_INF;
754 x->lft.hard_packet_limit = XFRM_INF;
755 x->replay_maxage = 0;
756 x->replay_maxdiff = 0;
757 x->pcpu_num = UINT_MAX;
758 spin_lock_init(&x->lock);
759 x->mode_data = NULL;
760 }
761 return x;
762 }
763 EXPORT_SYMBOL(xfrm_state_alloc);
764
765 #ifdef CONFIG_XFRM_OFFLOAD
xfrm_dev_state_delete(struct xfrm_state * x)766 void xfrm_dev_state_delete(struct xfrm_state *x)
767 {
768 struct xfrm_dev_offload *xso = &x->xso;
769 struct net_device *dev = READ_ONCE(xso->dev);
770
771 if (dev) {
772 dev->xfrmdev_ops->xdo_dev_state_delete(dev, x);
773 spin_lock_bh(&xfrm_state_dev_gc_lock);
774 hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list);
775 spin_unlock_bh(&xfrm_state_dev_gc_lock);
776 }
777 }
778 EXPORT_SYMBOL_GPL(xfrm_dev_state_delete);
779
xfrm_dev_state_free(struct xfrm_state * x)780 void xfrm_dev_state_free(struct xfrm_state *x)
781 {
782 struct xfrm_dev_offload *xso = &x->xso;
783 struct net_device *dev = READ_ONCE(xso->dev);
784
785 if (dev && dev->xfrmdev_ops) {
786 spin_lock_bh(&xfrm_state_dev_gc_lock);
787 if (!hlist_unhashed(&x->dev_gclist))
788 hlist_del(&x->dev_gclist);
789 spin_unlock_bh(&xfrm_state_dev_gc_lock);
790
791 if (dev->xfrmdev_ops->xdo_dev_state_free)
792 dev->xfrmdev_ops->xdo_dev_state_free(dev, x);
793 WRITE_ONCE(xso->dev, NULL);
794 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
795 netdev_put(dev, &xso->dev_tracker);
796 }
797 }
798 #endif
799
__xfrm_state_destroy(struct xfrm_state * x)800 void __xfrm_state_destroy(struct xfrm_state *x)
801 {
802 WARN_ON(x->km.state != XFRM_STATE_DEAD);
803
804 spin_lock_bh(&xfrm_state_gc_lock);
805 hlist_add_head(&x->gclist, &xfrm_state_gc_list);
806 spin_unlock_bh(&xfrm_state_gc_lock);
807 schedule_work(&xfrm_state_gc_work);
808 }
809 EXPORT_SYMBOL(__xfrm_state_destroy);
810
__xfrm_state_delete(struct xfrm_state * x)811 int __xfrm_state_delete(struct xfrm_state *x)
812 {
813 struct net *net = xs_net(x);
814 int err = -ESRCH;
815
816 if (x->km.state != XFRM_STATE_DEAD) {
817 x->km.state = XFRM_STATE_DEAD;
818
819 spin_lock(&net->xfrm.xfrm_state_lock);
820 list_del(&x->km.all);
821 hlist_del_init_rcu(&x->bydst);
822 hlist_del_init_rcu(&x->bysrc);
823 if (!hlist_unhashed(&x->byseq))
824 hlist_del_init_rcu(&x->byseq);
825 if (!hlist_unhashed(&x->state_cache))
826 hlist_del_rcu(&x->state_cache);
827 if (!hlist_unhashed(&x->state_cache_input))
828 hlist_del_rcu(&x->state_cache_input);
829
830 if (!hlist_unhashed(&x->byspi))
831 hlist_del_init_rcu(&x->byspi);
832 net->xfrm.state_num--;
833 xfrm_nat_keepalive_state_updated(x);
834 spin_unlock(&net->xfrm.xfrm_state_lock);
835
836 xfrm_dev_state_delete(x);
837
838 xfrm_state_delete_tunnel(x);
839
840 /* All xfrm_state objects are created by xfrm_state_alloc.
841 * The xfrm_state_alloc call gives a reference, and that
842 * is what we are dropping here.
843 */
844 xfrm_state_put(x);
845 err = 0;
846 }
847
848 return err;
849 }
850 EXPORT_SYMBOL(__xfrm_state_delete);
851
xfrm_state_delete(struct xfrm_state * x)852 int xfrm_state_delete(struct xfrm_state *x)
853 {
854 int err;
855
856 spin_lock_bh(&x->lock);
857 err = __xfrm_state_delete(x);
858 spin_unlock_bh(&x->lock);
859
860 return err;
861 }
862 EXPORT_SYMBOL(xfrm_state_delete);
863
864 #ifdef CONFIG_SECURITY_NETWORK_XFRM
865 static inline int
xfrm_state_flush_secctx_check(struct net * net,u8 proto,bool task_valid)866 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
867 {
868 int i, err = 0;
869
870 for (i = 0; i <= net->xfrm.state_hmask; i++) {
871 struct xfrm_state *x;
872
873 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
874 if (xfrm_id_proto_match(x->id.proto, proto) &&
875 (err = security_xfrm_state_delete(x)) != 0) {
876 xfrm_audit_state_delete(x, 0, task_valid);
877 return err;
878 }
879 }
880 }
881
882 return err;
883 }
884
885 static inline int
xfrm_dev_state_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)886 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
887 {
888 int i, err = 0;
889
890 for (i = 0; i <= net->xfrm.state_hmask; i++) {
891 struct xfrm_state *x;
892 struct xfrm_dev_offload *xso;
893
894 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
895 xso = &x->xso;
896
897 if (xso->dev == dev &&
898 (err = security_xfrm_state_delete(x)) != 0) {
899 xfrm_audit_state_delete(x, 0, task_valid);
900 return err;
901 }
902 }
903 }
904
905 return err;
906 }
907 #else
908 static inline int
xfrm_state_flush_secctx_check(struct net * net,u8 proto,bool task_valid)909 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
910 {
911 return 0;
912 }
913
914 static inline int
xfrm_dev_state_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)915 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
916 {
917 return 0;
918 }
919 #endif
920
xfrm_state_flush(struct net * net,u8 proto,bool task_valid)921 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid)
922 {
923 int i, err = 0, cnt = 0;
924
925 spin_lock_bh(&net->xfrm.xfrm_state_lock);
926 err = xfrm_state_flush_secctx_check(net, proto, task_valid);
927 if (err)
928 goto out;
929
930 err = -ESRCH;
931 for (i = 0; i <= net->xfrm.state_hmask; i++) {
932 struct xfrm_state *x;
933 restart:
934 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
935 if (!xfrm_state_kern(x) &&
936 xfrm_id_proto_match(x->id.proto, proto)) {
937 xfrm_state_hold(x);
938 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
939
940 err = xfrm_state_delete(x);
941 xfrm_audit_state_delete(x, err ? 0 : 1,
942 task_valid);
943 xfrm_state_put(x);
944 if (!err)
945 cnt++;
946
947 spin_lock_bh(&net->xfrm.xfrm_state_lock);
948 goto restart;
949 }
950 }
951 }
952 out:
953 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
954 if (cnt)
955 err = 0;
956
957 return err;
958 }
959 EXPORT_SYMBOL(xfrm_state_flush);
960
xfrm_dev_state_flush(struct net * net,struct net_device * dev,bool task_valid)961 int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid)
962 {
963 struct xfrm_state *x;
964 struct hlist_node *tmp;
965 struct xfrm_dev_offload *xso;
966 int i, err = 0, cnt = 0;
967
968 spin_lock_bh(&net->xfrm.xfrm_state_lock);
969 err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid);
970 if (err)
971 goto out;
972
973 err = -ESRCH;
974 for (i = 0; i <= net->xfrm.state_hmask; i++) {
975 restart:
976 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
977 xso = &x->xso;
978
979 if (!xfrm_state_kern(x) && xso->dev == dev) {
980 xfrm_state_hold(x);
981 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
982
983 err = xfrm_state_delete(x);
984 xfrm_dev_state_free(x);
985
986 xfrm_audit_state_delete(x, err ? 0 : 1,
987 task_valid);
988 xfrm_state_put(x);
989 if (!err)
990 cnt++;
991
992 spin_lock_bh(&net->xfrm.xfrm_state_lock);
993 goto restart;
994 }
995 }
996 }
997 if (cnt)
998 err = 0;
999
1000 out:
1001 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1002
1003 spin_lock_bh(&xfrm_state_dev_gc_lock);
1004 restart_gc:
1005 hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) {
1006 xso = &x->xso;
1007
1008 if (xso->dev == dev) {
1009 spin_unlock_bh(&xfrm_state_dev_gc_lock);
1010 xfrm_dev_state_free(x);
1011 spin_lock_bh(&xfrm_state_dev_gc_lock);
1012 goto restart_gc;
1013 }
1014
1015 }
1016 spin_unlock_bh(&xfrm_state_dev_gc_lock);
1017
1018 xfrm_flush_gc();
1019
1020 return err;
1021 }
1022 EXPORT_SYMBOL(xfrm_dev_state_flush);
1023
xfrm_sad_getinfo(struct net * net,struct xfrmk_sadinfo * si)1024 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
1025 {
1026 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1027 si->sadcnt = net->xfrm.state_num;
1028 si->sadhcnt = net->xfrm.state_hmask + 1;
1029 si->sadhmcnt = xfrm_state_hashmax;
1030 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1031 }
1032 EXPORT_SYMBOL(xfrm_sad_getinfo);
1033
1034 static void
__xfrm4_init_tempsel(struct xfrm_selector * sel,const struct flowi * fl)1035 __xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
1036 {
1037 const struct flowi4 *fl4 = &fl->u.ip4;
1038
1039 sel->daddr.a4 = fl4->daddr;
1040 sel->saddr.a4 = fl4->saddr;
1041 sel->dport = xfrm_flowi_dport(fl, &fl4->uli);
1042 sel->dport_mask = htons(0xffff);
1043 sel->sport = xfrm_flowi_sport(fl, &fl4->uli);
1044 sel->sport_mask = htons(0xffff);
1045 sel->family = AF_INET;
1046 sel->prefixlen_d = 32;
1047 sel->prefixlen_s = 32;
1048 sel->proto = fl4->flowi4_proto;
1049 sel->ifindex = fl4->flowi4_oif;
1050 }
1051
1052 static void
__xfrm6_init_tempsel(struct xfrm_selector * sel,const struct flowi * fl)1053 __xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
1054 {
1055 const struct flowi6 *fl6 = &fl->u.ip6;
1056
1057 /* Initialize temporary selector matching only to current session. */
1058 *(struct in6_addr *)&sel->daddr = fl6->daddr;
1059 *(struct in6_addr *)&sel->saddr = fl6->saddr;
1060 sel->dport = xfrm_flowi_dport(fl, &fl6->uli);
1061 sel->dport_mask = htons(0xffff);
1062 sel->sport = xfrm_flowi_sport(fl, &fl6->uli);
1063 sel->sport_mask = htons(0xffff);
1064 sel->family = AF_INET6;
1065 sel->prefixlen_d = 128;
1066 sel->prefixlen_s = 128;
1067 sel->proto = fl6->flowi6_proto;
1068 sel->ifindex = fl6->flowi6_oif;
1069 }
1070
1071 static void
xfrm_init_tempstate(struct xfrm_state * x,const struct flowi * fl,const struct xfrm_tmpl * tmpl,const xfrm_address_t * daddr,const xfrm_address_t * saddr,unsigned short family)1072 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
1073 const struct xfrm_tmpl *tmpl,
1074 const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1075 unsigned short family)
1076 {
1077 switch (family) {
1078 case AF_INET:
1079 __xfrm4_init_tempsel(&x->sel, fl);
1080 break;
1081 case AF_INET6:
1082 __xfrm6_init_tempsel(&x->sel, fl);
1083 break;
1084 }
1085
1086 x->id = tmpl->id;
1087
1088 switch (tmpl->encap_family) {
1089 case AF_INET:
1090 if (x->id.daddr.a4 == 0)
1091 x->id.daddr.a4 = daddr->a4;
1092 x->props.saddr = tmpl->saddr;
1093 if (x->props.saddr.a4 == 0)
1094 x->props.saddr.a4 = saddr->a4;
1095 break;
1096 case AF_INET6:
1097 if (ipv6_addr_any((struct in6_addr *)&x->id.daddr))
1098 memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));
1099 memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));
1100 if (ipv6_addr_any((struct in6_addr *)&x->props.saddr))
1101 memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));
1102 break;
1103 }
1104
1105 x->props.mode = tmpl->mode;
1106 x->props.reqid = tmpl->reqid;
1107 x->props.family = tmpl->encap_family;
1108 }
1109
1110 struct xfrm_hash_state_ptrs {
1111 const struct hlist_head *bydst;
1112 const struct hlist_head *bysrc;
1113 const struct hlist_head *byspi;
1114 unsigned int hmask;
1115 };
1116
xfrm_hash_ptrs_get(const struct net * net,struct xfrm_hash_state_ptrs * ptrs)1117 static void xfrm_hash_ptrs_get(const struct net *net, struct xfrm_hash_state_ptrs *ptrs)
1118 {
1119 unsigned int sequence;
1120
1121 do {
1122 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
1123
1124 ptrs->bydst = xfrm_state_deref_check(net->xfrm.state_bydst, net);
1125 ptrs->bysrc = xfrm_state_deref_check(net->xfrm.state_bysrc, net);
1126 ptrs->byspi = xfrm_state_deref_check(net->xfrm.state_byspi, net);
1127 ptrs->hmask = net->xfrm.state_hmask;
1128 } while (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence));
1129 }
1130
__xfrm_state_lookup_all(const struct xfrm_hash_state_ptrs * state_ptrs,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family,struct xfrm_dev_offload * xdo)1131 static struct xfrm_state *__xfrm_state_lookup_all(const struct xfrm_hash_state_ptrs *state_ptrs,
1132 u32 mark,
1133 const xfrm_address_t *daddr,
1134 __be32 spi, u8 proto,
1135 unsigned short family,
1136 struct xfrm_dev_offload *xdo)
1137 {
1138 unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);
1139 struct xfrm_state *x;
1140
1141 hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {
1142 #ifdef CONFIG_XFRM_OFFLOAD
1143 if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {
1144 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1145 /* HW states are in the head of list, there is
1146 * no need to iterate further.
1147 */
1148 break;
1149
1150 /* Packet offload: both policy and SA should
1151 * have same device.
1152 */
1153 if (xdo->dev != x->xso.dev)
1154 continue;
1155 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1156 /* Skip HW policy for SW lookups */
1157 continue;
1158 #endif
1159 if (x->props.family != family ||
1160 x->id.spi != spi ||
1161 x->id.proto != proto ||
1162 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1163 continue;
1164
1165 if ((mark & x->mark.m) != x->mark.v)
1166 continue;
1167 if (!xfrm_state_hold_rcu(x))
1168 continue;
1169 return x;
1170 }
1171
1172 return NULL;
1173 }
1174
__xfrm_state_lookup(const struct xfrm_hash_state_ptrs * state_ptrs,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)1175 static struct xfrm_state *__xfrm_state_lookup(const struct xfrm_hash_state_ptrs *state_ptrs,
1176 u32 mark,
1177 const xfrm_address_t *daddr,
1178 __be32 spi, u8 proto,
1179 unsigned short family)
1180 {
1181 unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);
1182 struct xfrm_state *x;
1183
1184 hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {
1185 if (x->props.family != family ||
1186 x->id.spi != spi ||
1187 x->id.proto != proto ||
1188 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1189 continue;
1190
1191 if ((mark & x->mark.m) != x->mark.v)
1192 continue;
1193 if (!xfrm_state_hold_rcu(x))
1194 continue;
1195 return x;
1196 }
1197
1198 return NULL;
1199 }
1200
xfrm_input_state_lookup(struct net * net,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)1201 struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark,
1202 const xfrm_address_t *daddr,
1203 __be32 spi, u8 proto,
1204 unsigned short family)
1205 {
1206 struct xfrm_hash_state_ptrs state_ptrs;
1207 struct hlist_head *state_cache_input;
1208 struct xfrm_state *x = NULL;
1209
1210 /* BH is always disabled on the input path. */
1211 lockdep_assert_in_softirq();
1212
1213 state_cache_input = raw_cpu_ptr(net->xfrm.state_cache_input);
1214
1215 hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) {
1216 if (x->props.family != family ||
1217 x->id.spi != spi ||
1218 x->id.proto != proto ||
1219 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1220 continue;
1221
1222 if ((mark & x->mark.m) != x->mark.v)
1223 continue;
1224 if (!xfrm_state_hold_rcu(x))
1225 continue;
1226 goto out;
1227 }
1228
1229 xfrm_hash_ptrs_get(net, &state_ptrs);
1230
1231 x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);
1232 if (x) {
1233 spin_lock(&net->xfrm.xfrm_state_lock);
1234 if (x->km.state != XFRM_STATE_VALID) {
1235 /*
1236 * The state is about to be destroyed.
1237 *
1238 * Don't add it to the cache but still
1239 * return it to the caller.
1240 */
1241 } else if (hlist_unhashed(&x->state_cache_input)) {
1242 hlist_add_head_rcu(&x->state_cache_input, state_cache_input);
1243 } else {
1244 hlist_del_rcu(&x->state_cache_input);
1245 hlist_add_head_rcu(&x->state_cache_input, state_cache_input);
1246 }
1247 spin_unlock(&net->xfrm.xfrm_state_lock);
1248 }
1249
1250 out:
1251 return x;
1252 }
1253 EXPORT_SYMBOL(xfrm_input_state_lookup);
1254
__xfrm_state_lookup_byaddr(const struct xfrm_hash_state_ptrs * state_ptrs,u32 mark,const xfrm_address_t * daddr,const xfrm_address_t * saddr,u8 proto,unsigned short family)1255 static struct xfrm_state *__xfrm_state_lookup_byaddr(const struct xfrm_hash_state_ptrs *state_ptrs,
1256 u32 mark,
1257 const xfrm_address_t *daddr,
1258 const xfrm_address_t *saddr,
1259 u8 proto, unsigned short family)
1260 {
1261 unsigned int h = __xfrm_src_hash(daddr, saddr, family, state_ptrs->hmask);
1262 struct xfrm_state *x;
1263
1264 hlist_for_each_entry_rcu(x, state_ptrs->bysrc + h, bysrc) {
1265 if (x->props.family != family ||
1266 x->id.proto != proto ||
1267 !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
1268 !xfrm_addr_equal(&x->props.saddr, saddr, family))
1269 continue;
1270
1271 if ((mark & x->mark.m) != x->mark.v)
1272 continue;
1273 if (!xfrm_state_hold_rcu(x))
1274 continue;
1275 return x;
1276 }
1277
1278 return NULL;
1279 }
1280
1281 static inline struct xfrm_state *
__xfrm_state_locate(struct xfrm_state * x,int use_spi,int family)1282 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)
1283 {
1284 struct xfrm_hash_state_ptrs state_ptrs;
1285 struct net *net = xs_net(x);
1286 u32 mark = x->mark.v & x->mark.m;
1287
1288 xfrm_hash_ptrs_get(net, &state_ptrs);
1289
1290 if (use_spi)
1291 return __xfrm_state_lookup(&state_ptrs, mark, &x->id.daddr,
1292 x->id.spi, x->id.proto, family);
1293 else
1294 return __xfrm_state_lookup_byaddr(&state_ptrs, mark,
1295 &x->id.daddr,
1296 &x->props.saddr,
1297 x->id.proto, family);
1298 }
1299
xfrm_hash_grow_check(struct net * net,int have_hash_collision)1300 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
1301 {
1302 if (have_hash_collision &&
1303 (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&
1304 net->xfrm.state_num > net->xfrm.state_hmask)
1305 schedule_work(&net->xfrm.state_hash_work);
1306 }
1307
xfrm_state_look_at(struct xfrm_policy * pol,struct xfrm_state * x,const struct flowi * fl,unsigned short family,struct xfrm_state ** best,int * acq_in_progress,int * error,unsigned int pcpu_id)1308 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
1309 const struct flowi *fl, unsigned short family,
1310 struct xfrm_state **best, int *acq_in_progress,
1311 int *error, unsigned int pcpu_id)
1312 {
1313 /* Resolution logic:
1314 * 1. There is a valid state with matching selector. Done.
1315 * 2. Valid state with inappropriate selector. Skip.
1316 *
1317 * Entering area of "sysdeps".
1318 *
1319 * 3. If state is not valid, selector is temporary, it selects
1320 * only session which triggered previous resolution. Key
1321 * manager will do something to install a state with proper
1322 * selector.
1323 */
1324 if (x->km.state == XFRM_STATE_VALID) {
1325 if ((x->sel.family &&
1326 (x->sel.family != family ||
1327 !xfrm_selector_match(&x->sel, fl, family))) ||
1328 !security_xfrm_state_pol_flow_match(x, pol,
1329 &fl->u.__fl_common))
1330 return;
1331
1332 if (x->pcpu_num != UINT_MAX && x->pcpu_num != pcpu_id)
1333 return;
1334
1335 if (!*best ||
1336 ((*best)->pcpu_num == UINT_MAX && x->pcpu_num == pcpu_id) ||
1337 (*best)->km.dying > x->km.dying ||
1338 ((*best)->km.dying == x->km.dying &&
1339 (*best)->curlft.add_time < x->curlft.add_time))
1340 *best = x;
1341 } else if (x->km.state == XFRM_STATE_ACQ) {
1342 if (!*best || x->pcpu_num == pcpu_id)
1343 *acq_in_progress = 1;
1344 } else if (x->km.state == XFRM_STATE_ERROR ||
1345 x->km.state == XFRM_STATE_EXPIRED) {
1346 if ((!x->sel.family ||
1347 (x->sel.family == family &&
1348 xfrm_selector_match(&x->sel, fl, family))) &&
1349 security_xfrm_state_pol_flow_match(x, pol,
1350 &fl->u.__fl_common))
1351 *error = -ESRCH;
1352 }
1353 }
1354
1355 struct xfrm_state *
xfrm_state_find(const xfrm_address_t * daddr,const xfrm_address_t * saddr,const struct flowi * fl,struct xfrm_tmpl * tmpl,struct xfrm_policy * pol,int * err,unsigned short family,u32 if_id)1356 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1357 const struct flowi *fl, struct xfrm_tmpl *tmpl,
1358 struct xfrm_policy *pol, int *err,
1359 unsigned short family, u32 if_id)
1360 {
1361 static xfrm_address_t saddr_wildcard = { };
1362 struct xfrm_hash_state_ptrs state_ptrs;
1363 struct net *net = xp_net(pol);
1364 unsigned int h, h_wildcard;
1365 struct xfrm_state *x, *x0, *to_put;
1366 int acquire_in_progress = 0;
1367 int error = 0;
1368 struct xfrm_state *best = NULL;
1369 u32 mark = pol->mark.v & pol->mark.m;
1370 unsigned short encap_family = tmpl->encap_family;
1371 unsigned int sequence;
1372 struct km_event c;
1373 unsigned int pcpu_id;
1374 bool cached = false;
1375
1376 /* We need the cpu id just as a lookup key,
1377 * we don't require it to be stable.
1378 */
1379 pcpu_id = raw_smp_processor_id();
1380
1381 to_put = NULL;
1382
1383 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
1384
1385 rcu_read_lock();
1386 xfrm_hash_ptrs_get(net, &state_ptrs);
1387
1388 hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {
1389 if (x->props.family == encap_family &&
1390 x->props.reqid == tmpl->reqid &&
1391 (mark & x->mark.m) == x->mark.v &&
1392 x->if_id == if_id &&
1393 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1394 xfrm_state_addr_check(x, daddr, saddr, encap_family) &&
1395 tmpl->mode == x->props.mode &&
1396 tmpl->id.proto == x->id.proto &&
1397 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1398 xfrm_state_look_at(pol, x, fl, encap_family,
1399 &best, &acquire_in_progress, &error, pcpu_id);
1400 }
1401
1402 if (best)
1403 goto cached;
1404
1405 hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {
1406 if (x->props.family == encap_family &&
1407 x->props.reqid == tmpl->reqid &&
1408 (mark & x->mark.m) == x->mark.v &&
1409 x->if_id == if_id &&
1410 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1411 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&
1412 tmpl->mode == x->props.mode &&
1413 tmpl->id.proto == x->id.proto &&
1414 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1415 xfrm_state_look_at(pol, x, fl, family,
1416 &best, &acquire_in_progress, &error, pcpu_id);
1417 }
1418
1419 cached:
1420 cached = true;
1421 if (best)
1422 goto found;
1423 else if (error)
1424 best = NULL;
1425 else if (acquire_in_progress) /* XXX: acquire_in_progress should not happen */
1426 WARN_ON(1);
1427
1428 h = __xfrm_dst_hash(daddr, saddr, tmpl->reqid, encap_family, state_ptrs.hmask);
1429 hlist_for_each_entry_rcu(x, state_ptrs.bydst + h, bydst) {
1430 #ifdef CONFIG_XFRM_OFFLOAD
1431 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1432 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1433 /* HW states are in the head of list, there is
1434 * no need to iterate further.
1435 */
1436 break;
1437
1438 /* Packet offload: both policy and SA should
1439 * have same device.
1440 */
1441 if (pol->xdo.dev != x->xso.dev)
1442 continue;
1443 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1444 /* Skip HW policy for SW lookups */
1445 continue;
1446 #endif
1447 if (x->props.family == encap_family &&
1448 x->props.reqid == tmpl->reqid &&
1449 (mark & x->mark.m) == x->mark.v &&
1450 x->if_id == if_id &&
1451 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1452 xfrm_state_addr_check(x, daddr, saddr, encap_family) &&
1453 tmpl->mode == x->props.mode &&
1454 tmpl->id.proto == x->id.proto &&
1455 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1456 xfrm_state_look_at(pol, x, fl, family,
1457 &best, &acquire_in_progress, &error, pcpu_id);
1458 }
1459 if (best || acquire_in_progress)
1460 goto found;
1461
1462 h_wildcard = __xfrm_dst_hash(daddr, &saddr_wildcard, tmpl->reqid,
1463 encap_family, state_ptrs.hmask);
1464 hlist_for_each_entry_rcu(x, state_ptrs.bydst + h_wildcard, bydst) {
1465 #ifdef CONFIG_XFRM_OFFLOAD
1466 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1467 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1468 /* HW states are in the head of list, there is
1469 * no need to iterate further.
1470 */
1471 break;
1472
1473 /* Packet offload: both policy and SA should
1474 * have same device.
1475 */
1476 if (pol->xdo.dev != x->xso.dev)
1477 continue;
1478 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1479 /* Skip HW policy for SW lookups */
1480 continue;
1481 #endif
1482 if (x->props.family == encap_family &&
1483 x->props.reqid == tmpl->reqid &&
1484 (mark & x->mark.m) == x->mark.v &&
1485 x->if_id == if_id &&
1486 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1487 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&
1488 tmpl->mode == x->props.mode &&
1489 tmpl->id.proto == x->id.proto &&
1490 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1491 xfrm_state_look_at(pol, x, fl, family,
1492 &best, &acquire_in_progress, &error, pcpu_id);
1493 }
1494
1495 found:
1496 if (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) ||
1497 (best && (best->pcpu_num == pcpu_id)))
1498 x = best;
1499
1500 if (!x && !error && !acquire_in_progress) {
1501 if (tmpl->id.spi &&
1502 (x0 = __xfrm_state_lookup_all(&state_ptrs, mark, daddr,
1503 tmpl->id.spi, tmpl->id.proto,
1504 encap_family,
1505 &pol->xdo)) != NULL) {
1506 to_put = x0;
1507 error = -EEXIST;
1508 goto out;
1509 }
1510
1511 c.net = net;
1512 /* If the KMs have no listeners (yet...), avoid allocating an SA
1513 * for each and every packet - garbage collection might not
1514 * handle the flood.
1515 */
1516 if (!km_is_alive(&c)) {
1517 error = -ESRCH;
1518 goto out;
1519 }
1520
1521 x = xfrm_state_alloc(net);
1522 if (x == NULL) {
1523 error = -ENOMEM;
1524 goto out;
1525 }
1526 /* Initialize temporary state matching only
1527 * to current session. */
1528 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);
1529 memcpy(&x->mark, &pol->mark, sizeof(x->mark));
1530 x->if_id = if_id;
1531 if ((pol->flags & XFRM_POLICY_CPU_ACQUIRE) && best)
1532 x->pcpu_num = pcpu_id;
1533
1534 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid);
1535 if (error) {
1536 x->km.state = XFRM_STATE_DEAD;
1537 to_put = x;
1538 x = NULL;
1539 goto out;
1540 }
1541 #ifdef CONFIG_XFRM_OFFLOAD
1542 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1543 struct xfrm_dev_offload *xdo = &pol->xdo;
1544 struct xfrm_dev_offload *xso = &x->xso;
1545 struct net_device *dev = xdo->dev;
1546
1547 xso->type = XFRM_DEV_OFFLOAD_PACKET;
1548 xso->dir = xdo->dir;
1549 xso->dev = dev;
1550 xso->ifindex = dev->ifindex;
1551 xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
1552 netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC);
1553 error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x,
1554 NULL);
1555 if (error) {
1556 xso->dir = 0;
1557 netdev_put(dev, &xso->dev_tracker);
1558 xso->dev = NULL;
1559 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
1560 x->km.state = XFRM_STATE_DEAD;
1561 to_put = x;
1562 x = NULL;
1563 goto out;
1564 }
1565 }
1566 #endif
1567 if (km_query(x, tmpl, pol) == 0) {
1568 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1569 x->km.state = XFRM_STATE_ACQ;
1570 x->dir = XFRM_SA_DIR_OUT;
1571 list_add(&x->km.all, &net->xfrm.state_all);
1572 h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);
1573 XFRM_STATE_INSERT(bydst, &x->bydst,
1574 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h,
1575 x->xso.type);
1576 h = xfrm_src_hash(net, daddr, saddr, encap_family);
1577 XFRM_STATE_INSERT(bysrc, &x->bysrc,
1578 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h,
1579 x->xso.type);
1580 INIT_HLIST_NODE(&x->state_cache);
1581 if (x->id.spi) {
1582 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);
1583 XFRM_STATE_INSERT(byspi, &x->byspi,
1584 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h,
1585 x->xso.type);
1586 }
1587 if (x->km.seq) {
1588 h = xfrm_seq_hash(net, x->km.seq);
1589 XFRM_STATE_INSERT(byseq, &x->byseq,
1590 xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h,
1591 x->xso.type);
1592 }
1593 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1594 hrtimer_start(&x->mtimer,
1595 ktime_set(net->xfrm.sysctl_acq_expires, 0),
1596 HRTIMER_MODE_REL_SOFT);
1597 net->xfrm.state_num++;
1598 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1599 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1600 } else {
1601 #ifdef CONFIG_XFRM_OFFLOAD
1602 struct xfrm_dev_offload *xso = &x->xso;
1603
1604 if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {
1605 xfrm_dev_state_delete(x);
1606 xfrm_dev_state_free(x);
1607 }
1608 #endif
1609 x->km.state = XFRM_STATE_DEAD;
1610 to_put = x;
1611 x = NULL;
1612 error = -ESRCH;
1613 }
1614
1615 /* Use the already installed 'fallback' while the CPU-specific
1616 * SA acquire is handled*/
1617 if (best)
1618 x = best;
1619 }
1620 out:
1621 if (x) {
1622 if (!xfrm_state_hold_rcu(x)) {
1623 *err = -EAGAIN;
1624 x = NULL;
1625 }
1626 } else {
1627 *err = acquire_in_progress ? -EAGAIN : error;
1628 }
1629
1630 if (x && x->km.state == XFRM_STATE_VALID && !cached &&
1631 (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) || x->pcpu_num == pcpu_id)) {
1632 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1633 if (hlist_unhashed(&x->state_cache))
1634 hlist_add_head_rcu(&x->state_cache, &pol->state_cache_list);
1635 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1636 }
1637
1638 rcu_read_unlock();
1639 if (to_put)
1640 xfrm_state_put(to_put);
1641
1642 if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {
1643 *err = -EAGAIN;
1644 if (x) {
1645 xfrm_state_put(x);
1646 x = NULL;
1647 }
1648 }
1649
1650 return x;
1651 }
1652
1653 struct xfrm_state *
xfrm_stateonly_find(struct net * net,u32 mark,u32 if_id,xfrm_address_t * daddr,xfrm_address_t * saddr,unsigned short family,u8 mode,u8 proto,u32 reqid)1654 xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id,
1655 xfrm_address_t *daddr, xfrm_address_t *saddr,
1656 unsigned short family, u8 mode, u8 proto, u32 reqid)
1657 {
1658 unsigned int h;
1659 struct xfrm_state *rx = NULL, *x = NULL;
1660
1661 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1662 h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1663 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
1664 if (x->props.family == family &&
1665 x->props.reqid == reqid &&
1666 (mark & x->mark.m) == x->mark.v &&
1667 x->if_id == if_id &&
1668 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1669 xfrm_state_addr_check(x, daddr, saddr, family) &&
1670 mode == x->props.mode &&
1671 proto == x->id.proto &&
1672 x->km.state == XFRM_STATE_VALID) {
1673 rx = x;
1674 break;
1675 }
1676 }
1677
1678 if (rx)
1679 xfrm_state_hold(rx);
1680 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1681
1682
1683 return rx;
1684 }
1685 EXPORT_SYMBOL(xfrm_stateonly_find);
1686
xfrm_state_lookup_byspi(struct net * net,__be32 spi,unsigned short family)1687 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
1688 unsigned short family)
1689 {
1690 struct xfrm_state *x;
1691 struct xfrm_state_walk *w;
1692
1693 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1694 list_for_each_entry(w, &net->xfrm.state_all, all) {
1695 x = container_of(w, struct xfrm_state, km);
1696 if (x->props.family != family ||
1697 x->id.spi != spi)
1698 continue;
1699
1700 xfrm_state_hold(x);
1701 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1702 return x;
1703 }
1704 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1705 return NULL;
1706 }
1707 EXPORT_SYMBOL(xfrm_state_lookup_byspi);
1708
xfrm_state_lookup_spi_proto(struct net * net,__be32 spi,u8 proto)1709 static struct xfrm_state *xfrm_state_lookup_spi_proto(struct net *net, __be32 spi, u8 proto)
1710 {
1711 struct xfrm_state *x;
1712 unsigned int i;
1713
1714 for (i = 0; i <= net->xfrm.state_hmask; i++) {
1715 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byspi, net) + i, byspi) {
1716 if (x->id.spi == spi && x->id.proto == proto)
1717 return x;
1718 }
1719 }
1720 return NULL;
1721 }
1722
__xfrm_state_insert(struct xfrm_state * x)1723 static void __xfrm_state_insert(struct xfrm_state *x)
1724 {
1725 struct net *net = xs_net(x);
1726 unsigned int h;
1727
1728 list_add(&x->km.all, &net->xfrm.state_all);
1729
1730 /* Sanitize mark before store */
1731 x->mark.v &= x->mark.m;
1732
1733 h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
1734 x->props.reqid, x->props.family);
1735 XFRM_STATE_INSERT(bydst, &x->bydst,
1736 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h,
1737 x->xso.type);
1738
1739 h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);
1740 XFRM_STATE_INSERT(bysrc, &x->bysrc,
1741 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h,
1742 x->xso.type);
1743
1744 if (x->id.spi) {
1745 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,
1746 x->props.family);
1747
1748 XFRM_STATE_INSERT(byspi, &x->byspi,
1749 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h,
1750 x->xso.type);
1751 }
1752
1753 if (x->km.seq) {
1754 h = xfrm_seq_hash(net, x->km.seq);
1755
1756 XFRM_STATE_INSERT(byseq, &x->byseq,
1757 xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h,
1758 x->xso.type);
1759 }
1760
1761 hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
1762 if (x->replay_maxage)
1763 mod_timer(&x->rtimer, jiffies + x->replay_maxage);
1764
1765 net->xfrm.state_num++;
1766
1767 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1768 xfrm_nat_keepalive_state_updated(x);
1769 }
1770
1771 /* net->xfrm.xfrm_state_lock is held */
__xfrm_state_bump_genids(struct xfrm_state * xnew)1772 static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
1773 {
1774 struct net *net = xs_net(xnew);
1775 unsigned short family = xnew->props.family;
1776 u32 reqid = xnew->props.reqid;
1777 struct xfrm_state *x;
1778 unsigned int h;
1779 u32 mark = xnew->mark.v & xnew->mark.m;
1780 u32 if_id = xnew->if_id;
1781 u32 cpu_id = xnew->pcpu_num;
1782
1783 h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);
1784 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
1785 if (x->props.family == family &&
1786 x->props.reqid == reqid &&
1787 x->if_id == if_id &&
1788 x->pcpu_num == cpu_id &&
1789 (mark & x->mark.m) == x->mark.v &&
1790 xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&
1791 xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))
1792 x->genid++;
1793 }
1794 }
1795
xfrm_state_insert(struct xfrm_state * x)1796 void xfrm_state_insert(struct xfrm_state *x)
1797 {
1798 struct net *net = xs_net(x);
1799
1800 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1801 __xfrm_state_bump_genids(x);
1802 __xfrm_state_insert(x);
1803 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1804 }
1805 EXPORT_SYMBOL(xfrm_state_insert);
1806
1807 /* net->xfrm.xfrm_state_lock is held */
__find_acq_core(struct net * net,const struct xfrm_mark * m,unsigned short family,u8 mode,u32 reqid,u32 if_id,u32 pcpu_num,u8 proto,const xfrm_address_t * daddr,const xfrm_address_t * saddr,int create)1808 static struct xfrm_state *__find_acq_core(struct net *net,
1809 const struct xfrm_mark *m,
1810 unsigned short family, u8 mode,
1811 u32 reqid, u32 if_id, u32 pcpu_num, u8 proto,
1812 const xfrm_address_t *daddr,
1813 const xfrm_address_t *saddr,
1814 int create)
1815 {
1816 unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1817 struct xfrm_state *x;
1818 u32 mark = m->v & m->m;
1819
1820 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
1821 if (x->props.reqid != reqid ||
1822 x->props.mode != mode ||
1823 x->props.family != family ||
1824 x->km.state != XFRM_STATE_ACQ ||
1825 x->id.spi != 0 ||
1826 x->id.proto != proto ||
1827 (mark & x->mark.m) != x->mark.v ||
1828 x->pcpu_num != pcpu_num ||
1829 !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
1830 !xfrm_addr_equal(&x->props.saddr, saddr, family))
1831 continue;
1832
1833 xfrm_state_hold(x);
1834 return x;
1835 }
1836
1837 if (!create)
1838 return NULL;
1839
1840 x = xfrm_state_alloc(net);
1841 if (likely(x)) {
1842 switch (family) {
1843 case AF_INET:
1844 x->sel.daddr.a4 = daddr->a4;
1845 x->sel.saddr.a4 = saddr->a4;
1846 x->sel.prefixlen_d = 32;
1847 x->sel.prefixlen_s = 32;
1848 x->props.saddr.a4 = saddr->a4;
1849 x->id.daddr.a4 = daddr->a4;
1850 break;
1851
1852 case AF_INET6:
1853 x->sel.daddr.in6 = daddr->in6;
1854 x->sel.saddr.in6 = saddr->in6;
1855 x->sel.prefixlen_d = 128;
1856 x->sel.prefixlen_s = 128;
1857 x->props.saddr.in6 = saddr->in6;
1858 x->id.daddr.in6 = daddr->in6;
1859 break;
1860 }
1861
1862 x->pcpu_num = pcpu_num;
1863 x->km.state = XFRM_STATE_ACQ;
1864 x->id.proto = proto;
1865 x->props.family = family;
1866 x->props.mode = mode;
1867 x->props.reqid = reqid;
1868 x->if_id = if_id;
1869 x->mark.v = m->v;
1870 x->mark.m = m->m;
1871 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1872 xfrm_state_hold(x);
1873 hrtimer_start(&x->mtimer,
1874 ktime_set(net->xfrm.sysctl_acq_expires, 0),
1875 HRTIMER_MODE_REL_SOFT);
1876 list_add(&x->km.all, &net->xfrm.state_all);
1877 XFRM_STATE_INSERT(bydst, &x->bydst,
1878 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h,
1879 x->xso.type);
1880 h = xfrm_src_hash(net, daddr, saddr, family);
1881 XFRM_STATE_INSERT(bysrc, &x->bysrc,
1882 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h,
1883 x->xso.type);
1884
1885 net->xfrm.state_num++;
1886
1887 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1888 }
1889
1890 return x;
1891 }
1892
1893 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num);
1894
xfrm_state_add(struct xfrm_state * x)1895 int xfrm_state_add(struct xfrm_state *x)
1896 {
1897 struct net *net = xs_net(x);
1898 struct xfrm_state *x1, *to_put;
1899 int family;
1900 int err;
1901 u32 mark = x->mark.v & x->mark.m;
1902 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
1903
1904 family = x->props.family;
1905
1906 to_put = NULL;
1907
1908 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1909
1910 x1 = __xfrm_state_locate(x, use_spi, family);
1911 if (x1) {
1912 to_put = x1;
1913 x1 = NULL;
1914 err = -EEXIST;
1915 goto out;
1916 }
1917
1918 if (use_spi && x->km.seq) {
1919 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq, x->pcpu_num);
1920 if (x1 && ((x1->id.proto != x->id.proto) ||
1921 !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) {
1922 to_put = x1;
1923 x1 = NULL;
1924 }
1925 }
1926
1927 if (use_spi && !x1)
1928 x1 = __find_acq_core(net, &x->mark, family, x->props.mode,
1929 x->props.reqid, x->if_id, x->pcpu_num, x->id.proto,
1930 &x->id.daddr, &x->props.saddr, 0);
1931
1932 __xfrm_state_bump_genids(x);
1933 __xfrm_state_insert(x);
1934 err = 0;
1935
1936 out:
1937 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1938
1939 if (x1) {
1940 xfrm_state_delete(x1);
1941 xfrm_state_put(x1);
1942 }
1943
1944 if (to_put)
1945 xfrm_state_put(to_put);
1946
1947 return err;
1948 }
1949 EXPORT_SYMBOL(xfrm_state_add);
1950
1951 #ifdef CONFIG_XFRM_MIGRATE
clone_security(struct xfrm_state * x,struct xfrm_sec_ctx * security)1952 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)
1953 {
1954 struct xfrm_user_sec_ctx *uctx;
1955 int size = sizeof(*uctx) + security->ctx_len;
1956 int err;
1957
1958 uctx = kmalloc(size, GFP_KERNEL);
1959 if (!uctx)
1960 return -ENOMEM;
1961
1962 uctx->exttype = XFRMA_SEC_CTX;
1963 uctx->len = size;
1964 uctx->ctx_doi = security->ctx_doi;
1965 uctx->ctx_alg = security->ctx_alg;
1966 uctx->ctx_len = security->ctx_len;
1967 memcpy(uctx + 1, security->ctx_str, security->ctx_len);
1968 err = security_xfrm_state_alloc(x, uctx);
1969 kfree(uctx);
1970 if (err)
1971 return err;
1972
1973 return 0;
1974 }
1975
xfrm_state_clone_and_setup(struct xfrm_state * orig,const struct xfrm_migrate * m)1976 static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig,
1977 const struct xfrm_migrate *m)
1978 {
1979 struct net *net = xs_net(orig);
1980 struct xfrm_state *x = xfrm_state_alloc(net);
1981 if (!x)
1982 goto out;
1983
1984 memcpy(&x->id, &orig->id, sizeof(x->id));
1985 if (m->msg_type == XFRM_MSG_MIGRATE_STATE) {
1986 if (m->flags & XFRM_MIGRATE_STATE_UPDATE_H2H_SEL) {
1987 u8 prefixlen = (m->new_family == AF_INET6) ? 128 : 32;
1988
1989 x->sel = orig->sel;
1990 x->sel.family = m->new_family;
1991 x->sel.prefixlen_d = prefixlen;
1992 x->sel.prefixlen_s = prefixlen;
1993 x->sel.daddr = m->new_daddr;
1994 x->sel.saddr = m->new_saddr;
1995 } else {
1996 x->sel = *m->new_sel;
1997 }
1998 } else {
1999 x->sel = orig->sel;
2000 }
2001 memcpy(&x->lft, &orig->lft, sizeof(x->lft));
2002 x->props.mode = orig->props.mode;
2003 x->props.replay_window = orig->props.replay_window;
2004
2005 if (orig->aalg) {
2006 x->aalg = xfrm_algo_auth_clone(orig->aalg);
2007 if (!x->aalg)
2008 goto error;
2009 }
2010 x->props.aalgo = orig->props.aalgo;
2011
2012 if (orig->aead) {
2013 x->aead = xfrm_algo_aead_clone(orig->aead);
2014 x->geniv = orig->geniv;
2015 if (!x->aead)
2016 goto error;
2017 }
2018 if (orig->ealg) {
2019 x->ealg = xfrm_algo_clone(orig->ealg);
2020 if (!x->ealg)
2021 goto error;
2022 }
2023 x->props.ealgo = orig->props.ealgo;
2024
2025 if (orig->calg) {
2026 x->calg = xfrm_algo_clone(orig->calg);
2027 if (!x->calg)
2028 goto error;
2029 }
2030 x->props.calgo = orig->props.calgo;
2031
2032 if (m->encap) {
2033 x->encap = kmemdup(m->encap, sizeof(*x->encap), GFP_KERNEL);
2034 if (!x->encap)
2035 goto error;
2036 x->mapping_maxage = m->mapping_maxage;
2037 x->nat_keepalive_interval = m->nat_keepalive_interval;
2038 }
2039
2040 if (orig->security)
2041 if (clone_security(x, orig->security))
2042 goto error;
2043
2044 if (orig->coaddr) {
2045 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),
2046 GFP_KERNEL);
2047 if (!x->coaddr)
2048 goto error;
2049 }
2050
2051 if (xfrm_replay_clone(x, orig))
2052 goto error;
2053
2054 x->mark = m->new_mark ? *m->new_mark : m->old_mark;
2055
2056 x->props.smark = m->smark;
2057
2058 x->props.flags = orig->props.flags;
2059 x->props.extra_flags = orig->props.extra_flags;
2060
2061 x->pcpu_num = orig->pcpu_num;
2062 x->if_id = orig->if_id;
2063 x->tfcpad = orig->tfcpad;
2064 x->replay_maxdiff = orig->replay_maxdiff;
2065 x->replay_maxage = orig->replay_maxage;
2066 x->km.state = orig->km.state;
2067 x->km.seq = orig->km.seq;
2068 x->lastused = orig->lastused;
2069 x->new_mapping = 0;
2070 x->new_mapping_sport = 0;
2071 x->dir = orig->dir;
2072
2073 x->mode_cbs = orig->mode_cbs;
2074 if (x->mode_cbs && x->mode_cbs->clone_state) {
2075 if (x->mode_cbs->clone_state(x, orig)) {
2076 if (!x->mode_data)
2077 x->mode_cbs = NULL;
2078 goto error;
2079 }
2080 }
2081
2082 x->props.reqid = m->new_reqid;
2083 x->props.family = m->new_family;
2084 memcpy(&x->id.daddr, &m->new_daddr, sizeof(x->id.daddr));
2085 memcpy(&x->props.saddr, &m->new_saddr, sizeof(x->props.saddr));
2086
2087 return x;
2088
2089 error:
2090 x->km.state = XFRM_STATE_DEAD;
2091 xfrm_state_put(x);
2092 out:
2093 return NULL;
2094 }
2095
xfrm_migrate_state_find(struct xfrm_migrate * m,struct net * net,u32 if_id)2096 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
2097 u32 if_id)
2098 {
2099 unsigned int h;
2100 struct xfrm_state *x = NULL;
2101
2102 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2103
2104 if (m->old_reqid) {
2105 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,
2106 m->old_reqid, m->old_family);
2107 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
2108 if (x->props.mode != m->mode ||
2109 x->id.proto != m->proto)
2110 continue;
2111 if (m->old_reqid && x->props.reqid != m->old_reqid)
2112 continue;
2113 if (if_id != 0 && x->if_id != if_id)
2114 continue;
2115 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
2116 m->old_family) ||
2117 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
2118 m->old_family))
2119 continue;
2120 xfrm_state_hold(x);
2121 break;
2122 }
2123 } else {
2124 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,
2125 m->old_family);
2126 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, bysrc) {
2127 if (x->props.mode != m->mode ||
2128 x->id.proto != m->proto)
2129 continue;
2130 if (if_id != 0 && x->if_id != if_id)
2131 continue;
2132 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
2133 m->old_family) ||
2134 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
2135 m->old_family))
2136 continue;
2137 xfrm_state_hold(x);
2138 break;
2139 }
2140 }
2141
2142 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2143
2144 return x;
2145 }
2146 EXPORT_SYMBOL(xfrm_migrate_state_find);
2147
xfrm_state_migrate_create(struct xfrm_state * x,const struct xfrm_migrate * m,struct net * net,struct netlink_ext_ack * extack)2148 struct xfrm_state *xfrm_state_migrate_create(struct xfrm_state *x,
2149 const struct xfrm_migrate *m,
2150 struct net *net,
2151 struct netlink_ext_ack *extack)
2152 {
2153 struct xfrm_state *xc;
2154
2155 xc = xfrm_state_clone_and_setup(x, m);
2156 if (!xc) {
2157 NL_SET_ERR_MSG(extack, "Failed to clone and setup state");
2158 return NULL;
2159 }
2160
2161 if (xfrm_init_state(xc, extack) < 0) {
2162 NL_SET_ERR_MSG_WEAK(extack, "Failed to initialize migrated state");
2163 goto error;
2164 }
2165
2166 /* configure the hardware if offload is requested */
2167 if (m->xuo && xfrm_dev_state_add(net, xc, m->xuo, extack))
2168 goto error;
2169
2170 return xc;
2171 error:
2172 xc->km.state = XFRM_STATE_DEAD;
2173 xfrm_state_put(xc);
2174 return NULL;
2175 }
2176 EXPORT_SYMBOL(xfrm_state_migrate_create);
2177
xfrm_state_migrate_install(const struct xfrm_state * x,struct xfrm_state * xc,const struct xfrm_migrate * m,struct netlink_ext_ack * extack)2178 int xfrm_state_migrate_install(const struct xfrm_state *x,
2179 struct xfrm_state *xc,
2180 const struct xfrm_migrate *m,
2181 struct netlink_ext_ack *extack)
2182 {
2183 if (m->new_family == m->old_family &&
2184 xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {
2185 /*
2186 * Care is needed when the destination address of the state is
2187 * to be updated as it is a part of triplet.
2188 */
2189 xfrm_state_insert(xc);
2190 } else {
2191 if (xfrm_state_add(xc) < 0) {
2192 NL_SET_ERR_MSG(extack, "Failed to add migrated state");
2193 if (m->xuo)
2194 xfrm_dev_state_delete(xc);
2195 xc->km.state = XFRM_STATE_DEAD;
2196 xfrm_state_put(xc);
2197 return -EEXIST;
2198 }
2199 }
2200
2201 return 0;
2202 }
2203 EXPORT_SYMBOL(xfrm_state_migrate_install);
2204
xfrm_state_migrate(struct xfrm_state * x,struct xfrm_migrate * m,struct net * net,struct netlink_ext_ack * extack)2205 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
2206 struct xfrm_migrate *m,
2207 struct net *net,
2208 struct netlink_ext_ack *extack)
2209 {
2210 struct xfrm_state *xc;
2211
2212 xc = xfrm_state_migrate_create(x, m, net, extack);
2213 if (!xc)
2214 return NULL;
2215
2216 xfrm_migrate_sync(xc, x);
2217
2218 if (xfrm_state_migrate_install(x, xc, m, extack) < 0)
2219 return NULL;
2220
2221 return xc;
2222 }
2223 EXPORT_SYMBOL(xfrm_state_migrate);
2224 #endif
2225
xfrm_state_update(struct xfrm_state * x)2226 int xfrm_state_update(struct xfrm_state *x)
2227 {
2228 struct xfrm_state *x1, *to_put;
2229 int err;
2230 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
2231 struct net *net = xs_net(x);
2232
2233 to_put = NULL;
2234
2235 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2236 x1 = __xfrm_state_locate(x, use_spi, x->props.family);
2237
2238 err = -ESRCH;
2239 if (!x1)
2240 goto out;
2241
2242 if (xfrm_state_kern(x1)) {
2243 to_put = x1;
2244 err = -EEXIST;
2245 goto out;
2246 }
2247
2248 if (x1->km.state == XFRM_STATE_ACQ) {
2249 if (x->dir && x1->dir != x->dir) {
2250 to_put = x1;
2251 goto out;
2252 }
2253
2254 __xfrm_state_insert(x);
2255 x = NULL;
2256 } else {
2257 if (x1->dir != x->dir) {
2258 to_put = x1;
2259 goto out;
2260 }
2261 }
2262 err = 0;
2263
2264 out:
2265 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2266
2267 if (to_put)
2268 xfrm_state_put(to_put);
2269
2270 if (err)
2271 return err;
2272
2273 if (!x) {
2274 xfrm_state_delete(x1);
2275 xfrm_state_put(x1);
2276 return 0;
2277 }
2278
2279 err = -EINVAL;
2280 spin_lock_bh(&x1->lock);
2281 if (likely(x1->km.state == XFRM_STATE_VALID)) {
2282 if (x->encap && x1->encap &&
2283 x->encap->encap_type == x1->encap->encap_type)
2284 memcpy(x1->encap, x->encap, sizeof(*x1->encap));
2285 else if (x->encap || x1->encap)
2286 goto fail;
2287
2288 if (x->coaddr && x1->coaddr) {
2289 memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));
2290 }
2291 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel)))
2292 memcpy(&x1->sel, &x->sel, sizeof(x1->sel));
2293 memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
2294 x1->km.dying = 0;
2295
2296 hrtimer_start(&x1->mtimer, ktime_set(1, 0),
2297 HRTIMER_MODE_REL_SOFT);
2298 if (READ_ONCE(x1->curlft.use_time))
2299 xfrm_state_check_expire(x1);
2300
2301 if (x->props.smark.m || x->props.smark.v || x->if_id) {
2302 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2303
2304 if (x->props.smark.m || x->props.smark.v)
2305 x1->props.smark = x->props.smark;
2306
2307 if (x->if_id)
2308 x1->if_id = x->if_id;
2309
2310 __xfrm_state_bump_genids(x1);
2311 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2312 }
2313
2314 err = 0;
2315 x->km.state = XFRM_STATE_DEAD;
2316 xfrm_dev_state_delete(x);
2317 __xfrm_state_put(x);
2318 }
2319
2320 fail:
2321 spin_unlock_bh(&x1->lock);
2322
2323 xfrm_state_put(x1);
2324
2325 return err;
2326 }
2327 EXPORT_SYMBOL(xfrm_state_update);
2328
xfrm_state_check_expire(struct xfrm_state * x)2329 int xfrm_state_check_expire(struct xfrm_state *x)
2330 {
2331 /* All counters which are needed to decide if state is expired
2332 * are handled by SW for non-packet offload modes. Simply skip
2333 * the following update and save extra boilerplate in drivers.
2334 */
2335 if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
2336 xfrm_dev_state_update_stats(x);
2337
2338 if (!READ_ONCE(x->curlft.use_time))
2339 WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());
2340
2341 if (x->curlft.bytes >= x->lft.hard_byte_limit ||
2342 x->curlft.packets >= x->lft.hard_packet_limit) {
2343 x->km.state = XFRM_STATE_EXPIRED;
2344 hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT);
2345 return -EINVAL;
2346 }
2347
2348 if (!x->km.dying &&
2349 (x->curlft.bytes >= x->lft.soft_byte_limit ||
2350 x->curlft.packets >= x->lft.soft_packet_limit)) {
2351 x->km.dying = 1;
2352 km_state_expired(x, 0, 0);
2353 }
2354 return 0;
2355 }
2356 EXPORT_SYMBOL(xfrm_state_check_expire);
2357
xfrm_state_update_stats(struct net * net)2358 void xfrm_state_update_stats(struct net *net)
2359 {
2360 struct xfrm_state *x;
2361 int i;
2362
2363 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2364 for (i = 0; i <= net->xfrm.state_hmask; i++) {
2365 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst)
2366 xfrm_dev_state_update_stats(x);
2367 }
2368 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2369 }
2370
2371 struct xfrm_state *
xfrm_state_lookup(struct net * net,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)2372 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,
2373 u8 proto, unsigned short family)
2374 {
2375 struct xfrm_hash_state_ptrs state_ptrs;
2376 struct xfrm_state *x;
2377
2378 rcu_read_lock();
2379 xfrm_hash_ptrs_get(net, &state_ptrs);
2380
2381 x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);
2382 rcu_read_unlock();
2383 return x;
2384 }
2385 EXPORT_SYMBOL(xfrm_state_lookup);
2386
2387 struct xfrm_state *
xfrm_state_lookup_byaddr(struct net * net,u32 mark,const xfrm_address_t * daddr,const xfrm_address_t * saddr,u8 proto,unsigned short family)2388 xfrm_state_lookup_byaddr(struct net *net, u32 mark,
2389 const xfrm_address_t *daddr, const xfrm_address_t *saddr,
2390 u8 proto, unsigned short family)
2391 {
2392 struct xfrm_hash_state_ptrs state_ptrs;
2393 struct xfrm_state *x;
2394
2395 rcu_read_lock();
2396
2397 xfrm_hash_ptrs_get(net, &state_ptrs);
2398
2399 x = __xfrm_state_lookup_byaddr(&state_ptrs, mark, daddr, saddr, proto, family);
2400 rcu_read_unlock();
2401 return x;
2402 }
2403 EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
2404
2405 struct xfrm_state *
xfrm_find_acq(struct net * net,const struct xfrm_mark * mark,u8 mode,u32 reqid,u32 if_id,u32 pcpu_num,u8 proto,const xfrm_address_t * daddr,const xfrm_address_t * saddr,int create,unsigned short family)2406 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
2407 u32 if_id, u32 pcpu_num, u8 proto, const xfrm_address_t *daddr,
2408 const xfrm_address_t *saddr, int create, unsigned short family)
2409 {
2410 struct xfrm_state *x;
2411
2412 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2413 x = __find_acq_core(net, mark, family, mode, reqid, if_id, pcpu_num,
2414 proto, daddr, saddr, create);
2415 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2416
2417 return x;
2418 }
2419 EXPORT_SYMBOL(xfrm_find_acq);
2420
2421 #ifdef CONFIG_XFRM_SUB_POLICY
2422 #if IS_ENABLED(CONFIG_IPV6)
2423 /* distribution counting sort function for xfrm_state and xfrm_tmpl */
2424 static void
__xfrm6_sort(void ** dst,void ** src,int n,int (* cmp)(const void * p),int maxclass)2425 __xfrm6_sort(void **dst, void **src, int n,
2426 int (*cmp)(const void *p), int maxclass)
2427 {
2428 int count[XFRM_MAX_DEPTH] = { };
2429 int class[XFRM_MAX_DEPTH];
2430 int i;
2431
2432 for (i = 0; i < n; i++) {
2433 int c = cmp(src[i]);
2434
2435 class[i] = c;
2436 count[c]++;
2437 }
2438
2439 for (i = 2; i < maxclass; i++)
2440 count[i] += count[i - 1];
2441
2442 for (i = 0; i < n; i++) {
2443 dst[count[class[i] - 1]++] = src[i];
2444 src[i] = NULL;
2445 }
2446 }
2447
2448 /* Rule for xfrm_state:
2449 *
2450 * rule 1: select IPsec transport except AH
2451 * rule 2: select MIPv6 RO or inbound trigger
2452 * rule 3: select IPsec transport AH
2453 * rule 4: select IPsec tunnel
2454 * rule 5: others
2455 */
__xfrm6_state_sort_cmp(const void * p)2456 static int __xfrm6_state_sort_cmp(const void *p)
2457 {
2458 const struct xfrm_state *v = p;
2459
2460 switch (v->props.mode) {
2461 case XFRM_MODE_TRANSPORT:
2462 if (v->id.proto != IPPROTO_AH)
2463 return 1;
2464 else
2465 return 3;
2466 #if IS_ENABLED(CONFIG_IPV6_MIP6)
2467 case XFRM_MODE_ROUTEOPTIMIZATION:
2468 case XFRM_MODE_IN_TRIGGER:
2469 return 2;
2470 #endif
2471 case XFRM_MODE_TUNNEL:
2472 case XFRM_MODE_BEET:
2473 case XFRM_MODE_IPTFS:
2474 return 4;
2475 }
2476 return 5;
2477 }
2478
2479 /* Rule for xfrm_tmpl:
2480 *
2481 * rule 1: select IPsec transport
2482 * rule 2: select MIPv6 RO or inbound trigger
2483 * rule 3: select IPsec tunnel
2484 * rule 4: others
2485 */
__xfrm6_tmpl_sort_cmp(const void * p)2486 static int __xfrm6_tmpl_sort_cmp(const void *p)
2487 {
2488 const struct xfrm_tmpl *v = p;
2489
2490 switch (v->mode) {
2491 case XFRM_MODE_TRANSPORT:
2492 return 1;
2493 #if IS_ENABLED(CONFIG_IPV6_MIP6)
2494 case XFRM_MODE_ROUTEOPTIMIZATION:
2495 case XFRM_MODE_IN_TRIGGER:
2496 return 2;
2497 #endif
2498 case XFRM_MODE_TUNNEL:
2499 case XFRM_MODE_BEET:
2500 case XFRM_MODE_IPTFS:
2501 return 3;
2502 }
2503 return 4;
2504 }
2505 #else
__xfrm6_state_sort_cmp(const void * p)2506 static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; }
__xfrm6_tmpl_sort_cmp(const void * p)2507 static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; }
2508
2509 static inline void
__xfrm6_sort(void ** dst,void ** src,int n,int (* cmp)(const void * p),int maxclass)2510 __xfrm6_sort(void **dst, void **src, int n,
2511 int (*cmp)(const void *p), int maxclass)
2512 {
2513 int i;
2514
2515 for (i = 0; i < n; i++)
2516 dst[i] = src[i];
2517 }
2518 #endif /* CONFIG_IPV6 */
2519
2520 void
xfrm_tmpl_sort(struct xfrm_tmpl ** dst,struct xfrm_tmpl ** src,int n,unsigned short family)2521 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
2522 unsigned short family)
2523 {
2524 int i;
2525
2526 if (family == AF_INET6)
2527 __xfrm6_sort((void **)dst, (void **)src, n,
2528 __xfrm6_tmpl_sort_cmp, 5);
2529 else
2530 for (i = 0; i < n; i++)
2531 dst[i] = src[i];
2532 }
2533
2534 void
xfrm_state_sort(struct xfrm_state ** dst,struct xfrm_state ** src,int n,unsigned short family)2535 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
2536 unsigned short family)
2537 {
2538 int i;
2539
2540 if (family == AF_INET6)
2541 __xfrm6_sort((void **)dst, (void **)src, n,
2542 __xfrm6_state_sort_cmp, 6);
2543 else
2544 for (i = 0; i < n; i++)
2545 dst[i] = src[i];
2546 }
2547 #endif
2548
2549 /* Silly enough, but I'm lazy to build resolution list */
2550
__xfrm_find_acq_byseq(struct net * net,u32 mark,u32 seq,u32 pcpu_num)2551 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num)
2552 {
2553 unsigned int h = xfrm_seq_hash(net, seq);
2554 struct xfrm_state *x;
2555
2556 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, byseq) {
2557 if (x->km.seq == seq &&
2558 (mark & x->mark.m) == x->mark.v &&
2559 x->pcpu_num == pcpu_num &&
2560 x->km.state == XFRM_STATE_ACQ) {
2561 xfrm_state_hold(x);
2562 return x;
2563 }
2564 }
2565
2566 return NULL;
2567 }
2568
xfrm_find_acq_byseq(struct net * net,u32 mark,u32 seq,u32 pcpu_num)2569 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num)
2570 {
2571 struct xfrm_state *x;
2572
2573 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2574 x = __xfrm_find_acq_byseq(net, mark, seq, pcpu_num);
2575 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2576 return x;
2577 }
2578 EXPORT_SYMBOL(xfrm_find_acq_byseq);
2579
xfrm_get_acqseq(void)2580 u32 xfrm_get_acqseq(void)
2581 {
2582 u32 res;
2583 static atomic_t acqseq;
2584
2585 do {
2586 res = atomic_inc_return(&acqseq);
2587 } while (!res);
2588
2589 return res;
2590 }
2591 EXPORT_SYMBOL(xfrm_get_acqseq);
2592
verify_spi_info(u8 proto,u32 min,u32 max,struct netlink_ext_ack * extack)2593 int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack)
2594 {
2595 switch (proto) {
2596 case IPPROTO_AH:
2597 case IPPROTO_ESP:
2598 break;
2599
2600 case IPPROTO_COMP:
2601 /* IPCOMP spi is 16-bits. */
2602 if (max >= 0x10000) {
2603 NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535");
2604 return -EINVAL;
2605 }
2606 break;
2607
2608 default:
2609 NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP");
2610 return -EINVAL;
2611 }
2612
2613 if (min > max) {
2614 NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max");
2615 return -EINVAL;
2616 }
2617
2618 return 0;
2619 }
2620 EXPORT_SYMBOL(verify_spi_info);
2621
xfrm_alloc_spi(struct xfrm_state * x,u32 low,u32 high,struct netlink_ext_ack * extack)2622 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
2623 struct netlink_ext_ack *extack)
2624 {
2625 struct net *net = xs_net(x);
2626 unsigned int h;
2627 struct xfrm_state *x0;
2628 int err = -ENOENT;
2629 u32 range = high - low + 1;
2630 __be32 newspi = 0;
2631
2632 spin_lock_bh(&x->lock);
2633 if (x->km.state == XFRM_STATE_DEAD) {
2634 NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state");
2635 goto unlock;
2636 }
2637
2638 err = 0;
2639 if (x->id.spi)
2640 goto unlock;
2641
2642 err = -ENOENT;
2643
2644 for (h = 0; h < range; h++) {
2645 u32 spi = (low == high) ? low : get_random_u32_inclusive(low, high);
2646 if (spi == 0)
2647 goto next;
2648 newspi = htonl(spi);
2649
2650 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2651 x0 = xfrm_state_lookup_spi_proto(net, newspi, x->id.proto);
2652 if (!x0) {
2653 x->id.spi = newspi;
2654 h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family);
2655 XFRM_STATE_INSERT(byspi, &x->byspi,
2656 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h,
2657 x->xso.type);
2658 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2659 err = 0;
2660 goto unlock;
2661 }
2662 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2663
2664 next:
2665 if (signal_pending(current)) {
2666 err = -ERESTARTSYS;
2667 goto unlock;
2668 }
2669
2670 if (low == high)
2671 break;
2672 }
2673
2674 if (err)
2675 NL_SET_ERR_MSG(extack, "No SPI available in the requested range");
2676
2677 unlock:
2678 spin_unlock_bh(&x->lock);
2679
2680 return err;
2681 }
2682 EXPORT_SYMBOL(xfrm_alloc_spi);
2683
__xfrm_state_filter_match(struct xfrm_state * x,struct xfrm_address_filter * filter)2684 static bool __xfrm_state_filter_match(struct xfrm_state *x,
2685 struct xfrm_address_filter *filter)
2686 {
2687 if (filter) {
2688 if ((filter->family == AF_INET ||
2689 filter->family == AF_INET6) &&
2690 x->props.family != filter->family)
2691 return false;
2692
2693 return addr_match(&x->props.saddr, &filter->saddr,
2694 filter->splen) &&
2695 addr_match(&x->id.daddr, &filter->daddr,
2696 filter->dplen);
2697 }
2698 return true;
2699 }
2700
xfrm_state_walk(struct net * net,struct xfrm_state_walk * walk,int (* func)(struct xfrm_state *,int,void *),void * data)2701 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
2702 int (*func)(struct xfrm_state *, int, void*),
2703 void *data)
2704 {
2705 struct xfrm_state *state;
2706 struct xfrm_state_walk *x;
2707 int err = 0;
2708
2709 if (walk->seq != 0 && list_empty(&walk->all))
2710 return 0;
2711
2712 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2713 if (list_empty(&walk->all))
2714 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);
2715 else
2716 x = list_first_entry(&walk->all, struct xfrm_state_walk, all);
2717 list_for_each_entry_from(x, &net->xfrm.state_all, all) {
2718 if (x->state == XFRM_STATE_DEAD)
2719 continue;
2720 state = container_of(x, struct xfrm_state, km);
2721 if (!xfrm_id_proto_match(state->id.proto, walk->proto))
2722 continue;
2723 if (!__xfrm_state_filter_match(state, walk->filter))
2724 continue;
2725 err = func(state, walk->seq, data);
2726 if (err) {
2727 list_move_tail(&walk->all, &x->all);
2728 goto out;
2729 }
2730 walk->seq++;
2731 }
2732 if (walk->seq == 0) {
2733 err = -ENOENT;
2734 goto out;
2735 }
2736 list_del_init(&walk->all);
2737 out:
2738 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2739 return err;
2740 }
2741 EXPORT_SYMBOL(xfrm_state_walk);
2742
xfrm_state_walk_init(struct xfrm_state_walk * walk,u8 proto,struct xfrm_address_filter * filter)2743 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
2744 struct xfrm_address_filter *filter)
2745 {
2746 INIT_LIST_HEAD(&walk->all);
2747 walk->proto = proto;
2748 walk->state = XFRM_STATE_DEAD;
2749 walk->seq = 0;
2750 walk->filter = filter;
2751 }
2752 EXPORT_SYMBOL(xfrm_state_walk_init);
2753
xfrm_state_walk_done(struct xfrm_state_walk * walk,struct net * net)2754 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)
2755 {
2756 kfree(walk->filter);
2757
2758 if (list_empty(&walk->all))
2759 return;
2760
2761 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2762 list_del(&walk->all);
2763 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2764 }
2765 EXPORT_SYMBOL(xfrm_state_walk_done);
2766
xfrm_replay_timer_handler(struct timer_list * t)2767 static void xfrm_replay_timer_handler(struct timer_list *t)
2768 {
2769 struct xfrm_state *x = timer_container_of(x, t, rtimer);
2770
2771 spin_lock(&x->lock);
2772
2773 if (x->km.state == XFRM_STATE_VALID) {
2774 if (xfrm_aevent_is_on(xs_net(x)))
2775 xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
2776 else
2777 x->xflags |= XFRM_TIME_DEFER;
2778 }
2779
2780 spin_unlock(&x->lock);
2781 }
2782
2783 static LIST_HEAD(xfrm_km_list);
2784
km_policy_notify(struct xfrm_policy * xp,int dir,const struct km_event * c)2785 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
2786 {
2787 struct xfrm_mgr *km;
2788
2789 rcu_read_lock();
2790 list_for_each_entry_rcu(km, &xfrm_km_list, list)
2791 if (km->notify_policy)
2792 km->notify_policy(xp, dir, c);
2793 rcu_read_unlock();
2794 }
2795
km_state_notify(struct xfrm_state * x,const struct km_event * c)2796 void km_state_notify(struct xfrm_state *x, const struct km_event *c)
2797 {
2798 struct xfrm_mgr *km;
2799 rcu_read_lock();
2800 list_for_each_entry_rcu(km, &xfrm_km_list, list)
2801 if (km->notify)
2802 km->notify(x, c);
2803 rcu_read_unlock();
2804 }
2805
2806 EXPORT_SYMBOL(km_policy_notify);
2807 EXPORT_SYMBOL(km_state_notify);
2808
km_state_expired(struct xfrm_state * x,int hard,u32 portid)2809 void km_state_expired(struct xfrm_state *x, int hard, u32 portid)
2810 {
2811 struct km_event c;
2812
2813 c.data.hard = hard;
2814 c.portid = portid;
2815 c.event = XFRM_MSG_EXPIRE;
2816 km_state_notify(x, &c);
2817 }
2818
2819 EXPORT_SYMBOL(km_state_expired);
2820 /*
2821 * We send to all registered managers regardless of failure
2822 * We are happy with one success
2823 */
km_query(struct xfrm_state * x,struct xfrm_tmpl * t,struct xfrm_policy * pol)2824 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
2825 {
2826 int err = -EINVAL, acqret;
2827 struct xfrm_mgr *km;
2828
2829 rcu_read_lock();
2830 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2831 acqret = km->acquire(x, t, pol);
2832 if (!acqret)
2833 err = acqret;
2834 }
2835 rcu_read_unlock();
2836 return err;
2837 }
2838 EXPORT_SYMBOL(km_query);
2839
__km_new_mapping(struct xfrm_state * x,xfrm_address_t * ipaddr,__be16 sport)2840 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2841 {
2842 int err = -EINVAL;
2843 struct xfrm_mgr *km;
2844
2845 rcu_read_lock();
2846 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2847 if (km->new_mapping)
2848 err = km->new_mapping(x, ipaddr, sport);
2849 if (!err)
2850 break;
2851 }
2852 rcu_read_unlock();
2853 return err;
2854 }
2855
km_new_mapping(struct xfrm_state * x,xfrm_address_t * ipaddr,__be16 sport)2856 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2857 {
2858 int ret = 0;
2859
2860 if (x->mapping_maxage) {
2861 if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||
2862 x->new_mapping_sport != sport) {
2863 x->new_mapping_sport = sport;
2864 x->new_mapping = jiffies / HZ;
2865 ret = __km_new_mapping(x, ipaddr, sport);
2866 }
2867 } else {
2868 ret = __km_new_mapping(x, ipaddr, sport);
2869 }
2870
2871 return ret;
2872 }
2873 EXPORT_SYMBOL(km_new_mapping);
2874
km_policy_expired(struct xfrm_policy * pol,int dir,int hard,u32 portid)2875 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
2876 {
2877 struct km_event c;
2878
2879 c.data.hard = hard;
2880 c.portid = portid;
2881 c.event = XFRM_MSG_POLEXPIRE;
2882 km_policy_notify(pol, dir, &c);
2883 }
2884 EXPORT_SYMBOL(km_policy_expired);
2885
2886 #ifdef CONFIG_XFRM_MIGRATE
km_migrate(const struct xfrm_selector * sel,u8 dir,u8 type,const struct xfrm_migrate * m,int num_migrate,const struct xfrm_kmaddress * k,struct net * net,const struct xfrm_encap_tmpl * encap)2887 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
2888 const struct xfrm_migrate *m, int num_migrate,
2889 const struct xfrm_kmaddress *k, struct net *net,
2890 const struct xfrm_encap_tmpl *encap)
2891 {
2892 int err = -EINVAL;
2893 int ret;
2894 struct xfrm_mgr *km;
2895
2896 rcu_read_lock();
2897 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2898 if (km->migrate) {
2899 ret = km->migrate(sel, dir, type, m, num_migrate, k,
2900 net, encap);
2901 if (!ret)
2902 err = ret;
2903 }
2904 }
2905 rcu_read_unlock();
2906 return err;
2907 }
2908 EXPORT_SYMBOL(km_migrate);
2909 #endif
2910
km_report(struct net * net,u8 proto,struct xfrm_selector * sel,xfrm_address_t * addr)2911 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)
2912 {
2913 int err = -EINVAL;
2914 int ret;
2915 struct xfrm_mgr *km;
2916
2917 rcu_read_lock();
2918 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2919 if (km->report) {
2920 ret = km->report(net, proto, sel, addr);
2921 if (!ret)
2922 err = ret;
2923 }
2924 }
2925 rcu_read_unlock();
2926 return err;
2927 }
2928 EXPORT_SYMBOL(km_report);
2929
km_is_alive(const struct km_event * c)2930 static bool km_is_alive(const struct km_event *c)
2931 {
2932 struct xfrm_mgr *km;
2933 bool is_alive = false;
2934
2935 rcu_read_lock();
2936 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2937 if (km->is_alive && km->is_alive(c)) {
2938 is_alive = true;
2939 break;
2940 }
2941 }
2942 rcu_read_unlock();
2943
2944 return is_alive;
2945 }
2946
2947 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2948 static DEFINE_SPINLOCK(xfrm_translator_lock);
2949 static struct xfrm_translator __rcu *xfrm_translator;
2950
xfrm_get_translator(void)2951 struct xfrm_translator *xfrm_get_translator(void)
2952 {
2953 struct xfrm_translator *xtr;
2954
2955 rcu_read_lock();
2956 xtr = rcu_dereference(xfrm_translator);
2957 if (unlikely(!xtr))
2958 goto out;
2959 if (!try_module_get(xtr->owner))
2960 xtr = NULL;
2961 out:
2962 rcu_read_unlock();
2963 return xtr;
2964 }
2965 EXPORT_SYMBOL_GPL(xfrm_get_translator);
2966
xfrm_put_translator(struct xfrm_translator * xtr)2967 void xfrm_put_translator(struct xfrm_translator *xtr)
2968 {
2969 module_put(xtr->owner);
2970 }
2971 EXPORT_SYMBOL_GPL(xfrm_put_translator);
2972
xfrm_register_translator(struct xfrm_translator * xtr)2973 int xfrm_register_translator(struct xfrm_translator *xtr)
2974 {
2975 int err = 0;
2976
2977 spin_lock_bh(&xfrm_translator_lock);
2978 if (unlikely(xfrm_translator != NULL))
2979 err = -EEXIST;
2980 else
2981 rcu_assign_pointer(xfrm_translator, xtr);
2982 spin_unlock_bh(&xfrm_translator_lock);
2983
2984 return err;
2985 }
2986 EXPORT_SYMBOL_GPL(xfrm_register_translator);
2987
xfrm_unregister_translator(struct xfrm_translator * xtr)2988 int xfrm_unregister_translator(struct xfrm_translator *xtr)
2989 {
2990 int err = 0;
2991
2992 spin_lock_bh(&xfrm_translator_lock);
2993 if (likely(xfrm_translator != NULL)) {
2994 if (rcu_access_pointer(xfrm_translator) != xtr)
2995 err = -EINVAL;
2996 else
2997 RCU_INIT_POINTER(xfrm_translator, NULL);
2998 }
2999 spin_unlock_bh(&xfrm_translator_lock);
3000 synchronize_rcu();
3001
3002 return err;
3003 }
3004 EXPORT_SYMBOL_GPL(xfrm_unregister_translator);
3005 #endif
3006
xfrm_user_policy(struct sock * sk,int optname,sockptr_t optval,int optlen)3007 int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
3008 {
3009 int err;
3010 u8 *data;
3011 struct xfrm_mgr *km;
3012 struct xfrm_policy *pol = NULL;
3013
3014 if (sockptr_is_null(optval) && !optlen) {
3015 xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL);
3016 xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL);
3017 sk_dst_reset(sk);
3018 return 0;
3019 }
3020
3021 if (optlen <= 0 || optlen > PAGE_SIZE)
3022 return -EMSGSIZE;
3023
3024 data = memdup_sockptr(optval, optlen);
3025 if (IS_ERR(data))
3026 return PTR_ERR(data);
3027
3028 if (IS_ENABLED(CONFIG_COMPAT_FOR_U64_ALIGNMENT) && in_compat_syscall()) {
3029 struct xfrm_translator *xtr = xfrm_get_translator();
3030
3031 if (!xtr) {
3032 kfree(data);
3033 return -EOPNOTSUPP;
3034 }
3035
3036 err = xtr->xlate_user_policy_sockptr(&data, optlen);
3037 xfrm_put_translator(xtr);
3038 if (err) {
3039 kfree(data);
3040 return err;
3041 }
3042 }
3043
3044 err = -EINVAL;
3045 rcu_read_lock();
3046 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
3047 pol = km->compile_policy(sk, optname, data,
3048 optlen, &err);
3049 if (err >= 0)
3050 break;
3051 }
3052 rcu_read_unlock();
3053
3054 if (err >= 0) {
3055 xfrm_sk_policy_insert(sk, err, pol);
3056 xfrm_pol_put(pol);
3057 sk_dst_reset(sk);
3058 err = 0;
3059 }
3060
3061 kfree(data);
3062 return err;
3063 }
3064 EXPORT_SYMBOL(xfrm_user_policy);
3065
3066 static DEFINE_SPINLOCK(xfrm_km_lock);
3067
xfrm_register_km(struct xfrm_mgr * km)3068 void xfrm_register_km(struct xfrm_mgr *km)
3069 {
3070 spin_lock_bh(&xfrm_km_lock);
3071 list_add_tail_rcu(&km->list, &xfrm_km_list);
3072 spin_unlock_bh(&xfrm_km_lock);
3073 }
3074 EXPORT_SYMBOL(xfrm_register_km);
3075
xfrm_unregister_km(struct xfrm_mgr * km)3076 void xfrm_unregister_km(struct xfrm_mgr *km)
3077 {
3078 spin_lock_bh(&xfrm_km_lock);
3079 list_del_rcu(&km->list);
3080 spin_unlock_bh(&xfrm_km_lock);
3081 synchronize_rcu();
3082 }
3083 EXPORT_SYMBOL(xfrm_unregister_km);
3084
xfrm_state_register_afinfo(struct xfrm_state_afinfo * afinfo)3085 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo)
3086 {
3087 int err = 0;
3088
3089 if (WARN_ON(afinfo->family >= NPROTO))
3090 return -EAFNOSUPPORT;
3091
3092 spin_lock_bh(&xfrm_state_afinfo_lock);
3093 if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))
3094 err = -EEXIST;
3095 else
3096 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);
3097 spin_unlock_bh(&xfrm_state_afinfo_lock);
3098 return err;
3099 }
3100 EXPORT_SYMBOL(xfrm_state_register_afinfo);
3101
xfrm_state_unregister_afinfo(struct xfrm_state_afinfo * afinfo)3102 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo)
3103 {
3104 int err = 0, family = afinfo->family;
3105
3106 if (WARN_ON(family >= NPROTO))
3107 return -EAFNOSUPPORT;
3108
3109 spin_lock_bh(&xfrm_state_afinfo_lock);
3110 if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {
3111 if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo)
3112 err = -EINVAL;
3113 else
3114 RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);
3115 }
3116 spin_unlock_bh(&xfrm_state_afinfo_lock);
3117 synchronize_rcu();
3118 return err;
3119 }
3120 EXPORT_SYMBOL(xfrm_state_unregister_afinfo);
3121
xfrm_state_afinfo_get_rcu(unsigned int family)3122 struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family)
3123 {
3124 if (unlikely(family >= NPROTO))
3125 return NULL;
3126
3127 return rcu_dereference(xfrm_state_afinfo[family]);
3128 }
3129 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu);
3130
xfrm_state_get_afinfo(unsigned int family)3131 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
3132 {
3133 struct xfrm_state_afinfo *afinfo;
3134 if (unlikely(family >= NPROTO))
3135 return NULL;
3136 rcu_read_lock();
3137 afinfo = rcu_dereference(xfrm_state_afinfo[family]);
3138 if (unlikely(!afinfo))
3139 rcu_read_unlock();
3140 return afinfo;
3141 }
3142
xfrm_flush_gc(void)3143 void xfrm_flush_gc(void)
3144 {
3145 flush_work(&xfrm_state_gc_work);
3146 }
3147 EXPORT_SYMBOL(xfrm_flush_gc);
3148
xfrm_state_delete_tunnel(struct xfrm_state * x)3149 static void xfrm_state_delete_tunnel(struct xfrm_state *x)
3150 {
3151 if (x->tunnel) {
3152 struct xfrm_state *t = x->tunnel;
3153
3154 if (atomic_dec_return(&t->tunnel_users) == 1)
3155 xfrm_state_delete(t);
3156 xfrm_state_put(t);
3157 x->tunnel = NULL;
3158 }
3159 }
3160
xfrm_state_mtu(struct xfrm_state * x,int mtu)3161 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
3162 {
3163 const struct xfrm_type *type = READ_ONCE(x->type);
3164 struct crypto_aead *aead;
3165 u32 blksize, net_adj = 0;
3166 u32 overhead, payload_mtu;
3167
3168 if (x->km.state != XFRM_STATE_VALID ||
3169 !type || type->proto != IPPROTO_ESP) {
3170 if (mtu <= x->props.header_len)
3171 return 1;
3172 return mtu - x->props.header_len;
3173 }
3174
3175 aead = x->data;
3176 blksize = ALIGN(crypto_aead_blocksize(aead), 4);
3177
3178 switch (x->props.mode) {
3179 case XFRM_MODE_TRANSPORT:
3180 case XFRM_MODE_BEET:
3181 if (x->props.family == AF_INET)
3182 net_adj = sizeof(struct iphdr);
3183 else if (x->props.family == AF_INET6)
3184 net_adj = sizeof(struct ipv6hdr);
3185 break;
3186 case XFRM_MODE_TUNNEL:
3187 break;
3188 default:
3189 if (x->mode_cbs && x->mode_cbs->get_inner_mtu)
3190 return x->mode_cbs->get_inner_mtu(x, mtu);
3191
3192 WARN_ON_ONCE(1);
3193 break;
3194 }
3195
3196 overhead = x->props.header_len + crypto_aead_authsize(aead) + net_adj;
3197 if (mtu <= overhead)
3198 return 1;
3199
3200 payload_mtu = mtu - overhead;
3201 payload_mtu &= ~(blksize - 1);
3202 if (payload_mtu <= 2)
3203 return 1;
3204
3205 return payload_mtu + net_adj - 2;
3206
3207 }
3208 EXPORT_SYMBOL_GPL(xfrm_state_mtu);
3209
__xfrm_init_state(struct xfrm_state * x,struct netlink_ext_ack * extack)3210 int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
3211 {
3212 const struct xfrm_mode *inner_mode;
3213 const struct xfrm_mode *outer_mode;
3214 int family = x->props.family;
3215 int err;
3216
3217 if (family == AF_INET &&
3218 (!x->dir || x->dir == XFRM_SA_DIR_OUT) &&
3219 READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
3220 x->props.flags |= XFRM_STATE_NOPMTUDISC;
3221
3222 err = -EPROTONOSUPPORT;
3223
3224 if (x->sel.family != AF_UNSPEC) {
3225 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);
3226 if (inner_mode == NULL) {
3227 NL_SET_ERR_MSG(extack, "Requested mode not found");
3228 goto error;
3229 }
3230
3231 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
3232 family != x->sel.family) {
3233 NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family");
3234 goto error;
3235 }
3236
3237 x->inner_mode = *inner_mode;
3238 } else {
3239 const struct xfrm_mode *inner_mode_iaf;
3240 int iafamily = AF_INET;
3241
3242 inner_mode = xfrm_get_mode(x->props.mode, x->props.family);
3243 if (inner_mode == NULL) {
3244 NL_SET_ERR_MSG(extack, "Requested mode not found");
3245 goto error;
3246 }
3247
3248 x->inner_mode = *inner_mode;
3249
3250 if (x->props.family == AF_INET)
3251 iafamily = AF_INET6;
3252
3253 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);
3254 if (inner_mode_iaf) {
3255 if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
3256 x->inner_mode_iaf = *inner_mode_iaf;
3257 }
3258 }
3259
3260 x->type = xfrm_get_type(x->id.proto, family);
3261 if (x->type == NULL) {
3262 NL_SET_ERR_MSG(extack, "Requested type not found");
3263 goto error;
3264 }
3265
3266 err = x->type->init_state(x, extack);
3267 if (err)
3268 goto error;
3269
3270 outer_mode = xfrm_get_mode(x->props.mode, family);
3271 if (!outer_mode) {
3272 NL_SET_ERR_MSG(extack, "Requested mode not found");
3273 err = -EPROTONOSUPPORT;
3274 goto error;
3275 }
3276
3277 x->outer_mode = *outer_mode;
3278 if (x->nat_keepalive_interval) {
3279 if (x->dir != XFRM_SA_DIR_OUT) {
3280 NL_SET_ERR_MSG(extack, "NAT keepalive is only supported for outbound SAs");
3281 err = -EINVAL;
3282 goto error;
3283 }
3284
3285 if (!x->encap || x->encap->encap_type != UDP_ENCAP_ESPINUDP) {
3286 NL_SET_ERR_MSG(extack,
3287 "NAT keepalive is only supported for UDP encapsulation");
3288 err = -EINVAL;
3289 goto error;
3290 }
3291 }
3292
3293 x->mode_cbs = xfrm_get_mode_cbs(x->props.mode);
3294 if (x->mode_cbs) {
3295 if (x->mode_cbs->init_state)
3296 err = x->mode_cbs->init_state(x);
3297 module_put(x->mode_cbs->owner);
3298 if (err && !x->mode_data)
3299 x->mode_cbs = NULL;
3300 }
3301 error:
3302 return err;
3303 }
3304
3305 EXPORT_SYMBOL(__xfrm_init_state);
3306
xfrm_init_state(struct xfrm_state * x,struct netlink_ext_ack * extack)3307 int xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
3308 {
3309 int err;
3310
3311 err = __xfrm_init_state(x, extack);
3312 if (err)
3313 return err;
3314
3315 err = xfrm_init_replay(x, NULL);
3316 if (err)
3317 return err;
3318
3319 x->km.state = XFRM_STATE_VALID;
3320 return 0;
3321 }
3322
3323 EXPORT_SYMBOL(xfrm_init_state);
3324
xfrm_state_init(struct net * net)3325 int __net_init xfrm_state_init(struct net *net)
3326 {
3327 struct hlist_head *ndst, *nsrc, *nspi, *nseq;
3328 unsigned int sz;
3329
3330 if (net_eq(net, &init_net))
3331 xfrm_state_cache = KMEM_CACHE(xfrm_state,
3332 SLAB_HWCACHE_ALIGN | SLAB_PANIC);
3333
3334 INIT_LIST_HEAD(&net->xfrm.state_all);
3335
3336 sz = sizeof(struct hlist_head) * 8;
3337
3338 ndst = xfrm_hash_alloc(sz);
3339 if (!ndst)
3340 goto out_bydst;
3341 rcu_assign_pointer(net->xfrm.state_bydst, ndst);
3342
3343 nsrc = xfrm_hash_alloc(sz);
3344 if (!nsrc)
3345 goto out_bysrc;
3346 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
3347
3348 nspi = xfrm_hash_alloc(sz);
3349 if (!nspi)
3350 goto out_byspi;
3351 rcu_assign_pointer(net->xfrm.state_byspi, nspi);
3352
3353 nseq = xfrm_hash_alloc(sz);
3354 if (!nseq)
3355 goto out_byseq;
3356 rcu_assign_pointer(net->xfrm.state_byseq, nseq);
3357
3358 net->xfrm.state_cache_input = alloc_percpu(struct hlist_head);
3359 if (!net->xfrm.state_cache_input)
3360 goto out_state_cache_input;
3361
3362 net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
3363
3364 net->xfrm.state_num = 0;
3365 INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
3366 spin_lock_init(&net->xfrm.xfrm_state_lock);
3367 seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,
3368 &net->xfrm.xfrm_state_lock);
3369 return 0;
3370
3371 out_state_cache_input:
3372 xfrm_hash_free(nseq, sz);
3373 out_byseq:
3374 xfrm_hash_free(nspi, sz);
3375 out_byspi:
3376 xfrm_hash_free(nsrc, sz);
3377 out_bysrc:
3378 xfrm_hash_free(ndst, sz);
3379 out_bydst:
3380 return -ENOMEM;
3381 }
3382
3383 #define xfrm_state_deref_netexit(table) \
3384 rcu_dereference_protected((table), true /* netns is going away */)
xfrm_state_fini(struct net * net)3385 void xfrm_state_fini(struct net *net)
3386 {
3387 unsigned int sz;
3388 int i;
3389
3390 flush_work(&net->xfrm.state_hash_work);
3391 xfrm_state_flush(net, 0, false);
3392 flush_work(&xfrm_state_gc_work);
3393
3394 WARN_ON(!list_empty(&net->xfrm.state_all));
3395
3396 for (i = 0; i <= net->xfrm.state_hmask; i++) {
3397 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byseq) + i));
3398 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byspi) + i));
3399 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bysrc) + i));
3400 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bydst) + i));
3401 }
3402
3403 sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
3404 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byseq), sz);
3405 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byspi), sz);
3406 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bysrc), sz);
3407 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bydst), sz);
3408 free_percpu(net->xfrm.state_cache_input);
3409 }
3410
3411 #ifdef CONFIG_AUDITSYSCALL
xfrm_audit_helper_sainfo(struct xfrm_state * x,struct audit_buffer * audit_buf)3412 static void xfrm_audit_helper_sainfo(struct xfrm_state *x,
3413 struct audit_buffer *audit_buf)
3414 {
3415 struct xfrm_sec_ctx *ctx = x->security;
3416 u32 spi = ntohl(x->id.spi);
3417
3418 if (ctx)
3419 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
3420 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
3421
3422 switch (x->props.family) {
3423 case AF_INET:
3424 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
3425 &x->props.saddr.a4, &x->id.daddr.a4);
3426 break;
3427 case AF_INET6:
3428 audit_log_format(audit_buf, " src=%pI6 dst=%pI6",
3429 x->props.saddr.a6, x->id.daddr.a6);
3430 break;
3431 }
3432
3433 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
3434 }
3435
xfrm_audit_helper_pktinfo(struct sk_buff * skb,u16 family,struct audit_buffer * audit_buf)3436 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family,
3437 struct audit_buffer *audit_buf)
3438 {
3439 const struct iphdr *iph4;
3440 const struct ipv6hdr *iph6;
3441
3442 switch (family) {
3443 case AF_INET:
3444 iph4 = ip_hdr(skb);
3445 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
3446 &iph4->saddr, &iph4->daddr);
3447 break;
3448 case AF_INET6:
3449 iph6 = ipv6_hdr(skb);
3450 audit_log_format(audit_buf,
3451 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
3452 &iph6->saddr, &iph6->daddr,
3453 iph6->flow_lbl[0] & 0x0f,
3454 iph6->flow_lbl[1],
3455 iph6->flow_lbl[2]);
3456 break;
3457 }
3458 }
3459
xfrm_audit_state_add(struct xfrm_state * x,int result,bool task_valid)3460 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)
3461 {
3462 struct audit_buffer *audit_buf;
3463
3464 audit_buf = xfrm_audit_start("SAD-add");
3465 if (audit_buf == NULL)
3466 return;
3467 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3468 xfrm_audit_helper_sainfo(x, audit_buf);
3469 audit_log_format(audit_buf, " res=%u", result);
3470 audit_log_end(audit_buf);
3471 }
3472 EXPORT_SYMBOL_GPL(xfrm_audit_state_add);
3473
xfrm_audit_state_delete(struct xfrm_state * x,int result,bool task_valid)3474 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)
3475 {
3476 struct audit_buffer *audit_buf;
3477
3478 audit_buf = xfrm_audit_start("SAD-delete");
3479 if (audit_buf == NULL)
3480 return;
3481 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3482 xfrm_audit_helper_sainfo(x, audit_buf);
3483 audit_log_format(audit_buf, " res=%u", result);
3484 audit_log_end(audit_buf);
3485 }
3486 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete);
3487
xfrm_audit_state_replay_overflow(struct xfrm_state * x,struct sk_buff * skb)3488 void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
3489 struct sk_buff *skb)
3490 {
3491 struct audit_buffer *audit_buf;
3492 u32 spi;
3493
3494 audit_buf = xfrm_audit_start("SA-replay-overflow");
3495 if (audit_buf == NULL)
3496 return;
3497 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3498 /* don't record the sequence number because it's inherent in this kind
3499 * of audit message */
3500 spi = ntohl(x->id.spi);
3501 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
3502 audit_log_end(audit_buf);
3503 }
3504 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow);
3505
xfrm_audit_state_replay(struct xfrm_state * x,struct sk_buff * skb,__be32 net_seq)3506 void xfrm_audit_state_replay(struct xfrm_state *x,
3507 struct sk_buff *skb, __be32 net_seq)
3508 {
3509 struct audit_buffer *audit_buf;
3510 u32 spi;
3511
3512 audit_buf = xfrm_audit_start("SA-replayed-pkt");
3513 if (audit_buf == NULL)
3514 return;
3515 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3516 spi = ntohl(x->id.spi);
3517 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3518 spi, spi, ntohl(net_seq));
3519 audit_log_end(audit_buf);
3520 }
3521 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay);
3522
xfrm_audit_state_notfound_simple(struct sk_buff * skb,u16 family)3523 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family)
3524 {
3525 struct audit_buffer *audit_buf;
3526
3527 audit_buf = xfrm_audit_start("SA-notfound");
3528 if (audit_buf == NULL)
3529 return;
3530 xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3531 audit_log_end(audit_buf);
3532 }
3533 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple);
3534
xfrm_audit_state_notfound(struct sk_buff * skb,u16 family,__be32 net_spi,__be32 net_seq)3535 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
3536 __be32 net_spi, __be32 net_seq)
3537 {
3538 struct audit_buffer *audit_buf;
3539 u32 spi;
3540
3541 audit_buf = xfrm_audit_start("SA-notfound");
3542 if (audit_buf == NULL)
3543 return;
3544 xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3545 spi = ntohl(net_spi);
3546 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3547 spi, spi, ntohl(net_seq));
3548 audit_log_end(audit_buf);
3549 }
3550 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound);
3551
xfrm_audit_state_icvfail(struct xfrm_state * x,struct sk_buff * skb,u8 proto)3552 void xfrm_audit_state_icvfail(struct xfrm_state *x,
3553 struct sk_buff *skb, u8 proto)
3554 {
3555 struct audit_buffer *audit_buf;
3556 __be32 net_spi;
3557 __be32 net_seq;
3558
3559 audit_buf = xfrm_audit_start("SA-icv-failure");
3560 if (audit_buf == NULL)
3561 return;
3562 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3563 if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) {
3564 u32 spi = ntohl(net_spi);
3565 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3566 spi, spi, ntohl(net_seq));
3567 }
3568 audit_log_end(audit_buf);
3569 }
3570 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail);
3571 #endif /* CONFIG_AUDITSYSCALL */
3572