1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * xfrm_state.c
4 *
5 * Changes:
6 * Mitsuru KANDA @USAGI
7 * Kazunori MIYAZAWA @USAGI
8 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
9 * IPv6 support
10 * YOSHIFUJI Hideaki @USAGI
11 * Split up af-specific functions
12 * Derek Atkins <derek@ihtfp.com>
13 * Add UDP Encapsulation
14 *
15 */
16
17 #include <linux/compat.h>
18 #include <linux/workqueue.h>
19 #include <net/xfrm.h>
20 #include <linux/pfkeyv2.h>
21 #include <linux/ipsec.h>
22 #include <linux/module.h>
23 #include <linux/cache.h>
24 #include <linux/audit.h>
25 #include <linux/uaccess.h>
26 #include <linux/ktime.h>
27 #include <linux/slab.h>
28 #include <linux/interrupt.h>
29 #include <linux/kernel.h>
30
31 #include <crypto/aead.h>
32
33 #include "xfrm_hash.h"
34
35 #define xfrm_state_deref_prot(table, net) \
36 rcu_dereference_protected((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
37 #define xfrm_state_deref_check(table, net) \
38 rcu_dereference_check((table), lockdep_is_held(&(net)->xfrm.xfrm_state_lock))
39
40 static void xfrm_state_gc_task(struct work_struct *work);
41
42 /* Each xfrm_state may be linked to two tables:
43
44 1. Hash table by (spi,daddr,ah/esp) to find SA by SPI. (input,ctl)
45 2. Hash table by (daddr,family,reqid) to find what SAs exist for given
46 destination/tunnel endpoint. (output)
47 */
48
49 static unsigned int xfrm_state_hashmax __read_mostly = 1 * 1024 * 1024;
50 static struct kmem_cache *xfrm_state_cache __ro_after_init;
51
52 static DECLARE_WORK(xfrm_state_gc_work, xfrm_state_gc_task);
53 static HLIST_HEAD(xfrm_state_gc_list);
54 static HLIST_HEAD(xfrm_state_dev_gc_list);
55
xfrm_state_hold_rcu(struct xfrm_state * x)56 static inline bool xfrm_state_hold_rcu(struct xfrm_state *x)
57 {
58 return refcount_inc_not_zero(&x->refcnt);
59 }
60
xfrm_dst_hash(struct net * net,const xfrm_address_t * daddr,const xfrm_address_t * saddr,u32 reqid,unsigned short family)61 static inline unsigned int xfrm_dst_hash(struct net *net,
62 const xfrm_address_t *daddr,
63 const xfrm_address_t *saddr,
64 u32 reqid,
65 unsigned short family)
66 {
67 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
68
69 return __xfrm_dst_hash(daddr, saddr, reqid, family, net->xfrm.state_hmask);
70 }
71
xfrm_src_hash(struct net * net,const xfrm_address_t * daddr,const xfrm_address_t * saddr,unsigned short family)72 static inline unsigned int xfrm_src_hash(struct net *net,
73 const xfrm_address_t *daddr,
74 const xfrm_address_t *saddr,
75 unsigned short family)
76 {
77 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
78
79 return __xfrm_src_hash(daddr, saddr, family, net->xfrm.state_hmask);
80 }
81
82 static inline unsigned int
xfrm_spi_hash(struct net * net,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)83 xfrm_spi_hash(struct net *net, const xfrm_address_t *daddr,
84 __be32 spi, u8 proto, unsigned short family)
85 {
86 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
87
88 return __xfrm_spi_hash(daddr, spi, proto, family, net->xfrm.state_hmask);
89 }
90
xfrm_seq_hash(struct net * net,u32 seq)91 static unsigned int xfrm_seq_hash(struct net *net, u32 seq)
92 {
93 lockdep_assert_held(&net->xfrm.xfrm_state_lock);
94
95 return __xfrm_seq_hash(seq, net->xfrm.state_hmask);
96 }
97
98 #define XFRM_STATE_INSERT(by, _n, _h, _type) \
99 { \
100 struct xfrm_state *_x = NULL; \
101 \
102 if (_type != XFRM_DEV_OFFLOAD_PACKET) { \
103 hlist_for_each_entry_rcu(_x, _h, by) { \
104 if (_x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
105 continue; \
106 break; \
107 } \
108 } \
109 \
110 if (!_x || _x->xso.type == XFRM_DEV_OFFLOAD_PACKET) \
111 /* SAD is empty or consist from HW SAs only */ \
112 hlist_add_head_rcu(_n, _h); \
113 else \
114 hlist_add_before_rcu(_n, &_x->by); \
115 }
116
xfrm_hash_transfer(struct hlist_head * list,struct hlist_head * ndsttable,struct hlist_head * nsrctable,struct hlist_head * nspitable,struct hlist_head * nseqtable,unsigned int nhashmask)117 static void xfrm_hash_transfer(struct hlist_head *list,
118 struct hlist_head *ndsttable,
119 struct hlist_head *nsrctable,
120 struct hlist_head *nspitable,
121 struct hlist_head *nseqtable,
122 unsigned int nhashmask)
123 {
124 struct hlist_node *tmp;
125 struct xfrm_state *x;
126
127 hlist_for_each_entry_safe(x, tmp, list, bydst) {
128 unsigned int h;
129
130 h = __xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
131 x->props.reqid, x->props.family,
132 nhashmask);
133 XFRM_STATE_INSERT(bydst, &x->bydst, ndsttable + h, x->xso.type);
134
135 h = __xfrm_src_hash(&x->id.daddr, &x->props.saddr,
136 x->props.family,
137 nhashmask);
138 XFRM_STATE_INSERT(bysrc, &x->bysrc, nsrctable + h, x->xso.type);
139
140 if (x->id.spi) {
141 h = __xfrm_spi_hash(&x->id.daddr, x->id.spi,
142 x->id.proto, x->props.family,
143 nhashmask);
144 XFRM_STATE_INSERT(byspi, &x->byspi, nspitable + h,
145 x->xso.type);
146 }
147
148 if (x->km.seq) {
149 h = __xfrm_seq_hash(x->km.seq, nhashmask);
150 XFRM_STATE_INSERT(byseq, &x->byseq, nseqtable + h,
151 x->xso.type);
152 }
153 }
154 }
155
xfrm_hash_new_size(unsigned int state_hmask)156 static unsigned long xfrm_hash_new_size(unsigned int state_hmask)
157 {
158 return ((state_hmask + 1) << 1) * sizeof(struct hlist_head);
159 }
160
xfrm_hash_resize(struct work_struct * work)161 static void xfrm_hash_resize(struct work_struct *work)
162 {
163 struct net *net = container_of(work, struct net, xfrm.state_hash_work);
164 struct hlist_head *ndst, *nsrc, *nspi, *nseq, *odst, *osrc, *ospi, *oseq;
165 unsigned long nsize, osize;
166 unsigned int nhashmask, ohashmask;
167 int i;
168
169 nsize = xfrm_hash_new_size(net->xfrm.state_hmask);
170 ndst = xfrm_hash_alloc(nsize);
171 if (!ndst)
172 return;
173 nsrc = xfrm_hash_alloc(nsize);
174 if (!nsrc) {
175 xfrm_hash_free(ndst, nsize);
176 return;
177 }
178 nspi = xfrm_hash_alloc(nsize);
179 if (!nspi) {
180 xfrm_hash_free(ndst, nsize);
181 xfrm_hash_free(nsrc, nsize);
182 return;
183 }
184 nseq = xfrm_hash_alloc(nsize);
185 if (!nseq) {
186 xfrm_hash_free(ndst, nsize);
187 xfrm_hash_free(nsrc, nsize);
188 xfrm_hash_free(nspi, nsize);
189 return;
190 }
191
192 spin_lock_bh(&net->xfrm.xfrm_state_lock);
193 write_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
194
195 nhashmask = (nsize / sizeof(struct hlist_head)) - 1U;
196 odst = xfrm_state_deref_prot(net->xfrm.state_bydst, net);
197 for (i = net->xfrm.state_hmask; i >= 0; i--)
198 xfrm_hash_transfer(odst + i, ndst, nsrc, nspi, nseq, nhashmask);
199
200 osrc = xfrm_state_deref_prot(net->xfrm.state_bysrc, net);
201 ospi = xfrm_state_deref_prot(net->xfrm.state_byspi, net);
202 oseq = xfrm_state_deref_prot(net->xfrm.state_byseq, net);
203 ohashmask = net->xfrm.state_hmask;
204
205 rcu_assign_pointer(net->xfrm.state_bydst, ndst);
206 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
207 rcu_assign_pointer(net->xfrm.state_byspi, nspi);
208 rcu_assign_pointer(net->xfrm.state_byseq, nseq);
209 net->xfrm.state_hmask = nhashmask;
210
211 write_seqcount_end(&net->xfrm.xfrm_state_hash_generation);
212 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
213
214 osize = (ohashmask + 1) * sizeof(struct hlist_head);
215
216 synchronize_rcu();
217
218 xfrm_hash_free(odst, osize);
219 xfrm_hash_free(osrc, osize);
220 xfrm_hash_free(ospi, osize);
221 xfrm_hash_free(oseq, osize);
222 }
223
224 static DEFINE_SPINLOCK(xfrm_state_afinfo_lock);
225 static struct xfrm_state_afinfo __rcu *xfrm_state_afinfo[NPROTO];
226
227 static DEFINE_SPINLOCK(xfrm_state_gc_lock);
228 static DEFINE_SPINLOCK(xfrm_state_dev_gc_lock);
229 static DEFINE_MUTEX(xfrm_state_gc_mutex);
230
231 int __xfrm_state_delete(struct xfrm_state *x);
232
233 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol);
234 static bool km_is_alive(const struct km_event *c);
235 void km_state_expired(struct xfrm_state *x, int hard, u32 portid);
236
xfrm_register_type(const struct xfrm_type * type,unsigned short family)237 int xfrm_register_type(const struct xfrm_type *type, unsigned short family)
238 {
239 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
240 int err = 0;
241
242 if (!afinfo)
243 return -EAFNOSUPPORT;
244
245 #define X(afi, T, name) do { \
246 WARN_ON((afi)->type_ ## name); \
247 (afi)->type_ ## name = (T); \
248 } while (0)
249
250 switch (type->proto) {
251 case IPPROTO_COMP:
252 X(afinfo, type, comp);
253 break;
254 case IPPROTO_AH:
255 X(afinfo, type, ah);
256 break;
257 case IPPROTO_ESP:
258 X(afinfo, type, esp);
259 break;
260 case IPPROTO_IPIP:
261 X(afinfo, type, ipip);
262 break;
263 case IPPROTO_DSTOPTS:
264 X(afinfo, type, dstopts);
265 break;
266 case IPPROTO_ROUTING:
267 X(afinfo, type, routing);
268 break;
269 case IPPROTO_IPV6:
270 X(afinfo, type, ipip6);
271 break;
272 default:
273 WARN_ON(1);
274 err = -EPROTONOSUPPORT;
275 break;
276 }
277 #undef X
278 rcu_read_unlock();
279 return err;
280 }
281 EXPORT_SYMBOL(xfrm_register_type);
282
xfrm_unregister_type(const struct xfrm_type * type,unsigned short family)283 void xfrm_unregister_type(const struct xfrm_type *type, unsigned short family)
284 {
285 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
286
287 if (unlikely(afinfo == NULL))
288 return;
289
290 #define X(afi, T, name) do { \
291 WARN_ON((afi)->type_ ## name != (T)); \
292 (afi)->type_ ## name = NULL; \
293 } while (0)
294
295 switch (type->proto) {
296 case IPPROTO_COMP:
297 X(afinfo, type, comp);
298 break;
299 case IPPROTO_AH:
300 X(afinfo, type, ah);
301 break;
302 case IPPROTO_ESP:
303 X(afinfo, type, esp);
304 break;
305 case IPPROTO_IPIP:
306 X(afinfo, type, ipip);
307 break;
308 case IPPROTO_DSTOPTS:
309 X(afinfo, type, dstopts);
310 break;
311 case IPPROTO_ROUTING:
312 X(afinfo, type, routing);
313 break;
314 case IPPROTO_IPV6:
315 X(afinfo, type, ipip6);
316 break;
317 default:
318 WARN_ON(1);
319 break;
320 }
321 #undef X
322 rcu_read_unlock();
323 }
324 EXPORT_SYMBOL(xfrm_unregister_type);
325
xfrm_get_type(u8 proto,unsigned short family)326 static const struct xfrm_type *xfrm_get_type(u8 proto, unsigned short family)
327 {
328 const struct xfrm_type *type = NULL;
329 struct xfrm_state_afinfo *afinfo;
330 int modload_attempted = 0;
331
332 retry:
333 afinfo = xfrm_state_get_afinfo(family);
334 if (unlikely(afinfo == NULL))
335 return NULL;
336
337 switch (proto) {
338 case IPPROTO_COMP:
339 type = afinfo->type_comp;
340 break;
341 case IPPROTO_AH:
342 type = afinfo->type_ah;
343 break;
344 case IPPROTO_ESP:
345 type = afinfo->type_esp;
346 break;
347 case IPPROTO_IPIP:
348 type = afinfo->type_ipip;
349 break;
350 case IPPROTO_DSTOPTS:
351 type = afinfo->type_dstopts;
352 break;
353 case IPPROTO_ROUTING:
354 type = afinfo->type_routing;
355 break;
356 case IPPROTO_IPV6:
357 type = afinfo->type_ipip6;
358 break;
359 default:
360 break;
361 }
362
363 if (unlikely(type && !try_module_get(type->owner)))
364 type = NULL;
365
366 rcu_read_unlock();
367
368 if (!type && !modload_attempted) {
369 request_module("xfrm-type-%d-%d", family, proto);
370 modload_attempted = 1;
371 goto retry;
372 }
373
374 return type;
375 }
376
xfrm_put_type(const struct xfrm_type * type)377 static void xfrm_put_type(const struct xfrm_type *type)
378 {
379 module_put(type->owner);
380 }
381
xfrm_register_type_offload(const struct xfrm_type_offload * type,unsigned short family)382 int xfrm_register_type_offload(const struct xfrm_type_offload *type,
383 unsigned short family)
384 {
385 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
386 int err = 0;
387
388 if (unlikely(afinfo == NULL))
389 return -EAFNOSUPPORT;
390
391 switch (type->proto) {
392 case IPPROTO_ESP:
393 WARN_ON(afinfo->type_offload_esp);
394 afinfo->type_offload_esp = type;
395 break;
396 default:
397 WARN_ON(1);
398 err = -EPROTONOSUPPORT;
399 break;
400 }
401
402 rcu_read_unlock();
403 return err;
404 }
405 EXPORT_SYMBOL(xfrm_register_type_offload);
406
xfrm_unregister_type_offload(const struct xfrm_type_offload * type,unsigned short family)407 void xfrm_unregister_type_offload(const struct xfrm_type_offload *type,
408 unsigned short family)
409 {
410 struct xfrm_state_afinfo *afinfo = xfrm_state_get_afinfo(family);
411
412 if (unlikely(afinfo == NULL))
413 return;
414
415 switch (type->proto) {
416 case IPPROTO_ESP:
417 WARN_ON(afinfo->type_offload_esp != type);
418 afinfo->type_offload_esp = NULL;
419 break;
420 default:
421 WARN_ON(1);
422 break;
423 }
424 rcu_read_unlock();
425 }
426 EXPORT_SYMBOL(xfrm_unregister_type_offload);
427
xfrm_set_type_offload(struct xfrm_state * x,bool try_load)428 void xfrm_set_type_offload(struct xfrm_state *x, bool try_load)
429 {
430 const struct xfrm_type_offload *type = NULL;
431 struct xfrm_state_afinfo *afinfo;
432
433 retry:
434 afinfo = xfrm_state_get_afinfo(x->props.family);
435 if (unlikely(afinfo == NULL))
436 goto out;
437
438 switch (x->id.proto) {
439 case IPPROTO_ESP:
440 type = afinfo->type_offload_esp;
441 break;
442 default:
443 break;
444 }
445
446 if ((type && !try_module_get(type->owner)))
447 type = NULL;
448
449 rcu_read_unlock();
450
451 if (!type && try_load) {
452 request_module("xfrm-offload-%d-%d", x->props.family,
453 x->id.proto);
454 try_load = false;
455 goto retry;
456 }
457
458 out:
459 x->type_offload = type;
460 }
461 EXPORT_SYMBOL(xfrm_set_type_offload);
462
463 static const struct xfrm_mode xfrm4_mode_map[XFRM_MODE_MAX] = {
464 [XFRM_MODE_BEET] = {
465 .encap = XFRM_MODE_BEET,
466 .flags = XFRM_MODE_FLAG_TUNNEL,
467 .family = AF_INET,
468 },
469 [XFRM_MODE_TRANSPORT] = {
470 .encap = XFRM_MODE_TRANSPORT,
471 .family = AF_INET,
472 },
473 [XFRM_MODE_TUNNEL] = {
474 .encap = XFRM_MODE_TUNNEL,
475 .flags = XFRM_MODE_FLAG_TUNNEL,
476 .family = AF_INET,
477 },
478 [XFRM_MODE_IPTFS] = {
479 .encap = XFRM_MODE_IPTFS,
480 .flags = XFRM_MODE_FLAG_TUNNEL,
481 .family = AF_INET,
482 },
483 };
484
485 static const struct xfrm_mode xfrm6_mode_map[XFRM_MODE_MAX] = {
486 [XFRM_MODE_BEET] = {
487 .encap = XFRM_MODE_BEET,
488 .flags = XFRM_MODE_FLAG_TUNNEL,
489 .family = AF_INET6,
490 },
491 [XFRM_MODE_ROUTEOPTIMIZATION] = {
492 .encap = XFRM_MODE_ROUTEOPTIMIZATION,
493 .family = AF_INET6,
494 },
495 [XFRM_MODE_TRANSPORT] = {
496 .encap = XFRM_MODE_TRANSPORT,
497 .family = AF_INET6,
498 },
499 [XFRM_MODE_TUNNEL] = {
500 .encap = XFRM_MODE_TUNNEL,
501 .flags = XFRM_MODE_FLAG_TUNNEL,
502 .family = AF_INET6,
503 },
504 [XFRM_MODE_IPTFS] = {
505 .encap = XFRM_MODE_IPTFS,
506 .flags = XFRM_MODE_FLAG_TUNNEL,
507 .family = AF_INET6,
508 },
509 };
510
xfrm_get_mode(unsigned int encap,int family)511 static const struct xfrm_mode *xfrm_get_mode(unsigned int encap, int family)
512 {
513 const struct xfrm_mode *mode;
514
515 if (unlikely(encap >= XFRM_MODE_MAX))
516 return NULL;
517
518 switch (family) {
519 case AF_INET:
520 mode = &xfrm4_mode_map[encap];
521 if (mode->family == family)
522 return mode;
523 break;
524 case AF_INET6:
525 mode = &xfrm6_mode_map[encap];
526 if (mode->family == family)
527 return mode;
528 break;
529 default:
530 break;
531 }
532
533 return NULL;
534 }
535
536 static const struct xfrm_mode_cbs __rcu *xfrm_mode_cbs_map[XFRM_MODE_MAX];
537 static DEFINE_SPINLOCK(xfrm_mode_cbs_map_lock);
538
xfrm_register_mode_cbs(u8 mode,const struct xfrm_mode_cbs * mode_cbs)539 int xfrm_register_mode_cbs(u8 mode, const struct xfrm_mode_cbs *mode_cbs)
540 {
541 if (mode >= XFRM_MODE_MAX)
542 return -EINVAL;
543
544 spin_lock_bh(&xfrm_mode_cbs_map_lock);
545 rcu_assign_pointer(xfrm_mode_cbs_map[mode], mode_cbs);
546 spin_unlock_bh(&xfrm_mode_cbs_map_lock);
547
548 return 0;
549 }
550 EXPORT_SYMBOL(xfrm_register_mode_cbs);
551
xfrm_unregister_mode_cbs(u8 mode)552 void xfrm_unregister_mode_cbs(u8 mode)
553 {
554 if (mode >= XFRM_MODE_MAX)
555 return;
556
557 spin_lock_bh(&xfrm_mode_cbs_map_lock);
558 RCU_INIT_POINTER(xfrm_mode_cbs_map[mode], NULL);
559 spin_unlock_bh(&xfrm_mode_cbs_map_lock);
560 synchronize_rcu();
561 }
562 EXPORT_SYMBOL(xfrm_unregister_mode_cbs);
563
xfrm_get_mode_cbs(u8 mode)564 static const struct xfrm_mode_cbs *xfrm_get_mode_cbs(u8 mode)
565 {
566 const struct xfrm_mode_cbs *cbs;
567 bool try_load = true;
568
569 if (mode >= XFRM_MODE_MAX)
570 return NULL;
571
572 retry:
573 rcu_read_lock();
574
575 cbs = rcu_dereference(xfrm_mode_cbs_map[mode]);
576 if (cbs && !try_module_get(cbs->owner))
577 cbs = NULL;
578
579 rcu_read_unlock();
580
581 if (mode == XFRM_MODE_IPTFS && !cbs && try_load) {
582 request_module("xfrm-iptfs");
583 try_load = false;
584 goto retry;
585 }
586
587 return cbs;
588 }
589
xfrm_state_free(struct xfrm_state * x)590 void xfrm_state_free(struct xfrm_state *x)
591 {
592 kmem_cache_free(xfrm_state_cache, x);
593 }
594 EXPORT_SYMBOL(xfrm_state_free);
595
596 static void xfrm_state_delete_tunnel(struct xfrm_state *x);
xfrm_state_gc_destroy(struct xfrm_state * x)597 static void xfrm_state_gc_destroy(struct xfrm_state *x)
598 {
599 if (x->mode_cbs && x->mode_cbs->destroy_state)
600 x->mode_cbs->destroy_state(x);
601 hrtimer_cancel(&x->mtimer);
602 timer_delete_sync(&x->rtimer);
603 kfree_sensitive(x->aead);
604 kfree_sensitive(x->aalg);
605 kfree_sensitive(x->ealg);
606 kfree(x->calg);
607 kfree(x->encap);
608 kfree(x->coaddr);
609 kfree(x->replay_esn);
610 kfree(x->preplay_esn);
611 xfrm_unset_type_offload(x);
612 xfrm_state_delete_tunnel(x);
613 if (x->type) {
614 x->type->destructor(x);
615 xfrm_put_type(x->type);
616 }
617 if (x->xfrag.page)
618 put_page(x->xfrag.page);
619 xfrm_dev_state_free(x);
620 security_xfrm_state_free(x);
621 xfrm_state_free(x);
622 }
623
xfrm_state_gc_task(struct work_struct * work)624 static void xfrm_state_gc_task(struct work_struct *work)
625 {
626 struct xfrm_state *x;
627 struct hlist_node *tmp;
628 struct hlist_head gc_list;
629
630 spin_lock_bh(&xfrm_state_gc_lock);
631 hlist_move_list(&xfrm_state_gc_list, &gc_list);
632 spin_unlock_bh(&xfrm_state_gc_lock);
633
634 synchronize_rcu();
635
636 mutex_lock(&xfrm_state_gc_mutex);
637 hlist_for_each_entry_safe(x, tmp, &gc_list, gclist)
638 xfrm_state_gc_destroy(x);
639 mutex_unlock(&xfrm_state_gc_mutex);
640 }
641
xfrm_timer_handler(struct hrtimer * me)642 static enum hrtimer_restart xfrm_timer_handler(struct hrtimer *me)
643 {
644 struct xfrm_state *x = container_of(me, struct xfrm_state, mtimer);
645 enum hrtimer_restart ret = HRTIMER_NORESTART;
646 time64_t now = ktime_get_real_seconds();
647 time64_t next = TIME64_MAX;
648 int warn = 0;
649 int err = 0;
650
651 spin_lock(&x->lock);
652 xfrm_dev_state_update_stats(x);
653
654 if (x->km.state == XFRM_STATE_DEAD)
655 goto out;
656 if (x->km.state == XFRM_STATE_EXPIRED)
657 goto expired;
658 if (x->lft.hard_add_expires_seconds) {
659 time64_t tmo = x->lft.hard_add_expires_seconds +
660 x->curlft.add_time - now;
661 if (tmo <= 0) {
662 if (x->xflags & XFRM_SOFT_EXPIRE) {
663 /* enter hard expire without soft expire first?!
664 * setting a new date could trigger this.
665 * workaround: fix x->curflt.add_time by below:
666 */
667 x->curlft.add_time = now - x->saved_tmo - 1;
668 tmo = x->lft.hard_add_expires_seconds - x->saved_tmo;
669 } else
670 goto expired;
671 }
672 if (tmo < next)
673 next = tmo;
674 }
675 if (x->lft.hard_use_expires_seconds) {
676 time64_t tmo = x->lft.hard_use_expires_seconds +
677 (READ_ONCE(x->curlft.use_time) ? : now) - now;
678 if (tmo <= 0)
679 goto expired;
680 if (tmo < next)
681 next = tmo;
682 }
683 if (x->km.dying)
684 goto resched;
685 if (x->lft.soft_add_expires_seconds) {
686 time64_t tmo = x->lft.soft_add_expires_seconds +
687 x->curlft.add_time - now;
688 if (tmo <= 0) {
689 warn = 1;
690 x->xflags &= ~XFRM_SOFT_EXPIRE;
691 } else if (tmo < next) {
692 next = tmo;
693 x->xflags |= XFRM_SOFT_EXPIRE;
694 x->saved_tmo = tmo;
695 }
696 }
697 if (x->lft.soft_use_expires_seconds) {
698 time64_t tmo = x->lft.soft_use_expires_seconds +
699 (READ_ONCE(x->curlft.use_time) ? : now) - now;
700 if (tmo <= 0)
701 warn = 1;
702 else if (tmo < next)
703 next = tmo;
704 }
705
706 x->km.dying = warn;
707 if (warn)
708 km_state_expired(x, 0, 0);
709 resched:
710 if (next != TIME64_MAX) {
711 hrtimer_forward_now(&x->mtimer, ktime_set(next, 0));
712 ret = HRTIMER_RESTART;
713 }
714
715 goto out;
716
717 expired:
718 if (x->km.state == XFRM_STATE_ACQ && x->id.spi == 0)
719 x->km.state = XFRM_STATE_EXPIRED;
720
721 err = __xfrm_state_delete(x);
722 if (!err)
723 km_state_expired(x, 1, 0);
724
725 xfrm_audit_state_delete(x, err ? 0 : 1, true);
726
727 out:
728 spin_unlock(&x->lock);
729 return ret;
730 }
731
732 static void xfrm_replay_timer_handler(struct timer_list *t);
733
xfrm_state_alloc(struct net * net)734 struct xfrm_state *xfrm_state_alloc(struct net *net)
735 {
736 struct xfrm_state *x;
737
738 x = kmem_cache_zalloc(xfrm_state_cache, GFP_ATOMIC);
739
740 if (x) {
741 write_pnet(&x->xs_net, net);
742 refcount_set(&x->refcnt, 1);
743 atomic_set(&x->tunnel_users, 0);
744 INIT_LIST_HEAD(&x->km.all);
745 INIT_HLIST_NODE(&x->state_cache);
746 INIT_HLIST_NODE(&x->bydst);
747 INIT_HLIST_NODE(&x->bysrc);
748 INIT_HLIST_NODE(&x->byspi);
749 INIT_HLIST_NODE(&x->byseq);
750 hrtimer_setup(&x->mtimer, xfrm_timer_handler, CLOCK_BOOTTIME,
751 HRTIMER_MODE_ABS_SOFT);
752 timer_setup(&x->rtimer, xfrm_replay_timer_handler, 0);
753 x->curlft.add_time = ktime_get_real_seconds();
754 x->lft.soft_byte_limit = XFRM_INF;
755 x->lft.soft_packet_limit = XFRM_INF;
756 x->lft.hard_byte_limit = XFRM_INF;
757 x->lft.hard_packet_limit = XFRM_INF;
758 x->replay_maxage = 0;
759 x->replay_maxdiff = 0;
760 x->pcpu_num = UINT_MAX;
761 spin_lock_init(&x->lock);
762 x->mode_data = NULL;
763 }
764 return x;
765 }
766 EXPORT_SYMBOL(xfrm_state_alloc);
767
768 #ifdef CONFIG_XFRM_OFFLOAD
xfrm_dev_state_delete(struct xfrm_state * x)769 void xfrm_dev_state_delete(struct xfrm_state *x)
770 {
771 struct xfrm_dev_offload *xso = &x->xso;
772 struct net_device *dev = READ_ONCE(xso->dev);
773
774 if (dev) {
775 dev->xfrmdev_ops->xdo_dev_state_delete(dev, x);
776 spin_lock_bh(&xfrm_state_dev_gc_lock);
777 hlist_add_head(&x->dev_gclist, &xfrm_state_dev_gc_list);
778 spin_unlock_bh(&xfrm_state_dev_gc_lock);
779 }
780 }
781 EXPORT_SYMBOL_GPL(xfrm_dev_state_delete);
782
xfrm_dev_state_free(struct xfrm_state * x)783 void xfrm_dev_state_free(struct xfrm_state *x)
784 {
785 struct xfrm_dev_offload *xso = &x->xso;
786 struct net_device *dev = READ_ONCE(xso->dev);
787
788 if (dev && dev->xfrmdev_ops) {
789 spin_lock_bh(&xfrm_state_dev_gc_lock);
790 if (!hlist_unhashed(&x->dev_gclist))
791 hlist_del(&x->dev_gclist);
792 spin_unlock_bh(&xfrm_state_dev_gc_lock);
793
794 if (dev->xfrmdev_ops->xdo_dev_state_free)
795 dev->xfrmdev_ops->xdo_dev_state_free(dev, x);
796 WRITE_ONCE(xso->dev, NULL);
797 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
798 netdev_put(dev, &xso->dev_tracker);
799 }
800 }
801 #endif
802
__xfrm_state_destroy(struct xfrm_state * x)803 void __xfrm_state_destroy(struct xfrm_state *x)
804 {
805 WARN_ON(x->km.state != XFRM_STATE_DEAD);
806
807 spin_lock_bh(&xfrm_state_gc_lock);
808 hlist_add_head(&x->gclist, &xfrm_state_gc_list);
809 spin_unlock_bh(&xfrm_state_gc_lock);
810 schedule_work(&xfrm_state_gc_work);
811 }
812 EXPORT_SYMBOL(__xfrm_state_destroy);
813
__xfrm_state_delete(struct xfrm_state * x)814 int __xfrm_state_delete(struct xfrm_state *x)
815 {
816 struct net *net = xs_net(x);
817 int err = -ESRCH;
818
819 if (x->km.state != XFRM_STATE_DEAD) {
820 x->km.state = XFRM_STATE_DEAD;
821
822 spin_lock(&net->xfrm.xfrm_state_lock);
823 list_del(&x->km.all);
824 hlist_del_init_rcu(&x->bydst);
825 hlist_del_init_rcu(&x->bysrc);
826 if (!hlist_unhashed(&x->byseq))
827 hlist_del_init_rcu(&x->byseq);
828 if (!hlist_unhashed(&x->state_cache))
829 hlist_del_init_rcu(&x->state_cache);
830 if (!hlist_unhashed(&x->state_cache_input))
831 hlist_del_init_rcu(&x->state_cache_input);
832
833 if (!hlist_unhashed(&x->byspi))
834 hlist_del_init_rcu(&x->byspi);
835 net->xfrm.state_num--;
836 xfrm_nat_keepalive_state_updated(x);
837 spin_unlock(&net->xfrm.xfrm_state_lock);
838
839 xfrm_dev_state_delete(x);
840
841 xfrm_state_delete_tunnel(x);
842
843 /* All xfrm_state objects are created by xfrm_state_alloc.
844 * The xfrm_state_alloc call gives a reference, and that
845 * is what we are dropping here.
846 */
847 xfrm_state_put(x);
848 err = 0;
849 }
850
851 return err;
852 }
853 EXPORT_SYMBOL(__xfrm_state_delete);
854
xfrm_state_delete(struct xfrm_state * x)855 int xfrm_state_delete(struct xfrm_state *x)
856 {
857 int err;
858
859 spin_lock_bh(&x->lock);
860 err = __xfrm_state_delete(x);
861 spin_unlock_bh(&x->lock);
862
863 return err;
864 }
865 EXPORT_SYMBOL(xfrm_state_delete);
866
867 #ifdef CONFIG_SECURITY_NETWORK_XFRM
868 static inline int
xfrm_state_flush_secctx_check(struct net * net,u8 proto,bool task_valid)869 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
870 {
871 int i, err = 0;
872
873 for (i = 0; i <= net->xfrm.state_hmask; i++) {
874 struct xfrm_state *x;
875
876 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
877 if (xfrm_id_proto_match(x->id.proto, proto) &&
878 (err = security_xfrm_state_delete(x)) != 0) {
879 xfrm_audit_state_delete(x, 0, task_valid);
880 return err;
881 }
882 }
883 }
884
885 return err;
886 }
887
888 static inline int
xfrm_dev_state_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)889 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
890 {
891 int i, err = 0;
892
893 for (i = 0; i <= net->xfrm.state_hmask; i++) {
894 struct xfrm_state *x;
895 struct xfrm_dev_offload *xso;
896
897 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
898 xso = &x->xso;
899
900 if (xso->dev == dev &&
901 (err = security_xfrm_state_delete(x)) != 0) {
902 xfrm_audit_state_delete(x, 0, task_valid);
903 return err;
904 }
905 }
906 }
907
908 return err;
909 }
910 #else
911 static inline int
xfrm_state_flush_secctx_check(struct net * net,u8 proto,bool task_valid)912 xfrm_state_flush_secctx_check(struct net *net, u8 proto, bool task_valid)
913 {
914 return 0;
915 }
916
917 static inline int
xfrm_dev_state_flush_secctx_check(struct net * net,struct net_device * dev,bool task_valid)918 xfrm_dev_state_flush_secctx_check(struct net *net, struct net_device *dev, bool task_valid)
919 {
920 return 0;
921 }
922 #endif
923
xfrm_state_flush(struct net * net,u8 proto,bool task_valid)924 int xfrm_state_flush(struct net *net, u8 proto, bool task_valid)
925 {
926 int i, err = 0, cnt = 0;
927
928 spin_lock_bh(&net->xfrm.xfrm_state_lock);
929 err = xfrm_state_flush_secctx_check(net, proto, task_valid);
930 if (err)
931 goto out;
932
933 err = -ESRCH;
934 for (i = 0; i <= net->xfrm.state_hmask; i++) {
935 struct xfrm_state *x;
936 restart:
937 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
938 if (!xfrm_state_kern(x) &&
939 xfrm_id_proto_match(x->id.proto, proto)) {
940 xfrm_state_hold(x);
941 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
942
943 err = xfrm_state_delete(x);
944 xfrm_audit_state_delete(x, err ? 0 : 1,
945 task_valid);
946 xfrm_state_put(x);
947 if (!err)
948 cnt++;
949
950 spin_lock_bh(&net->xfrm.xfrm_state_lock);
951 goto restart;
952 }
953 }
954 }
955 out:
956 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
957 if (cnt)
958 err = 0;
959
960 return err;
961 }
962 EXPORT_SYMBOL(xfrm_state_flush);
963
xfrm_dev_state_flush(struct net * net,struct net_device * dev,bool task_valid)964 int xfrm_dev_state_flush(struct net *net, struct net_device *dev, bool task_valid)
965 {
966 struct xfrm_state *x;
967 struct hlist_node *tmp;
968 struct xfrm_dev_offload *xso;
969 int i, err = 0, cnt = 0;
970
971 spin_lock_bh(&net->xfrm.xfrm_state_lock);
972 err = xfrm_dev_state_flush_secctx_check(net, dev, task_valid);
973 if (err)
974 goto out;
975
976 err = -ESRCH;
977 for (i = 0; i <= net->xfrm.state_hmask; i++) {
978 restart:
979 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst) {
980 xso = &x->xso;
981
982 if (!xfrm_state_kern(x) && xso->dev == dev) {
983 xfrm_state_hold(x);
984 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
985
986 err = xfrm_state_delete(x);
987 xfrm_dev_state_free(x);
988
989 xfrm_audit_state_delete(x, err ? 0 : 1,
990 task_valid);
991 xfrm_state_put(x);
992 if (!err)
993 cnt++;
994
995 spin_lock_bh(&net->xfrm.xfrm_state_lock);
996 goto restart;
997 }
998 }
999 }
1000 if (cnt)
1001 err = 0;
1002
1003 out:
1004 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1005
1006 mutex_lock(&xfrm_state_gc_mutex);
1007 spin_lock_bh(&xfrm_state_dev_gc_lock);
1008 restart_gc:
1009 hlist_for_each_entry_safe(x, tmp, &xfrm_state_dev_gc_list, dev_gclist) {
1010 xso = &x->xso;
1011
1012 if (xso->dev == dev) {
1013 spin_unlock_bh(&xfrm_state_dev_gc_lock);
1014 xfrm_dev_state_free(x);
1015 spin_lock_bh(&xfrm_state_dev_gc_lock);
1016 goto restart_gc;
1017 }
1018
1019 }
1020 spin_unlock_bh(&xfrm_state_dev_gc_lock);
1021 mutex_unlock(&xfrm_state_gc_mutex);
1022
1023 xfrm_flush_gc();
1024
1025 return err;
1026 }
1027 EXPORT_SYMBOL(xfrm_dev_state_flush);
1028
xfrm_sad_getinfo(struct net * net,struct xfrmk_sadinfo * si)1029 void xfrm_sad_getinfo(struct net *net, struct xfrmk_sadinfo *si)
1030 {
1031 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1032 si->sadcnt = net->xfrm.state_num;
1033 si->sadhcnt = net->xfrm.state_hmask + 1;
1034 si->sadhmcnt = xfrm_state_hashmax;
1035 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1036 }
1037 EXPORT_SYMBOL(xfrm_sad_getinfo);
1038
1039 static void
__xfrm4_init_tempsel(struct xfrm_selector * sel,const struct flowi * fl)1040 __xfrm4_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
1041 {
1042 const struct flowi4 *fl4 = &fl->u.ip4;
1043
1044 sel->daddr.a4 = fl4->daddr;
1045 sel->saddr.a4 = fl4->saddr;
1046 sel->dport = xfrm_flowi_dport(fl, &fl4->uli);
1047 sel->dport_mask = htons(0xffff);
1048 sel->sport = xfrm_flowi_sport(fl, &fl4->uli);
1049 sel->sport_mask = htons(0xffff);
1050 sel->family = AF_INET;
1051 sel->prefixlen_d = 32;
1052 sel->prefixlen_s = 32;
1053 sel->proto = fl4->flowi4_proto;
1054 sel->ifindex = fl4->flowi4_oif;
1055 }
1056
1057 static void
__xfrm6_init_tempsel(struct xfrm_selector * sel,const struct flowi * fl)1058 __xfrm6_init_tempsel(struct xfrm_selector *sel, const struct flowi *fl)
1059 {
1060 const struct flowi6 *fl6 = &fl->u.ip6;
1061
1062 /* Initialize temporary selector matching only to current session. */
1063 *(struct in6_addr *)&sel->daddr = fl6->daddr;
1064 *(struct in6_addr *)&sel->saddr = fl6->saddr;
1065 sel->dport = xfrm_flowi_dport(fl, &fl6->uli);
1066 sel->dport_mask = htons(0xffff);
1067 sel->sport = xfrm_flowi_sport(fl, &fl6->uli);
1068 sel->sport_mask = htons(0xffff);
1069 sel->family = AF_INET6;
1070 sel->prefixlen_d = 128;
1071 sel->prefixlen_s = 128;
1072 sel->proto = fl6->flowi6_proto;
1073 sel->ifindex = fl6->flowi6_oif;
1074 }
1075
1076 static void
xfrm_init_tempstate(struct xfrm_state * x,const struct flowi * fl,const struct xfrm_tmpl * tmpl,const xfrm_address_t * daddr,const xfrm_address_t * saddr,unsigned short family)1077 xfrm_init_tempstate(struct xfrm_state *x, const struct flowi *fl,
1078 const struct xfrm_tmpl *tmpl,
1079 const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1080 unsigned short family)
1081 {
1082 switch (family) {
1083 case AF_INET:
1084 __xfrm4_init_tempsel(&x->sel, fl);
1085 break;
1086 case AF_INET6:
1087 __xfrm6_init_tempsel(&x->sel, fl);
1088 break;
1089 }
1090
1091 x->id = tmpl->id;
1092
1093 switch (tmpl->encap_family) {
1094 case AF_INET:
1095 if (x->id.daddr.a4 == 0)
1096 x->id.daddr.a4 = daddr->a4;
1097 x->props.saddr = tmpl->saddr;
1098 if (x->props.saddr.a4 == 0)
1099 x->props.saddr.a4 = saddr->a4;
1100 break;
1101 case AF_INET6:
1102 if (ipv6_addr_any((struct in6_addr *)&x->id.daddr))
1103 memcpy(&x->id.daddr, daddr, sizeof(x->sel.daddr));
1104 memcpy(&x->props.saddr, &tmpl->saddr, sizeof(x->props.saddr));
1105 if (ipv6_addr_any((struct in6_addr *)&x->props.saddr))
1106 memcpy(&x->props.saddr, saddr, sizeof(x->props.saddr));
1107 break;
1108 }
1109
1110 x->props.mode = tmpl->mode;
1111 x->props.reqid = tmpl->reqid;
1112 x->props.family = tmpl->encap_family;
1113 }
1114
1115 struct xfrm_hash_state_ptrs {
1116 const struct hlist_head *bydst;
1117 const struct hlist_head *bysrc;
1118 const struct hlist_head *byspi;
1119 unsigned int hmask;
1120 };
1121
xfrm_hash_ptrs_get(const struct net * net,struct xfrm_hash_state_ptrs * ptrs)1122 static void xfrm_hash_ptrs_get(const struct net *net, struct xfrm_hash_state_ptrs *ptrs)
1123 {
1124 unsigned int sequence;
1125
1126 do {
1127 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
1128
1129 ptrs->bydst = xfrm_state_deref_check(net->xfrm.state_bydst, net);
1130 ptrs->bysrc = xfrm_state_deref_check(net->xfrm.state_bysrc, net);
1131 ptrs->byspi = xfrm_state_deref_check(net->xfrm.state_byspi, net);
1132 ptrs->hmask = net->xfrm.state_hmask;
1133 } while (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence));
1134 }
1135
__xfrm_state_lookup_all(const struct xfrm_hash_state_ptrs * state_ptrs,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family,struct xfrm_dev_offload * xdo)1136 static struct xfrm_state *__xfrm_state_lookup_all(const struct xfrm_hash_state_ptrs *state_ptrs,
1137 u32 mark,
1138 const xfrm_address_t *daddr,
1139 __be32 spi, u8 proto,
1140 unsigned short family,
1141 struct xfrm_dev_offload *xdo)
1142 {
1143 unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);
1144 struct xfrm_state *x;
1145
1146 hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {
1147 #ifdef CONFIG_XFRM_OFFLOAD
1148 if (xdo->type == XFRM_DEV_OFFLOAD_PACKET) {
1149 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1150 /* HW states are in the head of list, there is
1151 * no need to iterate further.
1152 */
1153 break;
1154
1155 /* Packet offload: both policy and SA should
1156 * have same device.
1157 */
1158 if (xdo->dev != x->xso.dev)
1159 continue;
1160 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1161 /* Skip HW policy for SW lookups */
1162 continue;
1163 #endif
1164 if (x->props.family != family ||
1165 x->id.spi != spi ||
1166 x->id.proto != proto ||
1167 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1168 continue;
1169
1170 if ((mark & x->mark.m) != x->mark.v)
1171 continue;
1172 if (!xfrm_state_hold_rcu(x))
1173 continue;
1174 return x;
1175 }
1176
1177 return NULL;
1178 }
1179
__xfrm_state_lookup(const struct xfrm_hash_state_ptrs * state_ptrs,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)1180 static struct xfrm_state *__xfrm_state_lookup(const struct xfrm_hash_state_ptrs *state_ptrs,
1181 u32 mark,
1182 const xfrm_address_t *daddr,
1183 __be32 spi, u8 proto,
1184 unsigned short family)
1185 {
1186 unsigned int h = __xfrm_spi_hash(daddr, spi, proto, family, state_ptrs->hmask);
1187 struct xfrm_state *x;
1188
1189 hlist_for_each_entry_rcu(x, state_ptrs->byspi + h, byspi) {
1190 if (x->props.family != family ||
1191 x->id.spi != spi ||
1192 x->id.proto != proto ||
1193 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1194 continue;
1195
1196 if ((mark & x->mark.m) != x->mark.v)
1197 continue;
1198 if (!xfrm_state_hold_rcu(x))
1199 continue;
1200 return x;
1201 }
1202
1203 return NULL;
1204 }
1205
xfrm_input_state_lookup(struct net * net,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)1206 struct xfrm_state *xfrm_input_state_lookup(struct net *net, u32 mark,
1207 const xfrm_address_t *daddr,
1208 __be32 spi, u8 proto,
1209 unsigned short family)
1210 {
1211 struct xfrm_hash_state_ptrs state_ptrs;
1212 struct hlist_head *state_cache_input;
1213 struct xfrm_state *x = NULL;
1214
1215 /* BH is always disabled on the input path. */
1216 lockdep_assert_in_softirq();
1217
1218 state_cache_input = raw_cpu_ptr(net->xfrm.state_cache_input);
1219
1220 hlist_for_each_entry_rcu(x, state_cache_input, state_cache_input) {
1221 if (x->props.family != family ||
1222 x->id.spi != spi ||
1223 x->id.proto != proto ||
1224 !xfrm_addr_equal(&x->id.daddr, daddr, family))
1225 continue;
1226
1227 if ((mark & x->mark.m) != x->mark.v)
1228 continue;
1229 if (!xfrm_state_hold_rcu(x))
1230 continue;
1231 goto out;
1232 }
1233
1234 xfrm_hash_ptrs_get(net, &state_ptrs);
1235
1236 x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);
1237 if (x) {
1238 spin_lock(&net->xfrm.xfrm_state_lock);
1239 if (x->km.state != XFRM_STATE_VALID) {
1240 /*
1241 * The state is about to be destroyed.
1242 *
1243 * Don't add it to the cache but still
1244 * return it to the caller.
1245 */
1246 } else if (hlist_unhashed(&x->state_cache_input)) {
1247 hlist_add_head_rcu(&x->state_cache_input, state_cache_input);
1248 } else {
1249 hlist_del_rcu(&x->state_cache_input);
1250 hlist_add_head_rcu(&x->state_cache_input, state_cache_input);
1251 }
1252 spin_unlock(&net->xfrm.xfrm_state_lock);
1253 }
1254
1255 out:
1256 return x;
1257 }
1258 EXPORT_SYMBOL(xfrm_input_state_lookup);
1259
__xfrm_state_lookup_byaddr(const struct xfrm_hash_state_ptrs * state_ptrs,u32 mark,const xfrm_address_t * daddr,const xfrm_address_t * saddr,u8 proto,unsigned short family)1260 static struct xfrm_state *__xfrm_state_lookup_byaddr(const struct xfrm_hash_state_ptrs *state_ptrs,
1261 u32 mark,
1262 const xfrm_address_t *daddr,
1263 const xfrm_address_t *saddr,
1264 u8 proto, unsigned short family)
1265 {
1266 unsigned int h = __xfrm_src_hash(daddr, saddr, family, state_ptrs->hmask);
1267 struct xfrm_state *x;
1268
1269 hlist_for_each_entry_rcu(x, state_ptrs->bysrc + h, bysrc) {
1270 if (x->props.family != family ||
1271 x->id.proto != proto ||
1272 !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
1273 !xfrm_addr_equal(&x->props.saddr, saddr, family))
1274 continue;
1275
1276 if ((mark & x->mark.m) != x->mark.v)
1277 continue;
1278 if (!xfrm_state_hold_rcu(x))
1279 continue;
1280 return x;
1281 }
1282
1283 return NULL;
1284 }
1285
1286 static inline struct xfrm_state *
__xfrm_state_locate(struct xfrm_state * x,int use_spi,int family)1287 __xfrm_state_locate(struct xfrm_state *x, int use_spi, int family)
1288 {
1289 struct xfrm_hash_state_ptrs state_ptrs;
1290 struct net *net = xs_net(x);
1291 u32 mark = x->mark.v & x->mark.m;
1292
1293 xfrm_hash_ptrs_get(net, &state_ptrs);
1294
1295 if (use_spi)
1296 return __xfrm_state_lookup(&state_ptrs, mark, &x->id.daddr,
1297 x->id.spi, x->id.proto, family);
1298 else
1299 return __xfrm_state_lookup_byaddr(&state_ptrs, mark,
1300 &x->id.daddr,
1301 &x->props.saddr,
1302 x->id.proto, family);
1303 }
1304
xfrm_hash_grow_check(struct net * net,int have_hash_collision)1305 static void xfrm_hash_grow_check(struct net *net, int have_hash_collision)
1306 {
1307 if (have_hash_collision &&
1308 (net->xfrm.state_hmask + 1) < xfrm_state_hashmax &&
1309 net->xfrm.state_num > net->xfrm.state_hmask)
1310 schedule_work(&net->xfrm.state_hash_work);
1311 }
1312
xfrm_state_look_at(struct xfrm_policy * pol,struct xfrm_state * x,const struct flowi * fl,unsigned short family,struct xfrm_state ** best,int * acq_in_progress,int * error,unsigned int pcpu_id)1313 static void xfrm_state_look_at(struct xfrm_policy *pol, struct xfrm_state *x,
1314 const struct flowi *fl, unsigned short family,
1315 struct xfrm_state **best, int *acq_in_progress,
1316 int *error, unsigned int pcpu_id)
1317 {
1318 /* Resolution logic:
1319 * 1. There is a valid state with matching selector. Done.
1320 * 2. Valid state with inappropriate selector. Skip.
1321 *
1322 * Entering area of "sysdeps".
1323 *
1324 * 3. If state is not valid, selector is temporary, it selects
1325 * only session which triggered previous resolution. Key
1326 * manager will do something to install a state with proper
1327 * selector.
1328 */
1329 if (x->km.state == XFRM_STATE_VALID) {
1330 if ((x->sel.family &&
1331 (x->sel.family != family ||
1332 !xfrm_selector_match(&x->sel, fl, family))) ||
1333 !security_xfrm_state_pol_flow_match(x, pol,
1334 &fl->u.__fl_common))
1335 return;
1336
1337 if (x->pcpu_num != UINT_MAX && x->pcpu_num != pcpu_id)
1338 return;
1339
1340 if (!*best ||
1341 ((*best)->pcpu_num == UINT_MAX && x->pcpu_num == pcpu_id) ||
1342 (*best)->km.dying > x->km.dying ||
1343 ((*best)->km.dying == x->km.dying &&
1344 (*best)->curlft.add_time < x->curlft.add_time))
1345 *best = x;
1346 } else if (x->km.state == XFRM_STATE_ACQ) {
1347 if (!*best || x->pcpu_num == pcpu_id)
1348 *acq_in_progress = 1;
1349 } else if (x->km.state == XFRM_STATE_ERROR ||
1350 x->km.state == XFRM_STATE_EXPIRED) {
1351 if ((!x->sel.family ||
1352 (x->sel.family == family &&
1353 xfrm_selector_match(&x->sel, fl, family))) &&
1354 security_xfrm_state_pol_flow_match(x, pol,
1355 &fl->u.__fl_common))
1356 *error = -ESRCH;
1357 }
1358 }
1359
1360 struct xfrm_state *
xfrm_state_find(const xfrm_address_t * daddr,const xfrm_address_t * saddr,const struct flowi * fl,struct xfrm_tmpl * tmpl,struct xfrm_policy * pol,int * err,unsigned short family,u32 if_id)1361 xfrm_state_find(const xfrm_address_t *daddr, const xfrm_address_t *saddr,
1362 const struct flowi *fl, struct xfrm_tmpl *tmpl,
1363 struct xfrm_policy *pol, int *err,
1364 unsigned short family, u32 if_id)
1365 {
1366 static xfrm_address_t saddr_wildcard = { };
1367 struct xfrm_hash_state_ptrs state_ptrs;
1368 struct net *net = xp_net(pol);
1369 unsigned int h, h_wildcard;
1370 struct xfrm_state *x, *x0, *to_put;
1371 int acquire_in_progress = 0;
1372 int error = 0;
1373 struct xfrm_state *best = NULL;
1374 u32 mark = pol->mark.v & pol->mark.m;
1375 unsigned short encap_family = tmpl->encap_family;
1376 unsigned int sequence;
1377 struct km_event c;
1378 unsigned int pcpu_id;
1379 bool cached = false;
1380
1381 /* We need the cpu id just as a lookup key,
1382 * we don't require it to be stable.
1383 */
1384 pcpu_id = raw_smp_processor_id();
1385
1386 to_put = NULL;
1387
1388 sequence = read_seqcount_begin(&net->xfrm.xfrm_state_hash_generation);
1389
1390 rcu_read_lock();
1391 xfrm_hash_ptrs_get(net, &state_ptrs);
1392
1393 hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {
1394 if (x->props.family == encap_family &&
1395 x->props.reqid == tmpl->reqid &&
1396 (mark & x->mark.m) == x->mark.v &&
1397 x->if_id == if_id &&
1398 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1399 xfrm_state_addr_check(x, daddr, saddr, encap_family) &&
1400 tmpl->mode == x->props.mode &&
1401 tmpl->id.proto == x->id.proto &&
1402 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1403 xfrm_state_look_at(pol, x, fl, encap_family,
1404 &best, &acquire_in_progress, &error, pcpu_id);
1405 }
1406
1407 if (best)
1408 goto cached;
1409
1410 hlist_for_each_entry_rcu(x, &pol->state_cache_list, state_cache) {
1411 if (x->props.family == encap_family &&
1412 x->props.reqid == tmpl->reqid &&
1413 (mark & x->mark.m) == x->mark.v &&
1414 x->if_id == if_id &&
1415 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1416 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&
1417 tmpl->mode == x->props.mode &&
1418 tmpl->id.proto == x->id.proto &&
1419 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1420 xfrm_state_look_at(pol, x, fl, family,
1421 &best, &acquire_in_progress, &error, pcpu_id);
1422 }
1423
1424 cached:
1425 cached = true;
1426 if (best)
1427 goto found;
1428 else if (error)
1429 best = NULL;
1430 else if (acquire_in_progress) /* XXX: acquire_in_progress should not happen */
1431 WARN_ON(1);
1432
1433 h = __xfrm_dst_hash(daddr, saddr, tmpl->reqid, encap_family, state_ptrs.hmask);
1434 hlist_for_each_entry_rcu(x, state_ptrs.bydst + h, bydst) {
1435 #ifdef CONFIG_XFRM_OFFLOAD
1436 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1437 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1438 /* HW states are in the head of list, there is
1439 * no need to iterate further.
1440 */
1441 break;
1442
1443 /* Packet offload: both policy and SA should
1444 * have same device.
1445 */
1446 if (pol->xdo.dev != x->xso.dev)
1447 continue;
1448 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1449 /* Skip HW policy for SW lookups */
1450 continue;
1451 #endif
1452 if (x->props.family == encap_family &&
1453 x->props.reqid == tmpl->reqid &&
1454 (mark & x->mark.m) == x->mark.v &&
1455 x->if_id == if_id &&
1456 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1457 xfrm_state_addr_check(x, daddr, saddr, encap_family) &&
1458 tmpl->mode == x->props.mode &&
1459 tmpl->id.proto == x->id.proto &&
1460 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1461 xfrm_state_look_at(pol, x, fl, family,
1462 &best, &acquire_in_progress, &error, pcpu_id);
1463 }
1464 if (best || acquire_in_progress)
1465 goto found;
1466
1467 h_wildcard = __xfrm_dst_hash(daddr, &saddr_wildcard, tmpl->reqid,
1468 encap_family, state_ptrs.hmask);
1469 hlist_for_each_entry_rcu(x, state_ptrs.bydst + h_wildcard, bydst) {
1470 #ifdef CONFIG_XFRM_OFFLOAD
1471 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1472 if (x->xso.type != XFRM_DEV_OFFLOAD_PACKET)
1473 /* HW states are in the head of list, there is
1474 * no need to iterate further.
1475 */
1476 break;
1477
1478 /* Packet offload: both policy and SA should
1479 * have same device.
1480 */
1481 if (pol->xdo.dev != x->xso.dev)
1482 continue;
1483 } else if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
1484 /* Skip HW policy for SW lookups */
1485 continue;
1486 #endif
1487 if (x->props.family == encap_family &&
1488 x->props.reqid == tmpl->reqid &&
1489 (mark & x->mark.m) == x->mark.v &&
1490 x->if_id == if_id &&
1491 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1492 xfrm_addr_equal(&x->id.daddr, daddr, encap_family) &&
1493 tmpl->mode == x->props.mode &&
1494 tmpl->id.proto == x->id.proto &&
1495 (tmpl->id.spi == x->id.spi || !tmpl->id.spi))
1496 xfrm_state_look_at(pol, x, fl, family,
1497 &best, &acquire_in_progress, &error, pcpu_id);
1498 }
1499
1500 found:
1501 if (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) ||
1502 (best && (best->pcpu_num == pcpu_id)))
1503 x = best;
1504
1505 if (!x && !error && !acquire_in_progress) {
1506 if (tmpl->id.spi &&
1507 (x0 = __xfrm_state_lookup_all(&state_ptrs, mark, daddr,
1508 tmpl->id.spi, tmpl->id.proto,
1509 encap_family,
1510 &pol->xdo)) != NULL) {
1511 to_put = x0;
1512 error = -EEXIST;
1513 goto out;
1514 }
1515
1516 c.net = net;
1517 /* If the KMs have no listeners (yet...), avoid allocating an SA
1518 * for each and every packet - garbage collection might not
1519 * handle the flood.
1520 */
1521 if (!km_is_alive(&c)) {
1522 error = -ESRCH;
1523 goto out;
1524 }
1525
1526 x = xfrm_state_alloc(net);
1527 if (x == NULL) {
1528 error = -ENOMEM;
1529 goto out;
1530 }
1531 /* Initialize temporary state matching only
1532 * to current session. */
1533 xfrm_init_tempstate(x, fl, tmpl, daddr, saddr, family);
1534 memcpy(&x->mark, &pol->mark, sizeof(x->mark));
1535 x->if_id = if_id;
1536 if ((pol->flags & XFRM_POLICY_CPU_ACQUIRE) && best)
1537 x->pcpu_num = pcpu_id;
1538
1539 error = security_xfrm_state_alloc_acquire(x, pol->security, fl->flowi_secid);
1540 if (error) {
1541 x->km.state = XFRM_STATE_DEAD;
1542 to_put = x;
1543 x = NULL;
1544 goto out;
1545 }
1546 #ifdef CONFIG_XFRM_OFFLOAD
1547 if (pol->xdo.type == XFRM_DEV_OFFLOAD_PACKET) {
1548 struct xfrm_dev_offload *xdo = &pol->xdo;
1549 struct xfrm_dev_offload *xso = &x->xso;
1550 struct net_device *dev = xdo->dev;
1551
1552 xso->type = XFRM_DEV_OFFLOAD_PACKET;
1553 xso->dir = xdo->dir;
1554 xso->dev = dev;
1555 xso->ifindex = dev->ifindex;
1556 xso->flags = XFRM_DEV_OFFLOAD_FLAG_ACQ;
1557 netdev_hold(dev, &xso->dev_tracker, GFP_ATOMIC);
1558 error = dev->xfrmdev_ops->xdo_dev_state_add(dev, x,
1559 NULL);
1560 if (error) {
1561 xso->dir = 0;
1562 netdev_put(dev, &xso->dev_tracker);
1563 xso->dev = NULL;
1564 xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
1565 x->km.state = XFRM_STATE_DEAD;
1566 to_put = x;
1567 x = NULL;
1568 goto out;
1569 }
1570 }
1571 #endif
1572 if (km_query(x, tmpl, pol) == 0) {
1573 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1574 x->km.state = XFRM_STATE_ACQ;
1575 x->dir = XFRM_SA_DIR_OUT;
1576 list_add(&x->km.all, &net->xfrm.state_all);
1577 h = xfrm_dst_hash(net, daddr, saddr, tmpl->reqid, encap_family);
1578 XFRM_STATE_INSERT(bydst, &x->bydst,
1579 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h,
1580 x->xso.type);
1581 h = xfrm_src_hash(net, daddr, saddr, encap_family);
1582 XFRM_STATE_INSERT(bysrc, &x->bysrc,
1583 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h,
1584 x->xso.type);
1585 INIT_HLIST_NODE(&x->state_cache);
1586 if (x->id.spi) {
1587 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto, encap_family);
1588 XFRM_STATE_INSERT(byspi, &x->byspi,
1589 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h,
1590 x->xso.type);
1591 }
1592 if (x->km.seq) {
1593 h = xfrm_seq_hash(net, x->km.seq);
1594 XFRM_STATE_INSERT(byseq, &x->byseq,
1595 xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h,
1596 x->xso.type);
1597 }
1598 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1599 hrtimer_start(&x->mtimer,
1600 ktime_set(net->xfrm.sysctl_acq_expires, 0),
1601 HRTIMER_MODE_REL_SOFT);
1602 net->xfrm.state_num++;
1603 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1604 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1605 } else {
1606 #ifdef CONFIG_XFRM_OFFLOAD
1607 struct xfrm_dev_offload *xso = &x->xso;
1608
1609 if (xso->type == XFRM_DEV_OFFLOAD_PACKET) {
1610 xfrm_dev_state_delete(x);
1611 xfrm_dev_state_free(x);
1612 }
1613 #endif
1614 x->km.state = XFRM_STATE_DEAD;
1615 to_put = x;
1616 x = NULL;
1617 error = -ESRCH;
1618 }
1619
1620 /* Use the already installed 'fallback' while the CPU-specific
1621 * SA acquire is handled*/
1622 if (best)
1623 x = best;
1624 }
1625 out:
1626 if (x) {
1627 if (!xfrm_state_hold_rcu(x)) {
1628 *err = -EAGAIN;
1629 x = NULL;
1630 }
1631 } else {
1632 *err = acquire_in_progress ? -EAGAIN : error;
1633 }
1634
1635 if (x && x->km.state == XFRM_STATE_VALID && !cached &&
1636 (!(pol->flags & XFRM_POLICY_CPU_ACQUIRE) || x->pcpu_num == pcpu_id)) {
1637 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1638 if (hlist_unhashed(&x->state_cache))
1639 hlist_add_head_rcu(&x->state_cache, &pol->state_cache_list);
1640 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1641 }
1642
1643 rcu_read_unlock();
1644 if (to_put)
1645 xfrm_state_put(to_put);
1646
1647 if (read_seqcount_retry(&net->xfrm.xfrm_state_hash_generation, sequence)) {
1648 *err = -EAGAIN;
1649 if (x) {
1650 xfrm_state_put(x);
1651 x = NULL;
1652 }
1653 }
1654
1655 return x;
1656 }
1657
1658 struct xfrm_state *
xfrm_stateonly_find(struct net * net,u32 mark,u32 if_id,xfrm_address_t * daddr,xfrm_address_t * saddr,unsigned short family,u8 mode,u8 proto,u32 reqid)1659 xfrm_stateonly_find(struct net *net, u32 mark, u32 if_id,
1660 xfrm_address_t *daddr, xfrm_address_t *saddr,
1661 unsigned short family, u8 mode, u8 proto, u32 reqid)
1662 {
1663 unsigned int h;
1664 struct xfrm_state *rx = NULL, *x = NULL;
1665
1666 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1667 h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1668 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
1669 if (x->props.family == family &&
1670 x->props.reqid == reqid &&
1671 (mark & x->mark.m) == x->mark.v &&
1672 x->if_id == if_id &&
1673 !(x->props.flags & XFRM_STATE_WILDRECV) &&
1674 xfrm_state_addr_check(x, daddr, saddr, family) &&
1675 mode == x->props.mode &&
1676 proto == x->id.proto &&
1677 x->km.state == XFRM_STATE_VALID) {
1678 rx = x;
1679 break;
1680 }
1681 }
1682
1683 if (rx)
1684 xfrm_state_hold(rx);
1685 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1686
1687
1688 return rx;
1689 }
1690 EXPORT_SYMBOL(xfrm_stateonly_find);
1691
xfrm_state_lookup_byspi(struct net * net,__be32 spi,unsigned short family)1692 struct xfrm_state *xfrm_state_lookup_byspi(struct net *net, __be32 spi,
1693 unsigned short family)
1694 {
1695 struct xfrm_state *x;
1696 struct xfrm_state_walk *w;
1697
1698 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1699 list_for_each_entry(w, &net->xfrm.state_all, all) {
1700 x = container_of(w, struct xfrm_state, km);
1701 if (x->props.family != family ||
1702 x->id.spi != spi)
1703 continue;
1704
1705 xfrm_state_hold(x);
1706 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1707 return x;
1708 }
1709 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1710 return NULL;
1711 }
1712 EXPORT_SYMBOL(xfrm_state_lookup_byspi);
1713
xfrm_state_lookup_spi_proto(struct net * net,__be32 spi,u8 proto)1714 static struct xfrm_state *xfrm_state_lookup_spi_proto(struct net *net, __be32 spi, u8 proto)
1715 {
1716 struct xfrm_state *x;
1717 unsigned int i;
1718
1719 for (i = 0; i <= net->xfrm.state_hmask; i++) {
1720 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byspi, net) + i, byspi) {
1721 if (x->id.spi == spi && x->id.proto == proto)
1722 return x;
1723 }
1724 }
1725 return NULL;
1726 }
1727
__xfrm_state_insert(struct xfrm_state * x)1728 static void __xfrm_state_insert(struct xfrm_state *x)
1729 {
1730 struct net *net = xs_net(x);
1731 unsigned int h;
1732
1733 list_add(&x->km.all, &net->xfrm.state_all);
1734
1735 /* Sanitize mark before store */
1736 x->mark.v &= x->mark.m;
1737
1738 h = xfrm_dst_hash(net, &x->id.daddr, &x->props.saddr,
1739 x->props.reqid, x->props.family);
1740 XFRM_STATE_INSERT(bydst, &x->bydst,
1741 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h,
1742 x->xso.type);
1743
1744 h = xfrm_src_hash(net, &x->id.daddr, &x->props.saddr, x->props.family);
1745 XFRM_STATE_INSERT(bysrc, &x->bysrc,
1746 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h,
1747 x->xso.type);
1748
1749 if (x->id.spi) {
1750 h = xfrm_spi_hash(net, &x->id.daddr, x->id.spi, x->id.proto,
1751 x->props.family);
1752
1753 XFRM_STATE_INSERT(byspi, &x->byspi,
1754 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h,
1755 x->xso.type);
1756 }
1757
1758 if (x->km.seq) {
1759 h = xfrm_seq_hash(net, x->km.seq);
1760
1761 XFRM_STATE_INSERT(byseq, &x->byseq,
1762 xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h,
1763 x->xso.type);
1764 }
1765
1766 hrtimer_start(&x->mtimer, ktime_set(1, 0), HRTIMER_MODE_REL_SOFT);
1767 if (x->replay_maxage)
1768 mod_timer(&x->rtimer, jiffies + x->replay_maxage);
1769
1770 net->xfrm.state_num++;
1771
1772 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1773 xfrm_nat_keepalive_state_updated(x);
1774 }
1775
1776 /* net->xfrm.xfrm_state_lock is held */
__xfrm_state_bump_genids(struct xfrm_state * xnew)1777 static void __xfrm_state_bump_genids(struct xfrm_state *xnew)
1778 {
1779 struct net *net = xs_net(xnew);
1780 unsigned short family = xnew->props.family;
1781 u32 reqid = xnew->props.reqid;
1782 struct xfrm_state *x;
1783 unsigned int h;
1784 u32 mark = xnew->mark.v & xnew->mark.m;
1785 u32 if_id = xnew->if_id;
1786 u32 cpu_id = xnew->pcpu_num;
1787
1788 h = xfrm_dst_hash(net, &xnew->id.daddr, &xnew->props.saddr, reqid, family);
1789 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
1790 if (x->props.family == family &&
1791 x->props.reqid == reqid &&
1792 x->if_id == if_id &&
1793 x->pcpu_num == cpu_id &&
1794 (mark & x->mark.m) == x->mark.v &&
1795 xfrm_addr_equal(&x->id.daddr, &xnew->id.daddr, family) &&
1796 xfrm_addr_equal(&x->props.saddr, &xnew->props.saddr, family))
1797 x->genid++;
1798 }
1799 }
1800
xfrm_state_insert(struct xfrm_state * x)1801 void xfrm_state_insert(struct xfrm_state *x)
1802 {
1803 struct net *net = xs_net(x);
1804
1805 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1806 __xfrm_state_bump_genids(x);
1807 __xfrm_state_insert(x);
1808 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1809 }
1810 EXPORT_SYMBOL(xfrm_state_insert);
1811
1812 /* net->xfrm.xfrm_state_lock is held */
__find_acq_core(struct net * net,const struct xfrm_mark * m,unsigned short family,u8 mode,u32 reqid,u32 if_id,u32 pcpu_num,u8 proto,const xfrm_address_t * daddr,const xfrm_address_t * saddr,int create)1813 static struct xfrm_state *__find_acq_core(struct net *net,
1814 const struct xfrm_mark *m,
1815 unsigned short family, u8 mode,
1816 u32 reqid, u32 if_id, u32 pcpu_num, u8 proto,
1817 const xfrm_address_t *daddr,
1818 const xfrm_address_t *saddr,
1819 int create)
1820 {
1821 unsigned int h = xfrm_dst_hash(net, daddr, saddr, reqid, family);
1822 struct xfrm_state *x;
1823 u32 mark = m->v & m->m;
1824
1825 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
1826 if (x->props.reqid != reqid ||
1827 x->props.mode != mode ||
1828 x->props.family != family ||
1829 x->km.state != XFRM_STATE_ACQ ||
1830 x->id.spi != 0 ||
1831 x->id.proto != proto ||
1832 (mark & x->mark.m) != x->mark.v ||
1833 x->pcpu_num != pcpu_num ||
1834 !xfrm_addr_equal(&x->id.daddr, daddr, family) ||
1835 !xfrm_addr_equal(&x->props.saddr, saddr, family))
1836 continue;
1837
1838 xfrm_state_hold(x);
1839 return x;
1840 }
1841
1842 if (!create)
1843 return NULL;
1844
1845 x = xfrm_state_alloc(net);
1846 if (likely(x)) {
1847 switch (family) {
1848 case AF_INET:
1849 x->sel.daddr.a4 = daddr->a4;
1850 x->sel.saddr.a4 = saddr->a4;
1851 x->sel.prefixlen_d = 32;
1852 x->sel.prefixlen_s = 32;
1853 x->props.saddr.a4 = saddr->a4;
1854 x->id.daddr.a4 = daddr->a4;
1855 break;
1856
1857 case AF_INET6:
1858 x->sel.daddr.in6 = daddr->in6;
1859 x->sel.saddr.in6 = saddr->in6;
1860 x->sel.prefixlen_d = 128;
1861 x->sel.prefixlen_s = 128;
1862 x->props.saddr.in6 = saddr->in6;
1863 x->id.daddr.in6 = daddr->in6;
1864 break;
1865 }
1866
1867 x->pcpu_num = pcpu_num;
1868 x->km.state = XFRM_STATE_ACQ;
1869 x->id.proto = proto;
1870 x->props.family = family;
1871 x->props.mode = mode;
1872 x->props.reqid = reqid;
1873 x->if_id = if_id;
1874 x->mark.v = m->v;
1875 x->mark.m = m->m;
1876 x->lft.hard_add_expires_seconds = net->xfrm.sysctl_acq_expires;
1877 xfrm_state_hold(x);
1878 hrtimer_start(&x->mtimer,
1879 ktime_set(net->xfrm.sysctl_acq_expires, 0),
1880 HRTIMER_MODE_REL_SOFT);
1881 list_add(&x->km.all, &net->xfrm.state_all);
1882 XFRM_STATE_INSERT(bydst, &x->bydst,
1883 xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h,
1884 x->xso.type);
1885 h = xfrm_src_hash(net, daddr, saddr, family);
1886 XFRM_STATE_INSERT(bysrc, &x->bysrc,
1887 xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h,
1888 x->xso.type);
1889
1890 net->xfrm.state_num++;
1891
1892 xfrm_hash_grow_check(net, x->bydst.next != NULL);
1893 }
1894
1895 return x;
1896 }
1897
1898 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num);
1899
xfrm_state_add(struct xfrm_state * x)1900 int xfrm_state_add(struct xfrm_state *x)
1901 {
1902 struct net *net = xs_net(x);
1903 struct xfrm_state *x1, *to_put;
1904 int family;
1905 int err;
1906 u32 mark = x->mark.v & x->mark.m;
1907 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
1908
1909 family = x->props.family;
1910
1911 to_put = NULL;
1912
1913 spin_lock_bh(&net->xfrm.xfrm_state_lock);
1914
1915 x1 = __xfrm_state_locate(x, use_spi, family);
1916 if (x1) {
1917 to_put = x1;
1918 x1 = NULL;
1919 err = -EEXIST;
1920 goto out;
1921 }
1922
1923 if (use_spi && x->km.seq) {
1924 x1 = __xfrm_find_acq_byseq(net, mark, x->km.seq, x->pcpu_num);
1925 if (x1 && ((x1->id.proto != x->id.proto) ||
1926 !xfrm_addr_equal(&x1->id.daddr, &x->id.daddr, family))) {
1927 to_put = x1;
1928 x1 = NULL;
1929 }
1930 }
1931
1932 if (use_spi && !x1)
1933 x1 = __find_acq_core(net, &x->mark, family, x->props.mode,
1934 x->props.reqid, x->if_id, x->pcpu_num, x->id.proto,
1935 &x->id.daddr, &x->props.saddr, 0);
1936
1937 __xfrm_state_bump_genids(x);
1938 __xfrm_state_insert(x);
1939 err = 0;
1940
1941 out:
1942 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
1943
1944 if (x1) {
1945 xfrm_state_delete(x1);
1946 xfrm_state_put(x1);
1947 }
1948
1949 if (to_put)
1950 xfrm_state_put(to_put);
1951
1952 return err;
1953 }
1954 EXPORT_SYMBOL(xfrm_state_add);
1955
1956 #ifdef CONFIG_XFRM_MIGRATE
clone_security(struct xfrm_state * x,struct xfrm_sec_ctx * security)1957 static inline int clone_security(struct xfrm_state *x, struct xfrm_sec_ctx *security)
1958 {
1959 struct xfrm_user_sec_ctx *uctx;
1960 int size = sizeof(*uctx) + security->ctx_len;
1961 int err;
1962
1963 uctx = kmalloc(size, GFP_KERNEL);
1964 if (!uctx)
1965 return -ENOMEM;
1966
1967 uctx->exttype = XFRMA_SEC_CTX;
1968 uctx->len = size;
1969 uctx->ctx_doi = security->ctx_doi;
1970 uctx->ctx_alg = security->ctx_alg;
1971 uctx->ctx_len = security->ctx_len;
1972 memcpy(uctx + 1, security->ctx_str, security->ctx_len);
1973 err = security_xfrm_state_alloc(x, uctx);
1974 kfree(uctx);
1975 if (err)
1976 return err;
1977
1978 return 0;
1979 }
1980
xfrm_state_clone_and_setup(struct xfrm_state * orig,const struct xfrm_migrate * m)1981 static struct xfrm_state *xfrm_state_clone_and_setup(struct xfrm_state *orig,
1982 const struct xfrm_migrate *m)
1983 {
1984 struct net *net = xs_net(orig);
1985 struct xfrm_state *x = xfrm_state_alloc(net);
1986 if (!x)
1987 goto out;
1988
1989 memcpy(&x->id, &orig->id, sizeof(x->id));
1990 if (m->msg_type == XFRM_MSG_MIGRATE_STATE) {
1991 if (m->flags & XFRM_MIGRATE_STATE_UPDATE_H2H_SEL) {
1992 u8 prefixlen = (m->new_family == AF_INET6) ? 128 : 32;
1993
1994 x->sel = orig->sel;
1995 x->sel.family = m->new_family;
1996 x->sel.prefixlen_d = prefixlen;
1997 x->sel.prefixlen_s = prefixlen;
1998 x->sel.daddr = m->new_daddr;
1999 x->sel.saddr = m->new_saddr;
2000 } else {
2001 x->sel = *m->new_sel;
2002 }
2003 } else {
2004 x->sel = orig->sel;
2005 }
2006 memcpy(&x->lft, &orig->lft, sizeof(x->lft));
2007 x->props.mode = orig->props.mode;
2008 x->props.replay_window = orig->props.replay_window;
2009
2010 if (orig->aalg) {
2011 x->aalg = xfrm_algo_auth_clone(orig->aalg);
2012 if (!x->aalg)
2013 goto error;
2014 }
2015 x->props.aalgo = orig->props.aalgo;
2016
2017 if (orig->aead) {
2018 x->aead = xfrm_algo_aead_clone(orig->aead);
2019 x->geniv = orig->geniv;
2020 if (!x->aead)
2021 goto error;
2022 }
2023 if (orig->ealg) {
2024 x->ealg = xfrm_algo_clone(orig->ealg);
2025 if (!x->ealg)
2026 goto error;
2027 }
2028 x->props.ealgo = orig->props.ealgo;
2029
2030 if (orig->calg) {
2031 x->calg = xfrm_algo_clone(orig->calg);
2032 if (!x->calg)
2033 goto error;
2034 }
2035 x->props.calgo = orig->props.calgo;
2036
2037 if (m->encap) {
2038 x->encap = kmemdup(m->encap, sizeof(*x->encap), GFP_KERNEL);
2039 if (!x->encap)
2040 goto error;
2041 x->mapping_maxage = m->mapping_maxage;
2042 x->nat_keepalive_interval = m->nat_keepalive_interval;
2043 }
2044
2045 if (orig->security)
2046 if (clone_security(x, orig->security))
2047 goto error;
2048
2049 if (orig->coaddr) {
2050 x->coaddr = kmemdup(orig->coaddr, sizeof(*x->coaddr),
2051 GFP_KERNEL);
2052 if (!x->coaddr)
2053 goto error;
2054 }
2055
2056 if (xfrm_replay_clone(x, orig))
2057 goto error;
2058
2059 x->mark = m->new_mark ? *m->new_mark : m->old_mark;
2060
2061 x->props.smark = m->smark;
2062
2063 x->props.flags = orig->props.flags;
2064 x->props.extra_flags = orig->props.extra_flags;
2065
2066 x->pcpu_num = orig->pcpu_num;
2067 x->if_id = orig->if_id;
2068 x->tfcpad = orig->tfcpad;
2069 x->replay_maxdiff = orig->replay_maxdiff;
2070 x->replay_maxage = orig->replay_maxage;
2071 x->km.state = orig->km.state;
2072 x->km.seq = orig->km.seq;
2073 x->lastused = orig->lastused;
2074 x->new_mapping = 0;
2075 x->new_mapping_sport = 0;
2076 x->dir = orig->dir;
2077
2078 x->mode_cbs = orig->mode_cbs;
2079 if (x->mode_cbs && x->mode_cbs->clone_state) {
2080 if (x->mode_cbs->clone_state(x, orig)) {
2081 if (!x->mode_data)
2082 x->mode_cbs = NULL;
2083 goto error;
2084 }
2085 }
2086
2087 x->props.reqid = m->new_reqid;
2088 x->props.family = m->new_family;
2089 memcpy(&x->id.daddr, &m->new_daddr, sizeof(x->id.daddr));
2090 memcpy(&x->props.saddr, &m->new_saddr, sizeof(x->props.saddr));
2091
2092 return x;
2093
2094 error:
2095 x->km.state = XFRM_STATE_DEAD;
2096 xfrm_state_put(x);
2097 out:
2098 return NULL;
2099 }
2100
xfrm_migrate_state_find(struct xfrm_migrate * m,struct net * net,u32 if_id)2101 struct xfrm_state *xfrm_migrate_state_find(struct xfrm_migrate *m, struct net *net,
2102 u32 if_id)
2103 {
2104 unsigned int h;
2105 struct xfrm_state *x = NULL;
2106
2107 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2108
2109 if (m->old_reqid) {
2110 h = xfrm_dst_hash(net, &m->old_daddr, &m->old_saddr,
2111 m->old_reqid, m->old_family);
2112 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + h, bydst) {
2113 if (x->props.mode != m->mode ||
2114 x->id.proto != m->proto)
2115 continue;
2116 if (m->old_reqid && x->props.reqid != m->old_reqid)
2117 continue;
2118 if (if_id != 0 && x->if_id != if_id)
2119 continue;
2120 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
2121 m->old_family) ||
2122 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
2123 m->old_family))
2124 continue;
2125 xfrm_state_hold(x);
2126 break;
2127 }
2128 } else {
2129 h = xfrm_src_hash(net, &m->old_daddr, &m->old_saddr,
2130 m->old_family);
2131 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bysrc, net) + h, bysrc) {
2132 if (x->props.mode != m->mode ||
2133 x->id.proto != m->proto)
2134 continue;
2135 if (if_id != 0 && x->if_id != if_id)
2136 continue;
2137 if (!xfrm_addr_equal(&x->id.daddr, &m->old_daddr,
2138 m->old_family) ||
2139 !xfrm_addr_equal(&x->props.saddr, &m->old_saddr,
2140 m->old_family))
2141 continue;
2142 xfrm_state_hold(x);
2143 break;
2144 }
2145 }
2146
2147 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2148
2149 return x;
2150 }
2151 EXPORT_SYMBOL(xfrm_migrate_state_find);
2152
xfrm_state_migrate_create(struct xfrm_state * x,const struct xfrm_migrate * m,struct net * net,struct netlink_ext_ack * extack)2153 struct xfrm_state *xfrm_state_migrate_create(struct xfrm_state *x,
2154 const struct xfrm_migrate *m,
2155 struct net *net,
2156 struct netlink_ext_ack *extack)
2157 {
2158 struct xfrm_state *xc;
2159
2160 xc = xfrm_state_clone_and_setup(x, m);
2161 if (!xc) {
2162 NL_SET_ERR_MSG(extack, "Failed to clone and setup state");
2163 return NULL;
2164 }
2165
2166 if (xfrm_init_state(xc, extack) < 0) {
2167 NL_SET_ERR_MSG_WEAK(extack, "Failed to initialize migrated state");
2168 goto error;
2169 }
2170
2171 /* configure the hardware if offload is requested */
2172 if (m->xuo && xfrm_dev_state_add(net, xc, m->xuo, extack))
2173 goto error;
2174
2175 return xc;
2176 error:
2177 xc->km.state = XFRM_STATE_DEAD;
2178 xfrm_state_put(xc);
2179 return NULL;
2180 }
2181 EXPORT_SYMBOL(xfrm_state_migrate_create);
2182
xfrm_state_migrate_install(const struct xfrm_state * x,struct xfrm_state * xc,const struct xfrm_migrate * m,struct netlink_ext_ack * extack)2183 int xfrm_state_migrate_install(const struct xfrm_state *x,
2184 struct xfrm_state *xc,
2185 const struct xfrm_migrate *m,
2186 struct netlink_ext_ack *extack)
2187 {
2188 if (m->new_family == m->old_family &&
2189 xfrm_addr_equal(&x->id.daddr, &m->new_daddr, m->new_family)) {
2190 /*
2191 * Care is needed when the destination address of the state is
2192 * to be updated as it is a part of triplet.
2193 */
2194 xfrm_state_insert(xc);
2195 } else {
2196 if (xfrm_state_add(xc) < 0) {
2197 NL_SET_ERR_MSG(extack, "Failed to add migrated state");
2198 if (m->xuo)
2199 xfrm_dev_state_delete(xc);
2200 xc->km.state = XFRM_STATE_DEAD;
2201 xfrm_state_put(xc);
2202 return -EEXIST;
2203 }
2204 }
2205
2206 return 0;
2207 }
2208 EXPORT_SYMBOL(xfrm_state_migrate_install);
2209
xfrm_state_migrate(struct xfrm_state * x,struct xfrm_migrate * m,struct net * net,struct netlink_ext_ack * extack)2210 struct xfrm_state *xfrm_state_migrate(struct xfrm_state *x,
2211 struct xfrm_migrate *m,
2212 struct net *net,
2213 struct netlink_ext_ack *extack)
2214 {
2215 struct xfrm_state *xc;
2216
2217 xc = xfrm_state_migrate_create(x, m, net, extack);
2218 if (!xc)
2219 return NULL;
2220
2221 xfrm_migrate_sync(xc, x);
2222
2223 if (xfrm_state_migrate_install(x, xc, m, extack) < 0)
2224 return NULL;
2225
2226 return xc;
2227 }
2228 EXPORT_SYMBOL(xfrm_state_migrate);
2229 #endif
2230
xfrm_state_update(struct xfrm_state * x)2231 int xfrm_state_update(struct xfrm_state *x)
2232 {
2233 struct xfrm_state *x1, *to_put;
2234 int err;
2235 int use_spi = xfrm_id_proto_match(x->id.proto, IPSEC_PROTO_ANY);
2236 struct net *net = xs_net(x);
2237
2238 to_put = NULL;
2239
2240 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2241 x1 = __xfrm_state_locate(x, use_spi, x->props.family);
2242
2243 err = -ESRCH;
2244 if (!x1)
2245 goto out;
2246
2247 if (xfrm_state_kern(x1)) {
2248 to_put = x1;
2249 err = -EEXIST;
2250 goto out;
2251 }
2252
2253 if (x1->km.state == XFRM_STATE_ACQ) {
2254 if (x->dir && x1->dir != x->dir) {
2255 to_put = x1;
2256 goto out;
2257 }
2258
2259 __xfrm_state_insert(x);
2260 x = NULL;
2261 } else {
2262 if (x1->dir != x->dir) {
2263 to_put = x1;
2264 goto out;
2265 }
2266 }
2267 err = 0;
2268
2269 out:
2270 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2271
2272 if (to_put)
2273 xfrm_state_put(to_put);
2274
2275 if (err)
2276 return err;
2277
2278 if (!x) {
2279 xfrm_state_delete(x1);
2280 xfrm_state_put(x1);
2281 return 0;
2282 }
2283
2284 err = -EINVAL;
2285 spin_lock_bh(&x1->lock);
2286 if (likely(x1->km.state == XFRM_STATE_VALID)) {
2287 if (x->encap && x1->encap &&
2288 x->encap->encap_type == x1->encap->encap_type)
2289 memcpy(x1->encap, x->encap, sizeof(*x1->encap));
2290 else if (x->encap || x1->encap)
2291 goto fail;
2292
2293 if (x->coaddr && x1->coaddr) {
2294 memcpy(x1->coaddr, x->coaddr, sizeof(*x1->coaddr));
2295 }
2296 if (!use_spi && memcmp(&x1->sel, &x->sel, sizeof(x1->sel)))
2297 memcpy(&x1->sel, &x->sel, sizeof(x1->sel));
2298 memcpy(&x1->lft, &x->lft, sizeof(x1->lft));
2299 x1->km.dying = 0;
2300
2301 hrtimer_start(&x1->mtimer, ktime_set(1, 0),
2302 HRTIMER_MODE_REL_SOFT);
2303 if (READ_ONCE(x1->curlft.use_time))
2304 xfrm_state_check_expire(x1);
2305
2306 if (x->props.smark.m || x->props.smark.v || x->if_id) {
2307 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2308
2309 if (x->props.smark.m || x->props.smark.v)
2310 x1->props.smark = x->props.smark;
2311
2312 if (x->if_id)
2313 x1->if_id = x->if_id;
2314
2315 __xfrm_state_bump_genids(x1);
2316 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2317 }
2318
2319 err = 0;
2320 x->km.state = XFRM_STATE_DEAD;
2321 xfrm_dev_state_delete(x);
2322 __xfrm_state_put(x);
2323 }
2324
2325 fail:
2326 spin_unlock_bh(&x1->lock);
2327
2328 xfrm_state_put(x1);
2329
2330 return err;
2331 }
2332 EXPORT_SYMBOL(xfrm_state_update);
2333
xfrm_state_check_expire(struct xfrm_state * x)2334 int xfrm_state_check_expire(struct xfrm_state *x)
2335 {
2336 /* All counters which are needed to decide if state is expired
2337 * are handled by SW for non-packet offload modes. Simply skip
2338 * the following update and save extra boilerplate in drivers.
2339 */
2340 if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET)
2341 xfrm_dev_state_update_stats(x);
2342
2343 if (!READ_ONCE(x->curlft.use_time))
2344 WRITE_ONCE(x->curlft.use_time, ktime_get_real_seconds());
2345
2346 if (x->curlft.bytes >= x->lft.hard_byte_limit ||
2347 x->curlft.packets >= x->lft.hard_packet_limit) {
2348 x->km.state = XFRM_STATE_EXPIRED;
2349 hrtimer_start(&x->mtimer, 0, HRTIMER_MODE_REL_SOFT);
2350 return -EINVAL;
2351 }
2352
2353 if (!x->km.dying &&
2354 (x->curlft.bytes >= x->lft.soft_byte_limit ||
2355 x->curlft.packets >= x->lft.soft_packet_limit)) {
2356 x->km.dying = 1;
2357 km_state_expired(x, 0, 0);
2358 }
2359 return 0;
2360 }
2361 EXPORT_SYMBOL(xfrm_state_check_expire);
2362
xfrm_state_update_stats(struct net * net)2363 void xfrm_state_update_stats(struct net *net)
2364 {
2365 struct xfrm_state *x;
2366 int i;
2367
2368 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2369 for (i = 0; i <= net->xfrm.state_hmask; i++) {
2370 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_bydst, net) + i, bydst)
2371 xfrm_dev_state_update_stats(x);
2372 }
2373 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2374 }
2375
2376 struct xfrm_state *
xfrm_state_lookup(struct net * net,u32 mark,const xfrm_address_t * daddr,__be32 spi,u8 proto,unsigned short family)2377 xfrm_state_lookup(struct net *net, u32 mark, const xfrm_address_t *daddr, __be32 spi,
2378 u8 proto, unsigned short family)
2379 {
2380 struct xfrm_hash_state_ptrs state_ptrs;
2381 struct xfrm_state *x;
2382
2383 rcu_read_lock();
2384 xfrm_hash_ptrs_get(net, &state_ptrs);
2385
2386 x = __xfrm_state_lookup(&state_ptrs, mark, daddr, spi, proto, family);
2387 rcu_read_unlock();
2388 return x;
2389 }
2390 EXPORT_SYMBOL(xfrm_state_lookup);
2391
2392 struct xfrm_state *
xfrm_state_lookup_byaddr(struct net * net,u32 mark,const xfrm_address_t * daddr,const xfrm_address_t * saddr,u8 proto,unsigned short family)2393 xfrm_state_lookup_byaddr(struct net *net, u32 mark,
2394 const xfrm_address_t *daddr, const xfrm_address_t *saddr,
2395 u8 proto, unsigned short family)
2396 {
2397 struct xfrm_hash_state_ptrs state_ptrs;
2398 struct xfrm_state *x;
2399
2400 rcu_read_lock();
2401
2402 xfrm_hash_ptrs_get(net, &state_ptrs);
2403
2404 x = __xfrm_state_lookup_byaddr(&state_ptrs, mark, daddr, saddr, proto, family);
2405 rcu_read_unlock();
2406 return x;
2407 }
2408 EXPORT_SYMBOL(xfrm_state_lookup_byaddr);
2409
2410 struct xfrm_state *
xfrm_find_acq(struct net * net,const struct xfrm_mark * mark,u8 mode,u32 reqid,u32 if_id,u32 pcpu_num,u8 proto,const xfrm_address_t * daddr,const xfrm_address_t * saddr,int create,unsigned short family)2411 xfrm_find_acq(struct net *net, const struct xfrm_mark *mark, u8 mode, u32 reqid,
2412 u32 if_id, u32 pcpu_num, u8 proto, const xfrm_address_t *daddr,
2413 const xfrm_address_t *saddr, int create, unsigned short family)
2414 {
2415 struct xfrm_state *x;
2416
2417 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2418 x = __find_acq_core(net, mark, family, mode, reqid, if_id, pcpu_num,
2419 proto, daddr, saddr, create);
2420 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2421
2422 return x;
2423 }
2424 EXPORT_SYMBOL(xfrm_find_acq);
2425
2426 #ifdef CONFIG_XFRM_SUB_POLICY
2427 #if IS_ENABLED(CONFIG_IPV6)
2428 /* distribution counting sort function for xfrm_state and xfrm_tmpl */
2429 static void
__xfrm6_sort(void ** dst,void ** src,int n,int (* cmp)(const void * p),int maxclass)2430 __xfrm6_sort(void **dst, void **src, int n,
2431 int (*cmp)(const void *p), int maxclass)
2432 {
2433 int count[XFRM_MAX_DEPTH] = { };
2434 int class[XFRM_MAX_DEPTH];
2435 int i;
2436
2437 for (i = 0; i < n; i++) {
2438 int c = cmp(src[i]);
2439
2440 class[i] = c;
2441 count[c]++;
2442 }
2443
2444 for (i = 2; i < maxclass; i++)
2445 count[i] += count[i - 1];
2446
2447 for (i = 0; i < n; i++) {
2448 dst[count[class[i] - 1]++] = src[i];
2449 src[i] = NULL;
2450 }
2451 }
2452
2453 /* Rule for xfrm_state:
2454 *
2455 * rule 1: select IPsec transport except AH
2456 * rule 2: select MIPv6 RO or inbound trigger
2457 * rule 3: select IPsec transport AH
2458 * rule 4: select IPsec tunnel
2459 * rule 5: others
2460 */
__xfrm6_state_sort_cmp(const void * p)2461 static int __xfrm6_state_sort_cmp(const void *p)
2462 {
2463 const struct xfrm_state *v = p;
2464
2465 switch (v->props.mode) {
2466 case XFRM_MODE_TRANSPORT:
2467 if (v->id.proto != IPPROTO_AH)
2468 return 1;
2469 else
2470 return 3;
2471 #if IS_ENABLED(CONFIG_IPV6_MIP6)
2472 case XFRM_MODE_ROUTEOPTIMIZATION:
2473 case XFRM_MODE_IN_TRIGGER:
2474 return 2;
2475 #endif
2476 case XFRM_MODE_TUNNEL:
2477 case XFRM_MODE_BEET:
2478 case XFRM_MODE_IPTFS:
2479 return 4;
2480 }
2481 return 5;
2482 }
2483
2484 /* Rule for xfrm_tmpl:
2485 *
2486 * rule 1: select IPsec transport
2487 * rule 2: select MIPv6 RO or inbound trigger
2488 * rule 3: select IPsec tunnel
2489 * rule 4: others
2490 */
__xfrm6_tmpl_sort_cmp(const void * p)2491 static int __xfrm6_tmpl_sort_cmp(const void *p)
2492 {
2493 const struct xfrm_tmpl *v = p;
2494
2495 switch (v->mode) {
2496 case XFRM_MODE_TRANSPORT:
2497 return 1;
2498 #if IS_ENABLED(CONFIG_IPV6_MIP6)
2499 case XFRM_MODE_ROUTEOPTIMIZATION:
2500 case XFRM_MODE_IN_TRIGGER:
2501 return 2;
2502 #endif
2503 case XFRM_MODE_TUNNEL:
2504 case XFRM_MODE_BEET:
2505 case XFRM_MODE_IPTFS:
2506 return 3;
2507 }
2508 return 4;
2509 }
2510 #else
__xfrm6_state_sort_cmp(const void * p)2511 static inline int __xfrm6_state_sort_cmp(const void *p) { return 5; }
__xfrm6_tmpl_sort_cmp(const void * p)2512 static inline int __xfrm6_tmpl_sort_cmp(const void *p) { return 4; }
2513
2514 static inline void
__xfrm6_sort(void ** dst,void ** src,int n,int (* cmp)(const void * p),int maxclass)2515 __xfrm6_sort(void **dst, void **src, int n,
2516 int (*cmp)(const void *p), int maxclass)
2517 {
2518 int i;
2519
2520 for (i = 0; i < n; i++)
2521 dst[i] = src[i];
2522 }
2523 #endif /* CONFIG_IPV6 */
2524
2525 void
xfrm_tmpl_sort(struct xfrm_tmpl ** dst,struct xfrm_tmpl ** src,int n,unsigned short family)2526 xfrm_tmpl_sort(struct xfrm_tmpl **dst, struct xfrm_tmpl **src, int n,
2527 unsigned short family)
2528 {
2529 int i;
2530
2531 if (family == AF_INET6)
2532 __xfrm6_sort((void **)dst, (void **)src, n,
2533 __xfrm6_tmpl_sort_cmp, 5);
2534 else
2535 for (i = 0; i < n; i++)
2536 dst[i] = src[i];
2537 }
2538
2539 void
xfrm_state_sort(struct xfrm_state ** dst,struct xfrm_state ** src,int n,unsigned short family)2540 xfrm_state_sort(struct xfrm_state **dst, struct xfrm_state **src, int n,
2541 unsigned short family)
2542 {
2543 int i;
2544
2545 if (family == AF_INET6)
2546 __xfrm6_sort((void **)dst, (void **)src, n,
2547 __xfrm6_state_sort_cmp, 6);
2548 else
2549 for (i = 0; i < n; i++)
2550 dst[i] = src[i];
2551 }
2552 #endif
2553
2554 /* Silly enough, but I'm lazy to build resolution list */
2555
__xfrm_find_acq_byseq(struct net * net,u32 mark,u32 seq,u32 pcpu_num)2556 static struct xfrm_state *__xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num)
2557 {
2558 unsigned int h = xfrm_seq_hash(net, seq);
2559 struct xfrm_state *x;
2560
2561 hlist_for_each_entry(x, xfrm_state_deref_prot(net->xfrm.state_byseq, net) + h, byseq) {
2562 if (x->km.seq == seq &&
2563 (mark & x->mark.m) == x->mark.v &&
2564 x->pcpu_num == pcpu_num &&
2565 x->km.state == XFRM_STATE_ACQ) {
2566 xfrm_state_hold(x);
2567 return x;
2568 }
2569 }
2570
2571 return NULL;
2572 }
2573
xfrm_find_acq_byseq(struct net * net,u32 mark,u32 seq,u32 pcpu_num)2574 struct xfrm_state *xfrm_find_acq_byseq(struct net *net, u32 mark, u32 seq, u32 pcpu_num)
2575 {
2576 struct xfrm_state *x;
2577
2578 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2579 x = __xfrm_find_acq_byseq(net, mark, seq, pcpu_num);
2580 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2581 return x;
2582 }
2583 EXPORT_SYMBOL(xfrm_find_acq_byseq);
2584
xfrm_get_acqseq(void)2585 u32 xfrm_get_acqseq(void)
2586 {
2587 u32 res;
2588 static atomic_t acqseq;
2589
2590 do {
2591 res = atomic_inc_return(&acqseq);
2592 } while (!res);
2593
2594 return res;
2595 }
2596 EXPORT_SYMBOL(xfrm_get_acqseq);
2597
verify_spi_info(u8 proto,u32 min,u32 max,struct netlink_ext_ack * extack)2598 int verify_spi_info(u8 proto, u32 min, u32 max, struct netlink_ext_ack *extack)
2599 {
2600 switch (proto) {
2601 case IPPROTO_AH:
2602 case IPPROTO_ESP:
2603 break;
2604
2605 case IPPROTO_COMP:
2606 /* IPCOMP spi is 16-bits. */
2607 if (max >= 0x10000) {
2608 NL_SET_ERR_MSG(extack, "IPCOMP SPI must be <= 65535");
2609 return -EINVAL;
2610 }
2611 break;
2612
2613 default:
2614 NL_SET_ERR_MSG(extack, "Invalid protocol, must be one of AH, ESP, IPCOMP");
2615 return -EINVAL;
2616 }
2617
2618 if (min > max) {
2619 NL_SET_ERR_MSG(extack, "Invalid SPI range: min > max");
2620 return -EINVAL;
2621 }
2622
2623 return 0;
2624 }
2625 EXPORT_SYMBOL(verify_spi_info);
2626
xfrm_alloc_spi(struct xfrm_state * x,u32 low,u32 high,struct netlink_ext_ack * extack)2627 int xfrm_alloc_spi(struct xfrm_state *x, u32 low, u32 high,
2628 struct netlink_ext_ack *extack)
2629 {
2630 struct net *net = xs_net(x);
2631 unsigned int h;
2632 struct xfrm_state *x0;
2633 int err = -ENOENT;
2634 u32 range = high - low + 1;
2635 __be32 newspi = 0;
2636
2637 spin_lock_bh(&x->lock);
2638 if (x->km.state == XFRM_STATE_DEAD) {
2639 NL_SET_ERR_MSG(extack, "Target ACQUIRE is in DEAD state");
2640 goto unlock;
2641 }
2642
2643 err = 0;
2644 if (x->id.spi)
2645 goto unlock;
2646
2647 err = -ENOENT;
2648
2649 for (h = 0; h < range; h++) {
2650 u32 spi = (low == high) ? low : get_random_u32_inclusive(low, high);
2651 if (spi == 0)
2652 goto next;
2653 newspi = htonl(spi);
2654
2655 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2656 x0 = xfrm_state_lookup_spi_proto(net, newspi, x->id.proto);
2657 if (!x0) {
2658 x->id.spi = newspi;
2659 h = xfrm_spi_hash(net, &x->id.daddr, newspi, x->id.proto, x->props.family);
2660 XFRM_STATE_INSERT(byspi, &x->byspi,
2661 xfrm_state_deref_prot(net->xfrm.state_byspi, net) + h,
2662 x->xso.type);
2663 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2664 err = 0;
2665 goto unlock;
2666 }
2667 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2668
2669 next:
2670 if (signal_pending(current)) {
2671 err = -ERESTARTSYS;
2672 goto unlock;
2673 }
2674
2675 if (low == high)
2676 break;
2677 }
2678
2679 if (err)
2680 NL_SET_ERR_MSG(extack, "No SPI available in the requested range");
2681
2682 unlock:
2683 spin_unlock_bh(&x->lock);
2684
2685 return err;
2686 }
2687 EXPORT_SYMBOL(xfrm_alloc_spi);
2688
__xfrm_state_filter_match(struct xfrm_state * x,struct xfrm_address_filter * filter)2689 static bool __xfrm_state_filter_match(struct xfrm_state *x,
2690 struct xfrm_address_filter *filter)
2691 {
2692 if (filter) {
2693 if ((filter->family == AF_INET ||
2694 filter->family == AF_INET6) &&
2695 x->props.family != filter->family)
2696 return false;
2697
2698 return addr_match(&x->props.saddr, &filter->saddr,
2699 filter->splen) &&
2700 addr_match(&x->id.daddr, &filter->daddr,
2701 filter->dplen);
2702 }
2703 return true;
2704 }
2705
xfrm_state_walk(struct net * net,struct xfrm_state_walk * walk,int (* func)(struct xfrm_state *,int,void *),void * data)2706 int xfrm_state_walk(struct net *net, struct xfrm_state_walk *walk,
2707 int (*func)(struct xfrm_state *, int, void*),
2708 void *data)
2709 {
2710 struct xfrm_state *state;
2711 struct xfrm_state_walk *x;
2712 int err = 0;
2713
2714 if (walk->seq != 0 && list_empty(&walk->all))
2715 return 0;
2716
2717 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2718 if (list_empty(&walk->all))
2719 x = list_first_entry(&net->xfrm.state_all, struct xfrm_state_walk, all);
2720 else
2721 x = list_first_entry(&walk->all, struct xfrm_state_walk, all);
2722 list_for_each_entry_from(x, &net->xfrm.state_all, all) {
2723 if (x->state == XFRM_STATE_DEAD)
2724 continue;
2725 state = container_of(x, struct xfrm_state, km);
2726 if (!xfrm_id_proto_match(state->id.proto, walk->proto))
2727 continue;
2728 if (!__xfrm_state_filter_match(state, walk->filter))
2729 continue;
2730 err = func(state, walk->seq, data);
2731 if (err) {
2732 list_move_tail(&walk->all, &x->all);
2733 goto out;
2734 }
2735 walk->seq++;
2736 }
2737 if (walk->seq == 0) {
2738 err = -ENOENT;
2739 goto out;
2740 }
2741 list_del_init(&walk->all);
2742 out:
2743 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2744 return err;
2745 }
2746 EXPORT_SYMBOL(xfrm_state_walk);
2747
xfrm_state_walk_init(struct xfrm_state_walk * walk,u8 proto,struct xfrm_address_filter * filter)2748 void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto,
2749 struct xfrm_address_filter *filter)
2750 {
2751 INIT_LIST_HEAD(&walk->all);
2752 walk->proto = proto;
2753 walk->state = XFRM_STATE_DEAD;
2754 walk->seq = 0;
2755 walk->filter = filter;
2756 }
2757 EXPORT_SYMBOL(xfrm_state_walk_init);
2758
xfrm_state_walk_done(struct xfrm_state_walk * walk,struct net * net)2759 void xfrm_state_walk_done(struct xfrm_state_walk *walk, struct net *net)
2760 {
2761 kfree(walk->filter);
2762
2763 if (list_empty(&walk->all))
2764 return;
2765
2766 spin_lock_bh(&net->xfrm.xfrm_state_lock);
2767 list_del(&walk->all);
2768 spin_unlock_bh(&net->xfrm.xfrm_state_lock);
2769 }
2770 EXPORT_SYMBOL(xfrm_state_walk_done);
2771
xfrm_replay_timer_handler(struct timer_list * t)2772 static void xfrm_replay_timer_handler(struct timer_list *t)
2773 {
2774 struct xfrm_state *x = timer_container_of(x, t, rtimer);
2775
2776 spin_lock(&x->lock);
2777
2778 if (x->km.state == XFRM_STATE_VALID) {
2779 if (xfrm_aevent_is_on(xs_net(x)))
2780 xfrm_replay_notify(x, XFRM_REPLAY_TIMEOUT);
2781 else
2782 x->xflags |= XFRM_TIME_DEFER;
2783 }
2784
2785 spin_unlock(&x->lock);
2786 }
2787
2788 static LIST_HEAD(xfrm_km_list);
2789
km_policy_notify(struct xfrm_policy * xp,int dir,const struct km_event * c)2790 void km_policy_notify(struct xfrm_policy *xp, int dir, const struct km_event *c)
2791 {
2792 struct xfrm_mgr *km;
2793
2794 rcu_read_lock();
2795 list_for_each_entry_rcu(km, &xfrm_km_list, list)
2796 if (km->notify_policy)
2797 km->notify_policy(xp, dir, c);
2798 rcu_read_unlock();
2799 }
2800
km_state_notify(struct xfrm_state * x,const struct km_event * c)2801 void km_state_notify(struct xfrm_state *x, const struct km_event *c)
2802 {
2803 struct xfrm_mgr *km;
2804 rcu_read_lock();
2805 list_for_each_entry_rcu(km, &xfrm_km_list, list)
2806 if (km->notify)
2807 km->notify(x, c);
2808 rcu_read_unlock();
2809 }
2810
2811 EXPORT_SYMBOL(km_policy_notify);
2812 EXPORT_SYMBOL(km_state_notify);
2813
km_state_expired(struct xfrm_state * x,int hard,u32 portid)2814 void km_state_expired(struct xfrm_state *x, int hard, u32 portid)
2815 {
2816 struct km_event c;
2817
2818 c.data.hard = hard;
2819 c.portid = portid;
2820 c.event = XFRM_MSG_EXPIRE;
2821 km_state_notify(x, &c);
2822 }
2823
2824 EXPORT_SYMBOL(km_state_expired);
2825 /*
2826 * We send to all registered managers regardless of failure
2827 * We are happy with one success
2828 */
km_query(struct xfrm_state * x,struct xfrm_tmpl * t,struct xfrm_policy * pol)2829 int km_query(struct xfrm_state *x, struct xfrm_tmpl *t, struct xfrm_policy *pol)
2830 {
2831 int err = -EINVAL, acqret;
2832 struct xfrm_mgr *km;
2833
2834 rcu_read_lock();
2835 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2836 acqret = km->acquire(x, t, pol);
2837 if (!acqret)
2838 err = acqret;
2839 }
2840 rcu_read_unlock();
2841 return err;
2842 }
2843 EXPORT_SYMBOL(km_query);
2844
__km_new_mapping(struct xfrm_state * x,xfrm_address_t * ipaddr,__be16 sport)2845 static int __km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2846 {
2847 int err = -EINVAL;
2848 struct xfrm_mgr *km;
2849
2850 rcu_read_lock();
2851 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2852 if (km->new_mapping)
2853 err = km->new_mapping(x, ipaddr, sport);
2854 if (!err)
2855 break;
2856 }
2857 rcu_read_unlock();
2858 return err;
2859 }
2860
km_new_mapping(struct xfrm_state * x,xfrm_address_t * ipaddr,__be16 sport)2861 int km_new_mapping(struct xfrm_state *x, xfrm_address_t *ipaddr, __be16 sport)
2862 {
2863 int ret = 0;
2864
2865 if (x->mapping_maxage) {
2866 if ((jiffies / HZ - x->new_mapping) > x->mapping_maxage ||
2867 x->new_mapping_sport != sport) {
2868 x->new_mapping_sport = sport;
2869 x->new_mapping = jiffies / HZ;
2870 ret = __km_new_mapping(x, ipaddr, sport);
2871 }
2872 } else {
2873 ret = __km_new_mapping(x, ipaddr, sport);
2874 }
2875
2876 return ret;
2877 }
2878 EXPORT_SYMBOL(km_new_mapping);
2879
km_policy_expired(struct xfrm_policy * pol,int dir,int hard,u32 portid)2880 void km_policy_expired(struct xfrm_policy *pol, int dir, int hard, u32 portid)
2881 {
2882 struct km_event c;
2883
2884 c.data.hard = hard;
2885 c.portid = portid;
2886 c.event = XFRM_MSG_POLEXPIRE;
2887 km_policy_notify(pol, dir, &c);
2888 }
2889 EXPORT_SYMBOL(km_policy_expired);
2890
2891 #ifdef CONFIG_XFRM_MIGRATE
km_migrate(const struct xfrm_selector * sel,u8 dir,u8 type,const struct xfrm_migrate * m,int num_migrate,const struct xfrm_kmaddress * k,struct net * net,const struct xfrm_encap_tmpl * encap)2892 int km_migrate(const struct xfrm_selector *sel, u8 dir, u8 type,
2893 const struct xfrm_migrate *m, int num_migrate,
2894 const struct xfrm_kmaddress *k, struct net *net,
2895 const struct xfrm_encap_tmpl *encap)
2896 {
2897 int err = -EINVAL;
2898 int ret;
2899 struct xfrm_mgr *km;
2900
2901 rcu_read_lock();
2902 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2903 if (km->migrate) {
2904 ret = km->migrate(sel, dir, type, m, num_migrate, k,
2905 net, encap);
2906 if (!ret)
2907 err = ret;
2908 }
2909 }
2910 rcu_read_unlock();
2911 return err;
2912 }
2913 EXPORT_SYMBOL(km_migrate);
2914 #endif
2915
km_report(struct net * net,u8 proto,struct xfrm_selector * sel,xfrm_address_t * addr)2916 int km_report(struct net *net, u8 proto, struct xfrm_selector *sel, xfrm_address_t *addr)
2917 {
2918 int err = -EINVAL;
2919 int ret;
2920 struct xfrm_mgr *km;
2921
2922 rcu_read_lock();
2923 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2924 if (km->report) {
2925 ret = km->report(net, proto, sel, addr);
2926 if (!ret)
2927 err = ret;
2928 }
2929 }
2930 rcu_read_unlock();
2931 return err;
2932 }
2933 EXPORT_SYMBOL(km_report);
2934
km_is_alive(const struct km_event * c)2935 static bool km_is_alive(const struct km_event *c)
2936 {
2937 struct xfrm_mgr *km;
2938 bool is_alive = false;
2939
2940 rcu_read_lock();
2941 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
2942 if (km->is_alive && km->is_alive(c)) {
2943 is_alive = true;
2944 break;
2945 }
2946 }
2947 rcu_read_unlock();
2948
2949 return is_alive;
2950 }
2951
2952 #if IS_ENABLED(CONFIG_XFRM_USER_COMPAT)
2953 static DEFINE_SPINLOCK(xfrm_translator_lock);
2954 static struct xfrm_translator __rcu *xfrm_translator;
2955
xfrm_get_translator(void)2956 struct xfrm_translator *xfrm_get_translator(void)
2957 {
2958 struct xfrm_translator *xtr;
2959
2960 rcu_read_lock();
2961 xtr = rcu_dereference(xfrm_translator);
2962 if (unlikely(!xtr))
2963 goto out;
2964 if (!try_module_get(xtr->owner))
2965 xtr = NULL;
2966 out:
2967 rcu_read_unlock();
2968 return xtr;
2969 }
2970 EXPORT_SYMBOL_GPL(xfrm_get_translator);
2971
xfrm_put_translator(struct xfrm_translator * xtr)2972 void xfrm_put_translator(struct xfrm_translator *xtr)
2973 {
2974 module_put(xtr->owner);
2975 }
2976 EXPORT_SYMBOL_GPL(xfrm_put_translator);
2977
xfrm_register_translator(struct xfrm_translator * xtr)2978 int xfrm_register_translator(struct xfrm_translator *xtr)
2979 {
2980 int err = 0;
2981
2982 spin_lock_bh(&xfrm_translator_lock);
2983 if (unlikely(xfrm_translator != NULL))
2984 err = -EEXIST;
2985 else
2986 rcu_assign_pointer(xfrm_translator, xtr);
2987 spin_unlock_bh(&xfrm_translator_lock);
2988
2989 return err;
2990 }
2991 EXPORT_SYMBOL_GPL(xfrm_register_translator);
2992
xfrm_unregister_translator(struct xfrm_translator * xtr)2993 int xfrm_unregister_translator(struct xfrm_translator *xtr)
2994 {
2995 int err = 0;
2996
2997 spin_lock_bh(&xfrm_translator_lock);
2998 if (likely(xfrm_translator != NULL)) {
2999 if (rcu_access_pointer(xfrm_translator) != xtr)
3000 err = -EINVAL;
3001 else
3002 RCU_INIT_POINTER(xfrm_translator, NULL);
3003 }
3004 spin_unlock_bh(&xfrm_translator_lock);
3005 synchronize_rcu();
3006
3007 return err;
3008 }
3009 EXPORT_SYMBOL_GPL(xfrm_unregister_translator);
3010 #endif
3011
xfrm_user_policy(struct sock * sk,int optname,sockptr_t optval,int optlen)3012 int xfrm_user_policy(struct sock *sk, int optname, sockptr_t optval, int optlen)
3013 {
3014 int err;
3015 u8 *data;
3016 struct xfrm_mgr *km;
3017 struct xfrm_policy *pol = NULL;
3018
3019 if (sockptr_is_null(optval) && !optlen) {
3020 xfrm_sk_policy_insert(sk, XFRM_POLICY_IN, NULL);
3021 xfrm_sk_policy_insert(sk, XFRM_POLICY_OUT, NULL);
3022 sk_dst_reset(sk);
3023 return 0;
3024 }
3025
3026 if (optlen <= 0 || optlen > PAGE_SIZE)
3027 return -EMSGSIZE;
3028
3029 data = memdup_sockptr(optval, optlen);
3030 if (IS_ERR(data))
3031 return PTR_ERR(data);
3032
3033 if (IS_ENABLED(CONFIG_COMPAT_FOR_U64_ALIGNMENT) && in_compat_syscall()) {
3034 struct xfrm_translator *xtr = xfrm_get_translator();
3035
3036 if (!xtr) {
3037 kfree(data);
3038 return -EOPNOTSUPP;
3039 }
3040
3041 err = xtr->xlate_user_policy_sockptr(&data, optlen);
3042 xfrm_put_translator(xtr);
3043 if (err) {
3044 kfree(data);
3045 return err;
3046 }
3047 }
3048
3049 err = -EINVAL;
3050 rcu_read_lock();
3051 list_for_each_entry_rcu(km, &xfrm_km_list, list) {
3052 pol = km->compile_policy(sk, optname, data,
3053 optlen, &err);
3054 if (err >= 0)
3055 break;
3056 }
3057 rcu_read_unlock();
3058
3059 if (err >= 0) {
3060 xfrm_sk_policy_insert(sk, err, pol);
3061 xfrm_pol_put(pol);
3062 sk_dst_reset(sk);
3063 err = 0;
3064 }
3065
3066 kfree(data);
3067 return err;
3068 }
3069 EXPORT_SYMBOL(xfrm_user_policy);
3070
3071 static DEFINE_SPINLOCK(xfrm_km_lock);
3072
xfrm_register_km(struct xfrm_mgr * km)3073 void xfrm_register_km(struct xfrm_mgr *km)
3074 {
3075 spin_lock_bh(&xfrm_km_lock);
3076 list_add_tail_rcu(&km->list, &xfrm_km_list);
3077 spin_unlock_bh(&xfrm_km_lock);
3078 }
3079 EXPORT_SYMBOL(xfrm_register_km);
3080
xfrm_unregister_km(struct xfrm_mgr * km)3081 void xfrm_unregister_km(struct xfrm_mgr *km)
3082 {
3083 spin_lock_bh(&xfrm_km_lock);
3084 list_del_rcu(&km->list);
3085 spin_unlock_bh(&xfrm_km_lock);
3086 synchronize_rcu();
3087 }
3088 EXPORT_SYMBOL(xfrm_unregister_km);
3089
xfrm_state_register_afinfo(struct xfrm_state_afinfo * afinfo)3090 int xfrm_state_register_afinfo(struct xfrm_state_afinfo *afinfo)
3091 {
3092 int err = 0;
3093
3094 if (WARN_ON(afinfo->family >= NPROTO))
3095 return -EAFNOSUPPORT;
3096
3097 spin_lock_bh(&xfrm_state_afinfo_lock);
3098 if (unlikely(xfrm_state_afinfo[afinfo->family] != NULL))
3099 err = -EEXIST;
3100 else
3101 rcu_assign_pointer(xfrm_state_afinfo[afinfo->family], afinfo);
3102 spin_unlock_bh(&xfrm_state_afinfo_lock);
3103 return err;
3104 }
3105 EXPORT_SYMBOL(xfrm_state_register_afinfo);
3106
xfrm_state_unregister_afinfo(struct xfrm_state_afinfo * afinfo)3107 int xfrm_state_unregister_afinfo(struct xfrm_state_afinfo *afinfo)
3108 {
3109 int err = 0, family = afinfo->family;
3110
3111 if (WARN_ON(family >= NPROTO))
3112 return -EAFNOSUPPORT;
3113
3114 spin_lock_bh(&xfrm_state_afinfo_lock);
3115 if (likely(xfrm_state_afinfo[afinfo->family] != NULL)) {
3116 if (rcu_access_pointer(xfrm_state_afinfo[family]) != afinfo)
3117 err = -EINVAL;
3118 else
3119 RCU_INIT_POINTER(xfrm_state_afinfo[afinfo->family], NULL);
3120 }
3121 spin_unlock_bh(&xfrm_state_afinfo_lock);
3122 synchronize_rcu();
3123 return err;
3124 }
3125 EXPORT_SYMBOL(xfrm_state_unregister_afinfo);
3126
xfrm_state_afinfo_get_rcu(unsigned int family)3127 struct xfrm_state_afinfo *xfrm_state_afinfo_get_rcu(unsigned int family)
3128 {
3129 if (unlikely(family >= NPROTO))
3130 return NULL;
3131
3132 return rcu_dereference(xfrm_state_afinfo[family]);
3133 }
3134 EXPORT_SYMBOL_GPL(xfrm_state_afinfo_get_rcu);
3135
xfrm_state_get_afinfo(unsigned int family)3136 struct xfrm_state_afinfo *xfrm_state_get_afinfo(unsigned int family)
3137 {
3138 struct xfrm_state_afinfo *afinfo;
3139 if (unlikely(family >= NPROTO))
3140 return NULL;
3141 rcu_read_lock();
3142 afinfo = rcu_dereference(xfrm_state_afinfo[family]);
3143 if (unlikely(!afinfo))
3144 rcu_read_unlock();
3145 return afinfo;
3146 }
3147
xfrm_flush_gc(void)3148 void xfrm_flush_gc(void)
3149 {
3150 flush_work(&xfrm_state_gc_work);
3151 }
3152 EXPORT_SYMBOL(xfrm_flush_gc);
3153
xfrm_state_delete_tunnel(struct xfrm_state * x)3154 static void xfrm_state_delete_tunnel(struct xfrm_state *x)
3155 {
3156 if (x->tunnel) {
3157 struct xfrm_state *t = x->tunnel;
3158
3159 if (atomic_dec_return(&t->tunnel_users) == 1)
3160 xfrm_state_delete(t);
3161 xfrm_state_put(t);
3162 x->tunnel = NULL;
3163 }
3164 }
3165
xfrm_state_mtu(struct xfrm_state * x,int mtu)3166 u32 xfrm_state_mtu(struct xfrm_state *x, int mtu)
3167 {
3168 const struct xfrm_type *type = READ_ONCE(x->type);
3169 struct crypto_aead *aead;
3170 u32 blksize, net_adj = 0;
3171 u32 overhead, payload_mtu;
3172
3173 if (x->km.state != XFRM_STATE_VALID ||
3174 !type || type->proto != IPPROTO_ESP) {
3175 if (mtu <= x->props.header_len)
3176 return 1;
3177 return mtu - x->props.header_len;
3178 }
3179
3180 aead = x->data;
3181 blksize = ALIGN(crypto_aead_blocksize(aead), 4);
3182
3183 switch (x->props.mode) {
3184 case XFRM_MODE_TRANSPORT:
3185 case XFRM_MODE_BEET:
3186 if (x->props.family == AF_INET)
3187 net_adj = sizeof(struct iphdr);
3188 else if (x->props.family == AF_INET6)
3189 net_adj = sizeof(struct ipv6hdr);
3190 break;
3191 case XFRM_MODE_TUNNEL:
3192 break;
3193 default:
3194 if (x->mode_cbs && x->mode_cbs->get_inner_mtu)
3195 return x->mode_cbs->get_inner_mtu(x, mtu);
3196
3197 WARN_ON_ONCE(1);
3198 break;
3199 }
3200
3201 overhead = x->props.header_len + crypto_aead_authsize(aead) + net_adj;
3202 if (mtu <= overhead)
3203 return 1;
3204
3205 payload_mtu = mtu - overhead;
3206 payload_mtu &= ~(blksize - 1);
3207 if (payload_mtu <= 2)
3208 return 1;
3209
3210 return payload_mtu + net_adj - 2;
3211
3212 }
3213 EXPORT_SYMBOL_GPL(xfrm_state_mtu);
3214
__xfrm_init_state(struct xfrm_state * x,struct netlink_ext_ack * extack)3215 int __xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
3216 {
3217 const struct xfrm_mode *inner_mode;
3218 const struct xfrm_mode *outer_mode;
3219 int family = x->props.family;
3220 int err;
3221
3222 if (family == AF_INET &&
3223 (!x->dir || x->dir == XFRM_SA_DIR_OUT) &&
3224 READ_ONCE(xs_net(x)->ipv4.sysctl_ip_no_pmtu_disc))
3225 x->props.flags |= XFRM_STATE_NOPMTUDISC;
3226
3227 err = -EPROTONOSUPPORT;
3228
3229 if (x->sel.family != AF_UNSPEC) {
3230 inner_mode = xfrm_get_mode(x->props.mode, x->sel.family);
3231 if (inner_mode == NULL) {
3232 NL_SET_ERR_MSG(extack, "Requested mode not found");
3233 goto error;
3234 }
3235
3236 if (!(inner_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
3237 family != x->sel.family) {
3238 NL_SET_ERR_MSG(extack, "Only tunnel modes can accommodate a change of family");
3239 goto error;
3240 }
3241
3242 x->inner_mode = *inner_mode;
3243 } else {
3244 const struct xfrm_mode *inner_mode_iaf;
3245 int iafamily = AF_INET;
3246
3247 inner_mode = xfrm_get_mode(x->props.mode, x->props.family);
3248 if (inner_mode == NULL) {
3249 NL_SET_ERR_MSG(extack, "Requested mode not found");
3250 goto error;
3251 }
3252
3253 x->inner_mode = *inner_mode;
3254
3255 if (x->props.family == AF_INET)
3256 iafamily = AF_INET6;
3257
3258 inner_mode_iaf = xfrm_get_mode(x->props.mode, iafamily);
3259 if (inner_mode_iaf) {
3260 if (inner_mode_iaf->flags & XFRM_MODE_FLAG_TUNNEL)
3261 x->inner_mode_iaf = *inner_mode_iaf;
3262 }
3263 }
3264
3265 x->type = xfrm_get_type(x->id.proto, family);
3266 if (x->type == NULL) {
3267 NL_SET_ERR_MSG(extack, "Requested type not found");
3268 goto error;
3269 }
3270
3271 err = x->type->init_state(x, extack);
3272 if (err)
3273 goto error;
3274
3275 outer_mode = xfrm_get_mode(x->props.mode, family);
3276 if (!outer_mode) {
3277 NL_SET_ERR_MSG(extack, "Requested mode not found");
3278 err = -EPROTONOSUPPORT;
3279 goto error;
3280 }
3281
3282 x->outer_mode = *outer_mode;
3283 if (x->nat_keepalive_interval) {
3284 if (x->dir != XFRM_SA_DIR_OUT) {
3285 NL_SET_ERR_MSG(extack, "NAT keepalive is only supported for outbound SAs");
3286 err = -EINVAL;
3287 goto error;
3288 }
3289
3290 if (!x->encap || x->encap->encap_type != UDP_ENCAP_ESPINUDP) {
3291 NL_SET_ERR_MSG(extack,
3292 "NAT keepalive is only supported for UDP encapsulation");
3293 err = -EINVAL;
3294 goto error;
3295 }
3296 }
3297
3298 x->mode_cbs = xfrm_get_mode_cbs(x->props.mode);
3299 if (x->mode_cbs) {
3300 if (x->mode_cbs->init_state)
3301 err = x->mode_cbs->init_state(x);
3302 module_put(x->mode_cbs->owner);
3303 if (err && !x->mode_data)
3304 x->mode_cbs = NULL;
3305 }
3306 error:
3307 return err;
3308 }
3309
3310 EXPORT_SYMBOL(__xfrm_init_state);
3311
xfrm_init_state(struct xfrm_state * x,struct netlink_ext_ack * extack)3312 int xfrm_init_state(struct xfrm_state *x, struct netlink_ext_ack *extack)
3313 {
3314 int err;
3315
3316 err = __xfrm_init_state(x, extack);
3317 if (err)
3318 return err;
3319
3320 err = xfrm_init_replay(x, NULL);
3321 if (err)
3322 return err;
3323
3324 x->km.state = XFRM_STATE_VALID;
3325 return 0;
3326 }
3327
3328 EXPORT_SYMBOL(xfrm_init_state);
3329
xfrm_state_init(struct net * net)3330 int __net_init xfrm_state_init(struct net *net)
3331 {
3332 struct hlist_head *ndst, *nsrc, *nspi, *nseq;
3333 unsigned int sz;
3334
3335 if (net_eq(net, &init_net))
3336 xfrm_state_cache = KMEM_CACHE(xfrm_state,
3337 SLAB_HWCACHE_ALIGN | SLAB_PANIC);
3338
3339 INIT_LIST_HEAD(&net->xfrm.state_all);
3340
3341 sz = sizeof(struct hlist_head) * 8;
3342
3343 ndst = xfrm_hash_alloc(sz);
3344 if (!ndst)
3345 goto out_bydst;
3346 rcu_assign_pointer(net->xfrm.state_bydst, ndst);
3347
3348 nsrc = xfrm_hash_alloc(sz);
3349 if (!nsrc)
3350 goto out_bysrc;
3351 rcu_assign_pointer(net->xfrm.state_bysrc, nsrc);
3352
3353 nspi = xfrm_hash_alloc(sz);
3354 if (!nspi)
3355 goto out_byspi;
3356 rcu_assign_pointer(net->xfrm.state_byspi, nspi);
3357
3358 nseq = xfrm_hash_alloc(sz);
3359 if (!nseq)
3360 goto out_byseq;
3361 rcu_assign_pointer(net->xfrm.state_byseq, nseq);
3362
3363 net->xfrm.state_cache_input = alloc_percpu(struct hlist_head);
3364 if (!net->xfrm.state_cache_input)
3365 goto out_state_cache_input;
3366
3367 net->xfrm.state_hmask = ((sz / sizeof(struct hlist_head)) - 1);
3368
3369 net->xfrm.state_num = 0;
3370 INIT_WORK(&net->xfrm.state_hash_work, xfrm_hash_resize);
3371 spin_lock_init(&net->xfrm.xfrm_state_lock);
3372 seqcount_spinlock_init(&net->xfrm.xfrm_state_hash_generation,
3373 &net->xfrm.xfrm_state_lock);
3374 return 0;
3375
3376 out_state_cache_input:
3377 xfrm_hash_free(nseq, sz);
3378 out_byseq:
3379 xfrm_hash_free(nspi, sz);
3380 out_byspi:
3381 xfrm_hash_free(nsrc, sz);
3382 out_bysrc:
3383 xfrm_hash_free(ndst, sz);
3384 out_bydst:
3385 return -ENOMEM;
3386 }
3387
3388 #define xfrm_state_deref_netexit(table) \
3389 rcu_dereference_protected((table), true /* netns is going away */)
xfrm_state_fini(struct net * net)3390 void xfrm_state_fini(struct net *net)
3391 {
3392 unsigned int sz;
3393 int i;
3394
3395 flush_work(&net->xfrm.state_hash_work);
3396 xfrm_state_flush(net, 0, false);
3397 flush_work(&xfrm_state_gc_work);
3398
3399 WARN_ON(!list_empty(&net->xfrm.state_all));
3400
3401 for (i = 0; i <= net->xfrm.state_hmask; i++) {
3402 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byseq) + i));
3403 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_byspi) + i));
3404 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bysrc) + i));
3405 WARN_ON(!hlist_empty(xfrm_state_deref_netexit(net->xfrm.state_bydst) + i));
3406 }
3407
3408 sz = (net->xfrm.state_hmask + 1) * sizeof(struct hlist_head);
3409 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byseq), sz);
3410 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_byspi), sz);
3411 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bysrc), sz);
3412 xfrm_hash_free(xfrm_state_deref_netexit(net->xfrm.state_bydst), sz);
3413 free_percpu(net->xfrm.state_cache_input);
3414 }
3415
3416 #ifdef CONFIG_AUDITSYSCALL
xfrm_audit_helper_sainfo(struct xfrm_state * x,struct audit_buffer * audit_buf)3417 static void xfrm_audit_helper_sainfo(struct xfrm_state *x,
3418 struct audit_buffer *audit_buf)
3419 {
3420 struct xfrm_sec_ctx *ctx = x->security;
3421 u32 spi = ntohl(x->id.spi);
3422
3423 if (ctx)
3424 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
3425 ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
3426
3427 switch (x->props.family) {
3428 case AF_INET:
3429 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
3430 &x->props.saddr.a4, &x->id.daddr.a4);
3431 break;
3432 case AF_INET6:
3433 audit_log_format(audit_buf, " src=%pI6 dst=%pI6",
3434 x->props.saddr.a6, x->id.daddr.a6);
3435 break;
3436 }
3437
3438 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
3439 }
3440
xfrm_audit_helper_pktinfo(struct sk_buff * skb,u16 family,struct audit_buffer * audit_buf)3441 static void xfrm_audit_helper_pktinfo(struct sk_buff *skb, u16 family,
3442 struct audit_buffer *audit_buf)
3443 {
3444 const struct iphdr *iph4;
3445 const struct ipv6hdr *iph6;
3446
3447 switch (family) {
3448 case AF_INET:
3449 iph4 = ip_hdr(skb);
3450 audit_log_format(audit_buf, " src=%pI4 dst=%pI4",
3451 &iph4->saddr, &iph4->daddr);
3452 break;
3453 case AF_INET6:
3454 iph6 = ipv6_hdr(skb);
3455 audit_log_format(audit_buf,
3456 " src=%pI6 dst=%pI6 flowlbl=0x%x%02x%02x",
3457 &iph6->saddr, &iph6->daddr,
3458 iph6->flow_lbl[0] & 0x0f,
3459 iph6->flow_lbl[1],
3460 iph6->flow_lbl[2]);
3461 break;
3462 }
3463 }
3464
xfrm_audit_state_add(struct xfrm_state * x,int result,bool task_valid)3465 void xfrm_audit_state_add(struct xfrm_state *x, int result, bool task_valid)
3466 {
3467 struct audit_buffer *audit_buf;
3468
3469 audit_buf = xfrm_audit_start("SAD-add");
3470 if (audit_buf == NULL)
3471 return;
3472 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3473 xfrm_audit_helper_sainfo(x, audit_buf);
3474 audit_log_format(audit_buf, " res=%u", result);
3475 audit_log_end(audit_buf);
3476 }
3477 EXPORT_SYMBOL_GPL(xfrm_audit_state_add);
3478
xfrm_audit_state_delete(struct xfrm_state * x,int result,bool task_valid)3479 void xfrm_audit_state_delete(struct xfrm_state *x, int result, bool task_valid)
3480 {
3481 struct audit_buffer *audit_buf;
3482
3483 audit_buf = xfrm_audit_start("SAD-delete");
3484 if (audit_buf == NULL)
3485 return;
3486 xfrm_audit_helper_usrinfo(task_valid, audit_buf);
3487 xfrm_audit_helper_sainfo(x, audit_buf);
3488 audit_log_format(audit_buf, " res=%u", result);
3489 audit_log_end(audit_buf);
3490 }
3491 EXPORT_SYMBOL_GPL(xfrm_audit_state_delete);
3492
xfrm_audit_state_replay_overflow(struct xfrm_state * x,struct sk_buff * skb)3493 void xfrm_audit_state_replay_overflow(struct xfrm_state *x,
3494 struct sk_buff *skb)
3495 {
3496 struct audit_buffer *audit_buf;
3497 u32 spi;
3498
3499 audit_buf = xfrm_audit_start("SA-replay-overflow");
3500 if (audit_buf == NULL)
3501 return;
3502 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3503 /* don't record the sequence number because it's inherent in this kind
3504 * of audit message */
3505 spi = ntohl(x->id.spi);
3506 audit_log_format(audit_buf, " spi=%u(0x%x)", spi, spi);
3507 audit_log_end(audit_buf);
3508 }
3509 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay_overflow);
3510
xfrm_audit_state_replay(struct xfrm_state * x,struct sk_buff * skb,__be32 net_seq)3511 void xfrm_audit_state_replay(struct xfrm_state *x,
3512 struct sk_buff *skb, __be32 net_seq)
3513 {
3514 struct audit_buffer *audit_buf;
3515 u32 spi;
3516
3517 audit_buf = xfrm_audit_start("SA-replayed-pkt");
3518 if (audit_buf == NULL)
3519 return;
3520 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3521 spi = ntohl(x->id.spi);
3522 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3523 spi, spi, ntohl(net_seq));
3524 audit_log_end(audit_buf);
3525 }
3526 EXPORT_SYMBOL_GPL(xfrm_audit_state_replay);
3527
xfrm_audit_state_notfound_simple(struct sk_buff * skb,u16 family)3528 void xfrm_audit_state_notfound_simple(struct sk_buff *skb, u16 family)
3529 {
3530 struct audit_buffer *audit_buf;
3531
3532 audit_buf = xfrm_audit_start("SA-notfound");
3533 if (audit_buf == NULL)
3534 return;
3535 xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3536 audit_log_end(audit_buf);
3537 }
3538 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound_simple);
3539
xfrm_audit_state_notfound(struct sk_buff * skb,u16 family,__be32 net_spi,__be32 net_seq)3540 void xfrm_audit_state_notfound(struct sk_buff *skb, u16 family,
3541 __be32 net_spi, __be32 net_seq)
3542 {
3543 struct audit_buffer *audit_buf;
3544 u32 spi;
3545
3546 audit_buf = xfrm_audit_start("SA-notfound");
3547 if (audit_buf == NULL)
3548 return;
3549 xfrm_audit_helper_pktinfo(skb, family, audit_buf);
3550 spi = ntohl(net_spi);
3551 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3552 spi, spi, ntohl(net_seq));
3553 audit_log_end(audit_buf);
3554 }
3555 EXPORT_SYMBOL_GPL(xfrm_audit_state_notfound);
3556
xfrm_audit_state_icvfail(struct xfrm_state * x,struct sk_buff * skb,u8 proto)3557 void xfrm_audit_state_icvfail(struct xfrm_state *x,
3558 struct sk_buff *skb, u8 proto)
3559 {
3560 struct audit_buffer *audit_buf;
3561 __be32 net_spi;
3562 __be32 net_seq;
3563
3564 audit_buf = xfrm_audit_start("SA-icv-failure");
3565 if (audit_buf == NULL)
3566 return;
3567 xfrm_audit_helper_pktinfo(skb, x->props.family, audit_buf);
3568 if (xfrm_parse_spi(skb, proto, &net_spi, &net_seq) == 0) {
3569 u32 spi = ntohl(net_spi);
3570 audit_log_format(audit_buf, " spi=%u(0x%x) seqno=%u",
3571 spi, spi, ntohl(net_seq));
3572 }
3573 audit_log_end(audit_buf);
3574 }
3575 EXPORT_SYMBOL_GPL(xfrm_audit_state_icvfail);
3576 #endif /* CONFIG_AUDITSYSCALL */
3577